ixv.c revision 1.170
1 /* $NetBSD: ixv.c,v 1.170 2021/12/14 05:37:44 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ixv.c,v 1.170 2021/12/14 05:37:44 msaitoh Exp $"); 39 40 #ifdef _KERNEL_OPT 41 #include "opt_inet.h" 42 #include "opt_inet6.h" 43 #include "opt_net_mpsafe.h" 44 #endif 45 46 #include "ixgbe.h" 47 48 /************************************************************************ 49 * Driver version 50 ************************************************************************/ 51 static const char ixv_driver_version[] = "2.0.1-k"; 52 /* XXX NetBSD: + 1.5.17 */ 53 54 /************************************************************************ 55 * PCI Device ID Table 56 * 57 * Used by probe to select devices to load on 58 * Last field stores an index into ixv_strings 59 * Last entry must be all 0s 60 * 61 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 62 ************************************************************************/ 63 static const ixgbe_vendor_info_t ixv_vendor_info_array[] = 64 { 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 70 /* required last entry */ 71 {0, 0, 0, 0, 0} 72 }; 73 74 /************************************************************************ 75 * Table of branding strings 76 ************************************************************************/ 77 static const char *ixv_strings[] = { 78 "Intel(R) PRO/10GbE Virtual Function Network Driver" 79 }; 80 81 /********************************************************************* 82 * Function prototypes 83 *********************************************************************/ 84 static int ixv_probe(device_t, cfdata_t, void *); 85 static void ixv_attach(device_t, device_t, void *); 86 static int ixv_detach(device_t, int); 87 #if 0 88 static int ixv_shutdown(device_t); 89 #endif 90 static int ixv_ifflags_cb(struct ethercom *); 91 static int ixv_ioctl(struct ifnet *, u_long, void *); 92 static int ixv_init(struct ifnet *); 93 static void ixv_init_locked(struct adapter *); 94 static void ixv_ifstop(struct ifnet *, int); 95 static void ixv_stop_locked(void *); 96 static void ixv_init_device_features(struct adapter *); 97 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 98 static int ixv_media_change(struct ifnet *); 99 static int ixv_allocate_pci_resources(struct adapter *, 100 const struct pci_attach_args *); 101 static void ixv_free_deferred_handlers(struct adapter *); 102 static int ixv_allocate_msix(struct adapter *, 103 const struct pci_attach_args *); 104 static int ixv_configure_interrupts(struct adapter *); 105 static void ixv_free_pci_resources(struct adapter *); 106 static void ixv_local_timer(void *); 107 static void ixv_handle_timer(struct work *, void *); 108 static int ixv_setup_interface(device_t, struct adapter *); 109 static void ixv_schedule_admin_tasklet(struct adapter *); 110 static int ixv_negotiate_api(struct adapter *); 111 112 static void ixv_initialize_transmit_units(struct adapter *); 113 static void ixv_initialize_receive_units(struct adapter *); 114 static void ixv_initialize_rss_mapping(struct adapter *); 115 static s32 ixv_check_link(struct adapter *); 116 117 static void ixv_enable_intr(struct adapter *); 118 static void ixv_disable_intr(struct adapter *); 119 static int ixv_set_rxfilter(struct adapter *); 120 static void ixv_update_link_status(struct adapter *); 121 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 122 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 123 static void ixv_configure_ivars(struct adapter *); 124 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 125 static void ixv_eitr_write(struct adapter *, uint32_t, uint32_t); 126 127 static void ixv_setup_vlan_tagging(struct adapter *); 128 static int ixv_setup_vlan_support(struct adapter *); 129 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool); 130 static int ixv_register_vlan(struct adapter *, u16); 131 static int ixv_unregister_vlan(struct adapter *, u16); 132 133 static void ixv_add_device_sysctls(struct adapter *); 134 static void ixv_save_stats(struct adapter *); 135 static void ixv_init_stats(struct adapter *); 136 static void ixv_update_stats(struct adapter *); 137 static void ixv_add_stats_sysctls(struct adapter *); 138 static void ixv_clear_evcnt(struct adapter *); 139 140 /* Sysctl handlers */ 141 static void ixv_set_sysctl_value(struct adapter *, const char *, 142 const char *, int *, int); 143 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 144 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 145 static int ixv_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 146 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 147 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 148 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 149 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 150 static int ixv_sysctl_rx_copy_len(SYSCTLFN_PROTO); 151 152 /* The MSI-X Interrupt handlers */ 153 static int ixv_msix_que(void *); 154 static int ixv_msix_mbx(void *); 155 156 /* Event handlers running on workqueue */ 157 static void ixv_handle_que(void *); 158 159 /* Deferred workqueue handlers */ 160 static void ixv_handle_admin(struct work *, void *); 161 static void ixv_handle_que_work(struct work *, void *); 162 163 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 164 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 165 166 /************************************************************************ 167 * NetBSD Device Interface Entry Points 168 ************************************************************************/ 169 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 170 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 171 DVF_DETACH_SHUTDOWN); 172 173 #if 0 174 static driver_t ixv_driver = { 175 "ixv", ixv_methods, sizeof(struct adapter), 176 }; 177 178 devclass_t ixv_devclass; 179 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 180 MODULE_DEPEND(ixv, pci, 1, 1, 1); 181 MODULE_DEPEND(ixv, ether, 1, 1, 1); 182 #endif 183 184 /* 185 * TUNEABLE PARAMETERS: 186 */ 187 188 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 189 static int ixv_num_queues = 0; 190 #define TUNABLE_INT(__x, __y) 191 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 192 193 /* 194 * AIM: Adaptive Interrupt Moderation 195 * which means that the interrupt rate 196 * is varied over time based on the 197 * traffic for that interrupt vector 198 */ 199 static bool ixv_enable_aim = false; 200 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 201 202 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 203 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 204 205 /* How many packets rxeof tries to clean at a time */ 206 static int ixv_rx_process_limit = 256; 207 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 208 209 /* How many packets txeof tries to clean at a time */ 210 static int ixv_tx_process_limit = 256; 211 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 212 213 /* Which packet processing uses workqueue or softint */ 214 static bool ixv_txrx_workqueue = false; 215 216 /* 217 * Number of TX descriptors per ring, 218 * setting higher than RX as this seems 219 * the better performing choice. 220 */ 221 static int ixv_txd = PERFORM_TXD; 222 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 223 224 /* Number of RX descriptors per ring */ 225 static int ixv_rxd = PERFORM_RXD; 226 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 227 228 /* Legacy Transmit (single queue) */ 229 static int ixv_enable_legacy_tx = 0; 230 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 231 232 #ifdef NET_MPSAFE 233 #define IXGBE_MPSAFE 1 234 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 235 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 236 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 237 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 238 #else 239 #define IXGBE_CALLOUT_FLAGS 0 240 #define IXGBE_SOFTINT_FLAGS 0 241 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 242 #define IXGBE_TASKLET_WQ_FLAGS 0 243 #endif 244 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 245 246 #if 0 247 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 248 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 249 #endif 250 251 /************************************************************************ 252 * ixv_probe - Device identification routine 253 * 254 * Determines if the driver should be loaded on 255 * adapter based on its PCI vendor/device ID. 256 * 257 * return BUS_PROBE_DEFAULT on success, positive on failure 258 ************************************************************************/ 259 static int 260 ixv_probe(device_t dev, cfdata_t cf, void *aux) 261 { 262 #ifdef __HAVE_PCI_MSI_MSIX 263 const struct pci_attach_args *pa = aux; 264 265 return (ixv_lookup(pa) != NULL) ? 1 : 0; 266 #else 267 return 0; 268 #endif 269 } /* ixv_probe */ 270 271 static const ixgbe_vendor_info_t * 272 ixv_lookup(const struct pci_attach_args *pa) 273 { 274 const ixgbe_vendor_info_t *ent; 275 pcireg_t subid; 276 277 INIT_DEBUGOUT("ixv_lookup: begin"); 278 279 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 280 return NULL; 281 282 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 283 284 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 285 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 286 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 287 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 288 (ent->subvendor_id == 0)) && 289 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 290 (ent->subdevice_id == 0))) { 291 return ent; 292 } 293 } 294 295 return NULL; 296 } 297 298 /************************************************************************ 299 * ixv_attach - Device initialization routine 300 * 301 * Called when the driver is being loaded. 302 * Identifies the type of hardware, allocates all resources 303 * and initializes the hardware. 304 * 305 * return 0 on success, positive on failure 306 ************************************************************************/ 307 static void 308 ixv_attach(device_t parent, device_t dev, void *aux) 309 { 310 struct adapter *adapter; 311 struct ixgbe_hw *hw; 312 int error = 0; 313 pcireg_t id, subid; 314 const ixgbe_vendor_info_t *ent; 315 const struct pci_attach_args *pa = aux; 316 const char *apivstr; 317 const char *str; 318 char wqname[MAXCOMLEN]; 319 char buf[256]; 320 321 INIT_DEBUGOUT("ixv_attach: begin"); 322 323 /* 324 * Make sure BUSMASTER is set, on a VM under 325 * KVM it may not be and will break things. 326 */ 327 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 328 329 /* Allocate, clear, and link in our adapter structure */ 330 adapter = device_private(dev); 331 adapter->hw.back = adapter; 332 adapter->dev = dev; 333 hw = &adapter->hw; 334 335 adapter->init_locked = ixv_init_locked; 336 adapter->stop_locked = ixv_stop_locked; 337 338 adapter->osdep.pc = pa->pa_pc; 339 adapter->osdep.tag = pa->pa_tag; 340 if (pci_dma64_available(pa)) 341 adapter->osdep.dmat = pa->pa_dmat64; 342 else 343 adapter->osdep.dmat = pa->pa_dmat; 344 adapter->osdep.attached = false; 345 346 ent = ixv_lookup(pa); 347 348 KASSERT(ent != NULL); 349 350 aprint_normal(": %s, Version - %s\n", 351 ixv_strings[ent->index], ixv_driver_version); 352 353 /* Core Lock Init */ 354 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 355 356 /* Do base PCI setup - map BAR0 */ 357 if (ixv_allocate_pci_resources(adapter, pa)) { 358 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 359 error = ENXIO; 360 goto err_out; 361 } 362 363 /* SYSCTL APIs */ 364 ixv_add_device_sysctls(adapter); 365 366 /* Set up the timer callout and workqueue */ 367 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 368 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 369 error = workqueue_create(&adapter->timer_wq, wqname, 370 ixv_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 371 IXGBE_TASKLET_WQ_FLAGS); 372 if (error) { 373 aprint_error_dev(dev, 374 "could not create timer workqueue (%d)\n", error); 375 goto err_out; 376 } 377 378 /* Save off the information about this board */ 379 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 380 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 381 hw->vendor_id = PCI_VENDOR(id); 382 hw->device_id = PCI_PRODUCT(id); 383 hw->revision_id = 384 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 385 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 386 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 387 388 /* A subset of set_mac_type */ 389 switch (hw->device_id) { 390 case IXGBE_DEV_ID_82599_VF: 391 hw->mac.type = ixgbe_mac_82599_vf; 392 str = "82599 VF"; 393 break; 394 case IXGBE_DEV_ID_X540_VF: 395 hw->mac.type = ixgbe_mac_X540_vf; 396 str = "X540 VF"; 397 break; 398 case IXGBE_DEV_ID_X550_VF: 399 hw->mac.type = ixgbe_mac_X550_vf; 400 str = "X550 VF"; 401 break; 402 case IXGBE_DEV_ID_X550EM_X_VF: 403 hw->mac.type = ixgbe_mac_X550EM_x_vf; 404 str = "X550EM X VF"; 405 break; 406 case IXGBE_DEV_ID_X550EM_A_VF: 407 hw->mac.type = ixgbe_mac_X550EM_a_vf; 408 str = "X550EM A VF"; 409 break; 410 default: 411 /* Shouldn't get here since probe succeeded */ 412 aprint_error_dev(dev, "Unknown device ID!\n"); 413 error = ENXIO; 414 goto err_out; 415 break; 416 } 417 aprint_normal_dev(dev, "device %s\n", str); 418 419 ixv_init_device_features(adapter); 420 421 /* Initialize the shared code */ 422 error = ixgbe_init_ops_vf(hw); 423 if (error) { 424 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 425 error = EIO; 426 goto err_out; 427 } 428 429 /* Setup the mailbox */ 430 ixgbe_init_mbx_params_vf(hw); 431 432 /* Set the right number of segments */ 433 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 434 adapter->num_segs = IXGBE_SCATTER_DEFAULT; 435 436 /* Reset mbox api to 1.0 */ 437 error = hw->mac.ops.reset_hw(hw); 438 if (error == IXGBE_ERR_RESET_FAILED) 439 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 440 else if (error) 441 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 442 error); 443 if (error) { 444 error = EIO; 445 goto err_out; 446 } 447 448 error = hw->mac.ops.init_hw(hw); 449 if (error) { 450 aprint_error_dev(dev, "...init_hw() failed!\n"); 451 error = EIO; 452 goto err_out; 453 } 454 455 /* Negotiate mailbox API version */ 456 error = ixv_negotiate_api(adapter); 457 if (error) 458 aprint_normal_dev(dev, 459 "MBX API negotiation failed during attach!\n"); 460 switch (hw->api_version) { 461 case ixgbe_mbox_api_10: 462 apivstr = "1.0"; 463 break; 464 case ixgbe_mbox_api_20: 465 apivstr = "2.0"; 466 break; 467 case ixgbe_mbox_api_11: 468 apivstr = "1.1"; 469 break; 470 case ixgbe_mbox_api_12: 471 apivstr = "1.2"; 472 break; 473 case ixgbe_mbox_api_13: 474 apivstr = "1.3"; 475 break; 476 default: 477 apivstr = "unknown"; 478 break; 479 } 480 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 481 482 /* If no mac address was assigned, make a random one */ 483 if (!ixv_check_ether_addr(hw->mac.addr)) { 484 u8 addr[ETHER_ADDR_LEN]; 485 uint64_t rndval = cprng_strong64(); 486 487 memcpy(addr, &rndval, sizeof(addr)); 488 addr[0] &= 0xFE; 489 addr[0] |= 0x02; 490 bcopy(addr, hw->mac.addr, sizeof(addr)); 491 } 492 493 /* Register for VLAN events */ 494 ether_set_vlan_cb(&adapter->osdep.ec, ixv_vlan_cb); 495 496 /* Sysctls for limiting the amount of work done in the taskqueues */ 497 ixv_set_sysctl_value(adapter, "rx_processing_limit", 498 "max number of rx packets to process", 499 &adapter->rx_process_limit, ixv_rx_process_limit); 500 501 ixv_set_sysctl_value(adapter, "tx_processing_limit", 502 "max number of tx packets to process", 503 &adapter->tx_process_limit, ixv_tx_process_limit); 504 505 /* Do descriptor calc and sanity checks */ 506 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 507 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 508 aprint_error_dev(dev, "TXD config issue, using default!\n"); 509 adapter->num_tx_desc = DEFAULT_TXD; 510 } else 511 adapter->num_tx_desc = ixv_txd; 512 513 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 514 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 515 aprint_error_dev(dev, "RXD config issue, using default!\n"); 516 adapter->num_rx_desc = DEFAULT_RXD; 517 } else 518 adapter->num_rx_desc = ixv_rxd; 519 520 /* Set default high limit of copying mbuf in rxeof */ 521 adapter->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 522 523 /* Setup MSI-X */ 524 error = ixv_configure_interrupts(adapter); 525 if (error) 526 goto err_out; 527 528 /* Allocate our TX/RX Queues */ 529 if (ixgbe_allocate_queues(adapter)) { 530 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 531 error = ENOMEM; 532 goto err_out; 533 } 534 535 /* hw.ix defaults init */ 536 adapter->enable_aim = ixv_enable_aim; 537 538 adapter->txrx_use_workqueue = ixv_txrx_workqueue; 539 540 error = ixv_allocate_msix(adapter, pa); 541 if (error) { 542 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n"); 543 goto err_late; 544 } 545 546 /* Setup OS specific network interface */ 547 error = ixv_setup_interface(dev, adapter); 548 if (error != 0) { 549 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 550 goto err_late; 551 } 552 553 /* Allocate multicast array memory */ 554 adapter->mta = malloc(sizeof(*adapter->mta) * 555 IXGBE_MAX_VF_MC, M_DEVBUF, M_WAITOK); 556 557 /* Do the stats setup */ 558 ixv_save_stats(adapter); 559 ixv_init_stats(adapter); 560 ixv_add_stats_sysctls(adapter); 561 562 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 563 ixgbe_netmap_attach(adapter); 564 565 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 566 aprint_verbose_dev(dev, "feature cap %s\n", buf); 567 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 568 aprint_verbose_dev(dev, "feature ena %s\n", buf); 569 570 INIT_DEBUGOUT("ixv_attach: end"); 571 adapter->osdep.attached = true; 572 573 return; 574 575 err_late: 576 ixgbe_free_queues(adapter); 577 err_out: 578 ixv_free_pci_resources(adapter); 579 IXGBE_CORE_LOCK_DESTROY(adapter); 580 581 return; 582 } /* ixv_attach */ 583 584 /************************************************************************ 585 * ixv_detach - Device removal routine 586 * 587 * Called when the driver is being removed. 588 * Stops the adapter and deallocates all the resources 589 * that were allocated for driver operation. 590 * 591 * return 0 on success, positive on failure 592 ************************************************************************/ 593 static int 594 ixv_detach(device_t dev, int flags) 595 { 596 struct adapter *adapter = device_private(dev); 597 struct ixgbe_hw *hw = &adapter->hw; 598 struct tx_ring *txr = adapter->tx_rings; 599 struct rx_ring *rxr = adapter->rx_rings; 600 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 601 602 INIT_DEBUGOUT("ixv_detach: begin"); 603 if (adapter->osdep.attached == false) 604 return 0; 605 606 /* Stop the interface. Callouts are stopped in it. */ 607 ixv_ifstop(adapter->ifp, 1); 608 609 if (VLAN_ATTACHED(&adapter->osdep.ec) && 610 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 611 aprint_error_dev(dev, "VLANs in use, detach first\n"); 612 return EBUSY; 613 } 614 615 ether_ifdetach(adapter->ifp); 616 callout_halt(&adapter->timer, NULL); 617 ixv_free_deferred_handlers(adapter); 618 619 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 620 netmap_detach(adapter->ifp); 621 622 ixv_free_pci_resources(adapter); 623 #if 0 /* XXX the NetBSD port is probably missing something here */ 624 bus_generic_detach(dev); 625 #endif 626 if_detach(adapter->ifp); 627 ifmedia_fini(&adapter->media); 628 if_percpuq_destroy(adapter->ipq); 629 630 sysctl_teardown(&adapter->sysctllog); 631 evcnt_detach(&adapter->efbig_tx_dma_setup); 632 evcnt_detach(&adapter->mbuf_defrag_failed); 633 evcnt_detach(&adapter->efbig2_tx_dma_setup); 634 evcnt_detach(&adapter->einval_tx_dma_setup); 635 evcnt_detach(&adapter->other_tx_dma_setup); 636 evcnt_detach(&adapter->eagain_tx_dma_setup); 637 evcnt_detach(&adapter->enomem_tx_dma_setup); 638 evcnt_detach(&adapter->watchdog_events); 639 evcnt_detach(&adapter->tso_err); 640 evcnt_detach(&adapter->admin_irqev); 641 evcnt_detach(&adapter->link_workev); 642 643 txr = adapter->tx_rings; 644 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 645 evcnt_detach(&adapter->queues[i].irqs); 646 evcnt_detach(&adapter->queues[i].handleq); 647 evcnt_detach(&adapter->queues[i].req); 648 evcnt_detach(&txr->no_desc_avail); 649 evcnt_detach(&txr->total_packets); 650 evcnt_detach(&txr->tso_tx); 651 #ifndef IXGBE_LEGACY_TX 652 evcnt_detach(&txr->pcq_drops); 653 #endif 654 655 evcnt_detach(&rxr->rx_packets); 656 evcnt_detach(&rxr->rx_bytes); 657 evcnt_detach(&rxr->rx_copies); 658 evcnt_detach(&rxr->no_mbuf); 659 evcnt_detach(&rxr->rx_discarded); 660 } 661 evcnt_detach(&stats->ipcs); 662 evcnt_detach(&stats->l4cs); 663 evcnt_detach(&stats->ipcs_bad); 664 evcnt_detach(&stats->l4cs_bad); 665 666 /* Packet Reception Stats */ 667 evcnt_detach(&stats->vfgorc); 668 evcnt_detach(&stats->vfgprc); 669 evcnt_detach(&stats->vfmprc); 670 671 /* Packet Transmission Stats */ 672 evcnt_detach(&stats->vfgotc); 673 evcnt_detach(&stats->vfgptc); 674 675 /* Mailbox Stats */ 676 evcnt_detach(&hw->mbx.stats.msgs_tx); 677 evcnt_detach(&hw->mbx.stats.msgs_rx); 678 evcnt_detach(&hw->mbx.stats.acks); 679 evcnt_detach(&hw->mbx.stats.reqs); 680 evcnt_detach(&hw->mbx.stats.rsts); 681 682 ixgbe_free_queues(adapter); 683 684 IXGBE_CORE_LOCK_DESTROY(adapter); 685 686 return (0); 687 } /* ixv_detach */ 688 689 /************************************************************************ 690 * ixv_init_locked - Init entry point 691 * 692 * Used in two ways: It is used by the stack as an init entry 693 * point in network interface structure. It is also used 694 * by the driver as a hw/sw initialization routine to get 695 * to a consistent state. 696 * 697 * return 0 on success, positive on failure 698 ************************************************************************/ 699 static void 700 ixv_init_locked(struct adapter *adapter) 701 { 702 struct ifnet *ifp = adapter->ifp; 703 device_t dev = adapter->dev; 704 struct ixgbe_hw *hw = &adapter->hw; 705 struct ix_queue *que; 706 int error = 0; 707 uint32_t mask; 708 int i; 709 710 INIT_DEBUGOUT("ixv_init_locked: begin"); 711 KASSERT(mutex_owned(&adapter->core_mtx)); 712 hw->adapter_stopped = FALSE; 713 hw->mac.ops.stop_adapter(hw); 714 callout_stop(&adapter->timer); 715 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 716 que->disabled_count = 0; 717 718 adapter->max_frame_size = 719 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 720 721 /* reprogram the RAR[0] in case user changed it. */ 722 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 723 724 /* Get the latest mac address, User can use a LAA */ 725 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 726 IXGBE_ETH_LENGTH_OF_ADDRESS); 727 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 728 729 /* Prepare transmit descriptors and buffers */ 730 if (ixgbe_setup_transmit_structures(adapter)) { 731 aprint_error_dev(dev, "Could not setup transmit structures\n"); 732 ixv_stop_locked(adapter); 733 return; 734 } 735 736 /* Reset VF and renegotiate mailbox API version */ 737 hw->mac.ops.reset_hw(hw); 738 hw->mac.ops.start_hw(hw); 739 error = ixv_negotiate_api(adapter); 740 if (error) 741 device_printf(dev, 742 "Mailbox API negotiation failed in init_locked!\n"); 743 744 ixv_initialize_transmit_units(adapter); 745 746 /* Setup Multicast table */ 747 ixv_set_rxfilter(adapter); 748 749 /* Use fixed buffer size, even for jumbo frames */ 750 adapter->rx_mbuf_sz = MCLBYTES; 751 752 /* Prepare receive descriptors and buffers */ 753 error = ixgbe_setup_receive_structures(adapter); 754 if (error) { 755 device_printf(dev, 756 "Could not setup receive structures (err = %d)\n", error); 757 ixv_stop_locked(adapter); 758 return; 759 } 760 761 /* Configure RX settings */ 762 ixv_initialize_receive_units(adapter); 763 764 /* Initialize variable holding task enqueue requests interrupts */ 765 adapter->task_requests = 0; 766 767 /* Set up VLAN offload and filter */ 768 ixv_setup_vlan_support(adapter); 769 770 /* Set up MSI-X routing */ 771 ixv_configure_ivars(adapter); 772 773 /* Set up auto-mask */ 774 mask = (1 << adapter->vector); 775 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 776 mask |= (1 << que->msix); 777 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 778 779 /* Set moderation on the Link interrupt */ 780 ixv_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 781 782 /* Stats init */ 783 ixv_init_stats(adapter); 784 785 /* Config/Enable Link */ 786 hw->mac.get_link_status = TRUE; 787 hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up, 788 FALSE); 789 790 /* Start watchdog */ 791 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 792 atomic_store_relaxed(&adapter->timer_pending, 0); 793 794 /* OK to schedule workqueues. */ 795 adapter->schedule_wqs_ok = true; 796 797 /* And now turn on interrupts */ 798 ixv_enable_intr(adapter); 799 800 /* Update saved flags. See ixgbe_ifflags_cb() */ 801 adapter->if_flags = ifp->if_flags; 802 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 803 804 /* Now inform the stack we're ready */ 805 ifp->if_flags |= IFF_RUNNING; 806 ifp->if_flags &= ~IFF_OACTIVE; 807 808 return; 809 } /* ixv_init_locked */ 810 811 /************************************************************************ 812 * ixv_enable_queue 813 ************************************************************************/ 814 static inline void 815 ixv_enable_queue(struct adapter *adapter, u32 vector) 816 { 817 struct ixgbe_hw *hw = &adapter->hw; 818 struct ix_queue *que = &adapter->queues[vector]; 819 u32 queue = 1UL << vector; 820 u32 mask; 821 822 mutex_enter(&que->dc_mtx); 823 if (que->disabled_count > 0 && --que->disabled_count > 0) 824 goto out; 825 826 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 827 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 828 out: 829 mutex_exit(&que->dc_mtx); 830 } /* ixv_enable_queue */ 831 832 /************************************************************************ 833 * ixv_disable_queue 834 ************************************************************************/ 835 static inline void 836 ixv_disable_queue(struct adapter *adapter, u32 vector) 837 { 838 struct ixgbe_hw *hw = &adapter->hw; 839 struct ix_queue *que = &adapter->queues[vector]; 840 u32 queue = 1UL << vector; 841 u32 mask; 842 843 mutex_enter(&que->dc_mtx); 844 if (que->disabled_count++ > 0) 845 goto out; 846 847 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 848 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 849 out: 850 mutex_exit(&que->dc_mtx); 851 } /* ixv_disable_queue */ 852 853 #if 0 854 static inline void 855 ixv_rearm_queues(struct adapter *adapter, u64 queues) 856 { 857 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 858 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 859 } /* ixv_rearm_queues */ 860 #endif 861 862 863 /************************************************************************ 864 * ixv_msix_que - MSI-X Queue Interrupt Service routine 865 ************************************************************************/ 866 static int 867 ixv_msix_que(void *arg) 868 { 869 struct ix_queue *que = arg; 870 struct adapter *adapter = que->adapter; 871 struct tx_ring *txr = que->txr; 872 struct rx_ring *rxr = que->rxr; 873 bool more; 874 u32 newitr = 0; 875 876 ixv_disable_queue(adapter, que->msix); 877 ++que->irqs.ev_count; 878 879 #ifdef __NetBSD__ 880 /* Don't run ixgbe_rxeof in interrupt context */ 881 more = true; 882 #else 883 more = ixgbe_rxeof(que); 884 #endif 885 886 IXGBE_TX_LOCK(txr); 887 ixgbe_txeof(txr); 888 IXGBE_TX_UNLOCK(txr); 889 890 /* Do AIM now? */ 891 892 if (adapter->enable_aim == false) 893 goto no_calc; 894 /* 895 * Do Adaptive Interrupt Moderation: 896 * - Write out last calculated setting 897 * - Calculate based on average size over 898 * the last interval. 899 */ 900 if (que->eitr_setting) 901 ixv_eitr_write(adapter, que->msix, que->eitr_setting); 902 903 que->eitr_setting = 0; 904 905 /* Idle, do nothing */ 906 if ((txr->bytes == 0) && (rxr->bytes == 0)) 907 goto no_calc; 908 909 if ((txr->bytes) && (txr->packets)) 910 newitr = txr->bytes/txr->packets; 911 if ((rxr->bytes) && (rxr->packets)) 912 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 913 newitr += 24; /* account for hardware frame, crc */ 914 915 /* set an upper boundary */ 916 newitr = uimin(newitr, 3000); 917 918 /* Be nice to the mid range */ 919 if ((newitr > 300) && (newitr < 1200)) 920 newitr = (newitr / 3); 921 else 922 newitr = (newitr / 2); 923 924 /* 925 * When RSC is used, ITR interval must be larger than RSC_DELAY. 926 * Currently, we use 2us for RSC_DELAY. The minimum value is always 927 * greater than 2us on 100M (and 10M?(not documented)), but it's not 928 * on 1G and higher. 929 */ 930 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 931 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 932 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 933 newitr = IXGBE_MIN_RSC_EITR_10G1G; 934 } 935 936 /* save for next interrupt */ 937 que->eitr_setting = newitr; 938 939 /* Reset state */ 940 txr->bytes = 0; 941 txr->packets = 0; 942 rxr->bytes = 0; 943 rxr->packets = 0; 944 945 no_calc: 946 if (more) 947 softint_schedule(que->que_si); 948 else /* Re-enable this interrupt */ 949 ixv_enable_queue(adapter, que->msix); 950 951 return 1; 952 } /* ixv_msix_que */ 953 954 /************************************************************************ 955 * ixv_msix_mbx 956 ************************************************************************/ 957 static int 958 ixv_msix_mbx(void *arg) 959 { 960 struct adapter *adapter = arg; 961 struct ixgbe_hw *hw = &adapter->hw; 962 963 ++adapter->admin_irqev.ev_count; 964 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 965 966 /* Link status change */ 967 hw->mac.get_link_status = TRUE; 968 atomic_or_32(&adapter->task_requests, IXGBE_REQUEST_TASK_MBX); 969 ixv_schedule_admin_tasklet(adapter); 970 971 return 1; 972 } /* ixv_msix_mbx */ 973 974 static void 975 ixv_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 976 { 977 978 /* 979 * Newer devices than 82598 have VF function, so this function is 980 * simple. 981 */ 982 itr |= IXGBE_EITR_CNT_WDIS; 983 984 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(index), itr); 985 } 986 987 988 /************************************************************************ 989 * ixv_media_status - Media Ioctl callback 990 * 991 * Called whenever the user queries the status of 992 * the interface using ifconfig. 993 ************************************************************************/ 994 static void 995 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 996 { 997 struct adapter *adapter = ifp->if_softc; 998 999 INIT_DEBUGOUT("ixv_media_status: begin"); 1000 ixv_update_link_status(adapter); 1001 1002 ifmr->ifm_status = IFM_AVALID; 1003 ifmr->ifm_active = IFM_ETHER; 1004 1005 if (adapter->link_active != LINK_STATE_UP) { 1006 ifmr->ifm_active |= IFM_NONE; 1007 return; 1008 } 1009 1010 ifmr->ifm_status |= IFM_ACTIVE; 1011 1012 switch (adapter->link_speed) { 1013 case IXGBE_LINK_SPEED_10GB_FULL: 1014 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1015 break; 1016 case IXGBE_LINK_SPEED_5GB_FULL: 1017 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1018 break; 1019 case IXGBE_LINK_SPEED_2_5GB_FULL: 1020 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1021 break; 1022 case IXGBE_LINK_SPEED_1GB_FULL: 1023 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1024 break; 1025 case IXGBE_LINK_SPEED_100_FULL: 1026 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1027 break; 1028 case IXGBE_LINK_SPEED_10_FULL: 1029 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1030 break; 1031 } 1032 1033 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1034 } /* ixv_media_status */ 1035 1036 /************************************************************************ 1037 * ixv_media_change - Media Ioctl callback 1038 * 1039 * Called when the user changes speed/duplex using 1040 * media/mediopt option with ifconfig. 1041 ************************************************************************/ 1042 static int 1043 ixv_media_change(struct ifnet *ifp) 1044 { 1045 struct adapter *adapter = ifp->if_softc; 1046 struct ifmedia *ifm = &adapter->media; 1047 1048 INIT_DEBUGOUT("ixv_media_change: begin"); 1049 1050 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1051 return (EINVAL); 1052 1053 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1054 case IFM_AUTO: 1055 break; 1056 default: 1057 device_printf(adapter->dev, "Only auto media type\n"); 1058 return (EINVAL); 1059 } 1060 1061 return (0); 1062 } /* ixv_media_change */ 1063 1064 static void 1065 ixv_schedule_admin_tasklet(struct adapter *adapter) 1066 { 1067 if (adapter->schedule_wqs_ok) { 1068 if (atomic_cas_uint(&adapter->admin_pending, 0, 1) == 0) 1069 workqueue_enqueue(adapter->admin_wq, 1070 &adapter->admin_wc, NULL); 1071 } 1072 } 1073 1074 /************************************************************************ 1075 * ixv_negotiate_api 1076 * 1077 * Negotiate the Mailbox API with the PF; 1078 * start with the most featured API first. 1079 ************************************************************************/ 1080 static int 1081 ixv_negotiate_api(struct adapter *adapter) 1082 { 1083 struct ixgbe_hw *hw = &adapter->hw; 1084 int mbx_api[] = { ixgbe_mbox_api_13, 1085 ixgbe_mbox_api_12, 1086 ixgbe_mbox_api_11, 1087 ixgbe_mbox_api_10, 1088 ixgbe_mbox_api_unknown }; 1089 int i = 0; 1090 1091 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1092 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) 1093 return (0); 1094 i++; 1095 } 1096 1097 return (EINVAL); 1098 } /* ixv_negotiate_api */ 1099 1100 1101 /************************************************************************ 1102 * ixv_set_rxfilter - Multicast Update 1103 * 1104 * Called whenever multicast address list is updated. 1105 ************************************************************************/ 1106 static int 1107 ixv_set_rxfilter(struct adapter *adapter) 1108 { 1109 struct ixgbe_mc_addr *mta; 1110 struct ifnet *ifp = adapter->ifp; 1111 struct ixgbe_hw *hw = &adapter->hw; 1112 u8 *update_ptr; 1113 int mcnt = 0; 1114 struct ethercom *ec = &adapter->osdep.ec; 1115 struct ether_multi *enm; 1116 struct ether_multistep step; 1117 bool overflow = false; 1118 int error, rc = 0; 1119 1120 KASSERT(mutex_owned(&adapter->core_mtx)); 1121 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin"); 1122 1123 mta = adapter->mta; 1124 bzero(mta, sizeof(*mta) * IXGBE_MAX_VF_MC); 1125 1126 /* 1: For PROMISC */ 1127 if (ifp->if_flags & IFF_PROMISC) { 1128 error = hw->mac.ops.update_xcast_mode(hw, 1129 IXGBEVF_XCAST_MODE_PROMISC); 1130 if (error == IXGBE_ERR_NOT_TRUSTED) { 1131 device_printf(adapter->dev, 1132 "this interface is not trusted\n"); 1133 error = EPERM; 1134 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1135 device_printf(adapter->dev, 1136 "the PF doesn't support promisc mode\n"); 1137 error = EOPNOTSUPP; 1138 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) { 1139 device_printf(adapter->dev, 1140 "the PF may not in promisc mode\n"); 1141 error = EINVAL; 1142 } else if (error) { 1143 device_printf(adapter->dev, 1144 "failed to set promisc mode. error = %d\n", 1145 error); 1146 error = EIO; 1147 } else 1148 return 0; 1149 rc = error; 1150 } 1151 1152 /* 2: For ALLMULTI or normal */ 1153 ETHER_LOCK(ec); 1154 ETHER_FIRST_MULTI(step, ec, enm); 1155 while (enm != NULL) { 1156 if ((mcnt >= IXGBE_MAX_VF_MC) || 1157 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1158 ETHER_ADDR_LEN) != 0)) { 1159 overflow = true; 1160 break; 1161 } 1162 bcopy(enm->enm_addrlo, 1163 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 1164 mcnt++; 1165 ETHER_NEXT_MULTI(step, enm); 1166 } 1167 ETHER_UNLOCK(ec); 1168 1169 /* 3: For ALLMULTI */ 1170 if (overflow) { 1171 error = hw->mac.ops.update_xcast_mode(hw, 1172 IXGBEVF_XCAST_MODE_ALLMULTI); 1173 if (error == IXGBE_ERR_NOT_TRUSTED) { 1174 device_printf(adapter->dev, 1175 "this interface is not trusted\n"); 1176 error = EPERM; 1177 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1178 device_printf(adapter->dev, 1179 "the PF doesn't support allmulti mode\n"); 1180 error = EOPNOTSUPP; 1181 } else if (error) { 1182 device_printf(adapter->dev, 1183 "number of Ethernet multicast addresses " 1184 "exceeds the limit (%d). error = %d\n", 1185 IXGBE_MAX_VF_MC, error); 1186 error = ENOSPC; 1187 } else { 1188 ETHER_LOCK(ec); 1189 ec->ec_flags |= ETHER_F_ALLMULTI; 1190 ETHER_UNLOCK(ec); 1191 return rc; /* Promisc might have failed */ 1192 } 1193 1194 if (rc == 0) 1195 rc = error; 1196 1197 /* Continue to update the multicast table as many as we can */ 1198 } 1199 1200 /* 4: For normal operation */ 1201 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 1202 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) { 1203 /* Normal operation */ 1204 ETHER_LOCK(ec); 1205 ec->ec_flags &= ~ETHER_F_ALLMULTI; 1206 ETHER_UNLOCK(ec); 1207 error = 0; 1208 } else if (error) { 1209 device_printf(adapter->dev, 1210 "failed to set Ethernet multicast address " 1211 "operation to normal. error = %d\n", error); 1212 } 1213 1214 update_ptr = (u8 *)mta; 1215 error = adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, 1216 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1217 if (rc == 0) 1218 rc = error; 1219 1220 return rc; 1221 } /* ixv_set_rxfilter */ 1222 1223 /************************************************************************ 1224 * ixv_mc_array_itr 1225 * 1226 * An iterator function needed by the multicast shared code. 1227 * It feeds the shared code routine the addresses in the 1228 * array of ixv_set_rxfilter() one by one. 1229 ************************************************************************/ 1230 static u8 * 1231 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1232 { 1233 struct ixgbe_mc_addr *mta; 1234 1235 mta = (struct ixgbe_mc_addr *)*update_ptr; 1236 1237 *vmdq = 0; 1238 *update_ptr = (u8*)(mta + 1); 1239 1240 return (mta->addr); 1241 } /* ixv_mc_array_itr */ 1242 1243 /************************************************************************ 1244 * ixv_local_timer - Timer routine 1245 * 1246 * Checks for link status, updates statistics, 1247 * and runs the watchdog check. 1248 ************************************************************************/ 1249 static void 1250 ixv_local_timer(void *arg) 1251 { 1252 struct adapter *adapter = arg; 1253 1254 if (adapter->schedule_wqs_ok) { 1255 if (atomic_cas_uint(&adapter->timer_pending, 0, 1) == 0) 1256 workqueue_enqueue(adapter->timer_wq, 1257 &adapter->timer_wc, NULL); 1258 } 1259 } 1260 1261 static void 1262 ixv_handle_timer(struct work *wk, void *context) 1263 { 1264 struct adapter *adapter = context; 1265 device_t dev = adapter->dev; 1266 struct ix_queue *que = adapter->queues; 1267 u64 queues = 0; 1268 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1269 int hung = 0; 1270 int i; 1271 1272 IXGBE_CORE_LOCK(adapter); 1273 1274 if (ixv_check_link(adapter)) { 1275 ixv_init_locked(adapter); 1276 IXGBE_CORE_UNLOCK(adapter); 1277 return; 1278 } 1279 1280 /* Stats Update */ 1281 ixv_update_stats(adapter); 1282 1283 /* Update some event counters */ 1284 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1285 que = adapter->queues; 1286 for (i = 0; i < adapter->num_queues; i++, que++) { 1287 struct tx_ring *txr = que->txr; 1288 1289 v0 += txr->q_efbig_tx_dma_setup; 1290 v1 += txr->q_mbuf_defrag_failed; 1291 v2 += txr->q_efbig2_tx_dma_setup; 1292 v3 += txr->q_einval_tx_dma_setup; 1293 v4 += txr->q_other_tx_dma_setup; 1294 v5 += txr->q_eagain_tx_dma_setup; 1295 v6 += txr->q_enomem_tx_dma_setup; 1296 v7 += txr->q_tso_err; 1297 } 1298 adapter->efbig_tx_dma_setup.ev_count = v0; 1299 adapter->mbuf_defrag_failed.ev_count = v1; 1300 adapter->efbig2_tx_dma_setup.ev_count = v2; 1301 adapter->einval_tx_dma_setup.ev_count = v3; 1302 adapter->other_tx_dma_setup.ev_count = v4; 1303 adapter->eagain_tx_dma_setup.ev_count = v5; 1304 adapter->enomem_tx_dma_setup.ev_count = v6; 1305 adapter->tso_err.ev_count = v7; 1306 1307 /* 1308 * Check the TX queues status 1309 * - mark hung queues so we don't schedule on them 1310 * - watchdog only if all queues show hung 1311 */ 1312 que = adapter->queues; 1313 for (i = 0; i < adapter->num_queues; i++, que++) { 1314 /* Keep track of queues with work for soft irq */ 1315 if (que->txr->busy) 1316 queues |= ((u64)1 << que->me); 1317 /* 1318 * Each time txeof runs without cleaning, but there 1319 * are uncleaned descriptors it increments busy. If 1320 * we get to the MAX we declare it hung. 1321 */ 1322 if (que->busy == IXGBE_QUEUE_HUNG) { 1323 ++hung; 1324 /* Mark the queue as inactive */ 1325 adapter->active_queues &= ~((u64)1 << que->me); 1326 continue; 1327 } else { 1328 /* Check if we've come back from hung */ 1329 if ((adapter->active_queues & ((u64)1 << que->me)) == 0) 1330 adapter->active_queues |= ((u64)1 << que->me); 1331 } 1332 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1333 device_printf(dev, 1334 "Warning queue %d appears to be hung!\n", i); 1335 que->txr->busy = IXGBE_QUEUE_HUNG; 1336 ++hung; 1337 } 1338 } 1339 1340 /* Only truly watchdog if all queues show hung */ 1341 if (hung == adapter->num_queues) 1342 goto watchdog; 1343 #if 0 1344 else if (queues != 0) { /* Force an IRQ on queues with work */ 1345 ixv_rearm_queues(adapter, queues); 1346 } 1347 #endif 1348 1349 atomic_store_relaxed(&adapter->timer_pending, 0); 1350 IXGBE_CORE_UNLOCK(adapter); 1351 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1352 1353 return; 1354 1355 watchdog: 1356 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1357 adapter->ifp->if_flags &= ~IFF_RUNNING; 1358 adapter->watchdog_events.ev_count++; 1359 ixv_init_locked(adapter); 1360 IXGBE_CORE_UNLOCK(adapter); 1361 } /* ixv_handle_timer */ 1362 1363 /************************************************************************ 1364 * ixv_update_link_status - Update OS on link state 1365 * 1366 * Note: Only updates the OS on the cached link state. 1367 * The real check of the hardware only happens with 1368 * a link interrupt. 1369 ************************************************************************/ 1370 static void 1371 ixv_update_link_status(struct adapter *adapter) 1372 { 1373 struct ifnet *ifp = adapter->ifp; 1374 device_t dev = adapter->dev; 1375 1376 KASSERT(mutex_owned(&adapter->core_mtx)); 1377 1378 if (adapter->link_up) { 1379 if (adapter->link_active != LINK_STATE_UP) { 1380 if (bootverbose) { 1381 const char *bpsmsg; 1382 1383 switch (adapter->link_speed) { 1384 case IXGBE_LINK_SPEED_10GB_FULL: 1385 bpsmsg = "10 Gbps"; 1386 break; 1387 case IXGBE_LINK_SPEED_5GB_FULL: 1388 bpsmsg = "5 Gbps"; 1389 break; 1390 case IXGBE_LINK_SPEED_2_5GB_FULL: 1391 bpsmsg = "2.5 Gbps"; 1392 break; 1393 case IXGBE_LINK_SPEED_1GB_FULL: 1394 bpsmsg = "1 Gbps"; 1395 break; 1396 case IXGBE_LINK_SPEED_100_FULL: 1397 bpsmsg = "100 Mbps"; 1398 break; 1399 case IXGBE_LINK_SPEED_10_FULL: 1400 bpsmsg = "10 Mbps"; 1401 break; 1402 default: 1403 bpsmsg = "unknown speed"; 1404 break; 1405 } 1406 device_printf(dev, "Link is up %s %s \n", 1407 bpsmsg, "Full Duplex"); 1408 } 1409 adapter->link_active = LINK_STATE_UP; 1410 if_link_state_change(ifp, LINK_STATE_UP); 1411 } 1412 } else { 1413 /* 1414 * Do it when link active changes to DOWN. i.e. 1415 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 1416 * b) LINK_STATE_UP -> LINK_STATE_DOWN 1417 */ 1418 if (adapter->link_active != LINK_STATE_DOWN) { 1419 if (bootverbose) 1420 device_printf(dev, "Link is Down\n"); 1421 if_link_state_change(ifp, LINK_STATE_DOWN); 1422 adapter->link_active = LINK_STATE_DOWN; 1423 } 1424 } 1425 } /* ixv_update_link_status */ 1426 1427 1428 /************************************************************************ 1429 * ixv_stop - Stop the hardware 1430 * 1431 * Disables all traffic on the adapter by issuing a 1432 * global reset on the MAC and deallocates TX/RX buffers. 1433 ************************************************************************/ 1434 static void 1435 ixv_ifstop(struct ifnet *ifp, int disable) 1436 { 1437 struct adapter *adapter = ifp->if_softc; 1438 1439 IXGBE_CORE_LOCK(adapter); 1440 ixv_stop_locked(adapter); 1441 IXGBE_CORE_UNLOCK(adapter); 1442 1443 workqueue_wait(adapter->admin_wq, &adapter->admin_wc); 1444 atomic_store_relaxed(&adapter->admin_pending, 0); 1445 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 1446 atomic_store_relaxed(&adapter->timer_pending, 0); 1447 } 1448 1449 static void 1450 ixv_stop_locked(void *arg) 1451 { 1452 struct ifnet *ifp; 1453 struct adapter *adapter = arg; 1454 struct ixgbe_hw *hw = &adapter->hw; 1455 1456 ifp = adapter->ifp; 1457 1458 KASSERT(mutex_owned(&adapter->core_mtx)); 1459 1460 INIT_DEBUGOUT("ixv_stop_locked: begin\n"); 1461 ixv_disable_intr(adapter); 1462 1463 /* Tell the stack that the interface is no longer active */ 1464 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1465 1466 hw->mac.ops.reset_hw(hw); 1467 adapter->hw.adapter_stopped = FALSE; 1468 hw->mac.ops.stop_adapter(hw); 1469 callout_stop(&adapter->timer); 1470 1471 /* Don't schedule workqueues. */ 1472 adapter->schedule_wqs_ok = false; 1473 1474 /* reprogram the RAR[0] in case user changed it. */ 1475 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1476 1477 return; 1478 } /* ixv_stop_locked */ 1479 1480 1481 /************************************************************************ 1482 * ixv_allocate_pci_resources 1483 ************************************************************************/ 1484 static int 1485 ixv_allocate_pci_resources(struct adapter *adapter, 1486 const struct pci_attach_args *pa) 1487 { 1488 pcireg_t memtype, csr; 1489 device_t dev = adapter->dev; 1490 bus_addr_t addr; 1491 int flags; 1492 1493 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1494 switch (memtype) { 1495 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1496 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1497 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1498 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1499 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1500 goto map_err; 1501 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1502 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1503 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1504 } 1505 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1506 adapter->osdep.mem_size, flags, 1507 &adapter->osdep.mem_bus_space_handle) != 0) { 1508 map_err: 1509 adapter->osdep.mem_size = 0; 1510 aprint_error_dev(dev, "unable to map BAR0\n"); 1511 return ENXIO; 1512 } 1513 /* 1514 * Enable address decoding for memory range in case it's not 1515 * set. 1516 */ 1517 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 1518 PCI_COMMAND_STATUS_REG); 1519 csr |= PCI_COMMAND_MEM_ENABLE; 1520 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 1521 csr); 1522 break; 1523 default: 1524 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1525 return ENXIO; 1526 } 1527 1528 /* Pick up the tuneable queues */ 1529 adapter->num_queues = ixv_num_queues; 1530 1531 return (0); 1532 } /* ixv_allocate_pci_resources */ 1533 1534 static void 1535 ixv_free_deferred_handlers(struct adapter *adapter) 1536 { 1537 struct ix_queue *que = adapter->queues; 1538 struct tx_ring *txr = adapter->tx_rings; 1539 int i; 1540 1541 for (i = 0; i < adapter->num_queues; i++, que++, txr++) { 1542 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 1543 if (txr->txr_si != NULL) 1544 softint_disestablish(txr->txr_si); 1545 } 1546 if (que->que_si != NULL) 1547 softint_disestablish(que->que_si); 1548 } 1549 if (adapter->txr_wq != NULL) 1550 workqueue_destroy(adapter->txr_wq); 1551 if (adapter->txr_wq_enqueued != NULL) 1552 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 1553 if (adapter->que_wq != NULL) 1554 workqueue_destroy(adapter->que_wq); 1555 1556 /* Drain the Mailbox(link) queue */ 1557 if (adapter->admin_wq != NULL) { 1558 workqueue_destroy(adapter->admin_wq); 1559 adapter->admin_wq = NULL; 1560 } 1561 if (adapter->timer_wq != NULL) { 1562 workqueue_destroy(adapter->timer_wq); 1563 adapter->timer_wq = NULL; 1564 } 1565 } /* ixv_free_deferred_handlers */ 1566 1567 /************************************************************************ 1568 * ixv_free_pci_resources 1569 ************************************************************************/ 1570 static void 1571 ixv_free_pci_resources(struct adapter * adapter) 1572 { 1573 struct ix_queue *que = adapter->queues; 1574 int rid; 1575 1576 /* 1577 * Release all msix queue resources: 1578 */ 1579 for (int i = 0; i < adapter->num_queues; i++, que++) { 1580 if (que->res != NULL) 1581 pci_intr_disestablish(adapter->osdep.pc, 1582 adapter->osdep.ihs[i]); 1583 } 1584 1585 1586 /* Clean the Mailbox interrupt last */ 1587 rid = adapter->vector; 1588 1589 if (adapter->osdep.ihs[rid] != NULL) { 1590 pci_intr_disestablish(adapter->osdep.pc, 1591 adapter->osdep.ihs[rid]); 1592 adapter->osdep.ihs[rid] = NULL; 1593 } 1594 1595 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1596 adapter->osdep.nintrs); 1597 1598 if (adapter->osdep.mem_size != 0) { 1599 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 1600 adapter->osdep.mem_bus_space_handle, 1601 adapter->osdep.mem_size); 1602 } 1603 1604 return; 1605 } /* ixv_free_pci_resources */ 1606 1607 /************************************************************************ 1608 * ixv_setup_interface 1609 * 1610 * Setup networking device structure and register an interface. 1611 ************************************************************************/ 1612 static int 1613 ixv_setup_interface(device_t dev, struct adapter *adapter) 1614 { 1615 struct ethercom *ec = &adapter->osdep.ec; 1616 struct ifnet *ifp; 1617 1618 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1619 1620 ifp = adapter->ifp = &ec->ec_if; 1621 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1622 ifp->if_baudrate = IF_Gbps(10); 1623 ifp->if_init = ixv_init; 1624 ifp->if_stop = ixv_ifstop; 1625 ifp->if_softc = adapter; 1626 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1627 #ifdef IXGBE_MPSAFE 1628 ifp->if_extflags = IFEF_MPSAFE; 1629 #endif 1630 ifp->if_ioctl = ixv_ioctl; 1631 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1632 #if 0 1633 ixv_start_locked = ixgbe_legacy_start_locked; 1634 #endif 1635 } else { 1636 ifp->if_transmit = ixgbe_mq_start; 1637 #if 0 1638 ixv_start_locked = ixgbe_mq_start_locked; 1639 #endif 1640 } 1641 ifp->if_start = ixgbe_legacy_start; 1642 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1643 IFQ_SET_READY(&ifp->if_snd); 1644 1645 if_initialize(ifp); 1646 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1647 ether_ifattach(ifp, adapter->hw.mac.addr); 1648 aprint_normal_dev(dev, "Ethernet address %s\n", 1649 ether_sprintf(adapter->hw.mac.addr)); 1650 /* 1651 * We use per TX queue softint, so if_deferred_start_init() isn't 1652 * used. 1653 */ 1654 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1655 1656 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1657 1658 /* 1659 * Tell the upper layer(s) we support long frames. 1660 */ 1661 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1662 1663 /* Set capability flags */ 1664 ifp->if_capabilities |= IFCAP_HWCSUM 1665 | IFCAP_TSOv4 1666 | IFCAP_TSOv6; 1667 ifp->if_capenable = 0; 1668 1669 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER 1670 | ETHERCAP_VLAN_HWTAGGING 1671 | ETHERCAP_VLAN_HWCSUM 1672 | ETHERCAP_JUMBO_MTU 1673 | ETHERCAP_VLAN_MTU; 1674 1675 /* Enable the above capabilities by default */ 1676 ec->ec_capenable = ec->ec_capabilities; 1677 1678 /* Don't enable LRO by default */ 1679 #if 0 1680 /* NetBSD doesn't support LRO yet */ 1681 ifp->if_capabilities |= IFCAP_LRO; 1682 #endif 1683 1684 /* 1685 * Specify the media types supported by this adapter and register 1686 * callbacks to update media and link information 1687 */ 1688 ec->ec_ifmedia = &adapter->media; 1689 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, ixv_media_change, 1690 ixv_media_status, &adapter->core_mtx); 1691 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1692 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1693 1694 if_register(ifp); 1695 1696 return 0; 1697 } /* ixv_setup_interface */ 1698 1699 1700 /************************************************************************ 1701 * ixv_initialize_transmit_units - Enable transmit unit. 1702 ************************************************************************/ 1703 static void 1704 ixv_initialize_transmit_units(struct adapter *adapter) 1705 { 1706 struct tx_ring *txr = adapter->tx_rings; 1707 struct ixgbe_hw *hw = &adapter->hw; 1708 int i; 1709 1710 for (i = 0; i < adapter->num_queues; i++, txr++) { 1711 u64 tdba = txr->txdma.dma_paddr; 1712 u32 txctrl, txdctl; 1713 int j = txr->me; 1714 1715 /* Set WTHRESH to 8, burst writeback */ 1716 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1717 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 1718 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1719 1720 /* Set the HW Tx Head and Tail indices */ 1721 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(j), 0); 1722 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(j), 0); 1723 1724 /* Set Tx Tail register */ 1725 txr->tail = IXGBE_VFTDT(j); 1726 1727 txr->txr_no_space = false; 1728 1729 /* Set Ring parameters */ 1730 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1731 (tdba & 0x00000000ffffffffULL)); 1732 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1733 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1734 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1735 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1736 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1737 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1738 1739 /* Now enable */ 1740 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1741 txdctl |= IXGBE_TXDCTL_ENABLE; 1742 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1743 } 1744 1745 return; 1746 } /* ixv_initialize_transmit_units */ 1747 1748 1749 /************************************************************************ 1750 * ixv_initialize_rss_mapping 1751 ************************************************************************/ 1752 static void 1753 ixv_initialize_rss_mapping(struct adapter *adapter) 1754 { 1755 struct ixgbe_hw *hw = &adapter->hw; 1756 u32 reta = 0, mrqc, rss_key[10]; 1757 int queue_id; 1758 int i, j; 1759 u32 rss_hash_config; 1760 1761 /* force use default RSS key. */ 1762 #ifdef __NetBSD__ 1763 rss_getkey((uint8_t *) &rss_key); 1764 #else 1765 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1766 /* Fetch the configured RSS key */ 1767 rss_getkey((uint8_t *)&rss_key); 1768 } else { 1769 /* set up random bits */ 1770 cprng_fast(&rss_key, sizeof(rss_key)); 1771 } 1772 #endif 1773 1774 /* Now fill out hash function seeds */ 1775 for (i = 0; i < 10; i++) 1776 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1777 1778 /* Set up the redirection table */ 1779 for (i = 0, j = 0; i < 64; i++, j++) { 1780 if (j == adapter->num_queues) 1781 j = 0; 1782 1783 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1784 /* 1785 * Fetch the RSS bucket id for the given indirection 1786 * entry. Cap it at the number of configured buckets 1787 * (which is num_queues.) 1788 */ 1789 queue_id = rss_get_indirection_to_bucket(i); 1790 queue_id = queue_id % adapter->num_queues; 1791 } else 1792 queue_id = j; 1793 1794 /* 1795 * The low 8 bits are for hash value (n+0); 1796 * The next 8 bits are for hash value (n+1), etc. 1797 */ 1798 reta >>= 8; 1799 reta |= ((uint32_t)queue_id) << 24; 1800 if ((i & 3) == 3) { 1801 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1802 reta = 0; 1803 } 1804 } 1805 1806 /* Perform hash on these packet types */ 1807 if (adapter->feat_en & IXGBE_FEATURE_RSS) 1808 rss_hash_config = rss_gethashconfig(); 1809 else { 1810 /* 1811 * Disable UDP - IP fragments aren't currently being handled 1812 * and so we end up with a mix of 2-tuple and 4-tuple 1813 * traffic. 1814 */ 1815 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1816 | RSS_HASHTYPE_RSS_TCP_IPV4 1817 | RSS_HASHTYPE_RSS_IPV6 1818 | RSS_HASHTYPE_RSS_TCP_IPV6; 1819 } 1820 1821 mrqc = IXGBE_MRQC_RSSEN; 1822 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1823 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1824 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1825 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1826 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1827 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1828 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1829 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1830 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1831 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX defined, but not supported\n", 1832 __func__); 1833 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1834 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined, but not supported\n", 1835 __func__); 1836 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1837 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1838 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1839 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1840 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1841 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined, but not supported\n", 1842 __func__); 1843 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1844 } /* ixv_initialize_rss_mapping */ 1845 1846 1847 /************************************************************************ 1848 * ixv_initialize_receive_units - Setup receive registers and features. 1849 ************************************************************************/ 1850 static void 1851 ixv_initialize_receive_units(struct adapter *adapter) 1852 { 1853 struct rx_ring *rxr = adapter->rx_rings; 1854 struct ixgbe_hw *hw = &adapter->hw; 1855 struct ifnet *ifp = adapter->ifp; 1856 u32 bufsz, psrtype; 1857 1858 if (ifp->if_mtu > ETHERMTU) 1859 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1860 else 1861 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1862 1863 psrtype = IXGBE_PSRTYPE_TCPHDR 1864 | IXGBE_PSRTYPE_UDPHDR 1865 | IXGBE_PSRTYPE_IPV4HDR 1866 | IXGBE_PSRTYPE_IPV6HDR 1867 | IXGBE_PSRTYPE_L2HDR; 1868 1869 if (adapter->num_queues > 1) 1870 psrtype |= 1 << 29; 1871 1872 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1873 1874 /* Tell PF our max_frame size */ 1875 if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) { 1876 device_printf(adapter->dev, "There is a problem with the PF setup. It is likely the receive unit for this VF will not function correctly.\n"); 1877 } 1878 1879 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 1880 u64 rdba = rxr->rxdma.dma_paddr; 1881 u32 reg, rxdctl; 1882 int j = rxr->me; 1883 1884 /* Disable the queue */ 1885 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1886 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1887 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1888 for (int k = 0; k < 10; k++) { 1889 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1890 IXGBE_RXDCTL_ENABLE) 1891 msec_delay(1); 1892 else 1893 break; 1894 } 1895 IXGBE_WRITE_BARRIER(hw); 1896 /* Setup the Base and Length of the Rx Descriptor Ring */ 1897 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1898 (rdba & 0x00000000ffffffffULL)); 1899 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1900 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1901 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1902 1903 /* Reset the ring indices */ 1904 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1905 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1906 1907 /* Set up the SRRCTL register */ 1908 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1909 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1910 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1911 reg |= bufsz; 1912 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1913 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1914 1915 /* Capture Rx Tail index */ 1916 rxr->tail = IXGBE_VFRDT(rxr->me); 1917 1918 /* Do the queue enabling last */ 1919 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1920 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1921 for (int k = 0; k < 10; k++) { 1922 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1923 IXGBE_RXDCTL_ENABLE) 1924 break; 1925 msec_delay(1); 1926 } 1927 IXGBE_WRITE_BARRIER(hw); 1928 1929 /* Set the Tail Pointer */ 1930 #ifdef DEV_NETMAP 1931 /* 1932 * In netmap mode, we must preserve the buffers made 1933 * available to userspace before the if_init() 1934 * (this is true by default on the TX side, because 1935 * init makes all buffers available to userspace). 1936 * 1937 * netmap_reset() and the device specific routines 1938 * (e.g. ixgbe_setup_receive_rings()) map these 1939 * buffers at the end of the NIC ring, so here we 1940 * must set the RDT (tail) register to make sure 1941 * they are not overwritten. 1942 * 1943 * In this driver the NIC ring starts at RDH = 0, 1944 * RDT points to the last slot available for reception (?), 1945 * so RDT = num_rx_desc - 1 means the whole ring is available. 1946 */ 1947 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 1948 (ifp->if_capenable & IFCAP_NETMAP)) { 1949 struct netmap_adapter *na = NA(adapter->ifp); 1950 struct netmap_kring *kring = na->rx_rings[i]; 1951 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1952 1953 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1954 } else 1955 #endif /* DEV_NETMAP */ 1956 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1957 adapter->num_rx_desc - 1); 1958 } 1959 1960 if (adapter->hw.mac.type >= ixgbe_mac_X550_vf) 1961 ixv_initialize_rss_mapping(adapter); 1962 } /* ixv_initialize_receive_units */ 1963 1964 /************************************************************************ 1965 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1966 * 1967 * Retrieves the TDH value from the hardware 1968 ************************************************************************/ 1969 static int 1970 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1971 { 1972 struct sysctlnode node = *rnode; 1973 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1974 uint32_t val; 1975 1976 if (!txr) 1977 return (0); 1978 1979 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDH(txr->me)); 1980 node.sysctl_data = &val; 1981 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1982 } /* ixv_sysctl_tdh_handler */ 1983 1984 /************************************************************************ 1985 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 1986 * 1987 * Retrieves the TDT value from the hardware 1988 ************************************************************************/ 1989 static int 1990 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 1991 { 1992 struct sysctlnode node = *rnode; 1993 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1994 uint32_t val; 1995 1996 if (!txr) 1997 return (0); 1998 1999 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDT(txr->me)); 2000 node.sysctl_data = &val; 2001 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2002 } /* ixv_sysctl_tdt_handler */ 2003 2004 /************************************************************************ 2005 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check 2006 * handler function 2007 * 2008 * Retrieves the next_to_check value 2009 ************************************************************************/ 2010 static int 2011 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2012 { 2013 struct sysctlnode node = *rnode; 2014 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2015 uint32_t val; 2016 2017 if (!rxr) 2018 return (0); 2019 2020 val = rxr->next_to_check; 2021 node.sysctl_data = &val; 2022 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2023 } /* ixv_sysctl_next_to_check_handler */ 2024 2025 /************************************************************************ 2026 * ixv_sysctl_next_to_refresh_handler - Receive Descriptor next to refresh 2027 * handler function 2028 * 2029 * Retrieves the next_to_refresh value 2030 ************************************************************************/ 2031 static int 2032 ixv_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2033 { 2034 struct sysctlnode node = *rnode; 2035 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2036 struct adapter *adapter; 2037 uint32_t val; 2038 2039 if (!rxr) 2040 return (0); 2041 2042 adapter = rxr->adapter; 2043 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2044 return (EPERM); 2045 2046 val = rxr->next_to_refresh; 2047 node.sysctl_data = &val; 2048 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2049 } /* ixv_sysctl_next_to_refresh_handler */ 2050 2051 /************************************************************************ 2052 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 2053 * 2054 * Retrieves the RDH value from the hardware 2055 ************************************************************************/ 2056 static int 2057 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 2058 { 2059 struct sysctlnode node = *rnode; 2060 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2061 uint32_t val; 2062 2063 if (!rxr) 2064 return (0); 2065 2066 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDH(rxr->me)); 2067 node.sysctl_data = &val; 2068 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2069 } /* ixv_sysctl_rdh_handler */ 2070 2071 /************************************************************************ 2072 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 2073 * 2074 * Retrieves the RDT value from the hardware 2075 ************************************************************************/ 2076 static int 2077 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 2078 { 2079 struct sysctlnode node = *rnode; 2080 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2081 uint32_t val; 2082 2083 if (!rxr) 2084 return (0); 2085 2086 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDT(rxr->me)); 2087 node.sysctl_data = &val; 2088 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2089 } /* ixv_sysctl_rdt_handler */ 2090 2091 static void 2092 ixv_setup_vlan_tagging(struct adapter *adapter) 2093 { 2094 struct ethercom *ec = &adapter->osdep.ec; 2095 struct ixgbe_hw *hw = &adapter->hw; 2096 struct rx_ring *rxr; 2097 u32 ctrl; 2098 int i; 2099 bool hwtagging; 2100 2101 /* Enable HW tagging only if any vlan is attached */ 2102 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2103 && VLAN_ATTACHED(ec); 2104 2105 /* Enable the queues */ 2106 for (i = 0; i < adapter->num_queues; i++) { 2107 rxr = &adapter->rx_rings[i]; 2108 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 2109 if (hwtagging) 2110 ctrl |= IXGBE_RXDCTL_VME; 2111 else 2112 ctrl &= ~IXGBE_RXDCTL_VME; 2113 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 2114 /* 2115 * Let Rx path know that it needs to store VLAN tag 2116 * as part of extra mbuf info. 2117 */ 2118 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2119 } 2120 } /* ixv_setup_vlan_tagging */ 2121 2122 /************************************************************************ 2123 * ixv_setup_vlan_support 2124 ************************************************************************/ 2125 static int 2126 ixv_setup_vlan_support(struct adapter *adapter) 2127 { 2128 struct ethercom *ec = &adapter->osdep.ec; 2129 struct ixgbe_hw *hw = &adapter->hw; 2130 u32 vid, vfta, retry; 2131 struct vlanid_list *vlanidp; 2132 int rv, error = 0; 2133 2134 /* 2135 * This function is called from both if_init and ifflags_cb() 2136 * on NetBSD. 2137 */ 2138 2139 /* 2140 * Part 1: 2141 * Setup VLAN HW tagging 2142 */ 2143 ixv_setup_vlan_tagging(adapter); 2144 2145 if (!VLAN_ATTACHED(ec)) 2146 return 0; 2147 2148 /* 2149 * Part 2: 2150 * Setup VLAN HW filter 2151 */ 2152 /* Cleanup shadow_vfta */ 2153 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 2154 adapter->shadow_vfta[i] = 0; 2155 /* Generate shadow_vfta from ec_vids */ 2156 ETHER_LOCK(ec); 2157 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2158 uint32_t idx; 2159 2160 idx = vlanidp->vid / 32; 2161 KASSERT(idx < IXGBE_VFTA_SIZE); 2162 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2163 } 2164 ETHER_UNLOCK(ec); 2165 2166 /* 2167 * A soft reset zero's out the VFTA, so 2168 * we need to repopulate it now. 2169 */ 2170 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2171 if (adapter->shadow_vfta[i] == 0) 2172 continue; 2173 vfta = adapter->shadow_vfta[i]; 2174 /* 2175 * Reconstruct the vlan id's 2176 * based on the bits set in each 2177 * of the array ints. 2178 */ 2179 for (int j = 0; j < 32; j++) { 2180 retry = 0; 2181 if ((vfta & ((u32)1 << j)) == 0) 2182 continue; 2183 vid = (i * 32) + j; 2184 2185 /* Call the shared code mailbox routine */ 2186 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE, 2187 FALSE)) != 0) { 2188 if (++retry > 5) { 2189 device_printf(adapter->dev, 2190 "%s: max retry exceeded\n", 2191 __func__); 2192 break; 2193 } 2194 } 2195 if (rv != 0) { 2196 device_printf(adapter->dev, 2197 "failed to set vlan %d\n", vid); 2198 error = EACCES; 2199 } 2200 } 2201 } 2202 return error; 2203 } /* ixv_setup_vlan_support */ 2204 2205 static int 2206 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2207 { 2208 struct ifnet *ifp = &ec->ec_if; 2209 struct adapter *adapter = ifp->if_softc; 2210 int rv; 2211 2212 if (set) 2213 rv = ixv_register_vlan(adapter, vid); 2214 else 2215 rv = ixv_unregister_vlan(adapter, vid); 2216 2217 if (rv != 0) 2218 return rv; 2219 2220 /* 2221 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2222 * or 0 to 1. 2223 */ 2224 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2225 ixv_setup_vlan_tagging(adapter); 2226 2227 return rv; 2228 } 2229 2230 /************************************************************************ 2231 * ixv_register_vlan 2232 * 2233 * Run via a vlan config EVENT, it enables us to use the 2234 * HW Filter table since we can get the vlan id. This just 2235 * creates the entry in the soft version of the VFTA, init 2236 * will repopulate the real table. 2237 ************************************************************************/ 2238 static int 2239 ixv_register_vlan(struct adapter *adapter, u16 vtag) 2240 { 2241 struct ixgbe_hw *hw = &adapter->hw; 2242 u16 index, bit; 2243 int error; 2244 2245 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2246 return EINVAL; 2247 IXGBE_CORE_LOCK(adapter); 2248 index = (vtag >> 5) & 0x7F; 2249 bit = vtag & 0x1F; 2250 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2251 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false); 2252 IXGBE_CORE_UNLOCK(adapter); 2253 2254 if (error != 0) { 2255 device_printf(adapter->dev, "failed to register vlan %hu\n", 2256 vtag); 2257 error = EACCES; 2258 } 2259 return error; 2260 } /* ixv_register_vlan */ 2261 2262 /************************************************************************ 2263 * ixv_unregister_vlan 2264 * 2265 * Run via a vlan unconfig EVENT, remove our entry 2266 * in the soft vfta. 2267 ************************************************************************/ 2268 static int 2269 ixv_unregister_vlan(struct adapter *adapter, u16 vtag) 2270 { 2271 struct ixgbe_hw *hw = &adapter->hw; 2272 u16 index, bit; 2273 int error; 2274 2275 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2276 return EINVAL; 2277 2278 IXGBE_CORE_LOCK(adapter); 2279 index = (vtag >> 5) & 0x7F; 2280 bit = vtag & 0x1F; 2281 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2282 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false); 2283 IXGBE_CORE_UNLOCK(adapter); 2284 2285 if (error != 0) { 2286 device_printf(adapter->dev, "failed to unregister vlan %hu\n", 2287 vtag); 2288 error = EIO; 2289 } 2290 return error; 2291 } /* ixv_unregister_vlan */ 2292 2293 /************************************************************************ 2294 * ixv_enable_intr 2295 ************************************************************************/ 2296 static void 2297 ixv_enable_intr(struct adapter *adapter) 2298 { 2299 struct ixgbe_hw *hw = &adapter->hw; 2300 struct ix_queue *que = adapter->queues; 2301 u32 mask; 2302 int i; 2303 2304 /* For VTEIAC */ 2305 mask = (1 << adapter->vector); 2306 for (i = 0; i < adapter->num_queues; i++, que++) 2307 mask |= (1 << que->msix); 2308 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2309 2310 /* For VTEIMS */ 2311 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 2312 que = adapter->queues; 2313 for (i = 0; i < adapter->num_queues; i++, que++) 2314 ixv_enable_queue(adapter, que->msix); 2315 2316 IXGBE_WRITE_FLUSH(hw); 2317 } /* ixv_enable_intr */ 2318 2319 /************************************************************************ 2320 * ixv_disable_intr 2321 ************************************************************************/ 2322 static void 2323 ixv_disable_intr(struct adapter *adapter) 2324 { 2325 struct ix_queue *que = adapter->queues; 2326 2327 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 2328 2329 /* disable interrupts other than queues */ 2330 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, adapter->vector); 2331 2332 for (int i = 0; i < adapter->num_queues; i++, que++) 2333 ixv_disable_queue(adapter, que->msix); 2334 2335 IXGBE_WRITE_FLUSH(&adapter->hw); 2336 } /* ixv_disable_intr */ 2337 2338 /************************************************************************ 2339 * ixv_set_ivar 2340 * 2341 * Setup the correct IVAR register for a particular MSI-X interrupt 2342 * - entry is the register array entry 2343 * - vector is the MSI-X vector for this queue 2344 * - type is RX/TX/MISC 2345 ************************************************************************/ 2346 static void 2347 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 2348 { 2349 struct ixgbe_hw *hw = &adapter->hw; 2350 u32 ivar, index; 2351 2352 vector |= IXGBE_IVAR_ALLOC_VAL; 2353 2354 if (type == -1) { /* MISC IVAR */ 2355 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2356 ivar &= ~0xFF; 2357 ivar |= vector; 2358 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2359 } else { /* RX/TX IVARS */ 2360 index = (16 * (entry & 1)) + (8 * type); 2361 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2362 ivar &= ~(0xffUL << index); 2363 ivar |= ((u32)vector << index); 2364 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2365 } 2366 } /* ixv_set_ivar */ 2367 2368 /************************************************************************ 2369 * ixv_configure_ivars 2370 ************************************************************************/ 2371 static void 2372 ixv_configure_ivars(struct adapter *adapter) 2373 { 2374 struct ix_queue *que = adapter->queues; 2375 2376 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2377 2378 for (int i = 0; i < adapter->num_queues; i++, que++) { 2379 /* First the RX queue entry */ 2380 ixv_set_ivar(adapter, i, que->msix, 0); 2381 /* ... and the TX */ 2382 ixv_set_ivar(adapter, i, que->msix, 1); 2383 /* Set an initial value in EITR */ 2384 ixv_eitr_write(adapter, que->msix, IXGBE_EITR_DEFAULT); 2385 } 2386 2387 /* For the mailbox interrupt */ 2388 ixv_set_ivar(adapter, 1, adapter->vector, -1); 2389 } /* ixv_configure_ivars */ 2390 2391 2392 /************************************************************************ 2393 * ixv_save_stats 2394 * 2395 * The VF stats registers never have a truly virgin 2396 * starting point, so this routine tries to make an 2397 * artificial one, marking ground zero on attach as 2398 * it were. 2399 ************************************************************************/ 2400 static void 2401 ixv_save_stats(struct adapter *adapter) 2402 { 2403 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2404 2405 if (stats->vfgprc.ev_count || stats->vfgptc.ev_count) { 2406 stats->saved_reset_vfgprc += 2407 stats->vfgprc.ev_count - stats->base_vfgprc; 2408 stats->saved_reset_vfgptc += 2409 stats->vfgptc.ev_count - stats->base_vfgptc; 2410 stats->saved_reset_vfgorc += 2411 stats->vfgorc.ev_count - stats->base_vfgorc; 2412 stats->saved_reset_vfgotc += 2413 stats->vfgotc.ev_count - stats->base_vfgotc; 2414 stats->saved_reset_vfmprc += 2415 stats->vfmprc.ev_count - stats->base_vfmprc; 2416 } 2417 } /* ixv_save_stats */ 2418 2419 /************************************************************************ 2420 * ixv_init_stats 2421 ************************************************************************/ 2422 static void 2423 ixv_init_stats(struct adapter *adapter) 2424 { 2425 struct ixgbe_hw *hw = &adapter->hw; 2426 2427 adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2428 adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2429 adapter->stats.vf.last_vfgorc |= 2430 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2431 2432 adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2433 adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2434 adapter->stats.vf.last_vfgotc |= 2435 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2436 2437 adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2438 2439 adapter->stats.vf.base_vfgprc = adapter->stats.vf.last_vfgprc; 2440 adapter->stats.vf.base_vfgorc = adapter->stats.vf.last_vfgorc; 2441 adapter->stats.vf.base_vfgptc = adapter->stats.vf.last_vfgptc; 2442 adapter->stats.vf.base_vfgotc = adapter->stats.vf.last_vfgotc; 2443 adapter->stats.vf.base_vfmprc = adapter->stats.vf.last_vfmprc; 2444 } /* ixv_init_stats */ 2445 2446 #define UPDATE_STAT_32(reg, last, count) \ 2447 { \ 2448 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2449 if (current < (last)) \ 2450 count.ev_count += 0x100000000LL; \ 2451 (last) = current; \ 2452 count.ev_count &= 0xFFFFFFFF00000000LL; \ 2453 count.ev_count |= current; \ 2454 } 2455 2456 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2457 { \ 2458 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2459 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2460 u64 current = ((cur_msb << 32) | cur_lsb); \ 2461 if (current < (last)) \ 2462 count.ev_count += 0x1000000000LL; \ 2463 (last) = current; \ 2464 count.ev_count &= 0xFFFFFFF000000000LL; \ 2465 count.ev_count |= current; \ 2466 } 2467 2468 /************************************************************************ 2469 * ixv_update_stats - Update the board statistics counters. 2470 ************************************************************************/ 2471 void 2472 ixv_update_stats(struct adapter *adapter) 2473 { 2474 struct ixgbe_hw *hw = &adapter->hw; 2475 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2476 2477 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2478 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2479 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2480 stats->vfgorc); 2481 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2482 stats->vfgotc); 2483 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2484 2485 /* VF doesn't count errors by hardware */ 2486 2487 } /* ixv_update_stats */ 2488 2489 /************************************************************************ 2490 * ixv_sysctl_interrupt_rate_handler 2491 ************************************************************************/ 2492 static int 2493 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2494 { 2495 struct sysctlnode node = *rnode; 2496 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2497 struct adapter *adapter = que->adapter; 2498 uint32_t reg, usec, rate; 2499 int error; 2500 2501 if (que == NULL) 2502 return 0; 2503 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_VTEITR(que->msix)); 2504 usec = ((reg & 0x0FF8) >> 3); 2505 if (usec > 0) 2506 rate = 500000 / usec; 2507 else 2508 rate = 0; 2509 node.sysctl_data = &rate; 2510 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2511 if (error || newp == NULL) 2512 return error; 2513 reg &= ~0xfff; /* default, no limitation */ 2514 if (rate > 0 && rate < 500000) { 2515 if (rate < 1000) 2516 rate = 1000; 2517 reg |= ((4000000 / rate) & 0xff8); 2518 /* 2519 * When RSC is used, ITR interval must be larger than 2520 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2521 * The minimum value is always greater than 2us on 100M 2522 * (and 10M?(not documented)), but it's not on 1G and higher. 2523 */ 2524 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2525 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2526 if ((adapter->num_queues > 1) 2527 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2528 return EINVAL; 2529 } 2530 ixv_max_interrupt_rate = rate; 2531 } else 2532 ixv_max_interrupt_rate = 0; 2533 ixv_eitr_write(adapter, que->msix, reg); 2534 2535 return (0); 2536 } /* ixv_sysctl_interrupt_rate_handler */ 2537 2538 const struct sysctlnode * 2539 ixv_sysctl_instance(struct adapter *adapter) 2540 { 2541 const char *dvname; 2542 struct sysctllog **log; 2543 int rc; 2544 const struct sysctlnode *rnode; 2545 2546 log = &adapter->sysctllog; 2547 dvname = device_xname(adapter->dev); 2548 2549 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2550 0, CTLTYPE_NODE, dvname, 2551 SYSCTL_DESCR("ixv information and settings"), 2552 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2553 goto err; 2554 2555 return rnode; 2556 err: 2557 device_printf(adapter->dev, 2558 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2559 return NULL; 2560 } 2561 2562 static void 2563 ixv_add_device_sysctls(struct adapter *adapter) 2564 { 2565 struct sysctllog **log; 2566 const struct sysctlnode *rnode, *cnode; 2567 device_t dev; 2568 2569 dev = adapter->dev; 2570 log = &adapter->sysctllog; 2571 2572 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2573 aprint_error_dev(dev, "could not create sysctl root\n"); 2574 return; 2575 } 2576 2577 if (sysctl_createv(log, 0, &rnode, &cnode, 2578 CTLFLAG_READWRITE, CTLTYPE_INT, "debug", 2579 SYSCTL_DESCR("Debug Info"), 2580 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2581 aprint_error_dev(dev, "could not create sysctl\n"); 2582 2583 if (sysctl_createv(log, 0, &rnode, &cnode, 2584 CTLFLAG_READWRITE, CTLTYPE_INT, 2585 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 2586 ixv_sysctl_rx_copy_len, 0, 2587 (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2588 aprint_error_dev(dev, "could not create sysctl\n"); 2589 2590 if (sysctl_createv(log, 0, &rnode, &cnode, 2591 CTLFLAG_READWRITE, CTLTYPE_BOOL, "enable_aim", 2592 SYSCTL_DESCR("Interrupt Moderation"), 2593 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2594 aprint_error_dev(dev, "could not create sysctl\n"); 2595 2596 if (sysctl_createv(log, 0, &rnode, &cnode, 2597 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 2598 SYSCTL_DESCR("Use workqueue for packet processing"), 2599 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) 2600 != 0) 2601 aprint_error_dev(dev, "could not create sysctl\n"); 2602 } 2603 2604 /************************************************************************ 2605 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2606 ************************************************************************/ 2607 static void 2608 ixv_add_stats_sysctls(struct adapter *adapter) 2609 { 2610 device_t dev = adapter->dev; 2611 struct tx_ring *txr = adapter->tx_rings; 2612 struct rx_ring *rxr = adapter->rx_rings; 2613 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2614 struct ixgbe_hw *hw = &adapter->hw; 2615 const struct sysctlnode *rnode, *cnode; 2616 struct sysctllog **log = &adapter->sysctllog; 2617 const char *xname = device_xname(dev); 2618 2619 /* Driver Statistics */ 2620 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2621 NULL, xname, "Driver tx dma soft fail EFBIG"); 2622 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2623 NULL, xname, "m_defrag() failed"); 2624 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2625 NULL, xname, "Driver tx dma hard fail EFBIG"); 2626 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2627 NULL, xname, "Driver tx dma hard fail EINVAL"); 2628 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 2629 NULL, xname, "Driver tx dma hard fail other"); 2630 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2631 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2632 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2633 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2634 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 2635 NULL, xname, "Watchdog timeouts"); 2636 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 2637 NULL, xname, "TSO errors"); 2638 evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, 2639 NULL, xname, "Admin MSI-X IRQ Handled"); 2640 evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, 2641 NULL, xname, "Admin event"); 2642 2643 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2644 snprintf(adapter->queues[i].evnamebuf, 2645 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 2646 xname, i); 2647 snprintf(adapter->queues[i].namebuf, 2648 sizeof(adapter->queues[i].namebuf), "q%d", i); 2649 2650 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2651 aprint_error_dev(dev, "could not create sysctl root\n"); 2652 break; 2653 } 2654 2655 if (sysctl_createv(log, 0, &rnode, &rnode, 2656 0, CTLTYPE_NODE, 2657 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2658 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2659 break; 2660 2661 if (sysctl_createv(log, 0, &rnode, &cnode, 2662 CTLFLAG_READWRITE, CTLTYPE_INT, 2663 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2664 ixv_sysctl_interrupt_rate_handler, 0, 2665 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2666 break; 2667 2668 if (sysctl_createv(log, 0, &rnode, &cnode, 2669 CTLFLAG_READONLY, CTLTYPE_INT, 2670 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2671 ixv_sysctl_tdh_handler, 0, (void *)txr, 2672 0, CTL_CREATE, CTL_EOL) != 0) 2673 break; 2674 2675 if (sysctl_createv(log, 0, &rnode, &cnode, 2676 CTLFLAG_READONLY, CTLTYPE_INT, 2677 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2678 ixv_sysctl_tdt_handler, 0, (void *)txr, 2679 0, CTL_CREATE, CTL_EOL) != 0) 2680 break; 2681 2682 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 2683 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 2684 evcnt_attach_dynamic(&adapter->queues[i].handleq, 2685 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 2686 "Handled queue in softint"); 2687 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 2688 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 2689 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2690 NULL, adapter->queues[i].evnamebuf, "TSO"); 2691 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2692 NULL, adapter->queues[i].evnamebuf, 2693 "TX Queue No Descriptor Available"); 2694 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2695 NULL, adapter->queues[i].evnamebuf, 2696 "Queue Packets Transmitted"); 2697 #ifndef IXGBE_LEGACY_TX 2698 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2699 NULL, adapter->queues[i].evnamebuf, 2700 "Packets dropped in pcq"); 2701 #endif 2702 2703 #ifdef LRO 2704 struct lro_ctrl *lro = &rxr->lro; 2705 #endif /* LRO */ 2706 2707 if (sysctl_createv(log, 0, &rnode, &cnode, 2708 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 2709 SYSCTL_DESCR("Receive Descriptor next to check"), 2710 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 2711 CTL_CREATE, CTL_EOL) != 0) 2712 break; 2713 2714 if (sysctl_createv(log, 0, &rnode, &cnode, 2715 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 2716 SYSCTL_DESCR("Receive Descriptor next to refresh"), 2717 ixv_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 2718 CTL_CREATE, CTL_EOL) != 0) 2719 break; 2720 2721 if (sysctl_createv(log, 0, &rnode, &cnode, 2722 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 2723 SYSCTL_DESCR("Receive Descriptor Head"), 2724 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2725 CTL_CREATE, CTL_EOL) != 0) 2726 break; 2727 2728 if (sysctl_createv(log, 0, &rnode, &cnode, 2729 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 2730 SYSCTL_DESCR("Receive Descriptor Tail"), 2731 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2732 CTL_CREATE, CTL_EOL) != 0) 2733 break; 2734 2735 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2736 NULL, adapter->queues[i].evnamebuf, 2737 "Queue Packets Received"); 2738 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2739 NULL, adapter->queues[i].evnamebuf, 2740 "Queue Bytes Received"); 2741 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2742 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 2743 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 2744 NULL, adapter->queues[i].evnamebuf, "Rx no mbuf"); 2745 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2746 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 2747 #ifdef LRO 2748 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2749 CTLFLAG_RD, &lro->lro_queued, 0, 2750 "LRO Queued"); 2751 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2752 CTLFLAG_RD, &lro->lro_flushed, 0, 2753 "LRO Flushed"); 2754 #endif /* LRO */ 2755 } 2756 2757 /* MAC stats get their own sub node */ 2758 2759 snprintf(stats->namebuf, 2760 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2761 2762 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2763 stats->namebuf, "rx csum offload - IP"); 2764 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2765 stats->namebuf, "rx csum offload - L4"); 2766 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2767 stats->namebuf, "rx csum offload - IP bad"); 2768 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2769 stats->namebuf, "rx csum offload - L4 bad"); 2770 2771 /* Packet Reception Stats */ 2772 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2773 xname, "Good Packets Received"); 2774 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2775 xname, "Good Octets Received"); 2776 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2777 xname, "Multicast Packets Received"); 2778 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2779 xname, "Good Packets Transmitted"); 2780 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2781 xname, "Good Octets Transmitted"); 2782 2783 /* Mailbox Stats */ 2784 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2785 xname, "message TXs"); 2786 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2787 xname, "message RXs"); 2788 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2789 xname, "ACKs"); 2790 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2791 xname, "REQs"); 2792 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2793 xname, "RSTs"); 2794 2795 } /* ixv_add_stats_sysctls */ 2796 2797 static void 2798 ixv_clear_evcnt(struct adapter *adapter) 2799 { 2800 struct tx_ring *txr = adapter->tx_rings; 2801 struct rx_ring *rxr = adapter->rx_rings; 2802 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2803 struct ixgbe_hw *hw = &adapter->hw; 2804 int i; 2805 2806 /* Driver Statistics */ 2807 adapter->efbig_tx_dma_setup.ev_count = 0; 2808 adapter->mbuf_defrag_failed.ev_count = 0; 2809 adapter->efbig2_tx_dma_setup.ev_count = 0; 2810 adapter->einval_tx_dma_setup.ev_count = 0; 2811 adapter->other_tx_dma_setup.ev_count = 0; 2812 adapter->eagain_tx_dma_setup.ev_count = 0; 2813 adapter->enomem_tx_dma_setup.ev_count = 0; 2814 adapter->watchdog_events.ev_count = 0; 2815 adapter->tso_err.ev_count = 0; 2816 adapter->admin_irqev.ev_count = 0; 2817 adapter->link_workev.ev_count = 0; 2818 2819 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2820 adapter->queues[i].irqs.ev_count = 0; 2821 adapter->queues[i].handleq.ev_count = 0; 2822 adapter->queues[i].req.ev_count = 0; 2823 txr->tso_tx.ev_count = 0; 2824 txr->no_desc_avail.ev_count = 0; 2825 txr->total_packets.ev_count = 0; 2826 #ifndef IXGBE_LEGACY_TX 2827 txr->pcq_drops.ev_count = 0; 2828 #endif 2829 txr->q_efbig_tx_dma_setup = 0; 2830 txr->q_mbuf_defrag_failed = 0; 2831 txr->q_efbig2_tx_dma_setup = 0; 2832 txr->q_einval_tx_dma_setup = 0; 2833 txr->q_other_tx_dma_setup = 0; 2834 txr->q_eagain_tx_dma_setup = 0; 2835 txr->q_enomem_tx_dma_setup = 0; 2836 txr->q_tso_err = 0; 2837 2838 rxr->rx_packets.ev_count = 0; 2839 rxr->rx_bytes.ev_count = 0; 2840 rxr->rx_copies.ev_count = 0; 2841 rxr->no_mbuf.ev_count = 0; 2842 rxr->rx_discarded.ev_count = 0; 2843 } 2844 2845 /* MAC stats get their own sub node */ 2846 2847 stats->ipcs.ev_count = 0; 2848 stats->l4cs.ev_count = 0; 2849 stats->ipcs_bad.ev_count = 0; 2850 stats->l4cs_bad.ev_count = 0; 2851 2852 /* Packet Reception Stats */ 2853 stats->vfgprc.ev_count = 0; 2854 stats->vfgorc.ev_count = 0; 2855 stats->vfmprc.ev_count = 0; 2856 stats->vfgptc.ev_count = 0; 2857 stats->vfgotc.ev_count = 0; 2858 2859 /* Mailbox Stats */ 2860 hw->mbx.stats.msgs_tx.ev_count = 0; 2861 hw->mbx.stats.msgs_rx.ev_count = 0; 2862 hw->mbx.stats.acks.ev_count = 0; 2863 hw->mbx.stats.reqs.ev_count = 0; 2864 hw->mbx.stats.rsts.ev_count = 0; 2865 2866 } /* ixv_clear_evcnt */ 2867 2868 /************************************************************************ 2869 * ixv_set_sysctl_value 2870 ************************************************************************/ 2871 static void 2872 ixv_set_sysctl_value(struct adapter *adapter, const char *name, 2873 const char *description, int *limit, int value) 2874 { 2875 device_t dev = adapter->dev; 2876 struct sysctllog **log; 2877 const struct sysctlnode *rnode, *cnode; 2878 2879 log = &adapter->sysctllog; 2880 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2881 aprint_error_dev(dev, "could not create sysctl root\n"); 2882 return; 2883 } 2884 if (sysctl_createv(log, 0, &rnode, &cnode, 2885 CTLFLAG_READWRITE, CTLTYPE_INT, 2886 name, SYSCTL_DESCR(description), 2887 NULL, 0, limit, 0, CTL_CREATE, CTL_EOL) != 0) 2888 aprint_error_dev(dev, "could not create sysctl\n"); 2889 *limit = value; 2890 } /* ixv_set_sysctl_value */ 2891 2892 /************************************************************************ 2893 * ixv_print_debug_info 2894 * 2895 * Called only when em_display_debug_stats is enabled. 2896 * Provides a way to take a look at important statistics 2897 * maintained by the driver and hardware. 2898 ************************************************************************/ 2899 static void 2900 ixv_print_debug_info(struct adapter *adapter) 2901 { 2902 device_t dev = adapter->dev; 2903 struct ix_queue *que = adapter->queues; 2904 struct rx_ring *rxr; 2905 struct tx_ring *txr; 2906 #ifdef LRO 2907 struct lro_ctrl *lro; 2908 #endif /* LRO */ 2909 2910 for (int i = 0; i < adapter->num_queues; i++, que++) { 2911 txr = que->txr; 2912 rxr = que->rxr; 2913 #ifdef LRO 2914 lro = &rxr->lro; 2915 #endif /* LRO */ 2916 device_printf(dev, "QUE(%d) IRQs Handled: %lu\n", 2917 que->msix, (long)que->irqs.ev_count); 2918 device_printf(dev, "RX(%d) Packets Received: %lld\n", 2919 rxr->me, (long long)rxr->rx_packets.ev_count); 2920 device_printf(dev, "RX(%d) Bytes Received: %lu\n", 2921 rxr->me, (long)rxr->rx_bytes.ev_count); 2922 #ifdef LRO 2923 device_printf(dev, "RX(%d) LRO Queued= %ju\n", 2924 rxr->me, (uintmax_t)lro->lro_queued); 2925 device_printf(dev, "RX(%d) LRO Flushed= %ju\n", 2926 rxr->me, (uintmax_t)lro->lro_flushed); 2927 #endif /* LRO */ 2928 device_printf(dev, "TX(%d) Packets Sent: %lu\n", 2929 txr->me, (long)txr->total_packets.ev_count); 2930 device_printf(dev, "TX(%d) NO Desc Avail: %lu\n", 2931 txr->me, (long)txr->no_desc_avail.ev_count); 2932 } 2933 2934 device_printf(dev, "Admin IRQ Handled: %lu\n", 2935 (long)adapter->admin_irqev.ev_count); 2936 device_printf(dev, "Admin work Handled: %lu\n", 2937 (long)adapter->link_workev.ev_count); 2938 } /* ixv_print_debug_info */ 2939 2940 /************************************************************************ 2941 * ixv_sysctl_debug 2942 ************************************************************************/ 2943 static int 2944 ixv_sysctl_debug(SYSCTLFN_ARGS) 2945 { 2946 struct sysctlnode node = *rnode; 2947 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2948 int error, result; 2949 2950 node.sysctl_data = &result; 2951 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2952 2953 if (error || newp == NULL) 2954 return error; 2955 2956 if (result == 1) 2957 ixv_print_debug_info(adapter); 2958 2959 return 0; 2960 } /* ixv_sysctl_debug */ 2961 2962 /************************************************************************ 2963 * ixv_sysctl_rx_copy_len 2964 ************************************************************************/ 2965 static int 2966 ixv_sysctl_rx_copy_len(SYSCTLFN_ARGS) 2967 { 2968 struct sysctlnode node = *rnode; 2969 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2970 int error; 2971 int result = adapter->rx_copy_len; 2972 2973 node.sysctl_data = &result; 2974 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2975 2976 if (error || newp == NULL) 2977 return error; 2978 2979 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 2980 return EINVAL; 2981 2982 adapter->rx_copy_len = result; 2983 2984 return 0; 2985 } /* ixgbe_sysctl_rx_copy_len */ 2986 2987 /************************************************************************ 2988 * ixv_init_device_features 2989 ************************************************************************/ 2990 static void 2991 ixv_init_device_features(struct adapter *adapter) 2992 { 2993 adapter->feat_cap = IXGBE_FEATURE_NETMAP 2994 | IXGBE_FEATURE_VF 2995 | IXGBE_FEATURE_RSS 2996 | IXGBE_FEATURE_LEGACY_TX; 2997 2998 /* A tad short on feature flags for VFs, atm. */ 2999 switch (adapter->hw.mac.type) { 3000 case ixgbe_mac_82599_vf: 3001 break; 3002 case ixgbe_mac_X540_vf: 3003 break; 3004 case ixgbe_mac_X550_vf: 3005 case ixgbe_mac_X550EM_x_vf: 3006 case ixgbe_mac_X550EM_a_vf: 3007 adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 3008 break; 3009 default: 3010 break; 3011 } 3012 3013 /* Enabled by default... */ 3014 /* Is a virtual function (VF) */ 3015 if (adapter->feat_cap & IXGBE_FEATURE_VF) 3016 adapter->feat_en |= IXGBE_FEATURE_VF; 3017 /* Netmap */ 3018 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 3019 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 3020 /* Receive-Side Scaling (RSS) */ 3021 if (adapter->feat_cap & IXGBE_FEATURE_RSS) 3022 adapter->feat_en |= IXGBE_FEATURE_RSS; 3023 /* Needs advanced context descriptor regardless of offloads req'd */ 3024 if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 3025 adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 3026 3027 /* Enabled via sysctl... */ 3028 /* Legacy (single queue) transmit */ 3029 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 3030 ixv_enable_legacy_tx) 3031 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 3032 } /* ixv_init_device_features */ 3033 3034 /************************************************************************ 3035 * ixv_shutdown - Shutdown entry point 3036 ************************************************************************/ 3037 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3038 static int 3039 ixv_shutdown(device_t dev) 3040 { 3041 struct adapter *adapter = device_private(dev); 3042 IXGBE_CORE_LOCK(adapter); 3043 ixv_stop_locked(adapter); 3044 IXGBE_CORE_UNLOCK(adapter); 3045 3046 return (0); 3047 } /* ixv_shutdown */ 3048 #endif 3049 3050 static int 3051 ixv_ifflags_cb(struct ethercom *ec) 3052 { 3053 struct ifnet *ifp = &ec->ec_if; 3054 struct adapter *adapter = ifp->if_softc; 3055 u_short saved_flags; 3056 u_short change; 3057 int rv = 0; 3058 3059 IXGBE_CORE_LOCK(adapter); 3060 3061 saved_flags = adapter->if_flags; 3062 change = ifp->if_flags ^ adapter->if_flags; 3063 if (change != 0) 3064 adapter->if_flags = ifp->if_flags; 3065 3066 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 3067 rv = ENETRESET; 3068 goto out; 3069 } else if ((change & IFF_PROMISC) != 0) { 3070 rv = ixv_set_rxfilter(adapter); 3071 if (rv != 0) { 3072 /* Restore previous */ 3073 adapter->if_flags = saved_flags; 3074 goto out; 3075 } 3076 } 3077 3078 /* Check for ec_capenable. */ 3079 change = ec->ec_capenable ^ adapter->ec_capenable; 3080 adapter->ec_capenable = ec->ec_capenable; 3081 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 3082 | ETHERCAP_VLAN_HWFILTER)) != 0) { 3083 rv = ENETRESET; 3084 goto out; 3085 } 3086 3087 /* 3088 * Special handling is not required for ETHERCAP_VLAN_MTU. 3089 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 3090 */ 3091 3092 /* Set up VLAN support and filter */ 3093 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 3094 rv = ixv_setup_vlan_support(adapter); 3095 3096 out: 3097 IXGBE_CORE_UNLOCK(adapter); 3098 3099 return rv; 3100 } 3101 3102 3103 /************************************************************************ 3104 * ixv_ioctl - Ioctl entry point 3105 * 3106 * Called when the user wants to configure the interface. 3107 * 3108 * return 0 on success, positive on failure 3109 ************************************************************************/ 3110 static int 3111 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 3112 { 3113 struct adapter *adapter = ifp->if_softc; 3114 struct ixgbe_hw *hw = &adapter->hw; 3115 struct ifcapreq *ifcr = data; 3116 int error; 3117 int l4csum_en; 3118 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3119 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 3120 3121 switch (command) { 3122 case SIOCSIFFLAGS: 3123 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 3124 break; 3125 case SIOCADDMULTI: { 3126 struct ether_multi *enm; 3127 struct ether_multistep step; 3128 struct ethercom *ec = &adapter->osdep.ec; 3129 bool overflow = false; 3130 int mcnt = 0; 3131 3132 /* 3133 * Check the number of multicast address. If it exceeds, 3134 * return ENOSPC. 3135 * Update this code when we support API 1.3. 3136 */ 3137 ETHER_LOCK(ec); 3138 ETHER_FIRST_MULTI(step, ec, enm); 3139 while (enm != NULL) { 3140 mcnt++; 3141 3142 /* 3143 * This code is before adding, so one room is required 3144 * at least. 3145 */ 3146 if (mcnt > (IXGBE_MAX_VF_MC - 1)) { 3147 overflow = true; 3148 break; 3149 } 3150 ETHER_NEXT_MULTI(step, enm); 3151 } 3152 ETHER_UNLOCK(ec); 3153 error = 0; 3154 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) { 3155 error = hw->mac.ops.update_xcast_mode(hw, 3156 IXGBEVF_XCAST_MODE_ALLMULTI); 3157 if (error == IXGBE_ERR_NOT_TRUSTED) { 3158 device_printf(adapter->dev, 3159 "this interface is not trusted\n"); 3160 error = EPERM; 3161 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 3162 device_printf(adapter->dev, 3163 "the PF doesn't support allmulti mode\n"); 3164 error = EOPNOTSUPP; 3165 } else if (error) { 3166 device_printf(adapter->dev, 3167 "number of Ethernet multicast addresses " 3168 "exceeds the limit (%d). error = %d\n", 3169 IXGBE_MAX_VF_MC, error); 3170 error = ENOSPC; 3171 } else 3172 ec->ec_flags |= ETHER_F_ALLMULTI; 3173 } 3174 if (error) 3175 return error; 3176 } 3177 /*FALLTHROUGH*/ 3178 case SIOCDELMULTI: 3179 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 3180 break; 3181 case SIOCSIFMEDIA: 3182 case SIOCGIFMEDIA: 3183 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 3184 break; 3185 case SIOCSIFCAP: 3186 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 3187 break; 3188 case SIOCSIFMTU: 3189 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 3190 break; 3191 case SIOCZIFDATA: 3192 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 3193 ixv_update_stats(adapter); 3194 ixv_clear_evcnt(adapter); 3195 break; 3196 default: 3197 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 3198 break; 3199 } 3200 3201 switch (command) { 3202 case SIOCSIFCAP: 3203 /* Layer-4 Rx checksum offload has to be turned on and 3204 * off as a unit. 3205 */ 3206 l4csum_en = ifcr->ifcr_capenable & l4csum; 3207 if (l4csum_en != l4csum && l4csum_en != 0) 3208 return EINVAL; 3209 /*FALLTHROUGH*/ 3210 case SIOCADDMULTI: 3211 case SIOCDELMULTI: 3212 case SIOCSIFFLAGS: 3213 case SIOCSIFMTU: 3214 default: 3215 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 3216 return error; 3217 if ((ifp->if_flags & IFF_RUNNING) == 0) 3218 ; 3219 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 3220 IXGBE_CORE_LOCK(adapter); 3221 ixv_init_locked(adapter); 3222 IXGBE_CORE_UNLOCK(adapter); 3223 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 3224 /* 3225 * Multicast list has changed; set the hardware filter 3226 * accordingly. 3227 */ 3228 IXGBE_CORE_LOCK(adapter); 3229 ixv_disable_intr(adapter); 3230 ixv_set_rxfilter(adapter); 3231 ixv_enable_intr(adapter); 3232 IXGBE_CORE_UNLOCK(adapter); 3233 } 3234 return 0; 3235 } 3236 } /* ixv_ioctl */ 3237 3238 /************************************************************************ 3239 * ixv_init 3240 ************************************************************************/ 3241 static int 3242 ixv_init(struct ifnet *ifp) 3243 { 3244 struct adapter *adapter = ifp->if_softc; 3245 3246 IXGBE_CORE_LOCK(adapter); 3247 ixv_init_locked(adapter); 3248 IXGBE_CORE_UNLOCK(adapter); 3249 3250 return 0; 3251 } /* ixv_init */ 3252 3253 /************************************************************************ 3254 * ixv_handle_que 3255 ************************************************************************/ 3256 static void 3257 ixv_handle_que(void *context) 3258 { 3259 struct ix_queue *que = context; 3260 struct adapter *adapter = que->adapter; 3261 struct tx_ring *txr = que->txr; 3262 struct ifnet *ifp = adapter->ifp; 3263 bool more; 3264 3265 que->handleq.ev_count++; 3266 3267 if (ifp->if_flags & IFF_RUNNING) { 3268 more = ixgbe_rxeof(que); 3269 IXGBE_TX_LOCK(txr); 3270 more |= ixgbe_txeof(txr); 3271 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 3272 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 3273 ixgbe_mq_start_locked(ifp, txr); 3274 /* Only for queue 0 */ 3275 /* NetBSD still needs this for CBQ */ 3276 if ((&adapter->queues[0] == que) 3277 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 3278 ixgbe_legacy_start_locked(ifp, txr); 3279 IXGBE_TX_UNLOCK(txr); 3280 if (more) { 3281 que->req.ev_count++; 3282 if (adapter->txrx_use_workqueue) { 3283 /* 3284 * "enqueued flag" is not required here 3285 * the same as ixg(4). See ixgbe_msix_que(). 3286 */ 3287 workqueue_enqueue(adapter->que_wq, 3288 &que->wq_cookie, curcpu()); 3289 } else 3290 softint_schedule(que->que_si); 3291 return; 3292 } 3293 } 3294 3295 /* Re-enable this interrupt */ 3296 ixv_enable_queue(adapter, que->msix); 3297 3298 return; 3299 } /* ixv_handle_que */ 3300 3301 /************************************************************************ 3302 * ixv_handle_que_work 3303 ************************************************************************/ 3304 static void 3305 ixv_handle_que_work(struct work *wk, void *context) 3306 { 3307 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 3308 3309 /* 3310 * "enqueued flag" is not required here the same as ixg(4). 3311 * See ixgbe_msix_que(). 3312 */ 3313 ixv_handle_que(que); 3314 } 3315 3316 /************************************************************************ 3317 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 3318 ************************************************************************/ 3319 static int 3320 ixv_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 3321 { 3322 device_t dev = adapter->dev; 3323 struct ix_queue *que = adapter->queues; 3324 struct tx_ring *txr = adapter->tx_rings; 3325 int error, msix_ctrl, rid, vector = 0; 3326 pci_chipset_tag_t pc; 3327 pcitag_t tag; 3328 char intrbuf[PCI_INTRSTR_LEN]; 3329 char wqname[MAXCOMLEN]; 3330 char intr_xname[32]; 3331 const char *intrstr = NULL; 3332 kcpuset_t *affinity; 3333 int cpu_id = 0; 3334 3335 pc = adapter->osdep.pc; 3336 tag = adapter->osdep.tag; 3337 3338 adapter->osdep.nintrs = adapter->num_queues + 1; 3339 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 3340 adapter->osdep.nintrs) != 0) { 3341 aprint_error_dev(dev, 3342 "failed to allocate MSI-X interrupt\n"); 3343 return (ENXIO); 3344 } 3345 3346 kcpuset_create(&affinity, false); 3347 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 3348 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 3349 device_xname(dev), i); 3350 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 3351 sizeof(intrbuf)); 3352 #ifdef IXGBE_MPSAFE 3353 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 3354 true); 3355 #endif 3356 /* Set the handler function */ 3357 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 3358 adapter->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 3359 intr_xname); 3360 if (que->res == NULL) { 3361 pci_intr_release(pc, adapter->osdep.intrs, 3362 adapter->osdep.nintrs); 3363 aprint_error_dev(dev, 3364 "Failed to register QUE handler\n"); 3365 kcpuset_destroy(affinity); 3366 return (ENXIO); 3367 } 3368 que->msix = vector; 3369 adapter->active_queues |= (u64)(1 << que->msix); 3370 3371 cpu_id = i; 3372 /* Round-robin affinity */ 3373 kcpuset_zero(affinity); 3374 kcpuset_set(affinity, cpu_id % ncpu); 3375 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 3376 NULL); 3377 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 3378 intrstr); 3379 if (error == 0) 3380 aprint_normal(", bound queue %d to cpu %d\n", 3381 i, cpu_id % ncpu); 3382 else 3383 aprint_normal("\n"); 3384 3385 #ifndef IXGBE_LEGACY_TX 3386 txr->txr_si 3387 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3388 ixgbe_deferred_mq_start, txr); 3389 #endif 3390 que->que_si 3391 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3392 ixv_handle_que, que); 3393 if (que->que_si == NULL) { 3394 aprint_error_dev(dev, 3395 "could not establish software interrupt\n"); 3396 } 3397 } 3398 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 3399 error = workqueue_create(&adapter->txr_wq, wqname, 3400 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3401 IXGBE_WORKQUEUE_FLAGS); 3402 if (error) { 3403 aprint_error_dev(dev, 3404 "couldn't create workqueue for deferred Tx\n"); 3405 } 3406 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 3407 3408 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 3409 error = workqueue_create(&adapter->que_wq, wqname, 3410 ixv_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3411 IXGBE_WORKQUEUE_FLAGS); 3412 if (error) { 3413 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 3414 } 3415 3416 /* and Mailbox */ 3417 cpu_id++; 3418 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 3419 adapter->vector = vector; 3420 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 3421 sizeof(intrbuf)); 3422 #ifdef IXGBE_MPSAFE 3423 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 3424 true); 3425 #endif 3426 /* Set the mbx handler function */ 3427 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 3428 adapter->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, adapter, 3429 intr_xname); 3430 if (adapter->osdep.ihs[vector] == NULL) { 3431 aprint_error_dev(dev, "Failed to register LINK handler\n"); 3432 kcpuset_destroy(affinity); 3433 return (ENXIO); 3434 } 3435 /* Round-robin affinity */ 3436 kcpuset_zero(affinity); 3437 kcpuset_set(affinity, cpu_id % ncpu); 3438 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity, 3439 NULL); 3440 3441 aprint_normal_dev(dev, 3442 "for link, interrupting at %s", intrstr); 3443 if (error == 0) 3444 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 3445 else 3446 aprint_normal("\n"); 3447 3448 /* Tasklets for Mailbox */ 3449 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 3450 error = workqueue_create(&adapter->admin_wq, wqname, 3451 ixv_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3452 IXGBE_TASKLET_WQ_FLAGS); 3453 if (error) { 3454 aprint_error_dev(dev, 3455 "could not create admin workqueue (%d)\n", error); 3456 goto err_out; 3457 } 3458 3459 /* 3460 * Due to a broken design QEMU will fail to properly 3461 * enable the guest for MSI-X unless the vectors in 3462 * the table are all set up, so we must rewrite the 3463 * ENABLE in the MSI-X control register again at this 3464 * point to cause it to successfully initialize us. 3465 */ 3466 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 3467 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3468 rid += PCI_MSIX_CTL; 3469 msix_ctrl = pci_conf_read(pc, tag, rid); 3470 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3471 pci_conf_write(pc, tag, rid, msix_ctrl); 3472 } 3473 3474 kcpuset_destroy(affinity); 3475 return (0); 3476 err_out: 3477 kcpuset_destroy(affinity); 3478 ixv_free_deferred_handlers(adapter); 3479 ixv_free_pci_resources(adapter); 3480 return (error); 3481 } /* ixv_allocate_msix */ 3482 3483 /************************************************************************ 3484 * ixv_configure_interrupts - Setup MSI-X resources 3485 * 3486 * Note: The VF device MUST use MSI-X, there is no fallback. 3487 ************************************************************************/ 3488 static int 3489 ixv_configure_interrupts(struct adapter *adapter) 3490 { 3491 device_t dev = adapter->dev; 3492 int want, queues, msgs; 3493 3494 /* Must have at least 2 MSI-X vectors */ 3495 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 3496 if (msgs < 2) { 3497 aprint_error_dev(dev, "MSIX config error\n"); 3498 return (ENXIO); 3499 } 3500 msgs = MIN(msgs, IXG_MAX_NINTR); 3501 3502 /* Figure out a reasonable auto config value */ 3503 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3504 3505 if (ixv_num_queues != 0) 3506 queues = ixv_num_queues; 3507 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3508 queues = IXGBE_VF_MAX_TX_QUEUES; 3509 3510 /* 3511 * Want vectors for the queues, 3512 * plus an additional for mailbox. 3513 */ 3514 want = queues + 1; 3515 if (msgs >= want) 3516 msgs = want; 3517 else { 3518 aprint_error_dev(dev, 3519 "MSI-X Configuration Problem, " 3520 "%d vectors but %d queues wanted!\n", 3521 msgs, want); 3522 return -1; 3523 } 3524 3525 adapter->msix_mem = (void *)1; /* XXX */ 3526 aprint_normal_dev(dev, 3527 "Using MSI-X interrupts with %d vectors\n", msgs); 3528 adapter->num_queues = queues; 3529 3530 return (0); 3531 } /* ixv_configure_interrupts */ 3532 3533 3534 /************************************************************************ 3535 * ixv_handle_admin - Tasklet handler for MSI-X MBX interrupts 3536 * 3537 * Done outside of interrupt context since the driver might sleep 3538 ************************************************************************/ 3539 static void 3540 ixv_handle_admin(struct work *wk, void *context) 3541 { 3542 struct adapter *adapter = context; 3543 struct ixgbe_hw *hw = &adapter->hw; 3544 3545 IXGBE_CORE_LOCK(adapter); 3546 3547 ++adapter->link_workev.ev_count; 3548 adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, 3549 &adapter->link_up, FALSE); 3550 ixv_update_link_status(adapter); 3551 3552 adapter->task_requests = 0; 3553 atomic_store_relaxed(&adapter->admin_pending, 0); 3554 3555 /* Re-enable interrupts */ 3556 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 3557 3558 IXGBE_CORE_UNLOCK(adapter); 3559 } /* ixv_handle_admin */ 3560 3561 /************************************************************************ 3562 * ixv_check_link - Used in the local timer to poll for link changes 3563 ************************************************************************/ 3564 static s32 3565 ixv_check_link(struct adapter *adapter) 3566 { 3567 s32 error; 3568 3569 KASSERT(mutex_owned(&adapter->core_mtx)); 3570 3571 adapter->hw.mac.get_link_status = TRUE; 3572 3573 error = adapter->hw.mac.ops.check_link(&adapter->hw, 3574 &adapter->link_speed, &adapter->link_up, FALSE); 3575 ixv_update_link_status(adapter); 3576 3577 return error; 3578 } /* ixv_check_link */ 3579
Indexes created Tue Sep 30 20:09:53 GMT 2025