ixv.c revision 1.174
1 /* $NetBSD: ixv.c,v 1.174 2022/01/18 09:58:40 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ixv.c,v 1.174 2022/01/18 09:58:40 msaitoh Exp $"); 39 40 #ifdef _KERNEL_OPT 41 #include "opt_inet.h" 42 #include "opt_inet6.h" 43 #include "opt_net_mpsafe.h" 44 #endif 45 46 #include "ixgbe.h" 47 48 /************************************************************************ 49 * Driver version 50 ************************************************************************/ 51 static const char ixv_driver_version[] = "2.0.1-k"; 52 /* XXX NetBSD: + 1.5.17 */ 53 54 /************************************************************************ 55 * PCI Device ID Table 56 * 57 * Used by probe to select devices to load on 58 * Last field stores an index into ixv_strings 59 * Last entry must be all 0s 60 * 61 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 62 ************************************************************************/ 63 static const ixgbe_vendor_info_t ixv_vendor_info_array[] = 64 { 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 70 /* required last entry */ 71 {0, 0, 0, 0, 0} 72 }; 73 74 /************************************************************************ 75 * Table of branding strings 76 ************************************************************************/ 77 static const char *ixv_strings[] = { 78 "Intel(R) PRO/10GbE Virtual Function Network Driver" 79 }; 80 81 /********************************************************************* 82 * Function prototypes 83 *********************************************************************/ 84 static int ixv_probe(device_t, cfdata_t, void *); 85 static void ixv_attach(device_t, device_t, void *); 86 static int ixv_detach(device_t, int); 87 #if 0 88 static int ixv_shutdown(device_t); 89 #endif 90 static int ixv_ifflags_cb(struct ethercom *); 91 static int ixv_ioctl(struct ifnet *, u_long, void *); 92 static int ixv_init(struct ifnet *); 93 static void ixv_init_locked(struct adapter *); 94 static void ixv_ifstop(struct ifnet *, int); 95 static void ixv_stop_locked(void *); 96 static void ixv_init_device_features(struct adapter *); 97 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 98 static int ixv_media_change(struct ifnet *); 99 static int ixv_allocate_pci_resources(struct adapter *, 100 const struct pci_attach_args *); 101 static void ixv_free_deferred_handlers(struct adapter *); 102 static int ixv_allocate_msix(struct adapter *, 103 const struct pci_attach_args *); 104 static int ixv_configure_interrupts(struct adapter *); 105 static void ixv_free_pci_resources(struct adapter *); 106 static void ixv_local_timer(void *); 107 static void ixv_handle_timer(struct work *, void *); 108 static int ixv_setup_interface(device_t, struct adapter *); 109 static void ixv_schedule_admin_tasklet(struct adapter *); 110 static int ixv_negotiate_api(struct adapter *); 111 112 static void ixv_initialize_transmit_units(struct adapter *); 113 static void ixv_initialize_receive_units(struct adapter *); 114 static void ixv_initialize_rss_mapping(struct adapter *); 115 static s32 ixv_check_link(struct adapter *); 116 117 static void ixv_enable_intr(struct adapter *); 118 static void ixv_disable_intr(struct adapter *); 119 static int ixv_set_rxfilter(struct adapter *); 120 static void ixv_update_link_status(struct adapter *); 121 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 122 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 123 static void ixv_configure_ivars(struct adapter *); 124 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 125 static void ixv_eitr_write(struct adapter *, uint32_t, uint32_t); 126 127 static void ixv_setup_vlan_tagging(struct adapter *); 128 static int ixv_setup_vlan_support(struct adapter *); 129 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool); 130 static int ixv_register_vlan(struct adapter *, u16); 131 static int ixv_unregister_vlan(struct adapter *, u16); 132 133 static void ixv_add_device_sysctls(struct adapter *); 134 static void ixv_save_stats(struct adapter *); 135 static void ixv_init_stats(struct adapter *); 136 static void ixv_update_stats(struct adapter *); 137 static void ixv_add_stats_sysctls(struct adapter *); 138 static void ixv_clear_evcnt(struct adapter *); 139 140 /* Sysctl handlers */ 141 static void ixv_set_sysctl_value(struct adapter *, const char *, 142 const char *, int *, int); 143 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 144 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 145 static int ixv_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 146 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 147 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 148 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 149 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 150 static int ixv_sysctl_rx_copy_len(SYSCTLFN_PROTO); 151 152 /* The MSI-X Interrupt handlers */ 153 static int ixv_msix_que(void *); 154 static int ixv_msix_mbx(void *); 155 156 /* Event handlers running on workqueue */ 157 static void ixv_handle_que(void *); 158 159 /* Deferred workqueue handlers */ 160 static void ixv_handle_admin(struct work *, void *); 161 static void ixv_handle_que_work(struct work *, void *); 162 163 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 164 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 165 166 /************************************************************************ 167 * NetBSD Device Interface Entry Points 168 ************************************************************************/ 169 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 170 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 171 DVF_DETACH_SHUTDOWN); 172 173 #if 0 174 static driver_t ixv_driver = { 175 "ixv", ixv_methods, sizeof(struct adapter), 176 }; 177 178 devclass_t ixv_devclass; 179 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 180 MODULE_DEPEND(ixv, pci, 1, 1, 1); 181 MODULE_DEPEND(ixv, ether, 1, 1, 1); 182 #endif 183 184 /* 185 * TUNEABLE PARAMETERS: 186 */ 187 188 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 189 static int ixv_num_queues = 0; 190 #define TUNABLE_INT(__x, __y) 191 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 192 193 /* 194 * AIM: Adaptive Interrupt Moderation 195 * which means that the interrupt rate 196 * is varied over time based on the 197 * traffic for that interrupt vector 198 */ 199 static bool ixv_enable_aim = false; 200 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 201 202 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 203 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 204 205 /* How many packets rxeof tries to clean at a time */ 206 static int ixv_rx_process_limit = 256; 207 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 208 209 /* How many packets txeof tries to clean at a time */ 210 static int ixv_tx_process_limit = 256; 211 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 212 213 /* Which packet processing uses workqueue or softint */ 214 static bool ixv_txrx_workqueue = false; 215 216 /* 217 * Number of TX descriptors per ring, 218 * setting higher than RX as this seems 219 * the better performing choice. 220 */ 221 static int ixv_txd = PERFORM_TXD; 222 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 223 224 /* Number of RX descriptors per ring */ 225 static int ixv_rxd = PERFORM_RXD; 226 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 227 228 /* Legacy Transmit (single queue) */ 229 static int ixv_enable_legacy_tx = 0; 230 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 231 232 #ifdef NET_MPSAFE 233 #define IXGBE_MPSAFE 1 234 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 235 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 236 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 237 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 238 #else 239 #define IXGBE_CALLOUT_FLAGS 0 240 #define IXGBE_SOFTINT_FLAGS 0 241 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 242 #define IXGBE_TASKLET_WQ_FLAGS 0 243 #endif 244 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 245 246 #if 0 247 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 248 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 249 #endif 250 251 /************************************************************************ 252 * ixv_probe - Device identification routine 253 * 254 * Determines if the driver should be loaded on 255 * adapter based on its PCI vendor/device ID. 256 * 257 * return BUS_PROBE_DEFAULT on success, positive on failure 258 ************************************************************************/ 259 static int 260 ixv_probe(device_t dev, cfdata_t cf, void *aux) 261 { 262 #ifdef __HAVE_PCI_MSI_MSIX 263 const struct pci_attach_args *pa = aux; 264 265 return (ixv_lookup(pa) != NULL) ? 1 : 0; 266 #else 267 return 0; 268 #endif 269 } /* ixv_probe */ 270 271 static const ixgbe_vendor_info_t * 272 ixv_lookup(const struct pci_attach_args *pa) 273 { 274 const ixgbe_vendor_info_t *ent; 275 pcireg_t subid; 276 277 INIT_DEBUGOUT("ixv_lookup: begin"); 278 279 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 280 return NULL; 281 282 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 283 284 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 285 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 286 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 287 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 288 (ent->subvendor_id == 0)) && 289 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 290 (ent->subdevice_id == 0))) { 291 return ent; 292 } 293 } 294 295 return NULL; 296 } 297 298 /************************************************************************ 299 * ixv_attach - Device initialization routine 300 * 301 * Called when the driver is being loaded. 302 * Identifies the type of hardware, allocates all resources 303 * and initializes the hardware. 304 * 305 * return 0 on success, positive on failure 306 ************************************************************************/ 307 static void 308 ixv_attach(device_t parent, device_t dev, void *aux) 309 { 310 struct adapter *adapter; 311 struct ixgbe_hw *hw; 312 int error = 0; 313 pcireg_t id, subid; 314 const ixgbe_vendor_info_t *ent; 315 const struct pci_attach_args *pa = aux; 316 const char *apivstr; 317 const char *str; 318 char wqname[MAXCOMLEN]; 319 char buf[256]; 320 321 INIT_DEBUGOUT("ixv_attach: begin"); 322 323 /* 324 * Make sure BUSMASTER is set, on a VM under 325 * KVM it may not be and will break things. 326 */ 327 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 328 329 /* Allocate, clear, and link in our adapter structure */ 330 adapter = device_private(dev); 331 adapter->hw.back = adapter; 332 adapter->dev = dev; 333 hw = &adapter->hw; 334 335 adapter->init_locked = ixv_init_locked; 336 adapter->stop_locked = ixv_stop_locked; 337 338 adapter->osdep.pc = pa->pa_pc; 339 adapter->osdep.tag = pa->pa_tag; 340 if (pci_dma64_available(pa)) 341 adapter->osdep.dmat = pa->pa_dmat64; 342 else 343 adapter->osdep.dmat = pa->pa_dmat; 344 adapter->osdep.attached = false; 345 346 ent = ixv_lookup(pa); 347 348 KASSERT(ent != NULL); 349 350 aprint_normal(": %s, Version - %s\n", 351 ixv_strings[ent->index], ixv_driver_version); 352 353 /* Core Lock Init */ 354 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 355 356 /* Do base PCI setup - map BAR0 */ 357 if (ixv_allocate_pci_resources(adapter, pa)) { 358 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 359 error = ENXIO; 360 goto err_out; 361 } 362 363 /* SYSCTL APIs */ 364 ixv_add_device_sysctls(adapter); 365 366 /* Set up the timer callout and workqueue */ 367 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 368 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 369 error = workqueue_create(&adapter->timer_wq, wqname, 370 ixv_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 371 IXGBE_TASKLET_WQ_FLAGS); 372 if (error) { 373 aprint_error_dev(dev, 374 "could not create timer workqueue (%d)\n", error); 375 goto err_out; 376 } 377 378 /* Save off the information about this board */ 379 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 380 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 381 hw->vendor_id = PCI_VENDOR(id); 382 hw->device_id = PCI_PRODUCT(id); 383 hw->revision_id = 384 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 385 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 386 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 387 388 /* A subset of set_mac_type */ 389 switch (hw->device_id) { 390 case IXGBE_DEV_ID_82599_VF: 391 hw->mac.type = ixgbe_mac_82599_vf; 392 str = "82599 VF"; 393 break; 394 case IXGBE_DEV_ID_X540_VF: 395 hw->mac.type = ixgbe_mac_X540_vf; 396 str = "X540 VF"; 397 break; 398 case IXGBE_DEV_ID_X550_VF: 399 hw->mac.type = ixgbe_mac_X550_vf; 400 str = "X550 VF"; 401 break; 402 case IXGBE_DEV_ID_X550EM_X_VF: 403 hw->mac.type = ixgbe_mac_X550EM_x_vf; 404 str = "X550EM X VF"; 405 break; 406 case IXGBE_DEV_ID_X550EM_A_VF: 407 hw->mac.type = ixgbe_mac_X550EM_a_vf; 408 str = "X550EM A VF"; 409 break; 410 default: 411 /* Shouldn't get here since probe succeeded */ 412 aprint_error_dev(dev, "Unknown device ID!\n"); 413 error = ENXIO; 414 goto err_out; 415 break; 416 } 417 aprint_normal_dev(dev, "device %s\n", str); 418 419 ixv_init_device_features(adapter); 420 421 /* Initialize the shared code */ 422 error = ixgbe_init_ops_vf(hw); 423 if (error) { 424 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 425 error = EIO; 426 goto err_out; 427 } 428 429 /* Setup the mailbox */ 430 ixgbe_init_mbx_params_vf(hw); 431 432 /* Set the right number of segments */ 433 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 434 adapter->num_segs = IXGBE_SCATTER_DEFAULT; 435 436 /* Reset mbox api to 1.0 */ 437 error = hw->mac.ops.reset_hw(hw); 438 if (error == IXGBE_ERR_RESET_FAILED) 439 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 440 else if (error) 441 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 442 error); 443 if (error) { 444 error = EIO; 445 goto err_out; 446 } 447 448 error = hw->mac.ops.init_hw(hw); 449 if (error) { 450 aprint_error_dev(dev, "...init_hw() failed!\n"); 451 error = EIO; 452 goto err_out; 453 } 454 455 /* Negotiate mailbox API version */ 456 error = ixv_negotiate_api(adapter); 457 if (error) 458 aprint_normal_dev(dev, 459 "MBX API negotiation failed during attach!\n"); 460 switch (hw->api_version) { 461 case ixgbe_mbox_api_10: 462 apivstr = "1.0"; 463 break; 464 case ixgbe_mbox_api_20: 465 apivstr = "2.0"; 466 break; 467 case ixgbe_mbox_api_11: 468 apivstr = "1.1"; 469 break; 470 case ixgbe_mbox_api_12: 471 apivstr = "1.2"; 472 break; 473 case ixgbe_mbox_api_13: 474 apivstr = "1.3"; 475 break; 476 case ixgbe_mbox_api_14: 477 apivstr = "1.4"; 478 break; 479 case ixgbe_mbox_api_15: 480 apivstr = "1.5"; 481 break; 482 default: 483 apivstr = "unknown"; 484 break; 485 } 486 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 487 488 /* If no mac address was assigned, make a random one */ 489 if (!ixv_check_ether_addr(hw->mac.addr)) { 490 u8 addr[ETHER_ADDR_LEN]; 491 uint64_t rndval = cprng_strong64(); 492 493 memcpy(addr, &rndval, sizeof(addr)); 494 addr[0] &= 0xFE; 495 addr[0] |= 0x02; 496 bcopy(addr, hw->mac.addr, sizeof(addr)); 497 } 498 499 /* Register for VLAN events */ 500 ether_set_vlan_cb(&adapter->osdep.ec, ixv_vlan_cb); 501 502 /* Sysctls for limiting the amount of work done in the taskqueues */ 503 ixv_set_sysctl_value(adapter, "rx_processing_limit", 504 "max number of rx packets to process", 505 &adapter->rx_process_limit, ixv_rx_process_limit); 506 507 ixv_set_sysctl_value(adapter, "tx_processing_limit", 508 "max number of tx packets to process", 509 &adapter->tx_process_limit, ixv_tx_process_limit); 510 511 /* Do descriptor calc and sanity checks */ 512 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 513 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 514 aprint_error_dev(dev, "TXD config issue, using default!\n"); 515 adapter->num_tx_desc = DEFAULT_TXD; 516 } else 517 adapter->num_tx_desc = ixv_txd; 518 519 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 520 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 521 aprint_error_dev(dev, "RXD config issue, using default!\n"); 522 adapter->num_rx_desc = DEFAULT_RXD; 523 } else 524 adapter->num_rx_desc = ixv_rxd; 525 526 /* Set default high limit of copying mbuf in rxeof */ 527 adapter->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 528 529 /* Setup MSI-X */ 530 error = ixv_configure_interrupts(adapter); 531 if (error) 532 goto err_out; 533 534 /* Allocate our TX/RX Queues */ 535 if (ixgbe_allocate_queues(adapter)) { 536 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 537 error = ENOMEM; 538 goto err_out; 539 } 540 541 /* hw.ix defaults init */ 542 adapter->enable_aim = ixv_enable_aim; 543 544 adapter->txrx_use_workqueue = ixv_txrx_workqueue; 545 546 error = ixv_allocate_msix(adapter, pa); 547 if (error) { 548 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n"); 549 goto err_late; 550 } 551 552 /* Setup OS specific network interface */ 553 error = ixv_setup_interface(dev, adapter); 554 if (error != 0) { 555 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 556 goto err_late; 557 } 558 559 /* Allocate multicast array memory */ 560 adapter->mta = malloc(sizeof(*adapter->mta) * 561 IXGBE_MAX_VF_MC, M_DEVBUF, M_WAITOK); 562 563 /* Do the stats setup */ 564 ixv_save_stats(adapter); 565 ixv_init_stats(adapter); 566 ixv_add_stats_sysctls(adapter); 567 568 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 569 ixgbe_netmap_attach(adapter); 570 571 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 572 aprint_verbose_dev(dev, "feature cap %s\n", buf); 573 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 574 aprint_verbose_dev(dev, "feature ena %s\n", buf); 575 576 INIT_DEBUGOUT("ixv_attach: end"); 577 adapter->osdep.attached = true; 578 579 return; 580 581 err_late: 582 ixgbe_free_queues(adapter); 583 err_out: 584 ixv_free_pci_resources(adapter); 585 IXGBE_CORE_LOCK_DESTROY(adapter); 586 587 return; 588 } /* ixv_attach */ 589 590 /************************************************************************ 591 * ixv_detach - Device removal routine 592 * 593 * Called when the driver is being removed. 594 * Stops the adapter and deallocates all the resources 595 * that were allocated for driver operation. 596 * 597 * return 0 on success, positive on failure 598 ************************************************************************/ 599 static int 600 ixv_detach(device_t dev, int flags) 601 { 602 struct adapter *adapter = device_private(dev); 603 struct ixgbe_hw *hw = &adapter->hw; 604 struct tx_ring *txr = adapter->tx_rings; 605 struct rx_ring *rxr = adapter->rx_rings; 606 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 607 608 INIT_DEBUGOUT("ixv_detach: begin"); 609 if (adapter->osdep.attached == false) 610 return 0; 611 612 /* Stop the interface. Callouts are stopped in it. */ 613 ixv_ifstop(adapter->ifp, 1); 614 615 if (VLAN_ATTACHED(&adapter->osdep.ec) && 616 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 617 aprint_error_dev(dev, "VLANs in use, detach first\n"); 618 return EBUSY; 619 } 620 621 ether_ifdetach(adapter->ifp); 622 callout_halt(&adapter->timer, NULL); 623 ixv_free_deferred_handlers(adapter); 624 625 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 626 netmap_detach(adapter->ifp); 627 628 ixv_free_pci_resources(adapter); 629 #if 0 /* XXX the NetBSD port is probably missing something here */ 630 bus_generic_detach(dev); 631 #endif 632 if_detach(adapter->ifp); 633 ifmedia_fini(&adapter->media); 634 if_percpuq_destroy(adapter->ipq); 635 636 sysctl_teardown(&adapter->sysctllog); 637 evcnt_detach(&adapter->efbig_tx_dma_setup); 638 evcnt_detach(&adapter->mbuf_defrag_failed); 639 evcnt_detach(&adapter->efbig2_tx_dma_setup); 640 evcnt_detach(&adapter->einval_tx_dma_setup); 641 evcnt_detach(&adapter->other_tx_dma_setup); 642 evcnt_detach(&adapter->eagain_tx_dma_setup); 643 evcnt_detach(&adapter->enomem_tx_dma_setup); 644 evcnt_detach(&adapter->watchdog_events); 645 evcnt_detach(&adapter->tso_err); 646 evcnt_detach(&adapter->admin_irqev); 647 evcnt_detach(&adapter->link_workev); 648 649 txr = adapter->tx_rings; 650 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 651 evcnt_detach(&adapter->queues[i].irqs); 652 evcnt_detach(&adapter->queues[i].handleq); 653 evcnt_detach(&adapter->queues[i].req); 654 evcnt_detach(&txr->no_desc_avail); 655 evcnt_detach(&txr->total_packets); 656 evcnt_detach(&txr->tso_tx); 657 #ifndef IXGBE_LEGACY_TX 658 evcnt_detach(&txr->pcq_drops); 659 #endif 660 661 evcnt_detach(&rxr->rx_packets); 662 evcnt_detach(&rxr->rx_bytes); 663 evcnt_detach(&rxr->rx_copies); 664 evcnt_detach(&rxr->no_mbuf); 665 evcnt_detach(&rxr->rx_discarded); 666 } 667 evcnt_detach(&stats->ipcs); 668 evcnt_detach(&stats->l4cs); 669 evcnt_detach(&stats->ipcs_bad); 670 evcnt_detach(&stats->l4cs_bad); 671 672 /* Packet Reception Stats */ 673 evcnt_detach(&stats->vfgorc); 674 evcnt_detach(&stats->vfgprc); 675 evcnt_detach(&stats->vfmprc); 676 677 /* Packet Transmission Stats */ 678 evcnt_detach(&stats->vfgotc); 679 evcnt_detach(&stats->vfgptc); 680 681 /* Mailbox Stats */ 682 evcnt_detach(&hw->mbx.stats.msgs_tx); 683 evcnt_detach(&hw->mbx.stats.msgs_rx); 684 evcnt_detach(&hw->mbx.stats.acks); 685 evcnt_detach(&hw->mbx.stats.reqs); 686 evcnt_detach(&hw->mbx.stats.rsts); 687 688 ixgbe_free_queues(adapter); 689 690 IXGBE_CORE_LOCK_DESTROY(adapter); 691 692 return (0); 693 } /* ixv_detach */ 694 695 /************************************************************************ 696 * ixv_init_locked - Init entry point 697 * 698 * Used in two ways: It is used by the stack as an init entry 699 * point in network interface structure. It is also used 700 * by the driver as a hw/sw initialization routine to get 701 * to a consistent state. 702 * 703 * return 0 on success, positive on failure 704 ************************************************************************/ 705 static void 706 ixv_init_locked(struct adapter *adapter) 707 { 708 struct ifnet *ifp = adapter->ifp; 709 device_t dev = adapter->dev; 710 struct ixgbe_hw *hw = &adapter->hw; 711 struct ix_queue *que; 712 int error = 0; 713 uint32_t mask; 714 int i; 715 716 INIT_DEBUGOUT("ixv_init_locked: begin"); 717 KASSERT(mutex_owned(&adapter->core_mtx)); 718 hw->adapter_stopped = FALSE; 719 hw->mac.ops.stop_adapter(hw); 720 callout_stop(&adapter->timer); 721 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 722 que->disabled_count = 0; 723 724 adapter->max_frame_size = 725 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 726 727 /* reprogram the RAR[0] in case user changed it. */ 728 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 729 730 /* Get the latest mac address, User can use a LAA */ 731 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 732 IXGBE_ETH_LENGTH_OF_ADDRESS); 733 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 734 735 /* Prepare transmit descriptors and buffers */ 736 if (ixgbe_setup_transmit_structures(adapter)) { 737 aprint_error_dev(dev, "Could not setup transmit structures\n"); 738 ixv_stop_locked(adapter); 739 return; 740 } 741 742 /* Reset VF and renegotiate mailbox API version */ 743 hw->mac.ops.reset_hw(hw); 744 hw->mac.ops.start_hw(hw); 745 error = ixv_negotiate_api(adapter); 746 if (error) 747 device_printf(dev, 748 "Mailbox API negotiation failed in init_locked!\n"); 749 750 ixv_initialize_transmit_units(adapter); 751 752 /* Setup Multicast table */ 753 ixv_set_rxfilter(adapter); 754 755 /* Use fixed buffer size, even for jumbo frames */ 756 adapter->rx_mbuf_sz = MCLBYTES; 757 758 /* Prepare receive descriptors and buffers */ 759 error = ixgbe_setup_receive_structures(adapter); 760 if (error) { 761 device_printf(dev, 762 "Could not setup receive structures (err = %d)\n", error); 763 ixv_stop_locked(adapter); 764 return; 765 } 766 767 /* Configure RX settings */ 768 ixv_initialize_receive_units(adapter); 769 770 /* Initialize variable holding task enqueue requests interrupts */ 771 adapter->task_requests = 0; 772 773 /* Set up VLAN offload and filter */ 774 ixv_setup_vlan_support(adapter); 775 776 /* Set up MSI-X routing */ 777 ixv_configure_ivars(adapter); 778 779 /* Set up auto-mask */ 780 mask = (1 << adapter->vector); 781 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 782 mask |= (1 << que->msix); 783 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 784 785 /* Set moderation on the Link interrupt */ 786 ixv_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 787 788 /* Stats init */ 789 ixv_init_stats(adapter); 790 791 /* Config/Enable Link */ 792 hw->mac.get_link_status = TRUE; 793 hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up, 794 FALSE); 795 796 /* Start watchdog */ 797 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 798 atomic_store_relaxed(&adapter->timer_pending, 0); 799 800 /* OK to schedule workqueues. */ 801 adapter->schedule_wqs_ok = true; 802 803 /* And now turn on interrupts */ 804 ixv_enable_intr(adapter); 805 806 /* Update saved flags. See ixgbe_ifflags_cb() */ 807 adapter->if_flags = ifp->if_flags; 808 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 809 810 /* Now inform the stack we're ready */ 811 ifp->if_flags |= IFF_RUNNING; 812 ifp->if_flags &= ~IFF_OACTIVE; 813 814 return; 815 } /* ixv_init_locked */ 816 817 /************************************************************************ 818 * ixv_enable_queue 819 ************************************************************************/ 820 static inline void 821 ixv_enable_queue(struct adapter *adapter, u32 vector) 822 { 823 struct ixgbe_hw *hw = &adapter->hw; 824 struct ix_queue *que = &adapter->queues[vector]; 825 u32 queue = 1UL << vector; 826 u32 mask; 827 828 mutex_enter(&que->dc_mtx); 829 if (que->disabled_count > 0 && --que->disabled_count > 0) 830 goto out; 831 832 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 833 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 834 out: 835 mutex_exit(&que->dc_mtx); 836 } /* ixv_enable_queue */ 837 838 /************************************************************************ 839 * ixv_disable_queue 840 ************************************************************************/ 841 static inline void 842 ixv_disable_queue(struct adapter *adapter, u32 vector) 843 { 844 struct ixgbe_hw *hw = &adapter->hw; 845 struct ix_queue *que = &adapter->queues[vector]; 846 u32 queue = 1UL << vector; 847 u32 mask; 848 849 mutex_enter(&que->dc_mtx); 850 if (que->disabled_count++ > 0) 851 goto out; 852 853 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 854 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 855 out: 856 mutex_exit(&que->dc_mtx); 857 } /* ixv_disable_queue */ 858 859 #if 0 860 static inline void 861 ixv_rearm_queues(struct adapter *adapter, u64 queues) 862 { 863 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 864 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 865 } /* ixv_rearm_queues */ 866 #endif 867 868 869 /************************************************************************ 870 * ixv_msix_que - MSI-X Queue Interrupt Service routine 871 ************************************************************************/ 872 static int 873 ixv_msix_que(void *arg) 874 { 875 struct ix_queue *que = arg; 876 struct adapter *adapter = que->adapter; 877 struct tx_ring *txr = que->txr; 878 struct rx_ring *rxr = que->rxr; 879 bool more; 880 u32 newitr = 0; 881 882 ixv_disable_queue(adapter, que->msix); 883 ++que->irqs.ev_count; 884 885 #ifdef __NetBSD__ 886 /* Don't run ixgbe_rxeof in interrupt context */ 887 more = true; 888 #else 889 more = ixgbe_rxeof(que); 890 #endif 891 892 IXGBE_TX_LOCK(txr); 893 ixgbe_txeof(txr); 894 IXGBE_TX_UNLOCK(txr); 895 896 /* Do AIM now? */ 897 898 if (adapter->enable_aim == false) 899 goto no_calc; 900 /* 901 * Do Adaptive Interrupt Moderation: 902 * - Write out last calculated setting 903 * - Calculate based on average size over 904 * the last interval. 905 */ 906 if (que->eitr_setting) 907 ixv_eitr_write(adapter, que->msix, que->eitr_setting); 908 909 que->eitr_setting = 0; 910 911 /* Idle, do nothing */ 912 if ((txr->bytes == 0) && (rxr->bytes == 0)) 913 goto no_calc; 914 915 if ((txr->bytes) && (txr->packets)) 916 newitr = txr->bytes/txr->packets; 917 if ((rxr->bytes) && (rxr->packets)) 918 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 919 newitr += 24; /* account for hardware frame, crc */ 920 921 /* set an upper boundary */ 922 newitr = uimin(newitr, 3000); 923 924 /* Be nice to the mid range */ 925 if ((newitr > 300) && (newitr < 1200)) 926 newitr = (newitr / 3); 927 else 928 newitr = (newitr / 2); 929 930 /* 931 * When RSC is used, ITR interval must be larger than RSC_DELAY. 932 * Currently, we use 2us for RSC_DELAY. The minimum value is always 933 * greater than 2us on 100M (and 10M?(not documented)), but it's not 934 * on 1G and higher. 935 */ 936 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 937 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 938 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 939 newitr = IXGBE_MIN_RSC_EITR_10G1G; 940 } 941 942 /* save for next interrupt */ 943 que->eitr_setting = newitr; 944 945 /* Reset state */ 946 txr->bytes = 0; 947 txr->packets = 0; 948 rxr->bytes = 0; 949 rxr->packets = 0; 950 951 no_calc: 952 if (more) 953 softint_schedule(que->que_si); 954 else /* Re-enable this interrupt */ 955 ixv_enable_queue(adapter, que->msix); 956 957 return 1; 958 } /* ixv_msix_que */ 959 960 /************************************************************************ 961 * ixv_msix_mbx 962 ************************************************************************/ 963 static int 964 ixv_msix_mbx(void *arg) 965 { 966 struct adapter *adapter = arg; 967 struct ixgbe_hw *hw = &adapter->hw; 968 969 ++adapter->admin_irqev.ev_count; 970 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 971 972 /* Link status change */ 973 hw->mac.get_link_status = TRUE; 974 atomic_or_32(&adapter->task_requests, IXGBE_REQUEST_TASK_MBX); 975 ixv_schedule_admin_tasklet(adapter); 976 977 return 1; 978 } /* ixv_msix_mbx */ 979 980 static void 981 ixv_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 982 { 983 984 /* 985 * Newer devices than 82598 have VF function, so this function is 986 * simple. 987 */ 988 itr |= IXGBE_EITR_CNT_WDIS; 989 990 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(index), itr); 991 } 992 993 994 /************************************************************************ 995 * ixv_media_status - Media Ioctl callback 996 * 997 * Called whenever the user queries the status of 998 * the interface using ifconfig. 999 ************************************************************************/ 1000 static void 1001 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1002 { 1003 struct adapter *adapter = ifp->if_softc; 1004 1005 INIT_DEBUGOUT("ixv_media_status: begin"); 1006 ixv_update_link_status(adapter); 1007 1008 ifmr->ifm_status = IFM_AVALID; 1009 ifmr->ifm_active = IFM_ETHER; 1010 1011 if (adapter->link_active != LINK_STATE_UP) { 1012 ifmr->ifm_active |= IFM_NONE; 1013 return; 1014 } 1015 1016 ifmr->ifm_status |= IFM_ACTIVE; 1017 1018 switch (adapter->link_speed) { 1019 case IXGBE_LINK_SPEED_10GB_FULL: 1020 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1021 break; 1022 case IXGBE_LINK_SPEED_5GB_FULL: 1023 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1024 break; 1025 case IXGBE_LINK_SPEED_2_5GB_FULL: 1026 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1027 break; 1028 case IXGBE_LINK_SPEED_1GB_FULL: 1029 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1030 break; 1031 case IXGBE_LINK_SPEED_100_FULL: 1032 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1033 break; 1034 case IXGBE_LINK_SPEED_10_FULL: 1035 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1036 break; 1037 } 1038 1039 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1040 } /* ixv_media_status */ 1041 1042 /************************************************************************ 1043 * ixv_media_change - Media Ioctl callback 1044 * 1045 * Called when the user changes speed/duplex using 1046 * media/mediopt option with ifconfig. 1047 ************************************************************************/ 1048 static int 1049 ixv_media_change(struct ifnet *ifp) 1050 { 1051 struct adapter *adapter = ifp->if_softc; 1052 struct ifmedia *ifm = &adapter->media; 1053 1054 INIT_DEBUGOUT("ixv_media_change: begin"); 1055 1056 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1057 return (EINVAL); 1058 1059 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1060 case IFM_AUTO: 1061 break; 1062 default: 1063 device_printf(adapter->dev, "Only auto media type\n"); 1064 return (EINVAL); 1065 } 1066 1067 return (0); 1068 } /* ixv_media_change */ 1069 1070 static void 1071 ixv_schedule_admin_tasklet(struct adapter *adapter) 1072 { 1073 if (adapter->schedule_wqs_ok) { 1074 if (atomic_cas_uint(&adapter->admin_pending, 0, 1) == 0) 1075 workqueue_enqueue(adapter->admin_wq, 1076 &adapter->admin_wc, NULL); 1077 } 1078 } 1079 1080 /************************************************************************ 1081 * ixv_negotiate_api 1082 * 1083 * Negotiate the Mailbox API with the PF; 1084 * start with the most featured API first. 1085 ************************************************************************/ 1086 static int 1087 ixv_negotiate_api(struct adapter *adapter) 1088 { 1089 struct ixgbe_hw *hw = &adapter->hw; 1090 int mbx_api[] = { ixgbe_mbox_api_15, 1091 ixgbe_mbox_api_13, 1092 ixgbe_mbox_api_12, 1093 ixgbe_mbox_api_11, 1094 ixgbe_mbox_api_10, 1095 ixgbe_mbox_api_unknown }; 1096 int i = 0; 1097 1098 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1099 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) { 1100 if (hw->api_version >= ixgbe_mbox_api_15) 1101 ixgbe_upgrade_mbx_params_vf(hw); 1102 return (0); 1103 } 1104 i++; 1105 } 1106 1107 return (EINVAL); 1108 } /* ixv_negotiate_api */ 1109 1110 1111 /************************************************************************ 1112 * ixv_set_rxfilter - Multicast Update 1113 * 1114 * Called whenever multicast address list is updated. 1115 ************************************************************************/ 1116 static int 1117 ixv_set_rxfilter(struct adapter *adapter) 1118 { 1119 struct ixgbe_mc_addr *mta; 1120 struct ifnet *ifp = adapter->ifp; 1121 struct ixgbe_hw *hw = &adapter->hw; 1122 u8 *update_ptr; 1123 int mcnt = 0; 1124 struct ethercom *ec = &adapter->osdep.ec; 1125 struct ether_multi *enm; 1126 struct ether_multistep step; 1127 bool overflow = false; 1128 int error, rc = 0; 1129 1130 KASSERT(mutex_owned(&adapter->core_mtx)); 1131 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin"); 1132 1133 mta = adapter->mta; 1134 bzero(mta, sizeof(*mta) * IXGBE_MAX_VF_MC); 1135 1136 /* 1: For PROMISC */ 1137 if (ifp->if_flags & IFF_PROMISC) { 1138 error = hw->mac.ops.update_xcast_mode(hw, 1139 IXGBEVF_XCAST_MODE_PROMISC); 1140 if (error == IXGBE_ERR_NOT_TRUSTED) { 1141 device_printf(adapter->dev, 1142 "this interface is not trusted\n"); 1143 error = EPERM; 1144 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1145 device_printf(adapter->dev, 1146 "the PF doesn't support promisc mode\n"); 1147 error = EOPNOTSUPP; 1148 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) { 1149 device_printf(adapter->dev, 1150 "the PF may not in promisc mode\n"); 1151 error = EINVAL; 1152 } else if (error) { 1153 device_printf(adapter->dev, 1154 "failed to set promisc mode. error = %d\n", 1155 error); 1156 error = EIO; 1157 } else 1158 return 0; 1159 rc = error; 1160 } 1161 1162 /* 2: For ALLMULTI or normal */ 1163 ETHER_LOCK(ec); 1164 ETHER_FIRST_MULTI(step, ec, enm); 1165 while (enm != NULL) { 1166 if ((mcnt >= IXGBE_MAX_VF_MC) || 1167 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1168 ETHER_ADDR_LEN) != 0)) { 1169 overflow = true; 1170 break; 1171 } 1172 bcopy(enm->enm_addrlo, 1173 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 1174 mcnt++; 1175 ETHER_NEXT_MULTI(step, enm); 1176 } 1177 ETHER_UNLOCK(ec); 1178 1179 /* 3: For ALLMULTI */ 1180 if (overflow) { 1181 error = hw->mac.ops.update_xcast_mode(hw, 1182 IXGBEVF_XCAST_MODE_ALLMULTI); 1183 if (error == IXGBE_ERR_NOT_TRUSTED) { 1184 device_printf(adapter->dev, 1185 "this interface is not trusted\n"); 1186 error = EPERM; 1187 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1188 device_printf(adapter->dev, 1189 "the PF doesn't support allmulti mode\n"); 1190 error = EOPNOTSUPP; 1191 } else if (error) { 1192 device_printf(adapter->dev, 1193 "number of Ethernet multicast addresses " 1194 "exceeds the limit (%d). error = %d\n", 1195 IXGBE_MAX_VF_MC, error); 1196 error = ENOSPC; 1197 } else { 1198 ETHER_LOCK(ec); 1199 ec->ec_flags |= ETHER_F_ALLMULTI; 1200 ETHER_UNLOCK(ec); 1201 return rc; /* Promisc might have failed */ 1202 } 1203 1204 if (rc == 0) 1205 rc = error; 1206 1207 /* Continue to update the multicast table as many as we can */ 1208 } 1209 1210 /* 4: For normal operation */ 1211 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 1212 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) { 1213 /* Normal operation */ 1214 ETHER_LOCK(ec); 1215 ec->ec_flags &= ~ETHER_F_ALLMULTI; 1216 ETHER_UNLOCK(ec); 1217 error = 0; 1218 } else if (error) { 1219 device_printf(adapter->dev, 1220 "failed to set Ethernet multicast address " 1221 "operation to normal. error = %d\n", error); 1222 } 1223 1224 update_ptr = (u8 *)mta; 1225 error = adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, 1226 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1227 if (rc == 0) 1228 rc = error; 1229 1230 return rc; 1231 } /* ixv_set_rxfilter */ 1232 1233 /************************************************************************ 1234 * ixv_mc_array_itr 1235 * 1236 * An iterator function needed by the multicast shared code. 1237 * It feeds the shared code routine the addresses in the 1238 * array of ixv_set_rxfilter() one by one. 1239 ************************************************************************/ 1240 static u8 * 1241 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1242 { 1243 struct ixgbe_mc_addr *mta; 1244 1245 mta = (struct ixgbe_mc_addr *)*update_ptr; 1246 1247 *vmdq = 0; 1248 *update_ptr = (u8*)(mta + 1); 1249 1250 return (mta->addr); 1251 } /* ixv_mc_array_itr */ 1252 1253 /************************************************************************ 1254 * ixv_local_timer - Timer routine 1255 * 1256 * Checks for link status, updates statistics, 1257 * and runs the watchdog check. 1258 ************************************************************************/ 1259 static void 1260 ixv_local_timer(void *arg) 1261 { 1262 struct adapter *adapter = arg; 1263 1264 if (adapter->schedule_wqs_ok) { 1265 if (atomic_cas_uint(&adapter->timer_pending, 0, 1) == 0) 1266 workqueue_enqueue(adapter->timer_wq, 1267 &adapter->timer_wc, NULL); 1268 } 1269 } 1270 1271 static void 1272 ixv_handle_timer(struct work *wk, void *context) 1273 { 1274 struct adapter *adapter = context; 1275 device_t dev = adapter->dev; 1276 struct ix_queue *que = adapter->queues; 1277 u64 queues = 0; 1278 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1279 int hung = 0; 1280 int i; 1281 1282 IXGBE_CORE_LOCK(adapter); 1283 1284 if (ixv_check_link(adapter)) { 1285 ixv_init_locked(adapter); 1286 IXGBE_CORE_UNLOCK(adapter); 1287 return; 1288 } 1289 1290 /* Stats Update */ 1291 ixv_update_stats(adapter); 1292 1293 /* Update some event counters */ 1294 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1295 que = adapter->queues; 1296 for (i = 0; i < adapter->num_queues; i++, que++) { 1297 struct tx_ring *txr = que->txr; 1298 1299 v0 += txr->q_efbig_tx_dma_setup; 1300 v1 += txr->q_mbuf_defrag_failed; 1301 v2 += txr->q_efbig2_tx_dma_setup; 1302 v3 += txr->q_einval_tx_dma_setup; 1303 v4 += txr->q_other_tx_dma_setup; 1304 v5 += txr->q_eagain_tx_dma_setup; 1305 v6 += txr->q_enomem_tx_dma_setup; 1306 v7 += txr->q_tso_err; 1307 } 1308 adapter->efbig_tx_dma_setup.ev_count = v0; 1309 adapter->mbuf_defrag_failed.ev_count = v1; 1310 adapter->efbig2_tx_dma_setup.ev_count = v2; 1311 adapter->einval_tx_dma_setup.ev_count = v3; 1312 adapter->other_tx_dma_setup.ev_count = v4; 1313 adapter->eagain_tx_dma_setup.ev_count = v5; 1314 adapter->enomem_tx_dma_setup.ev_count = v6; 1315 adapter->tso_err.ev_count = v7; 1316 1317 /* 1318 * Check the TX queues status 1319 * - mark hung queues so we don't schedule on them 1320 * - watchdog only if all queues show hung 1321 */ 1322 que = adapter->queues; 1323 for (i = 0; i < adapter->num_queues; i++, que++) { 1324 /* Keep track of queues with work for soft irq */ 1325 if (que->txr->busy) 1326 queues |= ((u64)1 << que->me); 1327 /* 1328 * Each time txeof runs without cleaning, but there 1329 * are uncleaned descriptors it increments busy. If 1330 * we get to the MAX we declare it hung. 1331 */ 1332 if (que->busy == IXGBE_QUEUE_HUNG) { 1333 ++hung; 1334 /* Mark the queue as inactive */ 1335 adapter->active_queues &= ~((u64)1 << que->me); 1336 continue; 1337 } else { 1338 /* Check if we've come back from hung */ 1339 if ((adapter->active_queues & ((u64)1 << que->me)) == 0) 1340 adapter->active_queues |= ((u64)1 << que->me); 1341 } 1342 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1343 device_printf(dev, 1344 "Warning queue %d appears to be hung!\n", i); 1345 que->txr->busy = IXGBE_QUEUE_HUNG; 1346 ++hung; 1347 } 1348 } 1349 1350 /* Only truly watchdog if all queues show hung */ 1351 if (hung == adapter->num_queues) 1352 goto watchdog; 1353 #if 0 1354 else if (queues != 0) { /* Force an IRQ on queues with work */ 1355 ixv_rearm_queues(adapter, queues); 1356 } 1357 #endif 1358 1359 atomic_store_relaxed(&adapter->timer_pending, 0); 1360 IXGBE_CORE_UNLOCK(adapter); 1361 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1362 1363 return; 1364 1365 watchdog: 1366 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1367 adapter->ifp->if_flags &= ~IFF_RUNNING; 1368 adapter->watchdog_events.ev_count++; 1369 ixv_init_locked(adapter); 1370 IXGBE_CORE_UNLOCK(adapter); 1371 } /* ixv_handle_timer */ 1372 1373 /************************************************************************ 1374 * ixv_update_link_status - Update OS on link state 1375 * 1376 * Note: Only updates the OS on the cached link state. 1377 * The real check of the hardware only happens with 1378 * a link interrupt. 1379 ************************************************************************/ 1380 static void 1381 ixv_update_link_status(struct adapter *adapter) 1382 { 1383 struct ifnet *ifp = adapter->ifp; 1384 device_t dev = adapter->dev; 1385 1386 KASSERT(mutex_owned(&adapter->core_mtx)); 1387 1388 if (adapter->link_up) { 1389 if (adapter->link_active != LINK_STATE_UP) { 1390 if (bootverbose) { 1391 const char *bpsmsg; 1392 1393 switch (adapter->link_speed) { 1394 case IXGBE_LINK_SPEED_10GB_FULL: 1395 bpsmsg = "10 Gbps"; 1396 break; 1397 case IXGBE_LINK_SPEED_5GB_FULL: 1398 bpsmsg = "5 Gbps"; 1399 break; 1400 case IXGBE_LINK_SPEED_2_5GB_FULL: 1401 bpsmsg = "2.5 Gbps"; 1402 break; 1403 case IXGBE_LINK_SPEED_1GB_FULL: 1404 bpsmsg = "1 Gbps"; 1405 break; 1406 case IXGBE_LINK_SPEED_100_FULL: 1407 bpsmsg = "100 Mbps"; 1408 break; 1409 case IXGBE_LINK_SPEED_10_FULL: 1410 bpsmsg = "10 Mbps"; 1411 break; 1412 default: 1413 bpsmsg = "unknown speed"; 1414 break; 1415 } 1416 device_printf(dev, "Link is up %s %s \n", 1417 bpsmsg, "Full Duplex"); 1418 } 1419 adapter->link_active = LINK_STATE_UP; 1420 if_link_state_change(ifp, LINK_STATE_UP); 1421 } 1422 } else { 1423 /* 1424 * Do it when link active changes to DOWN. i.e. 1425 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 1426 * b) LINK_STATE_UP -> LINK_STATE_DOWN 1427 */ 1428 if (adapter->link_active != LINK_STATE_DOWN) { 1429 if (bootverbose) 1430 device_printf(dev, "Link is Down\n"); 1431 if_link_state_change(ifp, LINK_STATE_DOWN); 1432 adapter->link_active = LINK_STATE_DOWN; 1433 } 1434 } 1435 } /* ixv_update_link_status */ 1436 1437 1438 /************************************************************************ 1439 * ixv_stop - Stop the hardware 1440 * 1441 * Disables all traffic on the adapter by issuing a 1442 * global reset on the MAC and deallocates TX/RX buffers. 1443 ************************************************************************/ 1444 static void 1445 ixv_ifstop(struct ifnet *ifp, int disable) 1446 { 1447 struct adapter *adapter = ifp->if_softc; 1448 1449 IXGBE_CORE_LOCK(adapter); 1450 ixv_stop_locked(adapter); 1451 IXGBE_CORE_UNLOCK(adapter); 1452 1453 workqueue_wait(adapter->admin_wq, &adapter->admin_wc); 1454 atomic_store_relaxed(&adapter->admin_pending, 0); 1455 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 1456 atomic_store_relaxed(&adapter->timer_pending, 0); 1457 } 1458 1459 static void 1460 ixv_stop_locked(void *arg) 1461 { 1462 struct ifnet *ifp; 1463 struct adapter *adapter = arg; 1464 struct ixgbe_hw *hw = &adapter->hw; 1465 1466 ifp = adapter->ifp; 1467 1468 KASSERT(mutex_owned(&adapter->core_mtx)); 1469 1470 INIT_DEBUGOUT("ixv_stop_locked: begin\n"); 1471 ixv_disable_intr(adapter); 1472 1473 /* Tell the stack that the interface is no longer active */ 1474 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1475 1476 hw->mac.ops.reset_hw(hw); 1477 adapter->hw.adapter_stopped = FALSE; 1478 hw->mac.ops.stop_adapter(hw); 1479 callout_stop(&adapter->timer); 1480 1481 /* Don't schedule workqueues. */ 1482 adapter->schedule_wqs_ok = false; 1483 1484 /* reprogram the RAR[0] in case user changed it. */ 1485 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1486 1487 return; 1488 } /* ixv_stop_locked */ 1489 1490 1491 /************************************************************************ 1492 * ixv_allocate_pci_resources 1493 ************************************************************************/ 1494 static int 1495 ixv_allocate_pci_resources(struct adapter *adapter, 1496 const struct pci_attach_args *pa) 1497 { 1498 pcireg_t memtype, csr; 1499 device_t dev = adapter->dev; 1500 bus_addr_t addr; 1501 int flags; 1502 1503 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1504 switch (memtype) { 1505 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1506 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1507 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1508 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1509 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1510 goto map_err; 1511 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1512 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1513 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1514 } 1515 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1516 adapter->osdep.mem_size, flags, 1517 &adapter->osdep.mem_bus_space_handle) != 0) { 1518 map_err: 1519 adapter->osdep.mem_size = 0; 1520 aprint_error_dev(dev, "unable to map BAR0\n"); 1521 return ENXIO; 1522 } 1523 /* 1524 * Enable address decoding for memory range in case it's not 1525 * set. 1526 */ 1527 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 1528 PCI_COMMAND_STATUS_REG); 1529 csr |= PCI_COMMAND_MEM_ENABLE; 1530 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 1531 csr); 1532 break; 1533 default: 1534 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1535 return ENXIO; 1536 } 1537 1538 /* Pick up the tuneable queues */ 1539 adapter->num_queues = ixv_num_queues; 1540 1541 return (0); 1542 } /* ixv_allocate_pci_resources */ 1543 1544 static void 1545 ixv_free_deferred_handlers(struct adapter *adapter) 1546 { 1547 struct ix_queue *que = adapter->queues; 1548 struct tx_ring *txr = adapter->tx_rings; 1549 int i; 1550 1551 for (i = 0; i < adapter->num_queues; i++, que++, txr++) { 1552 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 1553 if (txr->txr_si != NULL) 1554 softint_disestablish(txr->txr_si); 1555 } 1556 if (que->que_si != NULL) 1557 softint_disestablish(que->que_si); 1558 } 1559 if (adapter->txr_wq != NULL) 1560 workqueue_destroy(adapter->txr_wq); 1561 if (adapter->txr_wq_enqueued != NULL) 1562 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 1563 if (adapter->que_wq != NULL) 1564 workqueue_destroy(adapter->que_wq); 1565 1566 /* Drain the Mailbox(link) queue */ 1567 if (adapter->admin_wq != NULL) { 1568 workqueue_destroy(adapter->admin_wq); 1569 adapter->admin_wq = NULL; 1570 } 1571 if (adapter->timer_wq != NULL) { 1572 workqueue_destroy(adapter->timer_wq); 1573 adapter->timer_wq = NULL; 1574 } 1575 } /* ixv_free_deferred_handlers */ 1576 1577 /************************************************************************ 1578 * ixv_free_pci_resources 1579 ************************************************************************/ 1580 static void 1581 ixv_free_pci_resources(struct adapter * adapter) 1582 { 1583 struct ix_queue *que = adapter->queues; 1584 int rid; 1585 1586 /* 1587 * Release all msix queue resources: 1588 */ 1589 for (int i = 0; i < adapter->num_queues; i++, que++) { 1590 if (que->res != NULL) 1591 pci_intr_disestablish(adapter->osdep.pc, 1592 adapter->osdep.ihs[i]); 1593 } 1594 1595 1596 /* Clean the Mailbox interrupt last */ 1597 rid = adapter->vector; 1598 1599 if (adapter->osdep.ihs[rid] != NULL) { 1600 pci_intr_disestablish(adapter->osdep.pc, 1601 adapter->osdep.ihs[rid]); 1602 adapter->osdep.ihs[rid] = NULL; 1603 } 1604 1605 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1606 adapter->osdep.nintrs); 1607 1608 if (adapter->osdep.mem_size != 0) { 1609 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 1610 adapter->osdep.mem_bus_space_handle, 1611 adapter->osdep.mem_size); 1612 } 1613 1614 return; 1615 } /* ixv_free_pci_resources */ 1616 1617 /************************************************************************ 1618 * ixv_setup_interface 1619 * 1620 * Setup networking device structure and register an interface. 1621 ************************************************************************/ 1622 static int 1623 ixv_setup_interface(device_t dev, struct adapter *adapter) 1624 { 1625 struct ethercom *ec = &adapter->osdep.ec; 1626 struct ifnet *ifp; 1627 1628 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1629 1630 ifp = adapter->ifp = &ec->ec_if; 1631 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1632 ifp->if_baudrate = IF_Gbps(10); 1633 ifp->if_init = ixv_init; 1634 ifp->if_stop = ixv_ifstop; 1635 ifp->if_softc = adapter; 1636 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1637 #ifdef IXGBE_MPSAFE 1638 ifp->if_extflags = IFEF_MPSAFE; 1639 #endif 1640 ifp->if_ioctl = ixv_ioctl; 1641 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1642 #if 0 1643 ixv_start_locked = ixgbe_legacy_start_locked; 1644 #endif 1645 } else { 1646 ifp->if_transmit = ixgbe_mq_start; 1647 #if 0 1648 ixv_start_locked = ixgbe_mq_start_locked; 1649 #endif 1650 } 1651 ifp->if_start = ixgbe_legacy_start; 1652 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1653 IFQ_SET_READY(&ifp->if_snd); 1654 1655 if_initialize(ifp); 1656 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1657 ether_ifattach(ifp, adapter->hw.mac.addr); 1658 aprint_normal_dev(dev, "Ethernet address %s\n", 1659 ether_sprintf(adapter->hw.mac.addr)); 1660 /* 1661 * We use per TX queue softint, so if_deferred_start_init() isn't 1662 * used. 1663 */ 1664 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1665 1666 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1667 1668 /* 1669 * Tell the upper layer(s) we support long frames. 1670 */ 1671 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1672 1673 /* Set capability flags */ 1674 ifp->if_capabilities |= IFCAP_HWCSUM 1675 | IFCAP_TSOv4 1676 | IFCAP_TSOv6; 1677 ifp->if_capenable = 0; 1678 1679 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER 1680 | ETHERCAP_VLAN_HWTAGGING 1681 | ETHERCAP_VLAN_HWCSUM 1682 | ETHERCAP_JUMBO_MTU 1683 | ETHERCAP_VLAN_MTU; 1684 1685 /* Enable the above capabilities by default */ 1686 ec->ec_capenable = ec->ec_capabilities; 1687 1688 /* Don't enable LRO by default */ 1689 #if 0 1690 /* NetBSD doesn't support LRO yet */ 1691 ifp->if_capabilities |= IFCAP_LRO; 1692 #endif 1693 1694 /* 1695 * Specify the media types supported by this adapter and register 1696 * callbacks to update media and link information 1697 */ 1698 ec->ec_ifmedia = &adapter->media; 1699 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, ixv_media_change, 1700 ixv_media_status, &adapter->core_mtx); 1701 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1702 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1703 1704 if_register(ifp); 1705 1706 return 0; 1707 } /* ixv_setup_interface */ 1708 1709 1710 /************************************************************************ 1711 * ixv_initialize_transmit_units - Enable transmit unit. 1712 ************************************************************************/ 1713 static void 1714 ixv_initialize_transmit_units(struct adapter *adapter) 1715 { 1716 struct tx_ring *txr = adapter->tx_rings; 1717 struct ixgbe_hw *hw = &adapter->hw; 1718 int i; 1719 1720 for (i = 0; i < adapter->num_queues; i++, txr++) { 1721 u64 tdba = txr->txdma.dma_paddr; 1722 u32 txctrl, txdctl; 1723 int j = txr->me; 1724 1725 /* Set WTHRESH to 8, burst writeback */ 1726 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1727 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 1728 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1729 1730 /* Set the HW Tx Head and Tail indices */ 1731 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(j), 0); 1732 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(j), 0); 1733 1734 /* Set Tx Tail register */ 1735 txr->tail = IXGBE_VFTDT(j); 1736 1737 txr->txr_no_space = false; 1738 1739 /* Set Ring parameters */ 1740 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1741 (tdba & 0x00000000ffffffffULL)); 1742 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1743 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1744 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1745 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1746 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1747 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1748 1749 /* Now enable */ 1750 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1751 txdctl |= IXGBE_TXDCTL_ENABLE; 1752 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1753 } 1754 1755 return; 1756 } /* ixv_initialize_transmit_units */ 1757 1758 1759 /************************************************************************ 1760 * ixv_initialize_rss_mapping 1761 ************************************************************************/ 1762 static void 1763 ixv_initialize_rss_mapping(struct adapter *adapter) 1764 { 1765 struct ixgbe_hw *hw = &adapter->hw; 1766 u32 reta = 0, mrqc, rss_key[10]; 1767 int queue_id; 1768 int i, j; 1769 u32 rss_hash_config; 1770 1771 /* force use default RSS key. */ 1772 #ifdef __NetBSD__ 1773 rss_getkey((uint8_t *) &rss_key); 1774 #else 1775 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1776 /* Fetch the configured RSS key */ 1777 rss_getkey((uint8_t *)&rss_key); 1778 } else { 1779 /* set up random bits */ 1780 cprng_fast(&rss_key, sizeof(rss_key)); 1781 } 1782 #endif 1783 1784 /* Now fill out hash function seeds */ 1785 for (i = 0; i < 10; i++) 1786 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1787 1788 /* Set up the redirection table */ 1789 for (i = 0, j = 0; i < 64; i++, j++) { 1790 if (j == adapter->num_queues) 1791 j = 0; 1792 1793 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1794 /* 1795 * Fetch the RSS bucket id for the given indirection 1796 * entry. Cap it at the number of configured buckets 1797 * (which is num_queues.) 1798 */ 1799 queue_id = rss_get_indirection_to_bucket(i); 1800 queue_id = queue_id % adapter->num_queues; 1801 } else 1802 queue_id = j; 1803 1804 /* 1805 * The low 8 bits are for hash value (n+0); 1806 * The next 8 bits are for hash value (n+1), etc. 1807 */ 1808 reta >>= 8; 1809 reta |= ((uint32_t)queue_id) << 24; 1810 if ((i & 3) == 3) { 1811 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1812 reta = 0; 1813 } 1814 } 1815 1816 /* Perform hash on these packet types */ 1817 if (adapter->feat_en & IXGBE_FEATURE_RSS) 1818 rss_hash_config = rss_gethashconfig(); 1819 else { 1820 /* 1821 * Disable UDP - IP fragments aren't currently being handled 1822 * and so we end up with a mix of 2-tuple and 4-tuple 1823 * traffic. 1824 */ 1825 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1826 | RSS_HASHTYPE_RSS_TCP_IPV4 1827 | RSS_HASHTYPE_RSS_IPV6 1828 | RSS_HASHTYPE_RSS_TCP_IPV6; 1829 } 1830 1831 mrqc = IXGBE_MRQC_RSSEN; 1832 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1833 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1834 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1835 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1836 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1837 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1838 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1839 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1840 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1841 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX defined, but not supported\n", 1842 __func__); 1843 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1844 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined, but not supported\n", 1845 __func__); 1846 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1847 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1848 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1849 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1850 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1851 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined, but not supported\n", 1852 __func__); 1853 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1854 } /* ixv_initialize_rss_mapping */ 1855 1856 1857 /************************************************************************ 1858 * ixv_initialize_receive_units - Setup receive registers and features. 1859 ************************************************************************/ 1860 static void 1861 ixv_initialize_receive_units(struct adapter *adapter) 1862 { 1863 struct rx_ring *rxr = adapter->rx_rings; 1864 struct ixgbe_hw *hw = &adapter->hw; 1865 struct ifnet *ifp = adapter->ifp; 1866 u32 bufsz, psrtype; 1867 1868 if (ifp->if_mtu > ETHERMTU) 1869 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1870 else 1871 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1872 1873 psrtype = IXGBE_PSRTYPE_TCPHDR 1874 | IXGBE_PSRTYPE_UDPHDR 1875 | IXGBE_PSRTYPE_IPV4HDR 1876 | IXGBE_PSRTYPE_IPV6HDR 1877 | IXGBE_PSRTYPE_L2HDR; 1878 1879 if (adapter->num_queues > 1) 1880 psrtype |= 1 << 29; 1881 1882 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1883 1884 /* Tell PF our max_frame size */ 1885 if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) { 1886 device_printf(adapter->dev, "There is a problem with the PF setup. It is likely the receive unit for this VF will not function correctly.\n"); 1887 } 1888 1889 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 1890 u64 rdba = rxr->rxdma.dma_paddr; 1891 u32 reg, rxdctl; 1892 int j = rxr->me; 1893 1894 /* Disable the queue */ 1895 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1896 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1897 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1898 for (int k = 0; k < 10; k++) { 1899 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1900 IXGBE_RXDCTL_ENABLE) 1901 msec_delay(1); 1902 else 1903 break; 1904 } 1905 IXGBE_WRITE_BARRIER(hw); 1906 /* Setup the Base and Length of the Rx Descriptor Ring */ 1907 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1908 (rdba & 0x00000000ffffffffULL)); 1909 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1910 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1911 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1912 1913 /* Reset the ring indices */ 1914 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1915 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1916 1917 /* Set up the SRRCTL register */ 1918 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1919 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1920 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1921 reg |= bufsz; 1922 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1923 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1924 1925 /* Capture Rx Tail index */ 1926 rxr->tail = IXGBE_VFRDT(rxr->me); 1927 1928 /* Do the queue enabling last */ 1929 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1930 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1931 for (int k = 0; k < 10; k++) { 1932 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1933 IXGBE_RXDCTL_ENABLE) 1934 break; 1935 msec_delay(1); 1936 } 1937 IXGBE_WRITE_BARRIER(hw); 1938 1939 /* Set the Tail Pointer */ 1940 #ifdef DEV_NETMAP 1941 /* 1942 * In netmap mode, we must preserve the buffers made 1943 * available to userspace before the if_init() 1944 * (this is true by default on the TX side, because 1945 * init makes all buffers available to userspace). 1946 * 1947 * netmap_reset() and the device specific routines 1948 * (e.g. ixgbe_setup_receive_rings()) map these 1949 * buffers at the end of the NIC ring, so here we 1950 * must set the RDT (tail) register to make sure 1951 * they are not overwritten. 1952 * 1953 * In this driver the NIC ring starts at RDH = 0, 1954 * RDT points to the last slot available for reception (?), 1955 * so RDT = num_rx_desc - 1 means the whole ring is available. 1956 */ 1957 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 1958 (ifp->if_capenable & IFCAP_NETMAP)) { 1959 struct netmap_adapter *na = NA(adapter->ifp); 1960 struct netmap_kring *kring = na->rx_rings[i]; 1961 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1962 1963 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1964 } else 1965 #endif /* DEV_NETMAP */ 1966 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1967 adapter->num_rx_desc - 1); 1968 } 1969 1970 if (adapter->hw.mac.type >= ixgbe_mac_X550_vf) 1971 ixv_initialize_rss_mapping(adapter); 1972 } /* ixv_initialize_receive_units */ 1973 1974 /************************************************************************ 1975 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1976 * 1977 * Retrieves the TDH value from the hardware 1978 ************************************************************************/ 1979 static int 1980 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1981 { 1982 struct sysctlnode node = *rnode; 1983 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1984 uint32_t val; 1985 1986 if (!txr) 1987 return (0); 1988 1989 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDH(txr->me)); 1990 node.sysctl_data = &val; 1991 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1992 } /* ixv_sysctl_tdh_handler */ 1993 1994 /************************************************************************ 1995 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 1996 * 1997 * Retrieves the TDT value from the hardware 1998 ************************************************************************/ 1999 static int 2000 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 2001 { 2002 struct sysctlnode node = *rnode; 2003 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2004 uint32_t val; 2005 2006 if (!txr) 2007 return (0); 2008 2009 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDT(txr->me)); 2010 node.sysctl_data = &val; 2011 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2012 } /* ixv_sysctl_tdt_handler */ 2013 2014 /************************************************************************ 2015 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check 2016 * handler function 2017 * 2018 * Retrieves the next_to_check value 2019 ************************************************************************/ 2020 static int 2021 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2022 { 2023 struct sysctlnode node = *rnode; 2024 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2025 uint32_t val; 2026 2027 if (!rxr) 2028 return (0); 2029 2030 val = rxr->next_to_check; 2031 node.sysctl_data = &val; 2032 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2033 } /* ixv_sysctl_next_to_check_handler */ 2034 2035 /************************************************************************ 2036 * ixv_sysctl_next_to_refresh_handler - Receive Descriptor next to refresh 2037 * handler function 2038 * 2039 * Retrieves the next_to_refresh value 2040 ************************************************************************/ 2041 static int 2042 ixv_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2043 { 2044 struct sysctlnode node = *rnode; 2045 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2046 struct adapter *adapter; 2047 uint32_t val; 2048 2049 if (!rxr) 2050 return (0); 2051 2052 adapter = rxr->adapter; 2053 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2054 return (EPERM); 2055 2056 val = rxr->next_to_refresh; 2057 node.sysctl_data = &val; 2058 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2059 } /* ixv_sysctl_next_to_refresh_handler */ 2060 2061 /************************************************************************ 2062 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 2063 * 2064 * Retrieves the RDH value from the hardware 2065 ************************************************************************/ 2066 static int 2067 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 2068 { 2069 struct sysctlnode node = *rnode; 2070 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2071 uint32_t val; 2072 2073 if (!rxr) 2074 return (0); 2075 2076 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDH(rxr->me)); 2077 node.sysctl_data = &val; 2078 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2079 } /* ixv_sysctl_rdh_handler */ 2080 2081 /************************************************************************ 2082 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 2083 * 2084 * Retrieves the RDT value from the hardware 2085 ************************************************************************/ 2086 static int 2087 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 2088 { 2089 struct sysctlnode node = *rnode; 2090 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2091 uint32_t val; 2092 2093 if (!rxr) 2094 return (0); 2095 2096 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDT(rxr->me)); 2097 node.sysctl_data = &val; 2098 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2099 } /* ixv_sysctl_rdt_handler */ 2100 2101 static void 2102 ixv_setup_vlan_tagging(struct adapter *adapter) 2103 { 2104 struct ethercom *ec = &adapter->osdep.ec; 2105 struct ixgbe_hw *hw = &adapter->hw; 2106 struct rx_ring *rxr; 2107 u32 ctrl; 2108 int i; 2109 bool hwtagging; 2110 2111 /* Enable HW tagging only if any vlan is attached */ 2112 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2113 && VLAN_ATTACHED(ec); 2114 2115 /* Enable the queues */ 2116 for (i = 0; i < adapter->num_queues; i++) { 2117 rxr = &adapter->rx_rings[i]; 2118 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 2119 if (hwtagging) 2120 ctrl |= IXGBE_RXDCTL_VME; 2121 else 2122 ctrl &= ~IXGBE_RXDCTL_VME; 2123 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 2124 /* 2125 * Let Rx path know that it needs to store VLAN tag 2126 * as part of extra mbuf info. 2127 */ 2128 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2129 } 2130 } /* ixv_setup_vlan_tagging */ 2131 2132 /************************************************************************ 2133 * ixv_setup_vlan_support 2134 ************************************************************************/ 2135 static int 2136 ixv_setup_vlan_support(struct adapter *adapter) 2137 { 2138 struct ethercom *ec = &adapter->osdep.ec; 2139 struct ixgbe_hw *hw = &adapter->hw; 2140 u32 vid, vfta, retry; 2141 struct vlanid_list *vlanidp; 2142 int rv, error = 0; 2143 2144 /* 2145 * This function is called from both if_init and ifflags_cb() 2146 * on NetBSD. 2147 */ 2148 2149 /* 2150 * Part 1: 2151 * Setup VLAN HW tagging 2152 */ 2153 ixv_setup_vlan_tagging(adapter); 2154 2155 if (!VLAN_ATTACHED(ec)) 2156 return 0; 2157 2158 /* 2159 * Part 2: 2160 * Setup VLAN HW filter 2161 */ 2162 /* Cleanup shadow_vfta */ 2163 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 2164 adapter->shadow_vfta[i] = 0; 2165 /* Generate shadow_vfta from ec_vids */ 2166 ETHER_LOCK(ec); 2167 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2168 uint32_t idx; 2169 2170 idx = vlanidp->vid / 32; 2171 KASSERT(idx < IXGBE_VFTA_SIZE); 2172 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2173 } 2174 ETHER_UNLOCK(ec); 2175 2176 /* 2177 * A soft reset zero's out the VFTA, so 2178 * we need to repopulate it now. 2179 */ 2180 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2181 if (adapter->shadow_vfta[i] == 0) 2182 continue; 2183 vfta = adapter->shadow_vfta[i]; 2184 /* 2185 * Reconstruct the vlan id's 2186 * based on the bits set in each 2187 * of the array ints. 2188 */ 2189 for (int j = 0; j < 32; j++) { 2190 retry = 0; 2191 if ((vfta & ((u32)1 << j)) == 0) 2192 continue; 2193 vid = (i * 32) + j; 2194 2195 /* Call the shared code mailbox routine */ 2196 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE, 2197 FALSE)) != 0) { 2198 if (++retry > 5) { 2199 device_printf(adapter->dev, 2200 "%s: max retry exceeded\n", 2201 __func__); 2202 break; 2203 } 2204 } 2205 if (rv != 0) { 2206 device_printf(adapter->dev, 2207 "failed to set vlan %d\n", vid); 2208 error = EACCES; 2209 } 2210 } 2211 } 2212 return error; 2213 } /* ixv_setup_vlan_support */ 2214 2215 static int 2216 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2217 { 2218 struct ifnet *ifp = &ec->ec_if; 2219 struct adapter *adapter = ifp->if_softc; 2220 int rv; 2221 2222 if (set) 2223 rv = ixv_register_vlan(adapter, vid); 2224 else 2225 rv = ixv_unregister_vlan(adapter, vid); 2226 2227 if (rv != 0) 2228 return rv; 2229 2230 /* 2231 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2232 * or 0 to 1. 2233 */ 2234 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2235 ixv_setup_vlan_tagging(adapter); 2236 2237 return rv; 2238 } 2239 2240 /************************************************************************ 2241 * ixv_register_vlan 2242 * 2243 * Run via a vlan config EVENT, it enables us to use the 2244 * HW Filter table since we can get the vlan id. This just 2245 * creates the entry in the soft version of the VFTA, init 2246 * will repopulate the real table. 2247 ************************************************************************/ 2248 static int 2249 ixv_register_vlan(struct adapter *adapter, u16 vtag) 2250 { 2251 struct ixgbe_hw *hw = &adapter->hw; 2252 u16 index, bit; 2253 int error; 2254 2255 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2256 return EINVAL; 2257 IXGBE_CORE_LOCK(adapter); 2258 index = (vtag >> 5) & 0x7F; 2259 bit = vtag & 0x1F; 2260 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2261 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false); 2262 IXGBE_CORE_UNLOCK(adapter); 2263 2264 if (error != 0) { 2265 device_printf(adapter->dev, "failed to register vlan %hu\n", 2266 vtag); 2267 error = EACCES; 2268 } 2269 return error; 2270 } /* ixv_register_vlan */ 2271 2272 /************************************************************************ 2273 * ixv_unregister_vlan 2274 * 2275 * Run via a vlan unconfig EVENT, remove our entry 2276 * in the soft vfta. 2277 ************************************************************************/ 2278 static int 2279 ixv_unregister_vlan(struct adapter *adapter, u16 vtag) 2280 { 2281 struct ixgbe_hw *hw = &adapter->hw; 2282 u16 index, bit; 2283 int error; 2284 2285 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2286 return EINVAL; 2287 2288 IXGBE_CORE_LOCK(adapter); 2289 index = (vtag >> 5) & 0x7F; 2290 bit = vtag & 0x1F; 2291 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2292 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false); 2293 IXGBE_CORE_UNLOCK(adapter); 2294 2295 if (error != 0) { 2296 device_printf(adapter->dev, "failed to unregister vlan %hu\n", 2297 vtag); 2298 error = EIO; 2299 } 2300 return error; 2301 } /* ixv_unregister_vlan */ 2302 2303 /************************************************************************ 2304 * ixv_enable_intr 2305 ************************************************************************/ 2306 static void 2307 ixv_enable_intr(struct adapter *adapter) 2308 { 2309 struct ixgbe_hw *hw = &adapter->hw; 2310 struct ix_queue *que = adapter->queues; 2311 u32 mask; 2312 int i; 2313 2314 /* For VTEIAC */ 2315 mask = (1 << adapter->vector); 2316 for (i = 0; i < adapter->num_queues; i++, que++) 2317 mask |= (1 << que->msix); 2318 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2319 2320 /* For VTEIMS */ 2321 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 2322 que = adapter->queues; 2323 for (i = 0; i < adapter->num_queues; i++, que++) 2324 ixv_enable_queue(adapter, que->msix); 2325 2326 IXGBE_WRITE_FLUSH(hw); 2327 } /* ixv_enable_intr */ 2328 2329 /************************************************************************ 2330 * ixv_disable_intr 2331 ************************************************************************/ 2332 static void 2333 ixv_disable_intr(struct adapter *adapter) 2334 { 2335 struct ix_queue *que = adapter->queues; 2336 2337 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 2338 2339 /* disable interrupts other than queues */ 2340 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, adapter->vector); 2341 2342 for (int i = 0; i < adapter->num_queues; i++, que++) 2343 ixv_disable_queue(adapter, que->msix); 2344 2345 IXGBE_WRITE_FLUSH(&adapter->hw); 2346 } /* ixv_disable_intr */ 2347 2348 /************************************************************************ 2349 * ixv_set_ivar 2350 * 2351 * Setup the correct IVAR register for a particular MSI-X interrupt 2352 * - entry is the register array entry 2353 * - vector is the MSI-X vector for this queue 2354 * - type is RX/TX/MISC 2355 ************************************************************************/ 2356 static void 2357 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 2358 { 2359 struct ixgbe_hw *hw = &adapter->hw; 2360 u32 ivar, index; 2361 2362 vector |= IXGBE_IVAR_ALLOC_VAL; 2363 2364 if (type == -1) { /* MISC IVAR */ 2365 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2366 ivar &= ~0xFF; 2367 ivar |= vector; 2368 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2369 } else { /* RX/TX IVARS */ 2370 index = (16 * (entry & 1)) + (8 * type); 2371 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2372 ivar &= ~(0xffUL << index); 2373 ivar |= ((u32)vector << index); 2374 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2375 } 2376 } /* ixv_set_ivar */ 2377 2378 /************************************************************************ 2379 * ixv_configure_ivars 2380 ************************************************************************/ 2381 static void 2382 ixv_configure_ivars(struct adapter *adapter) 2383 { 2384 struct ix_queue *que = adapter->queues; 2385 2386 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2387 2388 for (int i = 0; i < adapter->num_queues; i++, que++) { 2389 /* First the RX queue entry */ 2390 ixv_set_ivar(adapter, i, que->msix, 0); 2391 /* ... and the TX */ 2392 ixv_set_ivar(adapter, i, que->msix, 1); 2393 /* Set an initial value in EITR */ 2394 ixv_eitr_write(adapter, que->msix, IXGBE_EITR_DEFAULT); 2395 } 2396 2397 /* For the mailbox interrupt */ 2398 ixv_set_ivar(adapter, 1, adapter->vector, -1); 2399 } /* ixv_configure_ivars */ 2400 2401 2402 /************************************************************************ 2403 * ixv_save_stats 2404 * 2405 * The VF stats registers never have a truly virgin 2406 * starting point, so this routine tries to make an 2407 * artificial one, marking ground zero on attach as 2408 * it were. 2409 ************************************************************************/ 2410 static void 2411 ixv_save_stats(struct adapter *adapter) 2412 { 2413 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2414 2415 if (stats->vfgprc.ev_count || stats->vfgptc.ev_count) { 2416 stats->saved_reset_vfgprc += 2417 stats->vfgprc.ev_count - stats->base_vfgprc; 2418 stats->saved_reset_vfgptc += 2419 stats->vfgptc.ev_count - stats->base_vfgptc; 2420 stats->saved_reset_vfgorc += 2421 stats->vfgorc.ev_count - stats->base_vfgorc; 2422 stats->saved_reset_vfgotc += 2423 stats->vfgotc.ev_count - stats->base_vfgotc; 2424 stats->saved_reset_vfmprc += 2425 stats->vfmprc.ev_count - stats->base_vfmprc; 2426 } 2427 } /* ixv_save_stats */ 2428 2429 /************************************************************************ 2430 * ixv_init_stats 2431 ************************************************************************/ 2432 static void 2433 ixv_init_stats(struct adapter *adapter) 2434 { 2435 struct ixgbe_hw *hw = &adapter->hw; 2436 2437 adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2438 adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2439 adapter->stats.vf.last_vfgorc |= 2440 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2441 2442 adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2443 adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2444 adapter->stats.vf.last_vfgotc |= 2445 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2446 2447 adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2448 2449 adapter->stats.vf.base_vfgprc = adapter->stats.vf.last_vfgprc; 2450 adapter->stats.vf.base_vfgorc = adapter->stats.vf.last_vfgorc; 2451 adapter->stats.vf.base_vfgptc = adapter->stats.vf.last_vfgptc; 2452 adapter->stats.vf.base_vfgotc = adapter->stats.vf.last_vfgotc; 2453 adapter->stats.vf.base_vfmprc = adapter->stats.vf.last_vfmprc; 2454 } /* ixv_init_stats */ 2455 2456 #define UPDATE_STAT_32(reg, last, count) \ 2457 { \ 2458 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2459 if (current < (last)) \ 2460 count.ev_count += 0x100000000LL; \ 2461 (last) = current; \ 2462 count.ev_count &= 0xFFFFFFFF00000000LL; \ 2463 count.ev_count |= current; \ 2464 } 2465 2466 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2467 { \ 2468 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2469 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2470 u64 current = ((cur_msb << 32) | cur_lsb); \ 2471 if (current < (last)) \ 2472 count.ev_count += 0x1000000000LL; \ 2473 (last) = current; \ 2474 count.ev_count &= 0xFFFFFFF000000000LL; \ 2475 count.ev_count |= current; \ 2476 } 2477 2478 /************************************************************************ 2479 * ixv_update_stats - Update the board statistics counters. 2480 ************************************************************************/ 2481 void 2482 ixv_update_stats(struct adapter *adapter) 2483 { 2484 struct ixgbe_hw *hw = &adapter->hw; 2485 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2486 2487 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2488 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2489 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2490 stats->vfgorc); 2491 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2492 stats->vfgotc); 2493 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2494 2495 /* VF doesn't count errors by hardware */ 2496 2497 } /* ixv_update_stats */ 2498 2499 /************************************************************************ 2500 * ixv_sysctl_interrupt_rate_handler 2501 ************************************************************************/ 2502 static int 2503 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2504 { 2505 struct sysctlnode node = *rnode; 2506 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2507 struct adapter *adapter = que->adapter; 2508 uint32_t reg, usec, rate; 2509 int error; 2510 2511 if (que == NULL) 2512 return 0; 2513 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_VTEITR(que->msix)); 2514 usec = ((reg & 0x0FF8) >> 3); 2515 if (usec > 0) 2516 rate = 500000 / usec; 2517 else 2518 rate = 0; 2519 node.sysctl_data = &rate; 2520 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2521 if (error || newp == NULL) 2522 return error; 2523 reg &= ~0xfff; /* default, no limitation */ 2524 if (rate > 0 && rate < 500000) { 2525 if (rate < 1000) 2526 rate = 1000; 2527 reg |= ((4000000 / rate) & 0xff8); 2528 /* 2529 * When RSC is used, ITR interval must be larger than 2530 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2531 * The minimum value is always greater than 2us on 100M 2532 * (and 10M?(not documented)), but it's not on 1G and higher. 2533 */ 2534 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2535 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2536 if ((adapter->num_queues > 1) 2537 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2538 return EINVAL; 2539 } 2540 ixv_max_interrupt_rate = rate; 2541 } else 2542 ixv_max_interrupt_rate = 0; 2543 ixv_eitr_write(adapter, que->msix, reg); 2544 2545 return (0); 2546 } /* ixv_sysctl_interrupt_rate_handler */ 2547 2548 const struct sysctlnode * 2549 ixv_sysctl_instance(struct adapter *adapter) 2550 { 2551 const char *dvname; 2552 struct sysctllog **log; 2553 int rc; 2554 const struct sysctlnode *rnode; 2555 2556 log = &adapter->sysctllog; 2557 dvname = device_xname(adapter->dev); 2558 2559 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2560 0, CTLTYPE_NODE, dvname, 2561 SYSCTL_DESCR("ixv information and settings"), 2562 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2563 goto err; 2564 2565 return rnode; 2566 err: 2567 device_printf(adapter->dev, 2568 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2569 return NULL; 2570 } 2571 2572 static void 2573 ixv_add_device_sysctls(struct adapter *adapter) 2574 { 2575 struct sysctllog **log; 2576 const struct sysctlnode *rnode, *cnode; 2577 device_t dev; 2578 2579 dev = adapter->dev; 2580 log = &adapter->sysctllog; 2581 2582 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2583 aprint_error_dev(dev, "could not create sysctl root\n"); 2584 return; 2585 } 2586 2587 if (sysctl_createv(log, 0, &rnode, &cnode, 2588 CTLFLAG_READWRITE, CTLTYPE_INT, "debug", 2589 SYSCTL_DESCR("Debug Info"), 2590 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2591 aprint_error_dev(dev, "could not create sysctl\n"); 2592 2593 if (sysctl_createv(log, 0, &rnode, &cnode, 2594 CTLFLAG_READWRITE, CTLTYPE_INT, 2595 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 2596 ixv_sysctl_rx_copy_len, 0, 2597 (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2598 aprint_error_dev(dev, "could not create sysctl\n"); 2599 2600 if (sysctl_createv(log, 0, &rnode, &cnode, 2601 CTLFLAG_READWRITE, CTLTYPE_BOOL, "enable_aim", 2602 SYSCTL_DESCR("Interrupt Moderation"), 2603 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2604 aprint_error_dev(dev, "could not create sysctl\n"); 2605 2606 if (sysctl_createv(log, 0, &rnode, &cnode, 2607 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 2608 SYSCTL_DESCR("Use workqueue for packet processing"), 2609 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) 2610 != 0) 2611 aprint_error_dev(dev, "could not create sysctl\n"); 2612 } 2613 2614 /************************************************************************ 2615 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2616 ************************************************************************/ 2617 static void 2618 ixv_add_stats_sysctls(struct adapter *adapter) 2619 { 2620 device_t dev = adapter->dev; 2621 struct tx_ring *txr = adapter->tx_rings; 2622 struct rx_ring *rxr = adapter->rx_rings; 2623 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2624 struct ixgbe_hw *hw = &adapter->hw; 2625 const struct sysctlnode *rnode, *cnode; 2626 struct sysctllog **log = &adapter->sysctllog; 2627 const char *xname = device_xname(dev); 2628 2629 /* Driver Statistics */ 2630 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2631 NULL, xname, "Driver tx dma soft fail EFBIG"); 2632 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2633 NULL, xname, "m_defrag() failed"); 2634 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2635 NULL, xname, "Driver tx dma hard fail EFBIG"); 2636 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2637 NULL, xname, "Driver tx dma hard fail EINVAL"); 2638 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 2639 NULL, xname, "Driver tx dma hard fail other"); 2640 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2641 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2642 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2643 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2644 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 2645 NULL, xname, "Watchdog timeouts"); 2646 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 2647 NULL, xname, "TSO errors"); 2648 evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, 2649 NULL, xname, "Admin MSI-X IRQ Handled"); 2650 evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, 2651 NULL, xname, "Admin event"); 2652 2653 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2654 snprintf(adapter->queues[i].evnamebuf, 2655 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 2656 xname, i); 2657 snprintf(adapter->queues[i].namebuf, 2658 sizeof(adapter->queues[i].namebuf), "q%d", i); 2659 2660 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2661 aprint_error_dev(dev, "could not create sysctl root\n"); 2662 break; 2663 } 2664 2665 if (sysctl_createv(log, 0, &rnode, &rnode, 2666 0, CTLTYPE_NODE, 2667 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2668 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2669 break; 2670 2671 if (sysctl_createv(log, 0, &rnode, &cnode, 2672 CTLFLAG_READWRITE, CTLTYPE_INT, 2673 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2674 ixv_sysctl_interrupt_rate_handler, 0, 2675 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2676 break; 2677 2678 if (sysctl_createv(log, 0, &rnode, &cnode, 2679 CTLFLAG_READONLY, CTLTYPE_INT, 2680 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2681 ixv_sysctl_tdh_handler, 0, (void *)txr, 2682 0, CTL_CREATE, CTL_EOL) != 0) 2683 break; 2684 2685 if (sysctl_createv(log, 0, &rnode, &cnode, 2686 CTLFLAG_READONLY, CTLTYPE_INT, 2687 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2688 ixv_sysctl_tdt_handler, 0, (void *)txr, 2689 0, CTL_CREATE, CTL_EOL) != 0) 2690 break; 2691 2692 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 2693 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 2694 evcnt_attach_dynamic(&adapter->queues[i].handleq, 2695 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 2696 "Handled queue in softint"); 2697 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 2698 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 2699 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2700 NULL, adapter->queues[i].evnamebuf, "TSO"); 2701 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2702 NULL, adapter->queues[i].evnamebuf, 2703 "TX Queue No Descriptor Available"); 2704 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2705 NULL, adapter->queues[i].evnamebuf, 2706 "Queue Packets Transmitted"); 2707 #ifndef IXGBE_LEGACY_TX 2708 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2709 NULL, adapter->queues[i].evnamebuf, 2710 "Packets dropped in pcq"); 2711 #endif 2712 2713 #ifdef LRO 2714 struct lro_ctrl *lro = &rxr->lro; 2715 #endif /* LRO */ 2716 2717 if (sysctl_createv(log, 0, &rnode, &cnode, 2718 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 2719 SYSCTL_DESCR("Receive Descriptor next to check"), 2720 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 2721 CTL_CREATE, CTL_EOL) != 0) 2722 break; 2723 2724 if (sysctl_createv(log, 0, &rnode, &cnode, 2725 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 2726 SYSCTL_DESCR("Receive Descriptor next to refresh"), 2727 ixv_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 2728 CTL_CREATE, CTL_EOL) != 0) 2729 break; 2730 2731 if (sysctl_createv(log, 0, &rnode, &cnode, 2732 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 2733 SYSCTL_DESCR("Receive Descriptor Head"), 2734 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2735 CTL_CREATE, CTL_EOL) != 0) 2736 break; 2737 2738 if (sysctl_createv(log, 0, &rnode, &cnode, 2739 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 2740 SYSCTL_DESCR("Receive Descriptor Tail"), 2741 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2742 CTL_CREATE, CTL_EOL) != 0) 2743 break; 2744 2745 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2746 NULL, adapter->queues[i].evnamebuf, 2747 "Queue Packets Received"); 2748 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2749 NULL, adapter->queues[i].evnamebuf, 2750 "Queue Bytes Received"); 2751 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2752 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 2753 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 2754 NULL, adapter->queues[i].evnamebuf, "Rx no mbuf"); 2755 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2756 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 2757 #ifdef LRO 2758 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2759 CTLFLAG_RD, &lro->lro_queued, 0, 2760 "LRO Queued"); 2761 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2762 CTLFLAG_RD, &lro->lro_flushed, 0, 2763 "LRO Flushed"); 2764 #endif /* LRO */ 2765 } 2766 2767 /* MAC stats get their own sub node */ 2768 2769 snprintf(stats->namebuf, 2770 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2771 2772 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2773 stats->namebuf, "rx csum offload - IP"); 2774 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2775 stats->namebuf, "rx csum offload - L4"); 2776 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2777 stats->namebuf, "rx csum offload - IP bad"); 2778 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2779 stats->namebuf, "rx csum offload - L4 bad"); 2780 2781 /* Packet Reception Stats */ 2782 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2783 xname, "Good Packets Received"); 2784 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2785 xname, "Good Octets Received"); 2786 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2787 xname, "Multicast Packets Received"); 2788 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2789 xname, "Good Packets Transmitted"); 2790 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2791 xname, "Good Octets Transmitted"); 2792 2793 /* Mailbox Stats */ 2794 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2795 xname, "message TXs"); 2796 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2797 xname, "message RXs"); 2798 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2799 xname, "ACKs"); 2800 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2801 xname, "REQs"); 2802 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2803 xname, "RSTs"); 2804 2805 } /* ixv_add_stats_sysctls */ 2806 2807 static void 2808 ixv_clear_evcnt(struct adapter *adapter) 2809 { 2810 struct tx_ring *txr = adapter->tx_rings; 2811 struct rx_ring *rxr = adapter->rx_rings; 2812 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2813 struct ixgbe_hw *hw = &adapter->hw; 2814 int i; 2815 2816 /* Driver Statistics */ 2817 adapter->efbig_tx_dma_setup.ev_count = 0; 2818 adapter->mbuf_defrag_failed.ev_count = 0; 2819 adapter->efbig2_tx_dma_setup.ev_count = 0; 2820 adapter->einval_tx_dma_setup.ev_count = 0; 2821 adapter->other_tx_dma_setup.ev_count = 0; 2822 adapter->eagain_tx_dma_setup.ev_count = 0; 2823 adapter->enomem_tx_dma_setup.ev_count = 0; 2824 adapter->watchdog_events.ev_count = 0; 2825 adapter->tso_err.ev_count = 0; 2826 adapter->admin_irqev.ev_count = 0; 2827 adapter->link_workev.ev_count = 0; 2828 2829 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2830 adapter->queues[i].irqs.ev_count = 0; 2831 adapter->queues[i].handleq.ev_count = 0; 2832 adapter->queues[i].req.ev_count = 0; 2833 txr->tso_tx.ev_count = 0; 2834 txr->no_desc_avail.ev_count = 0; 2835 txr->total_packets.ev_count = 0; 2836 #ifndef IXGBE_LEGACY_TX 2837 txr->pcq_drops.ev_count = 0; 2838 #endif 2839 txr->q_efbig_tx_dma_setup = 0; 2840 txr->q_mbuf_defrag_failed = 0; 2841 txr->q_efbig2_tx_dma_setup = 0; 2842 txr->q_einval_tx_dma_setup = 0; 2843 txr->q_other_tx_dma_setup = 0; 2844 txr->q_eagain_tx_dma_setup = 0; 2845 txr->q_enomem_tx_dma_setup = 0; 2846 txr->q_tso_err = 0; 2847 2848 rxr->rx_packets.ev_count = 0; 2849 rxr->rx_bytes.ev_count = 0; 2850 rxr->rx_copies.ev_count = 0; 2851 rxr->no_mbuf.ev_count = 0; 2852 rxr->rx_discarded.ev_count = 0; 2853 } 2854 2855 /* MAC stats get their own sub node */ 2856 2857 stats->ipcs.ev_count = 0; 2858 stats->l4cs.ev_count = 0; 2859 stats->ipcs_bad.ev_count = 0; 2860 stats->l4cs_bad.ev_count = 0; 2861 2862 /* Packet Reception Stats */ 2863 stats->vfgprc.ev_count = 0; 2864 stats->vfgorc.ev_count = 0; 2865 stats->vfmprc.ev_count = 0; 2866 stats->vfgptc.ev_count = 0; 2867 stats->vfgotc.ev_count = 0; 2868 2869 /* Mailbox Stats */ 2870 hw->mbx.stats.msgs_tx.ev_count = 0; 2871 hw->mbx.stats.msgs_rx.ev_count = 0; 2872 hw->mbx.stats.acks.ev_count = 0; 2873 hw->mbx.stats.reqs.ev_count = 0; 2874 hw->mbx.stats.rsts.ev_count = 0; 2875 2876 } /* ixv_clear_evcnt */ 2877 2878 /************************************************************************ 2879 * ixv_set_sysctl_value 2880 ************************************************************************/ 2881 static void 2882 ixv_set_sysctl_value(struct adapter *adapter, const char *name, 2883 const char *description, int *limit, int value) 2884 { 2885 device_t dev = adapter->dev; 2886 struct sysctllog **log; 2887 const struct sysctlnode *rnode, *cnode; 2888 2889 log = &adapter->sysctllog; 2890 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2891 aprint_error_dev(dev, "could not create sysctl root\n"); 2892 return; 2893 } 2894 if (sysctl_createv(log, 0, &rnode, &cnode, 2895 CTLFLAG_READWRITE, CTLTYPE_INT, 2896 name, SYSCTL_DESCR(description), 2897 NULL, 0, limit, 0, CTL_CREATE, CTL_EOL) != 0) 2898 aprint_error_dev(dev, "could not create sysctl\n"); 2899 *limit = value; 2900 } /* ixv_set_sysctl_value */ 2901 2902 /************************************************************************ 2903 * ixv_print_debug_info 2904 * 2905 * Provides a way to take a look at important statistics 2906 * maintained by the driver and hardware. 2907 ************************************************************************/ 2908 static void 2909 ixv_print_debug_info(struct adapter *adapter) 2910 { 2911 device_t dev = adapter->dev; 2912 struct ix_queue *que = adapter->queues; 2913 struct rx_ring *rxr; 2914 struct tx_ring *txr; 2915 #ifdef LRO 2916 struct lro_ctrl *lro; 2917 #endif /* LRO */ 2918 2919 for (int i = 0; i < adapter->num_queues; i++, que++) { 2920 txr = que->txr; 2921 rxr = que->rxr; 2922 #ifdef LRO 2923 lro = &rxr->lro; 2924 #endif /* LRO */ 2925 device_printf(dev, "QUE(%d) IRQs Handled: %lu\n", 2926 que->msix, (long)que->irqs.ev_count); 2927 device_printf(dev, "RX(%d) Packets Received: %lld\n", 2928 rxr->me, (long long)rxr->rx_packets.ev_count); 2929 device_printf(dev, "RX(%d) Bytes Received: %lu\n", 2930 rxr->me, (long)rxr->rx_bytes.ev_count); 2931 #ifdef LRO 2932 device_printf(dev, "RX(%d) LRO Queued= %ju\n", 2933 rxr->me, (uintmax_t)lro->lro_queued); 2934 device_printf(dev, "RX(%d) LRO Flushed= %ju\n", 2935 rxr->me, (uintmax_t)lro->lro_flushed); 2936 #endif /* LRO */ 2937 device_printf(dev, "TX(%d) Packets Sent: %lu\n", 2938 txr->me, (long)txr->total_packets.ev_count); 2939 device_printf(dev, "TX(%d) NO Desc Avail: %lu\n", 2940 txr->me, (long)txr->no_desc_avail.ev_count); 2941 } 2942 2943 device_printf(dev, "Admin IRQ Handled: %lu\n", 2944 (long)adapter->admin_irqev.ev_count); 2945 device_printf(dev, "Admin work Handled: %lu\n", 2946 (long)adapter->link_workev.ev_count); 2947 } /* ixv_print_debug_info */ 2948 2949 /************************************************************************ 2950 * ixv_sysctl_debug 2951 ************************************************************************/ 2952 static int 2953 ixv_sysctl_debug(SYSCTLFN_ARGS) 2954 { 2955 struct sysctlnode node = *rnode; 2956 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2957 int error, result = 0; 2958 2959 node.sysctl_data = &result; 2960 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2961 2962 if (error || newp == NULL) 2963 return error; 2964 2965 if (result == 1) 2966 ixv_print_debug_info(adapter); 2967 2968 return 0; 2969 } /* ixv_sysctl_debug */ 2970 2971 /************************************************************************ 2972 * ixv_sysctl_rx_copy_len 2973 ************************************************************************/ 2974 static int 2975 ixv_sysctl_rx_copy_len(SYSCTLFN_ARGS) 2976 { 2977 struct sysctlnode node = *rnode; 2978 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2979 int error; 2980 int result = adapter->rx_copy_len; 2981 2982 node.sysctl_data = &result; 2983 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2984 2985 if (error || newp == NULL) 2986 return error; 2987 2988 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 2989 return EINVAL; 2990 2991 adapter->rx_copy_len = result; 2992 2993 return 0; 2994 } /* ixgbe_sysctl_rx_copy_len */ 2995 2996 /************************************************************************ 2997 * ixv_init_device_features 2998 ************************************************************************/ 2999 static void 3000 ixv_init_device_features(struct adapter *adapter) 3001 { 3002 adapter->feat_cap = IXGBE_FEATURE_NETMAP 3003 | IXGBE_FEATURE_VF 3004 | IXGBE_FEATURE_RSS 3005 | IXGBE_FEATURE_LEGACY_TX; 3006 3007 /* A tad short on feature flags for VFs, atm. */ 3008 switch (adapter->hw.mac.type) { 3009 case ixgbe_mac_82599_vf: 3010 break; 3011 case ixgbe_mac_X540_vf: 3012 break; 3013 case ixgbe_mac_X550_vf: 3014 case ixgbe_mac_X550EM_x_vf: 3015 case ixgbe_mac_X550EM_a_vf: 3016 adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 3017 break; 3018 default: 3019 break; 3020 } 3021 3022 /* Enabled by default... */ 3023 /* Is a virtual function (VF) */ 3024 if (adapter->feat_cap & IXGBE_FEATURE_VF) 3025 adapter->feat_en |= IXGBE_FEATURE_VF; 3026 /* Netmap */ 3027 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 3028 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 3029 /* Receive-Side Scaling (RSS) */ 3030 if (adapter->feat_cap & IXGBE_FEATURE_RSS) 3031 adapter->feat_en |= IXGBE_FEATURE_RSS; 3032 /* Needs advanced context descriptor regardless of offloads req'd */ 3033 if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 3034 adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 3035 3036 /* Enabled via sysctl... */ 3037 /* Legacy (single queue) transmit */ 3038 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 3039 ixv_enable_legacy_tx) 3040 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 3041 } /* ixv_init_device_features */ 3042 3043 /************************************************************************ 3044 * ixv_shutdown - Shutdown entry point 3045 ************************************************************************/ 3046 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3047 static int 3048 ixv_shutdown(device_t dev) 3049 { 3050 struct adapter *adapter = device_private(dev); 3051 IXGBE_CORE_LOCK(adapter); 3052 ixv_stop_locked(adapter); 3053 IXGBE_CORE_UNLOCK(adapter); 3054 3055 return (0); 3056 } /* ixv_shutdown */ 3057 #endif 3058 3059 static int 3060 ixv_ifflags_cb(struct ethercom *ec) 3061 { 3062 struct ifnet *ifp = &ec->ec_if; 3063 struct adapter *adapter = ifp->if_softc; 3064 u_short saved_flags; 3065 u_short change; 3066 int rv = 0; 3067 3068 IXGBE_CORE_LOCK(adapter); 3069 3070 saved_flags = adapter->if_flags; 3071 change = ifp->if_flags ^ adapter->if_flags; 3072 if (change != 0) 3073 adapter->if_flags = ifp->if_flags; 3074 3075 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 3076 rv = ENETRESET; 3077 goto out; 3078 } else if ((change & IFF_PROMISC) != 0) { 3079 rv = ixv_set_rxfilter(adapter); 3080 if (rv != 0) { 3081 /* Restore previous */ 3082 adapter->if_flags = saved_flags; 3083 goto out; 3084 } 3085 } 3086 3087 /* Check for ec_capenable. */ 3088 change = ec->ec_capenable ^ adapter->ec_capenable; 3089 adapter->ec_capenable = ec->ec_capenable; 3090 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 3091 | ETHERCAP_VLAN_HWFILTER)) != 0) { 3092 rv = ENETRESET; 3093 goto out; 3094 } 3095 3096 /* 3097 * Special handling is not required for ETHERCAP_VLAN_MTU. 3098 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 3099 */ 3100 3101 /* Set up VLAN support and filter */ 3102 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 3103 rv = ixv_setup_vlan_support(adapter); 3104 3105 out: 3106 IXGBE_CORE_UNLOCK(adapter); 3107 3108 return rv; 3109 } 3110 3111 3112 /************************************************************************ 3113 * ixv_ioctl - Ioctl entry point 3114 * 3115 * Called when the user wants to configure the interface. 3116 * 3117 * return 0 on success, positive on failure 3118 ************************************************************************/ 3119 static int 3120 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 3121 { 3122 struct adapter *adapter = ifp->if_softc; 3123 struct ixgbe_hw *hw = &adapter->hw; 3124 struct ifcapreq *ifcr = data; 3125 int error; 3126 int l4csum_en; 3127 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3128 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 3129 3130 switch (command) { 3131 case SIOCSIFFLAGS: 3132 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 3133 break; 3134 case SIOCADDMULTI: { 3135 struct ether_multi *enm; 3136 struct ether_multistep step; 3137 struct ethercom *ec = &adapter->osdep.ec; 3138 bool overflow = false; 3139 int mcnt = 0; 3140 3141 /* 3142 * Check the number of multicast address. If it exceeds, 3143 * return ENOSPC. 3144 * Update this code when we support API 1.3. 3145 */ 3146 ETHER_LOCK(ec); 3147 ETHER_FIRST_MULTI(step, ec, enm); 3148 while (enm != NULL) { 3149 mcnt++; 3150 3151 /* 3152 * This code is before adding, so one room is required 3153 * at least. 3154 */ 3155 if (mcnt > (IXGBE_MAX_VF_MC - 1)) { 3156 overflow = true; 3157 break; 3158 } 3159 ETHER_NEXT_MULTI(step, enm); 3160 } 3161 ETHER_UNLOCK(ec); 3162 error = 0; 3163 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) { 3164 error = hw->mac.ops.update_xcast_mode(hw, 3165 IXGBEVF_XCAST_MODE_ALLMULTI); 3166 if (error == IXGBE_ERR_NOT_TRUSTED) { 3167 device_printf(adapter->dev, 3168 "this interface is not trusted\n"); 3169 error = EPERM; 3170 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 3171 device_printf(adapter->dev, 3172 "the PF doesn't support allmulti mode\n"); 3173 error = EOPNOTSUPP; 3174 } else if (error) { 3175 device_printf(adapter->dev, 3176 "number of Ethernet multicast addresses " 3177 "exceeds the limit (%d). error = %d\n", 3178 IXGBE_MAX_VF_MC, error); 3179 error = ENOSPC; 3180 } else 3181 ec->ec_flags |= ETHER_F_ALLMULTI; 3182 } 3183 if (error) 3184 return error; 3185 } 3186 /*FALLTHROUGH*/ 3187 case SIOCDELMULTI: 3188 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 3189 break; 3190 case SIOCSIFMEDIA: 3191 case SIOCGIFMEDIA: 3192 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 3193 break; 3194 case SIOCSIFCAP: 3195 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 3196 break; 3197 case SIOCSIFMTU: 3198 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 3199 break; 3200 case SIOCZIFDATA: 3201 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 3202 ixv_update_stats(adapter); 3203 ixv_clear_evcnt(adapter); 3204 break; 3205 default: 3206 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 3207 break; 3208 } 3209 3210 switch (command) { 3211 case SIOCSIFCAP: 3212 /* Layer-4 Rx checksum offload has to be turned on and 3213 * off as a unit. 3214 */ 3215 l4csum_en = ifcr->ifcr_capenable & l4csum; 3216 if (l4csum_en != l4csum && l4csum_en != 0) 3217 return EINVAL; 3218 /*FALLTHROUGH*/ 3219 case SIOCADDMULTI: 3220 case SIOCDELMULTI: 3221 case SIOCSIFFLAGS: 3222 case SIOCSIFMTU: 3223 default: 3224 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 3225 return error; 3226 if ((ifp->if_flags & IFF_RUNNING) == 0) 3227 ; 3228 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 3229 IXGBE_CORE_LOCK(adapter); 3230 ixv_init_locked(adapter); 3231 IXGBE_CORE_UNLOCK(adapter); 3232 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 3233 /* 3234 * Multicast list has changed; set the hardware filter 3235 * accordingly. 3236 */ 3237 IXGBE_CORE_LOCK(adapter); 3238 ixv_disable_intr(adapter); 3239 ixv_set_rxfilter(adapter); 3240 ixv_enable_intr(adapter); 3241 IXGBE_CORE_UNLOCK(adapter); 3242 } 3243 return 0; 3244 } 3245 } /* ixv_ioctl */ 3246 3247 /************************************************************************ 3248 * ixv_init 3249 ************************************************************************/ 3250 static int 3251 ixv_init(struct ifnet *ifp) 3252 { 3253 struct adapter *adapter = ifp->if_softc; 3254 3255 IXGBE_CORE_LOCK(adapter); 3256 ixv_init_locked(adapter); 3257 IXGBE_CORE_UNLOCK(adapter); 3258 3259 return 0; 3260 } /* ixv_init */ 3261 3262 /************************************************************************ 3263 * ixv_handle_que 3264 ************************************************************************/ 3265 static void 3266 ixv_handle_que(void *context) 3267 { 3268 struct ix_queue *que = context; 3269 struct adapter *adapter = que->adapter; 3270 struct tx_ring *txr = que->txr; 3271 struct ifnet *ifp = adapter->ifp; 3272 bool more; 3273 3274 que->handleq.ev_count++; 3275 3276 if (ifp->if_flags & IFF_RUNNING) { 3277 more = ixgbe_rxeof(que); 3278 IXGBE_TX_LOCK(txr); 3279 more |= ixgbe_txeof(txr); 3280 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 3281 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 3282 ixgbe_mq_start_locked(ifp, txr); 3283 /* Only for queue 0 */ 3284 /* NetBSD still needs this for CBQ */ 3285 if ((&adapter->queues[0] == que) 3286 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 3287 ixgbe_legacy_start_locked(ifp, txr); 3288 IXGBE_TX_UNLOCK(txr); 3289 if (more) { 3290 que->req.ev_count++; 3291 if (adapter->txrx_use_workqueue) { 3292 /* 3293 * "enqueued flag" is not required here 3294 * the same as ixg(4). See ixgbe_msix_que(). 3295 */ 3296 workqueue_enqueue(adapter->que_wq, 3297 &que->wq_cookie, curcpu()); 3298 } else 3299 softint_schedule(que->que_si); 3300 return; 3301 } 3302 } 3303 3304 /* Re-enable this interrupt */ 3305 ixv_enable_queue(adapter, que->msix); 3306 3307 return; 3308 } /* ixv_handle_que */ 3309 3310 /************************************************************************ 3311 * ixv_handle_que_work 3312 ************************************************************************/ 3313 static void 3314 ixv_handle_que_work(struct work *wk, void *context) 3315 { 3316 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 3317 3318 /* 3319 * "enqueued flag" is not required here the same as ixg(4). 3320 * See ixgbe_msix_que(). 3321 */ 3322 ixv_handle_que(que); 3323 } 3324 3325 /************************************************************************ 3326 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 3327 ************************************************************************/ 3328 static int 3329 ixv_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 3330 { 3331 device_t dev = adapter->dev; 3332 struct ix_queue *que = adapter->queues; 3333 struct tx_ring *txr = adapter->tx_rings; 3334 int error, msix_ctrl, rid, vector = 0; 3335 pci_chipset_tag_t pc; 3336 pcitag_t tag; 3337 char intrbuf[PCI_INTRSTR_LEN]; 3338 char wqname[MAXCOMLEN]; 3339 char intr_xname[32]; 3340 const char *intrstr = NULL; 3341 kcpuset_t *affinity; 3342 int cpu_id = 0; 3343 3344 pc = adapter->osdep.pc; 3345 tag = adapter->osdep.tag; 3346 3347 adapter->osdep.nintrs = adapter->num_queues + 1; 3348 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 3349 adapter->osdep.nintrs) != 0) { 3350 aprint_error_dev(dev, 3351 "failed to allocate MSI-X interrupt\n"); 3352 return (ENXIO); 3353 } 3354 3355 kcpuset_create(&affinity, false); 3356 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 3357 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 3358 device_xname(dev), i); 3359 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 3360 sizeof(intrbuf)); 3361 #ifdef IXGBE_MPSAFE 3362 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 3363 true); 3364 #endif 3365 /* Set the handler function */ 3366 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 3367 adapter->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 3368 intr_xname); 3369 if (que->res == NULL) { 3370 pci_intr_release(pc, adapter->osdep.intrs, 3371 adapter->osdep.nintrs); 3372 aprint_error_dev(dev, 3373 "Failed to register QUE handler\n"); 3374 kcpuset_destroy(affinity); 3375 return (ENXIO); 3376 } 3377 que->msix = vector; 3378 adapter->active_queues |= (u64)(1 << que->msix); 3379 3380 cpu_id = i; 3381 /* Round-robin affinity */ 3382 kcpuset_zero(affinity); 3383 kcpuset_set(affinity, cpu_id % ncpu); 3384 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 3385 NULL); 3386 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 3387 intrstr); 3388 if (error == 0) 3389 aprint_normal(", bound queue %d to cpu %d\n", 3390 i, cpu_id % ncpu); 3391 else 3392 aprint_normal("\n"); 3393 3394 #ifndef IXGBE_LEGACY_TX 3395 txr->txr_si 3396 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3397 ixgbe_deferred_mq_start, txr); 3398 #endif 3399 que->que_si 3400 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3401 ixv_handle_que, que); 3402 if (que->que_si == NULL) { 3403 aprint_error_dev(dev, 3404 "could not establish software interrupt\n"); 3405 } 3406 } 3407 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 3408 error = workqueue_create(&adapter->txr_wq, wqname, 3409 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3410 IXGBE_WORKQUEUE_FLAGS); 3411 if (error) { 3412 aprint_error_dev(dev, 3413 "couldn't create workqueue for deferred Tx\n"); 3414 } 3415 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 3416 3417 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 3418 error = workqueue_create(&adapter->que_wq, wqname, 3419 ixv_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3420 IXGBE_WORKQUEUE_FLAGS); 3421 if (error) { 3422 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 3423 } 3424 3425 /* and Mailbox */ 3426 cpu_id++; 3427 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 3428 adapter->vector = vector; 3429 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 3430 sizeof(intrbuf)); 3431 #ifdef IXGBE_MPSAFE 3432 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 3433 true); 3434 #endif 3435 /* Set the mbx handler function */ 3436 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 3437 adapter->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, adapter, 3438 intr_xname); 3439 if (adapter->osdep.ihs[vector] == NULL) { 3440 aprint_error_dev(dev, "Failed to register LINK handler\n"); 3441 kcpuset_destroy(affinity); 3442 return (ENXIO); 3443 } 3444 /* Round-robin affinity */ 3445 kcpuset_zero(affinity); 3446 kcpuset_set(affinity, cpu_id % ncpu); 3447 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity, 3448 NULL); 3449 3450 aprint_normal_dev(dev, 3451 "for link, interrupting at %s", intrstr); 3452 if (error == 0) 3453 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 3454 else 3455 aprint_normal("\n"); 3456 3457 /* Tasklets for Mailbox */ 3458 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 3459 error = workqueue_create(&adapter->admin_wq, wqname, 3460 ixv_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3461 IXGBE_TASKLET_WQ_FLAGS); 3462 if (error) { 3463 aprint_error_dev(dev, 3464 "could not create admin workqueue (%d)\n", error); 3465 goto err_out; 3466 } 3467 3468 /* 3469 * Due to a broken design QEMU will fail to properly 3470 * enable the guest for MSI-X unless the vectors in 3471 * the table are all set up, so we must rewrite the 3472 * ENABLE in the MSI-X control register again at this 3473 * point to cause it to successfully initialize us. 3474 */ 3475 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 3476 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3477 rid += PCI_MSIX_CTL; 3478 msix_ctrl = pci_conf_read(pc, tag, rid); 3479 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3480 pci_conf_write(pc, tag, rid, msix_ctrl); 3481 } 3482 3483 kcpuset_destroy(affinity); 3484 return (0); 3485 err_out: 3486 kcpuset_destroy(affinity); 3487 ixv_free_deferred_handlers(adapter); 3488 ixv_free_pci_resources(adapter); 3489 return (error); 3490 } /* ixv_allocate_msix */ 3491 3492 /************************************************************************ 3493 * ixv_configure_interrupts - Setup MSI-X resources 3494 * 3495 * Note: The VF device MUST use MSI-X, there is no fallback. 3496 ************************************************************************/ 3497 static int 3498 ixv_configure_interrupts(struct adapter *adapter) 3499 { 3500 device_t dev = adapter->dev; 3501 int want, queues, msgs; 3502 3503 /* Must have at least 2 MSI-X vectors */ 3504 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 3505 if (msgs < 2) { 3506 aprint_error_dev(dev, "MSIX config error\n"); 3507 return (ENXIO); 3508 } 3509 msgs = MIN(msgs, IXG_MAX_NINTR); 3510 3511 /* Figure out a reasonable auto config value */ 3512 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3513 3514 if (ixv_num_queues != 0) 3515 queues = ixv_num_queues; 3516 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3517 queues = IXGBE_VF_MAX_TX_QUEUES; 3518 3519 /* 3520 * Want vectors for the queues, 3521 * plus an additional for mailbox. 3522 */ 3523 want = queues + 1; 3524 if (msgs >= want) 3525 msgs = want; 3526 else { 3527 aprint_error_dev(dev, 3528 "MSI-X Configuration Problem, " 3529 "%d vectors but %d queues wanted!\n", 3530 msgs, want); 3531 return -1; 3532 } 3533 3534 adapter->msix_mem = (void *)1; /* XXX */ 3535 aprint_normal_dev(dev, 3536 "Using MSI-X interrupts with %d vectors\n", msgs); 3537 adapter->num_queues = queues; 3538 3539 return (0); 3540 } /* ixv_configure_interrupts */ 3541 3542 3543 /************************************************************************ 3544 * ixv_handle_admin - Tasklet handler for MSI-X MBX interrupts 3545 * 3546 * Done outside of interrupt context since the driver might sleep 3547 ************************************************************************/ 3548 static void 3549 ixv_handle_admin(struct work *wk, void *context) 3550 { 3551 struct adapter *adapter = context; 3552 struct ixgbe_hw *hw = &adapter->hw; 3553 3554 IXGBE_CORE_LOCK(adapter); 3555 3556 ++adapter->link_workev.ev_count; 3557 adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, 3558 &adapter->link_up, FALSE); 3559 ixv_update_link_status(adapter); 3560 3561 adapter->task_requests = 0; 3562 atomic_store_relaxed(&adapter->admin_pending, 0); 3563 3564 /* Re-enable interrupts */ 3565 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 3566 3567 IXGBE_CORE_UNLOCK(adapter); 3568 } /* ixv_handle_admin */ 3569 3570 /************************************************************************ 3571 * ixv_check_link - Used in the local timer to poll for link changes 3572 ************************************************************************/ 3573 static s32 3574 ixv_check_link(struct adapter *adapter) 3575 { 3576 s32 error; 3577 3578 KASSERT(mutex_owned(&adapter->core_mtx)); 3579 3580 adapter->hw.mac.get_link_status = TRUE; 3581 3582 error = adapter->hw.mac.ops.check_link(&adapter->hw, 3583 &adapter->link_speed, &adapter->link_up, FALSE); 3584 ixv_update_link_status(adapter); 3585 3586 return error; 3587 } /* ixv_check_link */ 3588
Indexes created Tue Sep 30 20:09:53 GMT 2025