ixv.c revision 1.178
1 /* $NetBSD: ixv.c,v 1.178 2022/01/25 01:56:22 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ixv.c,v 1.178 2022/01/25 01:56:22 msaitoh Exp $"); 39 40 #ifdef _KERNEL_OPT 41 #include "opt_inet.h" 42 #include "opt_inet6.h" 43 #include "opt_net_mpsafe.h" 44 #endif 45 46 #include "ixgbe.h" 47 48 /************************************************************************ 49 * Driver version 50 ************************************************************************/ 51 static const char ixv_driver_version[] = "2.0.1-k"; 52 /* XXX NetBSD: + 1.5.17 */ 53 54 /************************************************************************ 55 * PCI Device ID Table 56 * 57 * Used by probe to select devices to load on 58 * Last field stores an index into ixv_strings 59 * Last entry must be all 0s 60 * 61 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 62 ************************************************************************/ 63 static const ixgbe_vendor_info_t ixv_vendor_info_array[] = 64 { 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 70 /* required last entry */ 71 {0, 0, 0, 0, 0} 72 }; 73 74 /************************************************************************ 75 * Table of branding strings 76 ************************************************************************/ 77 static const char *ixv_strings[] = { 78 "Intel(R) PRO/10GbE Virtual Function Network Driver" 79 }; 80 81 /********************************************************************* 82 * Function prototypes 83 *********************************************************************/ 84 static int ixv_probe(device_t, cfdata_t, void *); 85 static void ixv_attach(device_t, device_t, void *); 86 static int ixv_detach(device_t, int); 87 #if 0 88 static int ixv_shutdown(device_t); 89 #endif 90 static int ixv_ifflags_cb(struct ethercom *); 91 static int ixv_ioctl(struct ifnet *, u_long, void *); 92 static int ixv_init(struct ifnet *); 93 static void ixv_init_locked(struct adapter *); 94 static void ixv_ifstop(struct ifnet *, int); 95 static void ixv_stop_locked(void *); 96 static void ixv_init_device_features(struct adapter *); 97 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 98 static int ixv_media_change(struct ifnet *); 99 static int ixv_allocate_pci_resources(struct adapter *, 100 const struct pci_attach_args *); 101 static void ixv_free_deferred_handlers(struct adapter *); 102 static int ixv_allocate_msix(struct adapter *, 103 const struct pci_attach_args *); 104 static int ixv_configure_interrupts(struct adapter *); 105 static void ixv_free_pci_resources(struct adapter *); 106 static void ixv_local_timer(void *); 107 static void ixv_handle_timer(struct work *, void *); 108 static int ixv_setup_interface(device_t, struct adapter *); 109 static void ixv_schedule_admin_tasklet(struct adapter *); 110 static int ixv_negotiate_api(struct adapter *); 111 112 static void ixv_initialize_transmit_units(struct adapter *); 113 static void ixv_initialize_receive_units(struct adapter *); 114 static void ixv_initialize_rss_mapping(struct adapter *); 115 static s32 ixv_check_link(struct adapter *); 116 117 static void ixv_enable_intr(struct adapter *); 118 static void ixv_disable_intr(struct adapter *); 119 static int ixv_set_rxfilter(struct adapter *); 120 static void ixv_update_link_status(struct adapter *); 121 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 122 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 123 static void ixv_configure_ivars(struct adapter *); 124 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 125 static void ixv_eitr_write(struct adapter *, uint32_t, uint32_t); 126 127 static void ixv_setup_vlan_tagging(struct adapter *); 128 static int ixv_setup_vlan_support(struct adapter *); 129 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool); 130 static int ixv_register_vlan(struct adapter *, u16); 131 static int ixv_unregister_vlan(struct adapter *, u16); 132 133 static void ixv_add_device_sysctls(struct adapter *); 134 static void ixv_init_stats(struct adapter *); 135 static void ixv_update_stats(struct adapter *); 136 static void ixv_add_stats_sysctls(struct adapter *); 137 static void ixv_clear_evcnt(struct adapter *); 138 139 /* Sysctl handlers */ 140 static void ixv_set_sysctl_value(struct adapter *, const char *, 141 const char *, int *, int); 142 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 143 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 144 static int ixv_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 145 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 146 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 147 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 148 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 149 static int ixv_sysctl_rx_copy_len(SYSCTLFN_PROTO); 150 151 /* The MSI-X Interrupt handlers */ 152 static int ixv_msix_que(void *); 153 static int ixv_msix_mbx(void *); 154 155 /* Event handlers running on workqueue */ 156 static void ixv_handle_que(void *); 157 158 /* Deferred workqueue handlers */ 159 static void ixv_handle_admin(struct work *, void *); 160 static void ixv_handle_que_work(struct work *, void *); 161 162 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 163 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 164 165 /************************************************************************ 166 * NetBSD Device Interface Entry Points 167 ************************************************************************/ 168 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 169 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 170 DVF_DETACH_SHUTDOWN); 171 172 #if 0 173 static driver_t ixv_driver = { 174 "ixv", ixv_methods, sizeof(struct adapter), 175 }; 176 177 devclass_t ixv_devclass; 178 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 179 MODULE_DEPEND(ixv, pci, 1, 1, 1); 180 MODULE_DEPEND(ixv, ether, 1, 1, 1); 181 #endif 182 183 /* 184 * TUNEABLE PARAMETERS: 185 */ 186 187 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 188 static int ixv_num_queues = 0; 189 #define TUNABLE_INT(__x, __y) 190 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 191 192 /* 193 * AIM: Adaptive Interrupt Moderation 194 * which means that the interrupt rate 195 * is varied over time based on the 196 * traffic for that interrupt vector 197 */ 198 static bool ixv_enable_aim = false; 199 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 200 201 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 202 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 203 204 /* How many packets rxeof tries to clean at a time */ 205 static int ixv_rx_process_limit = 256; 206 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 207 208 /* How many packets txeof tries to clean at a time */ 209 static int ixv_tx_process_limit = 256; 210 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 211 212 /* Which packet processing uses workqueue or softint */ 213 static bool ixv_txrx_workqueue = false; 214 215 /* 216 * Number of TX descriptors per ring, 217 * setting higher than RX as this seems 218 * the better performing choice. 219 */ 220 static int ixv_txd = PERFORM_TXD; 221 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 222 223 /* Number of RX descriptors per ring */ 224 static int ixv_rxd = PERFORM_RXD; 225 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 226 227 /* Legacy Transmit (single queue) */ 228 static int ixv_enable_legacy_tx = 0; 229 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 230 231 #ifdef NET_MPSAFE 232 #define IXGBE_MPSAFE 1 233 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 234 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 235 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 236 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 237 #else 238 #define IXGBE_CALLOUT_FLAGS 0 239 #define IXGBE_SOFTINT_FLAGS 0 240 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 241 #define IXGBE_TASKLET_WQ_FLAGS 0 242 #endif 243 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 244 245 #if 0 246 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 247 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 248 #endif 249 250 /************************************************************************ 251 * ixv_probe - Device identification routine 252 * 253 * Determines if the driver should be loaded on 254 * adapter based on its PCI vendor/device ID. 255 * 256 * return BUS_PROBE_DEFAULT on success, positive on failure 257 ************************************************************************/ 258 static int 259 ixv_probe(device_t dev, cfdata_t cf, void *aux) 260 { 261 #ifdef __HAVE_PCI_MSI_MSIX 262 const struct pci_attach_args *pa = aux; 263 264 return (ixv_lookup(pa) != NULL) ? 1 : 0; 265 #else 266 return 0; 267 #endif 268 } /* ixv_probe */ 269 270 static const ixgbe_vendor_info_t * 271 ixv_lookup(const struct pci_attach_args *pa) 272 { 273 const ixgbe_vendor_info_t *ent; 274 pcireg_t subid; 275 276 INIT_DEBUGOUT("ixv_lookup: begin"); 277 278 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 279 return NULL; 280 281 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 282 283 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 284 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 285 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 286 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 287 (ent->subvendor_id == 0)) && 288 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 289 (ent->subdevice_id == 0))) { 290 return ent; 291 } 292 } 293 294 return NULL; 295 } 296 297 /************************************************************************ 298 * ixv_attach - Device initialization routine 299 * 300 * Called when the driver is being loaded. 301 * Identifies the type of hardware, allocates all resources 302 * and initializes the hardware. 303 * 304 * return 0 on success, positive on failure 305 ************************************************************************/ 306 static void 307 ixv_attach(device_t parent, device_t dev, void *aux) 308 { 309 struct adapter *adapter; 310 struct ixgbe_hw *hw; 311 int error = 0; 312 pcireg_t id, subid; 313 const ixgbe_vendor_info_t *ent; 314 const struct pci_attach_args *pa = aux; 315 const char *apivstr; 316 const char *str; 317 char wqname[MAXCOMLEN]; 318 char buf[256]; 319 320 INIT_DEBUGOUT("ixv_attach: begin"); 321 322 /* 323 * Make sure BUSMASTER is set, on a VM under 324 * KVM it may not be and will break things. 325 */ 326 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 327 328 /* Allocate, clear, and link in our adapter structure */ 329 adapter = device_private(dev); 330 adapter->hw.back = adapter; 331 adapter->dev = dev; 332 hw = &adapter->hw; 333 334 adapter->init_locked = ixv_init_locked; 335 adapter->stop_locked = ixv_stop_locked; 336 337 adapter->osdep.pc = pa->pa_pc; 338 adapter->osdep.tag = pa->pa_tag; 339 if (pci_dma64_available(pa)) 340 adapter->osdep.dmat = pa->pa_dmat64; 341 else 342 adapter->osdep.dmat = pa->pa_dmat; 343 adapter->osdep.attached = false; 344 345 ent = ixv_lookup(pa); 346 347 KASSERT(ent != NULL); 348 349 aprint_normal(": %s, Version - %s\n", 350 ixv_strings[ent->index], ixv_driver_version); 351 352 /* Core Lock Init */ 353 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 354 355 /* Do base PCI setup - map BAR0 */ 356 if (ixv_allocate_pci_resources(adapter, pa)) { 357 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 358 error = ENXIO; 359 goto err_out; 360 } 361 362 /* SYSCTL APIs */ 363 ixv_add_device_sysctls(adapter); 364 365 /* Set up the timer callout and workqueue */ 366 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 367 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 368 error = workqueue_create(&adapter->timer_wq, wqname, 369 ixv_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 370 IXGBE_TASKLET_WQ_FLAGS); 371 if (error) { 372 aprint_error_dev(dev, 373 "could not create timer workqueue (%d)\n", error); 374 goto err_out; 375 } 376 377 /* Save off the information about this board */ 378 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 379 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 380 hw->vendor_id = PCI_VENDOR(id); 381 hw->device_id = PCI_PRODUCT(id); 382 hw->revision_id = 383 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 384 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 385 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 386 387 /* A subset of set_mac_type */ 388 switch (hw->device_id) { 389 case IXGBE_DEV_ID_82599_VF: 390 hw->mac.type = ixgbe_mac_82599_vf; 391 str = "82599 VF"; 392 break; 393 case IXGBE_DEV_ID_X540_VF: 394 hw->mac.type = ixgbe_mac_X540_vf; 395 str = "X540 VF"; 396 break; 397 case IXGBE_DEV_ID_X550_VF: 398 hw->mac.type = ixgbe_mac_X550_vf; 399 str = "X550 VF"; 400 break; 401 case IXGBE_DEV_ID_X550EM_X_VF: 402 hw->mac.type = ixgbe_mac_X550EM_x_vf; 403 str = "X550EM X VF"; 404 break; 405 case IXGBE_DEV_ID_X550EM_A_VF: 406 hw->mac.type = ixgbe_mac_X550EM_a_vf; 407 str = "X550EM A VF"; 408 break; 409 default: 410 /* Shouldn't get here since probe succeeded */ 411 aprint_error_dev(dev, "Unknown device ID!\n"); 412 error = ENXIO; 413 goto err_out; 414 break; 415 } 416 aprint_normal_dev(dev, "device %s\n", str); 417 418 ixv_init_device_features(adapter); 419 420 /* Initialize the shared code */ 421 error = ixgbe_init_ops_vf(hw); 422 if (error) { 423 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 424 error = EIO; 425 goto err_out; 426 } 427 428 /* Setup the mailbox */ 429 ixgbe_init_mbx_params_vf(hw); 430 431 /* Set the right number of segments */ 432 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 433 adapter->num_segs = IXGBE_SCATTER_DEFAULT; 434 435 /* Reset mbox api to 1.0 */ 436 error = hw->mac.ops.reset_hw(hw); 437 if (error == IXGBE_ERR_RESET_FAILED) 438 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 439 else if (error) 440 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 441 error); 442 if (error) { 443 error = EIO; 444 goto err_out; 445 } 446 447 error = hw->mac.ops.init_hw(hw); 448 if (error) { 449 aprint_error_dev(dev, "...init_hw() failed!\n"); 450 error = EIO; 451 goto err_out; 452 } 453 454 /* Negotiate mailbox API version */ 455 error = ixv_negotiate_api(adapter); 456 if (error) 457 aprint_normal_dev(dev, 458 "MBX API negotiation failed during attach!\n"); 459 switch (hw->api_version) { 460 case ixgbe_mbox_api_10: 461 apivstr = "1.0"; 462 break; 463 case ixgbe_mbox_api_20: 464 apivstr = "2.0"; 465 break; 466 case ixgbe_mbox_api_11: 467 apivstr = "1.1"; 468 break; 469 case ixgbe_mbox_api_12: 470 apivstr = "1.2"; 471 break; 472 case ixgbe_mbox_api_13: 473 apivstr = "1.3"; 474 break; 475 case ixgbe_mbox_api_14: 476 apivstr = "1.4"; 477 break; 478 case ixgbe_mbox_api_15: 479 apivstr = "1.5"; 480 break; 481 default: 482 apivstr = "unknown"; 483 break; 484 } 485 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 486 487 /* If no mac address was assigned, make a random one */ 488 if (!ixv_check_ether_addr(hw->mac.addr)) { 489 u8 addr[ETHER_ADDR_LEN]; 490 uint64_t rndval = cprng_strong64(); 491 492 memcpy(addr, &rndval, sizeof(addr)); 493 addr[0] &= 0xFE; 494 addr[0] |= 0x02; 495 bcopy(addr, hw->mac.addr, sizeof(addr)); 496 } 497 498 /* Register for VLAN events */ 499 ether_set_vlan_cb(&adapter->osdep.ec, ixv_vlan_cb); 500 501 /* Sysctls for limiting the amount of work done in the taskqueues */ 502 ixv_set_sysctl_value(adapter, "rx_processing_limit", 503 "max number of rx packets to process", 504 &adapter->rx_process_limit, ixv_rx_process_limit); 505 506 ixv_set_sysctl_value(adapter, "tx_processing_limit", 507 "max number of tx packets to process", 508 &adapter->tx_process_limit, ixv_tx_process_limit); 509 510 /* Do descriptor calc and sanity checks */ 511 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 512 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 513 aprint_error_dev(dev, "TXD config issue, using default!\n"); 514 adapter->num_tx_desc = DEFAULT_TXD; 515 } else 516 adapter->num_tx_desc = ixv_txd; 517 518 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 519 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 520 aprint_error_dev(dev, "RXD config issue, using default!\n"); 521 adapter->num_rx_desc = DEFAULT_RXD; 522 } else 523 adapter->num_rx_desc = ixv_rxd; 524 525 /* Set default high limit of copying mbuf in rxeof */ 526 adapter->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 527 528 /* Setup MSI-X */ 529 error = ixv_configure_interrupts(adapter); 530 if (error) 531 goto err_out; 532 533 /* Allocate our TX/RX Queues */ 534 if (ixgbe_allocate_queues(adapter)) { 535 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 536 error = ENOMEM; 537 goto err_out; 538 } 539 540 /* hw.ix defaults init */ 541 adapter->enable_aim = ixv_enable_aim; 542 543 adapter->txrx_use_workqueue = ixv_txrx_workqueue; 544 545 error = ixv_allocate_msix(adapter, pa); 546 if (error) { 547 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n"); 548 goto err_late; 549 } 550 551 /* Setup OS specific network interface */ 552 error = ixv_setup_interface(dev, adapter); 553 if (error != 0) { 554 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 555 goto err_late; 556 } 557 558 /* Allocate multicast array memory */ 559 adapter->mta = malloc(sizeof(*adapter->mta) * 560 IXGBE_MAX_VF_MC, M_DEVBUF, M_WAITOK); 561 562 /* Do the stats setup */ 563 ixv_init_stats(adapter); 564 ixv_add_stats_sysctls(adapter); 565 566 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 567 ixgbe_netmap_attach(adapter); 568 569 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 570 aprint_verbose_dev(dev, "feature cap %s\n", buf); 571 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 572 aprint_verbose_dev(dev, "feature ena %s\n", buf); 573 574 INIT_DEBUGOUT("ixv_attach: end"); 575 adapter->osdep.attached = true; 576 577 return; 578 579 err_late: 580 ixgbe_free_queues(adapter); 581 err_out: 582 ixv_free_pci_resources(adapter); 583 IXGBE_CORE_LOCK_DESTROY(adapter); 584 585 return; 586 } /* ixv_attach */ 587 588 /************************************************************************ 589 * ixv_detach - Device removal routine 590 * 591 * Called when the driver is being removed. 592 * Stops the adapter and deallocates all the resources 593 * that were allocated for driver operation. 594 * 595 * return 0 on success, positive on failure 596 ************************************************************************/ 597 static int 598 ixv_detach(device_t dev, int flags) 599 { 600 struct adapter *adapter = device_private(dev); 601 struct ixgbe_hw *hw = &adapter->hw; 602 struct tx_ring *txr = adapter->tx_rings; 603 struct rx_ring *rxr = adapter->rx_rings; 604 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 605 606 INIT_DEBUGOUT("ixv_detach: begin"); 607 if (adapter->osdep.attached == false) 608 return 0; 609 610 /* Stop the interface. Callouts are stopped in it. */ 611 ixv_ifstop(adapter->ifp, 1); 612 613 if (VLAN_ATTACHED(&adapter->osdep.ec) && 614 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 615 aprint_error_dev(dev, "VLANs in use, detach first\n"); 616 return EBUSY; 617 } 618 619 ether_ifdetach(adapter->ifp); 620 callout_halt(&adapter->timer, NULL); 621 ixv_free_deferred_handlers(adapter); 622 623 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 624 netmap_detach(adapter->ifp); 625 626 ixv_free_pci_resources(adapter); 627 #if 0 /* XXX the NetBSD port is probably missing something here */ 628 bus_generic_detach(dev); 629 #endif 630 if_detach(adapter->ifp); 631 ifmedia_fini(&adapter->media); 632 if_percpuq_destroy(adapter->ipq); 633 634 sysctl_teardown(&adapter->sysctllog); 635 evcnt_detach(&adapter->efbig_tx_dma_setup); 636 evcnt_detach(&adapter->mbuf_defrag_failed); 637 evcnt_detach(&adapter->efbig2_tx_dma_setup); 638 evcnt_detach(&adapter->einval_tx_dma_setup); 639 evcnt_detach(&adapter->other_tx_dma_setup); 640 evcnt_detach(&adapter->eagain_tx_dma_setup); 641 evcnt_detach(&adapter->enomem_tx_dma_setup); 642 evcnt_detach(&adapter->watchdog_events); 643 evcnt_detach(&adapter->tso_err); 644 evcnt_detach(&adapter->admin_irqev); 645 evcnt_detach(&adapter->link_workev); 646 647 txr = adapter->tx_rings; 648 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 649 evcnt_detach(&adapter->queues[i].irqs); 650 evcnt_detach(&adapter->queues[i].handleq); 651 evcnt_detach(&adapter->queues[i].req); 652 evcnt_detach(&txr->no_desc_avail); 653 evcnt_detach(&txr->total_packets); 654 evcnt_detach(&txr->tso_tx); 655 #ifndef IXGBE_LEGACY_TX 656 evcnt_detach(&txr->pcq_drops); 657 #endif 658 659 evcnt_detach(&rxr->rx_packets); 660 evcnt_detach(&rxr->rx_bytes); 661 evcnt_detach(&rxr->rx_copies); 662 evcnt_detach(&rxr->no_mbuf); 663 evcnt_detach(&rxr->rx_discarded); 664 } 665 evcnt_detach(&stats->ipcs); 666 evcnt_detach(&stats->l4cs); 667 evcnt_detach(&stats->ipcs_bad); 668 evcnt_detach(&stats->l4cs_bad); 669 670 /* Packet Reception Stats */ 671 evcnt_detach(&stats->vfgorc); 672 evcnt_detach(&stats->vfgprc); 673 evcnt_detach(&stats->vfmprc); 674 675 /* Packet Transmission Stats */ 676 evcnt_detach(&stats->vfgotc); 677 evcnt_detach(&stats->vfgptc); 678 679 /* Mailbox Stats */ 680 evcnt_detach(&hw->mbx.stats.msgs_tx); 681 evcnt_detach(&hw->mbx.stats.msgs_rx); 682 evcnt_detach(&hw->mbx.stats.acks); 683 evcnt_detach(&hw->mbx.stats.reqs); 684 evcnt_detach(&hw->mbx.stats.rsts); 685 686 ixgbe_free_queues(adapter); 687 688 IXGBE_CORE_LOCK_DESTROY(adapter); 689 690 return (0); 691 } /* ixv_detach */ 692 693 /************************************************************************ 694 * ixv_init_locked - Init entry point 695 * 696 * Used in two ways: It is used by the stack as an init entry 697 * point in network interface structure. It is also used 698 * by the driver as a hw/sw initialization routine to get 699 * to a consistent state. 700 * 701 * return 0 on success, positive on failure 702 ************************************************************************/ 703 static void 704 ixv_init_locked(struct adapter *adapter) 705 { 706 struct ifnet *ifp = adapter->ifp; 707 device_t dev = adapter->dev; 708 struct ixgbe_hw *hw = &adapter->hw; 709 struct ix_queue *que; 710 int error = 0; 711 uint32_t mask; 712 int i; 713 714 INIT_DEBUGOUT("ixv_init_locked: begin"); 715 KASSERT(mutex_owned(&adapter->core_mtx)); 716 hw->adapter_stopped = FALSE; 717 hw->mac.ops.stop_adapter(hw); 718 callout_stop(&adapter->timer); 719 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 720 que->disabled_count = 0; 721 722 adapter->max_frame_size = 723 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 724 725 /* reprogram the RAR[0] in case user changed it. */ 726 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 727 728 /* Get the latest mac address, User can use a LAA */ 729 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 730 IXGBE_ETH_LENGTH_OF_ADDRESS); 731 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 732 733 /* Prepare transmit descriptors and buffers */ 734 if (ixgbe_setup_transmit_structures(adapter)) { 735 aprint_error_dev(dev, "Could not setup transmit structures\n"); 736 ixv_stop_locked(adapter); 737 return; 738 } 739 740 /* Reset VF and renegotiate mailbox API version */ 741 hw->mac.ops.reset_hw(hw); 742 hw->mac.ops.start_hw(hw); 743 error = ixv_negotiate_api(adapter); 744 if (error) 745 device_printf(dev, 746 "Mailbox API negotiation failed in init_locked!\n"); 747 748 ixv_initialize_transmit_units(adapter); 749 750 /* Setup Multicast table */ 751 ixv_set_rxfilter(adapter); 752 753 /* Use fixed buffer size, even for jumbo frames */ 754 adapter->rx_mbuf_sz = MCLBYTES; 755 756 /* Prepare receive descriptors and buffers */ 757 error = ixgbe_setup_receive_structures(adapter); 758 if (error) { 759 device_printf(dev, 760 "Could not setup receive structures (err = %d)\n", error); 761 ixv_stop_locked(adapter); 762 return; 763 } 764 765 /* Configure RX settings */ 766 ixv_initialize_receive_units(adapter); 767 768 /* Initialize variable holding task enqueue requests interrupts */ 769 adapter->task_requests = 0; 770 771 /* Set up VLAN offload and filter */ 772 ixv_setup_vlan_support(adapter); 773 774 /* Set up MSI-X routing */ 775 ixv_configure_ivars(adapter); 776 777 /* Set up auto-mask */ 778 mask = (1 << adapter->vector); 779 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 780 mask |= (1 << que->msix); 781 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 782 783 /* Set moderation on the Link interrupt */ 784 ixv_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 785 786 /* Stats init */ 787 ixv_init_stats(adapter); 788 789 /* Config/Enable Link */ 790 hw->mac.get_link_status = TRUE; 791 hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up, 792 FALSE); 793 794 /* Start watchdog */ 795 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 796 atomic_store_relaxed(&adapter->timer_pending, 0); 797 798 /* OK to schedule workqueues. */ 799 adapter->schedule_wqs_ok = true; 800 801 /* And now turn on interrupts */ 802 ixv_enable_intr(adapter); 803 804 /* Update saved flags. See ixgbe_ifflags_cb() */ 805 adapter->if_flags = ifp->if_flags; 806 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 807 808 /* Now inform the stack we're ready */ 809 ifp->if_flags |= IFF_RUNNING; 810 ifp->if_flags &= ~IFF_OACTIVE; 811 812 return; 813 } /* ixv_init_locked */ 814 815 /************************************************************************ 816 * ixv_enable_queue 817 ************************************************************************/ 818 static inline void 819 ixv_enable_queue(struct adapter *adapter, u32 vector) 820 { 821 struct ixgbe_hw *hw = &adapter->hw; 822 struct ix_queue *que = &adapter->queues[vector]; 823 u32 queue = 1UL << vector; 824 u32 mask; 825 826 mutex_enter(&que->dc_mtx); 827 if (que->disabled_count > 0 && --que->disabled_count > 0) 828 goto out; 829 830 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 831 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 832 out: 833 mutex_exit(&que->dc_mtx); 834 } /* ixv_enable_queue */ 835 836 /************************************************************************ 837 * ixv_disable_queue 838 ************************************************************************/ 839 static inline void 840 ixv_disable_queue(struct adapter *adapter, u32 vector) 841 { 842 struct ixgbe_hw *hw = &adapter->hw; 843 struct ix_queue *que = &adapter->queues[vector]; 844 u32 queue = 1UL << vector; 845 u32 mask; 846 847 mutex_enter(&que->dc_mtx); 848 if (que->disabled_count++ > 0) 849 goto out; 850 851 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 852 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 853 out: 854 mutex_exit(&que->dc_mtx); 855 } /* ixv_disable_queue */ 856 857 #if 0 858 static inline void 859 ixv_rearm_queues(struct adapter *adapter, u64 queues) 860 { 861 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 862 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 863 } /* ixv_rearm_queues */ 864 #endif 865 866 867 /************************************************************************ 868 * ixv_msix_que - MSI-X Queue Interrupt Service routine 869 ************************************************************************/ 870 static int 871 ixv_msix_que(void *arg) 872 { 873 struct ix_queue *que = arg; 874 struct adapter *adapter = que->adapter; 875 struct tx_ring *txr = que->txr; 876 struct rx_ring *rxr = que->rxr; 877 bool more; 878 u32 newitr = 0; 879 880 ixv_disable_queue(adapter, que->msix); 881 IXGBE_EVC_ADD(&que->irqs, 1); 882 883 #ifdef __NetBSD__ 884 /* Don't run ixgbe_rxeof in interrupt context */ 885 more = true; 886 #else 887 more = ixgbe_rxeof(que); 888 #endif 889 890 IXGBE_TX_LOCK(txr); 891 ixgbe_txeof(txr); 892 IXGBE_TX_UNLOCK(txr); 893 894 /* Do AIM now? */ 895 896 if (adapter->enable_aim == false) 897 goto no_calc; 898 /* 899 * Do Adaptive Interrupt Moderation: 900 * - Write out last calculated setting 901 * - Calculate based on average size over 902 * the last interval. 903 */ 904 if (que->eitr_setting) 905 ixv_eitr_write(adapter, que->msix, que->eitr_setting); 906 907 que->eitr_setting = 0; 908 909 /* Idle, do nothing */ 910 if ((txr->bytes == 0) && (rxr->bytes == 0)) 911 goto no_calc; 912 913 if ((txr->bytes) && (txr->packets)) 914 newitr = txr->bytes/txr->packets; 915 if ((rxr->bytes) && (rxr->packets)) 916 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 917 newitr += 24; /* account for hardware frame, crc */ 918 919 /* set an upper boundary */ 920 newitr = uimin(newitr, 3000); 921 922 /* Be nice to the mid range */ 923 if ((newitr > 300) && (newitr < 1200)) 924 newitr = (newitr / 3); 925 else 926 newitr = (newitr / 2); 927 928 /* 929 * When RSC is used, ITR interval must be larger than RSC_DELAY. 930 * Currently, we use 2us for RSC_DELAY. The minimum value is always 931 * greater than 2us on 100M (and 10M?(not documented)), but it's not 932 * on 1G and higher. 933 */ 934 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 935 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 936 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 937 newitr = IXGBE_MIN_RSC_EITR_10G1G; 938 } 939 940 /* save for next interrupt */ 941 que->eitr_setting = newitr; 942 943 /* Reset state */ 944 txr->bytes = 0; 945 txr->packets = 0; 946 rxr->bytes = 0; 947 rxr->packets = 0; 948 949 no_calc: 950 if (more) 951 softint_schedule(que->que_si); 952 else /* Re-enable this interrupt */ 953 ixv_enable_queue(adapter, que->msix); 954 955 return 1; 956 } /* ixv_msix_que */ 957 958 /************************************************************************ 959 * ixv_msix_mbx 960 ************************************************************************/ 961 static int 962 ixv_msix_mbx(void *arg) 963 { 964 struct adapter *adapter = arg; 965 struct ixgbe_hw *hw = &adapter->hw; 966 967 IXGBE_EVC_ADD(&adapter->admin_irqev, 1); 968 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 969 970 /* Link status change */ 971 hw->mac.get_link_status = TRUE; 972 atomic_or_32(&adapter->task_requests, IXGBE_REQUEST_TASK_MBX); 973 ixv_schedule_admin_tasklet(adapter); 974 975 return 1; 976 } /* ixv_msix_mbx */ 977 978 static void 979 ixv_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 980 { 981 982 /* 983 * Newer devices than 82598 have VF function, so this function is 984 * simple. 985 */ 986 itr |= IXGBE_EITR_CNT_WDIS; 987 988 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(index), itr); 989 } 990 991 992 /************************************************************************ 993 * ixv_media_status - Media Ioctl callback 994 * 995 * Called whenever the user queries the status of 996 * the interface using ifconfig. 997 ************************************************************************/ 998 static void 999 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1000 { 1001 struct adapter *adapter = ifp->if_softc; 1002 1003 INIT_DEBUGOUT("ixv_media_status: begin"); 1004 ixv_update_link_status(adapter); 1005 1006 ifmr->ifm_status = IFM_AVALID; 1007 ifmr->ifm_active = IFM_ETHER; 1008 1009 if (adapter->link_active != LINK_STATE_UP) { 1010 ifmr->ifm_active |= IFM_NONE; 1011 return; 1012 } 1013 1014 ifmr->ifm_status |= IFM_ACTIVE; 1015 1016 switch (adapter->link_speed) { 1017 case IXGBE_LINK_SPEED_10GB_FULL: 1018 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1019 break; 1020 case IXGBE_LINK_SPEED_5GB_FULL: 1021 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1022 break; 1023 case IXGBE_LINK_SPEED_2_5GB_FULL: 1024 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1025 break; 1026 case IXGBE_LINK_SPEED_1GB_FULL: 1027 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1028 break; 1029 case IXGBE_LINK_SPEED_100_FULL: 1030 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1031 break; 1032 case IXGBE_LINK_SPEED_10_FULL: 1033 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1034 break; 1035 } 1036 1037 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1038 } /* ixv_media_status */ 1039 1040 /************************************************************************ 1041 * ixv_media_change - Media Ioctl callback 1042 * 1043 * Called when the user changes speed/duplex using 1044 * media/mediopt option with ifconfig. 1045 ************************************************************************/ 1046 static int 1047 ixv_media_change(struct ifnet *ifp) 1048 { 1049 struct adapter *adapter = ifp->if_softc; 1050 struct ifmedia *ifm = &adapter->media; 1051 1052 INIT_DEBUGOUT("ixv_media_change: begin"); 1053 1054 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1055 return (EINVAL); 1056 1057 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1058 case IFM_AUTO: 1059 break; 1060 default: 1061 device_printf(adapter->dev, "Only auto media type\n"); 1062 return (EINVAL); 1063 } 1064 1065 return (0); 1066 } /* ixv_media_change */ 1067 1068 static void 1069 ixv_schedule_admin_tasklet(struct adapter *adapter) 1070 { 1071 if (adapter->schedule_wqs_ok) { 1072 if (atomic_cas_uint(&adapter->admin_pending, 0, 1) == 0) 1073 workqueue_enqueue(adapter->admin_wq, 1074 &adapter->admin_wc, NULL); 1075 } 1076 } 1077 1078 /************************************************************************ 1079 * ixv_negotiate_api 1080 * 1081 * Negotiate the Mailbox API with the PF; 1082 * start with the most featured API first. 1083 ************************************************************************/ 1084 static int 1085 ixv_negotiate_api(struct adapter *adapter) 1086 { 1087 struct ixgbe_hw *hw = &adapter->hw; 1088 int mbx_api[] = { ixgbe_mbox_api_15, 1089 ixgbe_mbox_api_13, 1090 ixgbe_mbox_api_12, 1091 ixgbe_mbox_api_11, 1092 ixgbe_mbox_api_10, 1093 ixgbe_mbox_api_unknown }; 1094 int i = 0; 1095 1096 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1097 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) { 1098 if (hw->api_version >= ixgbe_mbox_api_15) 1099 ixgbe_upgrade_mbx_params_vf(hw); 1100 return (0); 1101 } 1102 i++; 1103 } 1104 1105 return (EINVAL); 1106 } /* ixv_negotiate_api */ 1107 1108 1109 /************************************************************************ 1110 * ixv_set_rxfilter - Multicast Update 1111 * 1112 * Called whenever multicast address list is updated. 1113 ************************************************************************/ 1114 static int 1115 ixv_set_rxfilter(struct adapter *adapter) 1116 { 1117 struct ixgbe_mc_addr *mta; 1118 struct ifnet *ifp = adapter->ifp; 1119 struct ixgbe_hw *hw = &adapter->hw; 1120 u8 *update_ptr; 1121 int mcnt = 0; 1122 struct ethercom *ec = &adapter->osdep.ec; 1123 struct ether_multi *enm; 1124 struct ether_multistep step; 1125 bool overflow = false; 1126 int error, rc = 0; 1127 1128 KASSERT(mutex_owned(&adapter->core_mtx)); 1129 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin"); 1130 1131 mta = adapter->mta; 1132 bzero(mta, sizeof(*mta) * IXGBE_MAX_VF_MC); 1133 1134 /* 1: For PROMISC */ 1135 if (ifp->if_flags & IFF_PROMISC) { 1136 error = hw->mac.ops.update_xcast_mode(hw, 1137 IXGBEVF_XCAST_MODE_PROMISC); 1138 if (error == IXGBE_ERR_NOT_TRUSTED) { 1139 device_printf(adapter->dev, 1140 "this interface is not trusted\n"); 1141 error = EPERM; 1142 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1143 device_printf(adapter->dev, 1144 "the PF doesn't support promisc mode\n"); 1145 error = EOPNOTSUPP; 1146 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) { 1147 device_printf(adapter->dev, 1148 "the PF may not in promisc mode\n"); 1149 error = EINVAL; 1150 } else if (error) { 1151 device_printf(adapter->dev, 1152 "failed to set promisc mode. error = %d\n", 1153 error); 1154 error = EIO; 1155 } else 1156 return 0; 1157 rc = error; 1158 } 1159 1160 /* 2: For ALLMULTI or normal */ 1161 ETHER_LOCK(ec); 1162 ETHER_FIRST_MULTI(step, ec, enm); 1163 while (enm != NULL) { 1164 if ((mcnt >= IXGBE_MAX_VF_MC) || 1165 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1166 ETHER_ADDR_LEN) != 0)) { 1167 overflow = true; 1168 break; 1169 } 1170 bcopy(enm->enm_addrlo, 1171 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 1172 mcnt++; 1173 ETHER_NEXT_MULTI(step, enm); 1174 } 1175 ETHER_UNLOCK(ec); 1176 1177 /* 3: For ALLMULTI */ 1178 if (overflow) { 1179 error = hw->mac.ops.update_xcast_mode(hw, 1180 IXGBEVF_XCAST_MODE_ALLMULTI); 1181 if (error == IXGBE_ERR_NOT_TRUSTED) { 1182 device_printf(adapter->dev, 1183 "this interface is not trusted\n"); 1184 error = EPERM; 1185 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1186 device_printf(adapter->dev, 1187 "the PF doesn't support allmulti mode\n"); 1188 error = EOPNOTSUPP; 1189 } else if (error) { 1190 device_printf(adapter->dev, 1191 "number of Ethernet multicast addresses " 1192 "exceeds the limit (%d). error = %d\n", 1193 IXGBE_MAX_VF_MC, error); 1194 error = ENOSPC; 1195 } else { 1196 ETHER_LOCK(ec); 1197 ec->ec_flags |= ETHER_F_ALLMULTI; 1198 ETHER_UNLOCK(ec); 1199 return rc; /* Promisc might have failed */ 1200 } 1201 1202 if (rc == 0) 1203 rc = error; 1204 1205 /* Continue to update the multicast table as many as we can */ 1206 } 1207 1208 /* 4: For normal operation */ 1209 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 1210 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) { 1211 /* Normal operation */ 1212 ETHER_LOCK(ec); 1213 ec->ec_flags &= ~ETHER_F_ALLMULTI; 1214 ETHER_UNLOCK(ec); 1215 error = 0; 1216 } else if (error) { 1217 device_printf(adapter->dev, 1218 "failed to set Ethernet multicast address " 1219 "operation to normal. error = %d\n", error); 1220 } 1221 1222 update_ptr = (u8 *)mta; 1223 error = adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, 1224 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1225 if (rc == 0) 1226 rc = error; 1227 1228 return rc; 1229 } /* ixv_set_rxfilter */ 1230 1231 /************************************************************************ 1232 * ixv_mc_array_itr 1233 * 1234 * An iterator function needed by the multicast shared code. 1235 * It feeds the shared code routine the addresses in the 1236 * array of ixv_set_rxfilter() one by one. 1237 ************************************************************************/ 1238 static u8 * 1239 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1240 { 1241 struct ixgbe_mc_addr *mta; 1242 1243 mta = (struct ixgbe_mc_addr *)*update_ptr; 1244 1245 *vmdq = 0; 1246 *update_ptr = (u8*)(mta + 1); 1247 1248 return (mta->addr); 1249 } /* ixv_mc_array_itr */ 1250 1251 /************************************************************************ 1252 * ixv_local_timer - Timer routine 1253 * 1254 * Checks for link status, updates statistics, 1255 * and runs the watchdog check. 1256 ************************************************************************/ 1257 static void 1258 ixv_local_timer(void *arg) 1259 { 1260 struct adapter *adapter = arg; 1261 1262 if (adapter->schedule_wqs_ok) { 1263 if (atomic_cas_uint(&adapter->timer_pending, 0, 1) == 0) 1264 workqueue_enqueue(adapter->timer_wq, 1265 &adapter->timer_wc, NULL); 1266 } 1267 } 1268 1269 static void 1270 ixv_handle_timer(struct work *wk, void *context) 1271 { 1272 struct adapter *adapter = context; 1273 device_t dev = adapter->dev; 1274 struct ix_queue *que = adapter->queues; 1275 u64 queues = 0; 1276 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1277 int hung = 0; 1278 int i; 1279 1280 IXGBE_CORE_LOCK(adapter); 1281 1282 if (ixv_check_link(adapter)) { 1283 ixv_init_locked(adapter); 1284 IXGBE_CORE_UNLOCK(adapter); 1285 return; 1286 } 1287 1288 /* Stats Update */ 1289 ixv_update_stats(adapter); 1290 1291 /* Update some event counters */ 1292 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1293 que = adapter->queues; 1294 for (i = 0; i < adapter->num_queues; i++, que++) { 1295 struct tx_ring *txr = que->txr; 1296 1297 v0 += txr->q_efbig_tx_dma_setup; 1298 v1 += txr->q_mbuf_defrag_failed; 1299 v2 += txr->q_efbig2_tx_dma_setup; 1300 v3 += txr->q_einval_tx_dma_setup; 1301 v4 += txr->q_other_tx_dma_setup; 1302 v5 += txr->q_eagain_tx_dma_setup; 1303 v6 += txr->q_enomem_tx_dma_setup; 1304 v7 += txr->q_tso_err; 1305 } 1306 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, v0); 1307 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, v1); 1308 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, v2); 1309 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, v3); 1310 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, v4); 1311 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, v5); 1312 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, v6); 1313 IXGBE_EVC_STORE(&adapter->tso_err, v7); 1314 1315 /* 1316 * Check the TX queues status 1317 * - mark hung queues so we don't schedule on them 1318 * - watchdog only if all queues show hung 1319 */ 1320 que = adapter->queues; 1321 for (i = 0; i < adapter->num_queues; i++, que++) { 1322 /* Keep track of queues with work for soft irq */ 1323 if (que->txr->busy) 1324 queues |= ((u64)1 << que->me); 1325 /* 1326 * Each time txeof runs without cleaning, but there 1327 * are uncleaned descriptors it increments busy. If 1328 * we get to the MAX we declare it hung. 1329 */ 1330 if (que->busy == IXGBE_QUEUE_HUNG) { 1331 ++hung; 1332 /* Mark the queue as inactive */ 1333 adapter->active_queues &= ~((u64)1 << que->me); 1334 continue; 1335 } else { 1336 /* Check if we've come back from hung */ 1337 if ((adapter->active_queues & ((u64)1 << que->me)) == 0) 1338 adapter->active_queues |= ((u64)1 << que->me); 1339 } 1340 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1341 device_printf(dev, 1342 "Warning queue %d appears to be hung!\n", i); 1343 que->txr->busy = IXGBE_QUEUE_HUNG; 1344 ++hung; 1345 } 1346 } 1347 1348 /* Only truly watchdog if all queues show hung */ 1349 if (hung == adapter->num_queues) 1350 goto watchdog; 1351 #if 0 1352 else if (queues != 0) { /* Force an IRQ on queues with work */ 1353 ixv_rearm_queues(adapter, queues); 1354 } 1355 #endif 1356 1357 atomic_store_relaxed(&adapter->timer_pending, 0); 1358 IXGBE_CORE_UNLOCK(adapter); 1359 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1360 1361 return; 1362 1363 watchdog: 1364 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1365 adapter->ifp->if_flags &= ~IFF_RUNNING; 1366 IXGBE_EVC_ADD(&adapter->watchdog_events, 1); 1367 ixv_init_locked(adapter); 1368 IXGBE_CORE_UNLOCK(adapter); 1369 } /* ixv_handle_timer */ 1370 1371 /************************************************************************ 1372 * ixv_update_link_status - Update OS on link state 1373 * 1374 * Note: Only updates the OS on the cached link state. 1375 * The real check of the hardware only happens with 1376 * a link interrupt. 1377 ************************************************************************/ 1378 static void 1379 ixv_update_link_status(struct adapter *adapter) 1380 { 1381 struct ifnet *ifp = adapter->ifp; 1382 device_t dev = adapter->dev; 1383 1384 KASSERT(mutex_owned(&adapter->core_mtx)); 1385 1386 if (adapter->link_up) { 1387 if (adapter->link_active != LINK_STATE_UP) { 1388 if (bootverbose) { 1389 const char *bpsmsg; 1390 1391 switch (adapter->link_speed) { 1392 case IXGBE_LINK_SPEED_10GB_FULL: 1393 bpsmsg = "10 Gbps"; 1394 break; 1395 case IXGBE_LINK_SPEED_5GB_FULL: 1396 bpsmsg = "5 Gbps"; 1397 break; 1398 case IXGBE_LINK_SPEED_2_5GB_FULL: 1399 bpsmsg = "2.5 Gbps"; 1400 break; 1401 case IXGBE_LINK_SPEED_1GB_FULL: 1402 bpsmsg = "1 Gbps"; 1403 break; 1404 case IXGBE_LINK_SPEED_100_FULL: 1405 bpsmsg = "100 Mbps"; 1406 break; 1407 case IXGBE_LINK_SPEED_10_FULL: 1408 bpsmsg = "10 Mbps"; 1409 break; 1410 default: 1411 bpsmsg = "unknown speed"; 1412 break; 1413 } 1414 device_printf(dev, "Link is up %s %s \n", 1415 bpsmsg, "Full Duplex"); 1416 } 1417 adapter->link_active = LINK_STATE_UP; 1418 if_link_state_change(ifp, LINK_STATE_UP); 1419 } 1420 } else { 1421 /* 1422 * Do it when link active changes to DOWN. i.e. 1423 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 1424 * b) LINK_STATE_UP -> LINK_STATE_DOWN 1425 */ 1426 if (adapter->link_active != LINK_STATE_DOWN) { 1427 if (bootverbose) 1428 device_printf(dev, "Link is Down\n"); 1429 if_link_state_change(ifp, LINK_STATE_DOWN); 1430 adapter->link_active = LINK_STATE_DOWN; 1431 } 1432 } 1433 } /* ixv_update_link_status */ 1434 1435 1436 /************************************************************************ 1437 * ixv_stop - Stop the hardware 1438 * 1439 * Disables all traffic on the adapter by issuing a 1440 * global reset on the MAC and deallocates TX/RX buffers. 1441 ************************************************************************/ 1442 static void 1443 ixv_ifstop(struct ifnet *ifp, int disable) 1444 { 1445 struct adapter *adapter = ifp->if_softc; 1446 1447 IXGBE_CORE_LOCK(adapter); 1448 ixv_stop_locked(adapter); 1449 IXGBE_CORE_UNLOCK(adapter); 1450 1451 workqueue_wait(adapter->admin_wq, &adapter->admin_wc); 1452 atomic_store_relaxed(&adapter->admin_pending, 0); 1453 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 1454 atomic_store_relaxed(&adapter->timer_pending, 0); 1455 } 1456 1457 static void 1458 ixv_stop_locked(void *arg) 1459 { 1460 struct ifnet *ifp; 1461 struct adapter *adapter = arg; 1462 struct ixgbe_hw *hw = &adapter->hw; 1463 1464 ifp = adapter->ifp; 1465 1466 KASSERT(mutex_owned(&adapter->core_mtx)); 1467 1468 INIT_DEBUGOUT("ixv_stop_locked: begin\n"); 1469 ixv_disable_intr(adapter); 1470 1471 /* Tell the stack that the interface is no longer active */ 1472 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1473 1474 hw->mac.ops.reset_hw(hw); 1475 adapter->hw.adapter_stopped = FALSE; 1476 hw->mac.ops.stop_adapter(hw); 1477 callout_stop(&adapter->timer); 1478 1479 /* Don't schedule workqueues. */ 1480 adapter->schedule_wqs_ok = false; 1481 1482 /* reprogram the RAR[0] in case user changed it. */ 1483 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1484 1485 return; 1486 } /* ixv_stop_locked */ 1487 1488 1489 /************************************************************************ 1490 * ixv_allocate_pci_resources 1491 ************************************************************************/ 1492 static int 1493 ixv_allocate_pci_resources(struct adapter *adapter, 1494 const struct pci_attach_args *pa) 1495 { 1496 pcireg_t memtype, csr; 1497 device_t dev = adapter->dev; 1498 bus_addr_t addr; 1499 int flags; 1500 1501 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1502 switch (memtype) { 1503 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1504 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1505 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1506 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1507 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1508 goto map_err; 1509 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1510 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1511 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1512 } 1513 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1514 adapter->osdep.mem_size, flags, 1515 &adapter->osdep.mem_bus_space_handle) != 0) { 1516 map_err: 1517 adapter->osdep.mem_size = 0; 1518 aprint_error_dev(dev, "unable to map BAR0\n"); 1519 return ENXIO; 1520 } 1521 /* 1522 * Enable address decoding for memory range in case it's not 1523 * set. 1524 */ 1525 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 1526 PCI_COMMAND_STATUS_REG); 1527 csr |= PCI_COMMAND_MEM_ENABLE; 1528 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 1529 csr); 1530 break; 1531 default: 1532 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1533 return ENXIO; 1534 } 1535 1536 /* Pick up the tuneable queues */ 1537 adapter->num_queues = ixv_num_queues; 1538 1539 return (0); 1540 } /* ixv_allocate_pci_resources */ 1541 1542 static void 1543 ixv_free_deferred_handlers(struct adapter *adapter) 1544 { 1545 struct ix_queue *que = adapter->queues; 1546 struct tx_ring *txr = adapter->tx_rings; 1547 int i; 1548 1549 for (i = 0; i < adapter->num_queues; i++, que++, txr++) { 1550 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 1551 if (txr->txr_si != NULL) 1552 softint_disestablish(txr->txr_si); 1553 } 1554 if (que->que_si != NULL) 1555 softint_disestablish(que->que_si); 1556 } 1557 if (adapter->txr_wq != NULL) 1558 workqueue_destroy(adapter->txr_wq); 1559 if (adapter->txr_wq_enqueued != NULL) 1560 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 1561 if (adapter->que_wq != NULL) 1562 workqueue_destroy(adapter->que_wq); 1563 1564 /* Drain the Mailbox(link) queue */ 1565 if (adapter->admin_wq != NULL) { 1566 workqueue_destroy(adapter->admin_wq); 1567 adapter->admin_wq = NULL; 1568 } 1569 if (adapter->timer_wq != NULL) { 1570 workqueue_destroy(adapter->timer_wq); 1571 adapter->timer_wq = NULL; 1572 } 1573 } /* ixv_free_deferred_handlers */ 1574 1575 /************************************************************************ 1576 * ixv_free_pci_resources 1577 ************************************************************************/ 1578 static void 1579 ixv_free_pci_resources(struct adapter * adapter) 1580 { 1581 struct ix_queue *que = adapter->queues; 1582 int rid; 1583 1584 /* 1585 * Release all msix queue resources: 1586 */ 1587 for (int i = 0; i < adapter->num_queues; i++, que++) { 1588 if (que->res != NULL) 1589 pci_intr_disestablish(adapter->osdep.pc, 1590 adapter->osdep.ihs[i]); 1591 } 1592 1593 1594 /* Clean the Mailbox interrupt last */ 1595 rid = adapter->vector; 1596 1597 if (adapter->osdep.ihs[rid] != NULL) { 1598 pci_intr_disestablish(adapter->osdep.pc, 1599 adapter->osdep.ihs[rid]); 1600 adapter->osdep.ihs[rid] = NULL; 1601 } 1602 1603 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1604 adapter->osdep.nintrs); 1605 1606 if (adapter->osdep.mem_size != 0) { 1607 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 1608 adapter->osdep.mem_bus_space_handle, 1609 adapter->osdep.mem_size); 1610 } 1611 1612 return; 1613 } /* ixv_free_pci_resources */ 1614 1615 /************************************************************************ 1616 * ixv_setup_interface 1617 * 1618 * Setup networking device structure and register an interface. 1619 ************************************************************************/ 1620 static int 1621 ixv_setup_interface(device_t dev, struct adapter *adapter) 1622 { 1623 struct ethercom *ec = &adapter->osdep.ec; 1624 struct ifnet *ifp; 1625 1626 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1627 1628 ifp = adapter->ifp = &ec->ec_if; 1629 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1630 ifp->if_baudrate = IF_Gbps(10); 1631 ifp->if_init = ixv_init; 1632 ifp->if_stop = ixv_ifstop; 1633 ifp->if_softc = adapter; 1634 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1635 #ifdef IXGBE_MPSAFE 1636 ifp->if_extflags = IFEF_MPSAFE; 1637 #endif 1638 ifp->if_ioctl = ixv_ioctl; 1639 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1640 #if 0 1641 ixv_start_locked = ixgbe_legacy_start_locked; 1642 #endif 1643 } else { 1644 ifp->if_transmit = ixgbe_mq_start; 1645 #if 0 1646 ixv_start_locked = ixgbe_mq_start_locked; 1647 #endif 1648 } 1649 ifp->if_start = ixgbe_legacy_start; 1650 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1651 IFQ_SET_READY(&ifp->if_snd); 1652 1653 if_initialize(ifp); 1654 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1655 ether_ifattach(ifp, adapter->hw.mac.addr); 1656 aprint_normal_dev(dev, "Ethernet address %s\n", 1657 ether_sprintf(adapter->hw.mac.addr)); 1658 /* 1659 * We use per TX queue softint, so if_deferred_start_init() isn't 1660 * used. 1661 */ 1662 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1663 1664 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1665 1666 /* 1667 * Tell the upper layer(s) we support long frames. 1668 */ 1669 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1670 1671 /* Set capability flags */ 1672 ifp->if_capabilities |= IFCAP_HWCSUM 1673 | IFCAP_TSOv4 1674 | IFCAP_TSOv6; 1675 ifp->if_capenable = 0; 1676 1677 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER 1678 | ETHERCAP_VLAN_HWTAGGING 1679 | ETHERCAP_VLAN_HWCSUM 1680 | ETHERCAP_JUMBO_MTU 1681 | ETHERCAP_VLAN_MTU; 1682 1683 /* Enable the above capabilities by default */ 1684 ec->ec_capenable = ec->ec_capabilities; 1685 1686 /* Don't enable LRO by default */ 1687 #if 0 1688 /* NetBSD doesn't support LRO yet */ 1689 ifp->if_capabilities |= IFCAP_LRO; 1690 #endif 1691 1692 /* 1693 * Specify the media types supported by this adapter and register 1694 * callbacks to update media and link information 1695 */ 1696 ec->ec_ifmedia = &adapter->media; 1697 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, ixv_media_change, 1698 ixv_media_status, &adapter->core_mtx); 1699 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1700 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1701 1702 if_register(ifp); 1703 1704 return 0; 1705 } /* ixv_setup_interface */ 1706 1707 1708 /************************************************************************ 1709 * ixv_initialize_transmit_units - Enable transmit unit. 1710 ************************************************************************/ 1711 static void 1712 ixv_initialize_transmit_units(struct adapter *adapter) 1713 { 1714 struct tx_ring *txr = adapter->tx_rings; 1715 struct ixgbe_hw *hw = &adapter->hw; 1716 int i; 1717 1718 for (i = 0; i < adapter->num_queues; i++, txr++) { 1719 u64 tdba = txr->txdma.dma_paddr; 1720 u32 txctrl, txdctl; 1721 int j = txr->me; 1722 1723 /* Set WTHRESH to 8, burst writeback */ 1724 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1725 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 1726 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1727 1728 /* Set the HW Tx Head and Tail indices */ 1729 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(j), 0); 1730 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(j), 0); 1731 1732 /* Set Tx Tail register */ 1733 txr->tail = IXGBE_VFTDT(j); 1734 1735 txr->txr_no_space = false; 1736 1737 /* Set Ring parameters */ 1738 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1739 (tdba & 0x00000000ffffffffULL)); 1740 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1741 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1742 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1743 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1744 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1745 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1746 1747 /* Now enable */ 1748 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1749 txdctl |= IXGBE_TXDCTL_ENABLE; 1750 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1751 } 1752 1753 return; 1754 } /* ixv_initialize_transmit_units */ 1755 1756 1757 /************************************************************************ 1758 * ixv_initialize_rss_mapping 1759 ************************************************************************/ 1760 static void 1761 ixv_initialize_rss_mapping(struct adapter *adapter) 1762 { 1763 struct ixgbe_hw *hw = &adapter->hw; 1764 u32 reta = 0, mrqc, rss_key[10]; 1765 int queue_id; 1766 int i, j; 1767 u32 rss_hash_config; 1768 1769 /* force use default RSS key. */ 1770 #ifdef __NetBSD__ 1771 rss_getkey((uint8_t *) &rss_key); 1772 #else 1773 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1774 /* Fetch the configured RSS key */ 1775 rss_getkey((uint8_t *)&rss_key); 1776 } else { 1777 /* set up random bits */ 1778 cprng_fast(&rss_key, sizeof(rss_key)); 1779 } 1780 #endif 1781 1782 /* Now fill out hash function seeds */ 1783 for (i = 0; i < 10; i++) 1784 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1785 1786 /* Set up the redirection table */ 1787 for (i = 0, j = 0; i < 64; i++, j++) { 1788 if (j == adapter->num_queues) 1789 j = 0; 1790 1791 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1792 /* 1793 * Fetch the RSS bucket id for the given indirection 1794 * entry. Cap it at the number of configured buckets 1795 * (which is num_queues.) 1796 */ 1797 queue_id = rss_get_indirection_to_bucket(i); 1798 queue_id = queue_id % adapter->num_queues; 1799 } else 1800 queue_id = j; 1801 1802 /* 1803 * The low 8 bits are for hash value (n+0); 1804 * The next 8 bits are for hash value (n+1), etc. 1805 */ 1806 reta >>= 8; 1807 reta |= ((uint32_t)queue_id) << 24; 1808 if ((i & 3) == 3) { 1809 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1810 reta = 0; 1811 } 1812 } 1813 1814 /* Perform hash on these packet types */ 1815 if (adapter->feat_en & IXGBE_FEATURE_RSS) 1816 rss_hash_config = rss_gethashconfig(); 1817 else { 1818 /* 1819 * Disable UDP - IP fragments aren't currently being handled 1820 * and so we end up with a mix of 2-tuple and 4-tuple 1821 * traffic. 1822 */ 1823 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1824 | RSS_HASHTYPE_RSS_TCP_IPV4 1825 | RSS_HASHTYPE_RSS_IPV6 1826 | RSS_HASHTYPE_RSS_TCP_IPV6; 1827 } 1828 1829 mrqc = IXGBE_MRQC_RSSEN; 1830 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1831 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1832 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1833 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1834 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1835 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1836 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1837 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1838 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1839 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX defined, but not supported\n", 1840 __func__); 1841 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1842 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined, but not supported\n", 1843 __func__); 1844 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1845 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1846 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1847 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1848 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1849 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined, but not supported\n", 1850 __func__); 1851 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1852 } /* ixv_initialize_rss_mapping */ 1853 1854 1855 /************************************************************************ 1856 * ixv_initialize_receive_units - Setup receive registers and features. 1857 ************************************************************************/ 1858 static void 1859 ixv_initialize_receive_units(struct adapter *adapter) 1860 { 1861 struct rx_ring *rxr = adapter->rx_rings; 1862 struct ixgbe_hw *hw = &adapter->hw; 1863 struct ifnet *ifp = adapter->ifp; 1864 u32 bufsz, psrtype; 1865 1866 if (ifp->if_mtu > ETHERMTU) 1867 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1868 else 1869 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1870 1871 psrtype = IXGBE_PSRTYPE_TCPHDR 1872 | IXGBE_PSRTYPE_UDPHDR 1873 | IXGBE_PSRTYPE_IPV4HDR 1874 | IXGBE_PSRTYPE_IPV6HDR 1875 | IXGBE_PSRTYPE_L2HDR; 1876 1877 if (adapter->num_queues > 1) 1878 psrtype |= 1 << 29; 1879 1880 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1881 1882 /* Tell PF our max_frame size */ 1883 if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) { 1884 device_printf(adapter->dev, "There is a problem with the PF setup. It is likely the receive unit for this VF will not function correctly.\n"); 1885 } 1886 1887 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 1888 u64 rdba = rxr->rxdma.dma_paddr; 1889 u32 reg, rxdctl; 1890 int j = rxr->me; 1891 1892 /* Disable the queue */ 1893 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1894 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1895 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1896 for (int k = 0; k < 10; k++) { 1897 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1898 IXGBE_RXDCTL_ENABLE) 1899 msec_delay(1); 1900 else 1901 break; 1902 } 1903 IXGBE_WRITE_BARRIER(hw); 1904 /* Setup the Base and Length of the Rx Descriptor Ring */ 1905 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1906 (rdba & 0x00000000ffffffffULL)); 1907 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1908 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1909 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1910 1911 /* Reset the ring indices */ 1912 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1913 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1914 1915 /* Set up the SRRCTL register */ 1916 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1917 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1918 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1919 reg |= bufsz; 1920 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1921 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1922 1923 /* Capture Rx Tail index */ 1924 rxr->tail = IXGBE_VFRDT(rxr->me); 1925 1926 /* Do the queue enabling last */ 1927 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1928 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1929 for (int k = 0; k < 10; k++) { 1930 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1931 IXGBE_RXDCTL_ENABLE) 1932 break; 1933 msec_delay(1); 1934 } 1935 IXGBE_WRITE_BARRIER(hw); 1936 1937 /* Set the Tail Pointer */ 1938 #ifdef DEV_NETMAP 1939 /* 1940 * In netmap mode, we must preserve the buffers made 1941 * available to userspace before the if_init() 1942 * (this is true by default on the TX side, because 1943 * init makes all buffers available to userspace). 1944 * 1945 * netmap_reset() and the device specific routines 1946 * (e.g. ixgbe_setup_receive_rings()) map these 1947 * buffers at the end of the NIC ring, so here we 1948 * must set the RDT (tail) register to make sure 1949 * they are not overwritten. 1950 * 1951 * In this driver the NIC ring starts at RDH = 0, 1952 * RDT points to the last slot available for reception (?), 1953 * so RDT = num_rx_desc - 1 means the whole ring is available. 1954 */ 1955 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 1956 (ifp->if_capenable & IFCAP_NETMAP)) { 1957 struct netmap_adapter *na = NA(adapter->ifp); 1958 struct netmap_kring *kring = na->rx_rings[i]; 1959 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1960 1961 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1962 } else 1963 #endif /* DEV_NETMAP */ 1964 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1965 adapter->num_rx_desc - 1); 1966 } 1967 1968 if (adapter->hw.mac.type >= ixgbe_mac_X550_vf) 1969 ixv_initialize_rss_mapping(adapter); 1970 } /* ixv_initialize_receive_units */ 1971 1972 /************************************************************************ 1973 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1974 * 1975 * Retrieves the TDH value from the hardware 1976 ************************************************************************/ 1977 static int 1978 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1979 { 1980 struct sysctlnode node = *rnode; 1981 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1982 uint32_t val; 1983 1984 if (!txr) 1985 return (0); 1986 1987 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDH(txr->me)); 1988 node.sysctl_data = &val; 1989 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1990 } /* ixv_sysctl_tdh_handler */ 1991 1992 /************************************************************************ 1993 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 1994 * 1995 * Retrieves the TDT value from the hardware 1996 ************************************************************************/ 1997 static int 1998 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 1999 { 2000 struct sysctlnode node = *rnode; 2001 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2002 uint32_t val; 2003 2004 if (!txr) 2005 return (0); 2006 2007 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDT(txr->me)); 2008 node.sysctl_data = &val; 2009 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2010 } /* ixv_sysctl_tdt_handler */ 2011 2012 /************************************************************************ 2013 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check 2014 * handler function 2015 * 2016 * Retrieves the next_to_check value 2017 ************************************************************************/ 2018 static int 2019 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2020 { 2021 struct sysctlnode node = *rnode; 2022 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2023 uint32_t val; 2024 2025 if (!rxr) 2026 return (0); 2027 2028 val = rxr->next_to_check; 2029 node.sysctl_data = &val; 2030 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2031 } /* ixv_sysctl_next_to_check_handler */ 2032 2033 /************************************************************************ 2034 * ixv_sysctl_next_to_refresh_handler - Receive Descriptor next to refresh 2035 * handler function 2036 * 2037 * Retrieves the next_to_refresh value 2038 ************************************************************************/ 2039 static int 2040 ixv_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2041 { 2042 struct sysctlnode node = *rnode; 2043 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2044 struct adapter *adapter; 2045 uint32_t val; 2046 2047 if (!rxr) 2048 return (0); 2049 2050 adapter = rxr->adapter; 2051 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2052 return (EPERM); 2053 2054 val = rxr->next_to_refresh; 2055 node.sysctl_data = &val; 2056 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2057 } /* ixv_sysctl_next_to_refresh_handler */ 2058 2059 /************************************************************************ 2060 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 2061 * 2062 * Retrieves the RDH value from the hardware 2063 ************************************************************************/ 2064 static int 2065 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 2066 { 2067 struct sysctlnode node = *rnode; 2068 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2069 uint32_t val; 2070 2071 if (!rxr) 2072 return (0); 2073 2074 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDH(rxr->me)); 2075 node.sysctl_data = &val; 2076 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2077 } /* ixv_sysctl_rdh_handler */ 2078 2079 /************************************************************************ 2080 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 2081 * 2082 * Retrieves the RDT value from the hardware 2083 ************************************************************************/ 2084 static int 2085 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 2086 { 2087 struct sysctlnode node = *rnode; 2088 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2089 uint32_t val; 2090 2091 if (!rxr) 2092 return (0); 2093 2094 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDT(rxr->me)); 2095 node.sysctl_data = &val; 2096 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2097 } /* ixv_sysctl_rdt_handler */ 2098 2099 static void 2100 ixv_setup_vlan_tagging(struct adapter *adapter) 2101 { 2102 struct ethercom *ec = &adapter->osdep.ec; 2103 struct ixgbe_hw *hw = &adapter->hw; 2104 struct rx_ring *rxr; 2105 u32 ctrl; 2106 int i; 2107 bool hwtagging; 2108 2109 /* Enable HW tagging only if any vlan is attached */ 2110 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2111 && VLAN_ATTACHED(ec); 2112 2113 /* Enable the queues */ 2114 for (i = 0; i < adapter->num_queues; i++) { 2115 rxr = &adapter->rx_rings[i]; 2116 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 2117 if (hwtagging) 2118 ctrl |= IXGBE_RXDCTL_VME; 2119 else 2120 ctrl &= ~IXGBE_RXDCTL_VME; 2121 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 2122 /* 2123 * Let Rx path know that it needs to store VLAN tag 2124 * as part of extra mbuf info. 2125 */ 2126 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2127 } 2128 } /* ixv_setup_vlan_tagging */ 2129 2130 /************************************************************************ 2131 * ixv_setup_vlan_support 2132 ************************************************************************/ 2133 static int 2134 ixv_setup_vlan_support(struct adapter *adapter) 2135 { 2136 struct ethercom *ec = &adapter->osdep.ec; 2137 struct ixgbe_hw *hw = &adapter->hw; 2138 u32 vid, vfta, retry; 2139 struct vlanid_list *vlanidp; 2140 int rv, error = 0; 2141 2142 /* 2143 * This function is called from both if_init and ifflags_cb() 2144 * on NetBSD. 2145 */ 2146 2147 /* 2148 * Part 1: 2149 * Setup VLAN HW tagging 2150 */ 2151 ixv_setup_vlan_tagging(adapter); 2152 2153 if (!VLAN_ATTACHED(ec)) 2154 return 0; 2155 2156 /* 2157 * Part 2: 2158 * Setup VLAN HW filter 2159 */ 2160 /* Cleanup shadow_vfta */ 2161 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 2162 adapter->shadow_vfta[i] = 0; 2163 /* Generate shadow_vfta from ec_vids */ 2164 ETHER_LOCK(ec); 2165 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2166 uint32_t idx; 2167 2168 idx = vlanidp->vid / 32; 2169 KASSERT(idx < IXGBE_VFTA_SIZE); 2170 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2171 } 2172 ETHER_UNLOCK(ec); 2173 2174 /* 2175 * A soft reset zero's out the VFTA, so 2176 * we need to repopulate it now. 2177 */ 2178 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2179 if (adapter->shadow_vfta[i] == 0) 2180 continue; 2181 vfta = adapter->shadow_vfta[i]; 2182 /* 2183 * Reconstruct the vlan id's 2184 * based on the bits set in each 2185 * of the array ints. 2186 */ 2187 for (int j = 0; j < 32; j++) { 2188 retry = 0; 2189 if ((vfta & ((u32)1 << j)) == 0) 2190 continue; 2191 vid = (i * 32) + j; 2192 2193 /* Call the shared code mailbox routine */ 2194 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE, 2195 FALSE)) != 0) { 2196 if (++retry > 5) { 2197 device_printf(adapter->dev, 2198 "%s: max retry exceeded\n", 2199 __func__); 2200 break; 2201 } 2202 } 2203 if (rv != 0) { 2204 device_printf(adapter->dev, 2205 "failed to set vlan %d\n", vid); 2206 error = EACCES; 2207 } 2208 } 2209 } 2210 return error; 2211 } /* ixv_setup_vlan_support */ 2212 2213 static int 2214 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2215 { 2216 struct ifnet *ifp = &ec->ec_if; 2217 struct adapter *adapter = ifp->if_softc; 2218 int rv; 2219 2220 if (set) 2221 rv = ixv_register_vlan(adapter, vid); 2222 else 2223 rv = ixv_unregister_vlan(adapter, vid); 2224 2225 if (rv != 0) 2226 return rv; 2227 2228 /* 2229 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2230 * or 0 to 1. 2231 */ 2232 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2233 ixv_setup_vlan_tagging(adapter); 2234 2235 return rv; 2236 } 2237 2238 /************************************************************************ 2239 * ixv_register_vlan 2240 * 2241 * Run via a vlan config EVENT, it enables us to use the 2242 * HW Filter table since we can get the vlan id. This just 2243 * creates the entry in the soft version of the VFTA, init 2244 * will repopulate the real table. 2245 ************************************************************************/ 2246 static int 2247 ixv_register_vlan(struct adapter *adapter, u16 vtag) 2248 { 2249 struct ixgbe_hw *hw = &adapter->hw; 2250 u16 index, bit; 2251 int error; 2252 2253 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2254 return EINVAL; 2255 IXGBE_CORE_LOCK(adapter); 2256 index = (vtag >> 5) & 0x7F; 2257 bit = vtag & 0x1F; 2258 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2259 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false); 2260 IXGBE_CORE_UNLOCK(adapter); 2261 2262 if (error != 0) { 2263 device_printf(adapter->dev, "failed to register vlan %hu\n", 2264 vtag); 2265 error = EACCES; 2266 } 2267 return error; 2268 } /* ixv_register_vlan */ 2269 2270 /************************************************************************ 2271 * ixv_unregister_vlan 2272 * 2273 * Run via a vlan unconfig EVENT, remove our entry 2274 * in the soft vfta. 2275 ************************************************************************/ 2276 static int 2277 ixv_unregister_vlan(struct adapter *adapter, u16 vtag) 2278 { 2279 struct ixgbe_hw *hw = &adapter->hw; 2280 u16 index, bit; 2281 int error; 2282 2283 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2284 return EINVAL; 2285 2286 IXGBE_CORE_LOCK(adapter); 2287 index = (vtag >> 5) & 0x7F; 2288 bit = vtag & 0x1F; 2289 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2290 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false); 2291 IXGBE_CORE_UNLOCK(adapter); 2292 2293 if (error != 0) { 2294 device_printf(adapter->dev, "failed to unregister vlan %hu\n", 2295 vtag); 2296 error = EIO; 2297 } 2298 return error; 2299 } /* ixv_unregister_vlan */ 2300 2301 /************************************************************************ 2302 * ixv_enable_intr 2303 ************************************************************************/ 2304 static void 2305 ixv_enable_intr(struct adapter *adapter) 2306 { 2307 struct ixgbe_hw *hw = &adapter->hw; 2308 struct ix_queue *que = adapter->queues; 2309 u32 mask; 2310 int i; 2311 2312 /* For VTEIAC */ 2313 mask = (1 << adapter->vector); 2314 for (i = 0; i < adapter->num_queues; i++, que++) 2315 mask |= (1 << que->msix); 2316 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2317 2318 /* For VTEIMS */ 2319 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 2320 que = adapter->queues; 2321 for (i = 0; i < adapter->num_queues; i++, que++) 2322 ixv_enable_queue(adapter, que->msix); 2323 2324 IXGBE_WRITE_FLUSH(hw); 2325 } /* ixv_enable_intr */ 2326 2327 /************************************************************************ 2328 * ixv_disable_intr 2329 ************************************************************************/ 2330 static void 2331 ixv_disable_intr(struct adapter *adapter) 2332 { 2333 struct ix_queue *que = adapter->queues; 2334 2335 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 2336 2337 /* disable interrupts other than queues */ 2338 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, adapter->vector); 2339 2340 for (int i = 0; i < adapter->num_queues; i++, que++) 2341 ixv_disable_queue(adapter, que->msix); 2342 2343 IXGBE_WRITE_FLUSH(&adapter->hw); 2344 } /* ixv_disable_intr */ 2345 2346 /************************************************************************ 2347 * ixv_set_ivar 2348 * 2349 * Setup the correct IVAR register for a particular MSI-X interrupt 2350 * - entry is the register array entry 2351 * - vector is the MSI-X vector for this queue 2352 * - type is RX/TX/MISC 2353 ************************************************************************/ 2354 static void 2355 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 2356 { 2357 struct ixgbe_hw *hw = &adapter->hw; 2358 u32 ivar, index; 2359 2360 vector |= IXGBE_IVAR_ALLOC_VAL; 2361 2362 if (type == -1) { /* MISC IVAR */ 2363 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2364 ivar &= ~0xFF; 2365 ivar |= vector; 2366 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2367 } else { /* RX/TX IVARS */ 2368 index = (16 * (entry & 1)) + (8 * type); 2369 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2370 ivar &= ~(0xffUL << index); 2371 ivar |= ((u32)vector << index); 2372 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2373 } 2374 } /* ixv_set_ivar */ 2375 2376 /************************************************************************ 2377 * ixv_configure_ivars 2378 ************************************************************************/ 2379 static void 2380 ixv_configure_ivars(struct adapter *adapter) 2381 { 2382 struct ix_queue *que = adapter->queues; 2383 2384 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2385 2386 for (int i = 0; i < adapter->num_queues; i++, que++) { 2387 /* First the RX queue entry */ 2388 ixv_set_ivar(adapter, i, que->msix, 0); 2389 /* ... and the TX */ 2390 ixv_set_ivar(adapter, i, que->msix, 1); 2391 /* Set an initial value in EITR */ 2392 ixv_eitr_write(adapter, que->msix, IXGBE_EITR_DEFAULT); 2393 } 2394 2395 /* For the mailbox interrupt */ 2396 ixv_set_ivar(adapter, 1, adapter->vector, -1); 2397 } /* ixv_configure_ivars */ 2398 2399 2400 /************************************************************************ 2401 * ixv_init_stats 2402 * 2403 * The VF stats registers never have a truly virgin 2404 * starting point, so this routine save initial vaules to 2405 * last_<REGNAME>. 2406 ************************************************************************/ 2407 static void 2408 ixv_init_stats(struct adapter *adapter) 2409 { 2410 struct ixgbe_hw *hw = &adapter->hw; 2411 2412 adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2413 adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2414 adapter->stats.vf.last_vfgorc |= 2415 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2416 2417 adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2418 adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2419 adapter->stats.vf.last_vfgotc |= 2420 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2421 2422 adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2423 } /* ixv_init_stats */ 2424 2425 #define UPDATE_STAT_32(reg, last, count) \ 2426 { \ 2427 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2428 IXGBE_EVC_ADD(&count, current - (last)); \ 2429 (last) = current; \ 2430 } 2431 2432 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2433 { \ 2434 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2435 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2436 u64 current = ((cur_msb << 32) | cur_lsb); \ 2437 if (current < (last)) \ 2438 IXGBE_EVC_ADD(&count, current + __BIT(36) - (last)); \ 2439 else \ 2440 IXGBE_EVC_ADD(&count, current - (last)); \ 2441 (last) = current; \ 2442 } 2443 2444 /************************************************************************ 2445 * ixv_update_stats - Update the board statistics counters. 2446 ************************************************************************/ 2447 void 2448 ixv_update_stats(struct adapter *adapter) 2449 { 2450 struct ixgbe_hw *hw = &adapter->hw; 2451 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2452 2453 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2454 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2455 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2456 stats->vfgorc); 2457 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2458 stats->vfgotc); 2459 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2460 2461 /* VF doesn't count errors by hardware */ 2462 2463 } /* ixv_update_stats */ 2464 2465 /************************************************************************ 2466 * ixv_sysctl_interrupt_rate_handler 2467 ************************************************************************/ 2468 static int 2469 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2470 { 2471 struct sysctlnode node = *rnode; 2472 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2473 struct adapter *adapter = que->adapter; 2474 uint32_t reg, usec, rate; 2475 int error; 2476 2477 if (que == NULL) 2478 return 0; 2479 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_VTEITR(que->msix)); 2480 usec = ((reg & 0x0FF8) >> 3); 2481 if (usec > 0) 2482 rate = 500000 / usec; 2483 else 2484 rate = 0; 2485 node.sysctl_data = &rate; 2486 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2487 if (error || newp == NULL) 2488 return error; 2489 reg &= ~0xfff; /* default, no limitation */ 2490 if (rate > 0 && rate < 500000) { 2491 if (rate < 1000) 2492 rate = 1000; 2493 reg |= ((4000000 / rate) & 0xff8); 2494 /* 2495 * When RSC is used, ITR interval must be larger than 2496 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2497 * The minimum value is always greater than 2us on 100M 2498 * (and 10M?(not documented)), but it's not on 1G and higher. 2499 */ 2500 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2501 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2502 if ((adapter->num_queues > 1) 2503 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2504 return EINVAL; 2505 } 2506 ixv_max_interrupt_rate = rate; 2507 } else 2508 ixv_max_interrupt_rate = 0; 2509 ixv_eitr_write(adapter, que->msix, reg); 2510 2511 return (0); 2512 } /* ixv_sysctl_interrupt_rate_handler */ 2513 2514 const struct sysctlnode * 2515 ixv_sysctl_instance(struct adapter *adapter) 2516 { 2517 const char *dvname; 2518 struct sysctllog **log; 2519 int rc; 2520 const struct sysctlnode *rnode; 2521 2522 log = &adapter->sysctllog; 2523 dvname = device_xname(adapter->dev); 2524 2525 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2526 0, CTLTYPE_NODE, dvname, 2527 SYSCTL_DESCR("ixv information and settings"), 2528 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2529 goto err; 2530 2531 return rnode; 2532 err: 2533 device_printf(adapter->dev, 2534 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2535 return NULL; 2536 } 2537 2538 static void 2539 ixv_add_device_sysctls(struct adapter *adapter) 2540 { 2541 struct sysctllog **log; 2542 const struct sysctlnode *rnode, *cnode; 2543 device_t dev; 2544 2545 dev = adapter->dev; 2546 log = &adapter->sysctllog; 2547 2548 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2549 aprint_error_dev(dev, "could not create sysctl root\n"); 2550 return; 2551 } 2552 2553 if (sysctl_createv(log, 0, &rnode, &cnode, 2554 CTLFLAG_READWRITE, CTLTYPE_INT, "debug", 2555 SYSCTL_DESCR("Debug Info"), 2556 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2557 aprint_error_dev(dev, "could not create sysctl\n"); 2558 2559 if (sysctl_createv(log, 0, &rnode, &cnode, 2560 CTLFLAG_READWRITE, CTLTYPE_INT, 2561 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 2562 ixv_sysctl_rx_copy_len, 0, 2563 (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2564 aprint_error_dev(dev, "could not create sysctl\n"); 2565 2566 if (sysctl_createv(log, 0, &rnode, &cnode, 2567 CTLFLAG_READWRITE, CTLTYPE_BOOL, "enable_aim", 2568 SYSCTL_DESCR("Interrupt Moderation"), 2569 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2570 aprint_error_dev(dev, "could not create sysctl\n"); 2571 2572 if (sysctl_createv(log, 0, &rnode, &cnode, 2573 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 2574 SYSCTL_DESCR("Use workqueue for packet processing"), 2575 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) 2576 != 0) 2577 aprint_error_dev(dev, "could not create sysctl\n"); 2578 } 2579 2580 /************************************************************************ 2581 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2582 ************************************************************************/ 2583 static void 2584 ixv_add_stats_sysctls(struct adapter *adapter) 2585 { 2586 device_t dev = adapter->dev; 2587 struct tx_ring *txr = adapter->tx_rings; 2588 struct rx_ring *rxr = adapter->rx_rings; 2589 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2590 struct ixgbe_hw *hw = &adapter->hw; 2591 const struct sysctlnode *rnode, *cnode; 2592 struct sysctllog **log = &adapter->sysctllog; 2593 const char *xname = device_xname(dev); 2594 2595 /* Driver Statistics */ 2596 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2597 NULL, xname, "Driver tx dma soft fail EFBIG"); 2598 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2599 NULL, xname, "m_defrag() failed"); 2600 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2601 NULL, xname, "Driver tx dma hard fail EFBIG"); 2602 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2603 NULL, xname, "Driver tx dma hard fail EINVAL"); 2604 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 2605 NULL, xname, "Driver tx dma hard fail other"); 2606 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2607 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2608 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2609 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2610 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 2611 NULL, xname, "Watchdog timeouts"); 2612 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 2613 NULL, xname, "TSO errors"); 2614 evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, 2615 NULL, xname, "Admin MSI-X IRQ Handled"); 2616 evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, 2617 NULL, xname, "Admin event"); 2618 2619 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2620 snprintf(adapter->queues[i].evnamebuf, 2621 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 2622 xname, i); 2623 snprintf(adapter->queues[i].namebuf, 2624 sizeof(adapter->queues[i].namebuf), "q%d", i); 2625 2626 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2627 aprint_error_dev(dev, "could not create sysctl root\n"); 2628 break; 2629 } 2630 2631 if (sysctl_createv(log, 0, &rnode, &rnode, 2632 0, CTLTYPE_NODE, 2633 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2634 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2635 break; 2636 2637 if (sysctl_createv(log, 0, &rnode, &cnode, 2638 CTLFLAG_READWRITE, CTLTYPE_INT, 2639 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2640 ixv_sysctl_interrupt_rate_handler, 0, 2641 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2642 break; 2643 2644 if (sysctl_createv(log, 0, &rnode, &cnode, 2645 CTLFLAG_READONLY, CTLTYPE_INT, 2646 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2647 ixv_sysctl_tdh_handler, 0, (void *)txr, 2648 0, CTL_CREATE, CTL_EOL) != 0) 2649 break; 2650 2651 if (sysctl_createv(log, 0, &rnode, &cnode, 2652 CTLFLAG_READONLY, CTLTYPE_INT, 2653 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2654 ixv_sysctl_tdt_handler, 0, (void *)txr, 2655 0, CTL_CREATE, CTL_EOL) != 0) 2656 break; 2657 2658 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 2659 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 2660 evcnt_attach_dynamic(&adapter->queues[i].handleq, 2661 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 2662 "Handled queue in softint"); 2663 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 2664 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 2665 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2666 NULL, adapter->queues[i].evnamebuf, "TSO"); 2667 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2668 NULL, adapter->queues[i].evnamebuf, 2669 "TX Queue No Descriptor Available"); 2670 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2671 NULL, adapter->queues[i].evnamebuf, 2672 "Queue Packets Transmitted"); 2673 #ifndef IXGBE_LEGACY_TX 2674 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2675 NULL, adapter->queues[i].evnamebuf, 2676 "Packets dropped in pcq"); 2677 #endif 2678 2679 #ifdef LRO 2680 struct lro_ctrl *lro = &rxr->lro; 2681 #endif /* LRO */ 2682 2683 if (sysctl_createv(log, 0, &rnode, &cnode, 2684 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 2685 SYSCTL_DESCR("Receive Descriptor next to check"), 2686 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 2687 CTL_CREATE, CTL_EOL) != 0) 2688 break; 2689 2690 if (sysctl_createv(log, 0, &rnode, &cnode, 2691 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 2692 SYSCTL_DESCR("Receive Descriptor next to refresh"), 2693 ixv_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 2694 CTL_CREATE, CTL_EOL) != 0) 2695 break; 2696 2697 if (sysctl_createv(log, 0, &rnode, &cnode, 2698 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 2699 SYSCTL_DESCR("Receive Descriptor Head"), 2700 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2701 CTL_CREATE, CTL_EOL) != 0) 2702 break; 2703 2704 if (sysctl_createv(log, 0, &rnode, &cnode, 2705 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 2706 SYSCTL_DESCR("Receive Descriptor Tail"), 2707 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2708 CTL_CREATE, CTL_EOL) != 0) 2709 break; 2710 2711 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2712 NULL, adapter->queues[i].evnamebuf, 2713 "Queue Packets Received"); 2714 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2715 NULL, adapter->queues[i].evnamebuf, 2716 "Queue Bytes Received"); 2717 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2718 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 2719 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 2720 NULL, adapter->queues[i].evnamebuf, "Rx no mbuf"); 2721 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2722 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 2723 #ifdef LRO 2724 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2725 CTLFLAG_RD, &lro->lro_queued, 0, 2726 "LRO Queued"); 2727 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2728 CTLFLAG_RD, &lro->lro_flushed, 0, 2729 "LRO Flushed"); 2730 #endif /* LRO */ 2731 } 2732 2733 /* MAC stats get their own sub node */ 2734 2735 snprintf(stats->namebuf, 2736 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2737 2738 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2739 stats->namebuf, "rx csum offload - IP"); 2740 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2741 stats->namebuf, "rx csum offload - L4"); 2742 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2743 stats->namebuf, "rx csum offload - IP bad"); 2744 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2745 stats->namebuf, "rx csum offload - L4 bad"); 2746 2747 /* Packet Reception Stats */ 2748 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2749 xname, "Good Packets Received"); 2750 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2751 xname, "Good Octets Received"); 2752 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2753 xname, "Multicast Packets Received"); 2754 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2755 xname, "Good Packets Transmitted"); 2756 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2757 xname, "Good Octets Transmitted"); 2758 2759 /* Mailbox Stats */ 2760 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2761 xname, "message TXs"); 2762 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2763 xname, "message RXs"); 2764 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2765 xname, "ACKs"); 2766 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2767 xname, "REQs"); 2768 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2769 xname, "RSTs"); 2770 2771 } /* ixv_add_stats_sysctls */ 2772 2773 static void 2774 ixv_clear_evcnt(struct adapter *adapter) 2775 { 2776 struct tx_ring *txr = adapter->tx_rings; 2777 struct rx_ring *rxr = adapter->rx_rings; 2778 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2779 struct ixgbe_hw *hw = &adapter->hw; 2780 int i; 2781 2782 /* Driver Statistics */ 2783 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, 0); 2784 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, 0); 2785 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, 0); 2786 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, 0); 2787 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, 0); 2788 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, 0); 2789 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, 0); 2790 IXGBE_EVC_STORE(&adapter->watchdog_events, 0); 2791 IXGBE_EVC_STORE(&adapter->tso_err, 0); 2792 IXGBE_EVC_STORE(&adapter->admin_irqev, 0); 2793 IXGBE_EVC_STORE(&adapter->link_workev, 0); 2794 2795 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2796 IXGBE_EVC_STORE(&adapter->queues[i].irqs, 0); 2797 IXGBE_EVC_STORE(&adapter->queues[i].handleq, 0); 2798 IXGBE_EVC_STORE(&adapter->queues[i].req, 0); 2799 IXGBE_EVC_STORE(&txr->tso_tx, 0); 2800 IXGBE_EVC_STORE(&txr->no_desc_avail, 0); 2801 IXGBE_EVC_STORE(&txr->total_packets, 0); 2802 #ifndef IXGBE_LEGACY_TX 2803 IXGBE_EVC_STORE(&txr->pcq_drops, 0); 2804 #endif 2805 txr->q_efbig_tx_dma_setup = 0; 2806 txr->q_mbuf_defrag_failed = 0; 2807 txr->q_efbig2_tx_dma_setup = 0; 2808 txr->q_einval_tx_dma_setup = 0; 2809 txr->q_other_tx_dma_setup = 0; 2810 txr->q_eagain_tx_dma_setup = 0; 2811 txr->q_enomem_tx_dma_setup = 0; 2812 txr->q_tso_err = 0; 2813 2814 IXGBE_EVC_STORE(&rxr->rx_packets, 0); 2815 IXGBE_EVC_STORE(&rxr->rx_bytes, 0); 2816 IXGBE_EVC_STORE(&rxr->rx_copies, 0); 2817 IXGBE_EVC_STORE(&rxr->no_mbuf, 0); 2818 IXGBE_EVC_STORE(&rxr->rx_discarded, 0); 2819 } 2820 2821 /* MAC stats get their own sub node */ 2822 2823 IXGBE_EVC_STORE(&stats->ipcs, 0); 2824 IXGBE_EVC_STORE(&stats->l4cs, 0); 2825 IXGBE_EVC_STORE(&stats->ipcs_bad, 0); 2826 IXGBE_EVC_STORE(&stats->l4cs_bad, 0); 2827 2828 /* 2829 * Packet Reception Stats. 2830 * Call ixv_init_stats() to save last VF counters' values. 2831 */ 2832 ixv_init_stats(adapter); 2833 IXGBE_EVC_STORE(&stats->vfgprc, 0); 2834 IXGBE_EVC_STORE(&stats->vfgorc, 0); 2835 IXGBE_EVC_STORE(&stats->vfmprc, 0); 2836 IXGBE_EVC_STORE(&stats->vfgptc, 0); 2837 IXGBE_EVC_STORE(&stats->vfgotc, 0); 2838 2839 /* Mailbox Stats */ 2840 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_tx, 0); 2841 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_rx, 0); 2842 IXGBE_EVC_STORE(&hw->mbx.stats.acks, 0); 2843 IXGBE_EVC_STORE(&hw->mbx.stats.reqs, 0); 2844 IXGBE_EVC_STORE(&hw->mbx.stats.rsts, 0); 2845 2846 } /* ixv_clear_evcnt */ 2847 2848 /************************************************************************ 2849 * ixv_set_sysctl_value 2850 ************************************************************************/ 2851 static void 2852 ixv_set_sysctl_value(struct adapter *adapter, const char *name, 2853 const char *description, int *limit, int value) 2854 { 2855 device_t dev = adapter->dev; 2856 struct sysctllog **log; 2857 const struct sysctlnode *rnode, *cnode; 2858 2859 log = &adapter->sysctllog; 2860 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2861 aprint_error_dev(dev, "could not create sysctl root\n"); 2862 return; 2863 } 2864 if (sysctl_createv(log, 0, &rnode, &cnode, 2865 CTLFLAG_READWRITE, CTLTYPE_INT, 2866 name, SYSCTL_DESCR(description), 2867 NULL, 0, limit, 0, CTL_CREATE, CTL_EOL) != 0) 2868 aprint_error_dev(dev, "could not create sysctl\n"); 2869 *limit = value; 2870 } /* ixv_set_sysctl_value */ 2871 2872 #define PRINTQS(adapter, regname) \ 2873 do { \ 2874 struct ixgbe_hw *_hw = &(adapter)->hw; \ 2875 int _i; \ 2876 \ 2877 printf("%s: %s", device_xname((adapter)->dev), #regname); \ 2878 for (_i = 0; _i < (adapter)->num_queues; _i++) { \ 2879 printf((_i == 0) ? "\t" : " "); \ 2880 printf("%08x", IXGBE_READ_REG(_hw, \ 2881 IXGBE_##regname(_i))); \ 2882 } \ 2883 printf("\n"); \ 2884 } while (0) 2885 2886 /************************************************************************ 2887 * ixv_print_debug_info 2888 * 2889 * Provides a way to take a look at important statistics 2890 * maintained by the driver and hardware. 2891 ************************************************************************/ 2892 static void 2893 ixv_print_debug_info(struct adapter *adapter) 2894 { 2895 device_t dev = adapter->dev; 2896 struct ixgbe_hw *hw = &adapter->hw; 2897 int i; 2898 2899 device_printf(dev, "queue:"); 2900 for (i = 0; i < adapter->num_queues; i++) { 2901 printf((i == 0) ? "\t" : " "); 2902 printf("%8d", i); 2903 } 2904 printf("\n"); 2905 PRINTQS(adapter, VFRDBAL); 2906 PRINTQS(adapter, VFRDBAH); 2907 PRINTQS(adapter, VFRDLEN); 2908 PRINTQS(adapter, VFSRRCTL); 2909 PRINTQS(adapter, VFRDH); 2910 PRINTQS(adapter, VFRDT); 2911 PRINTQS(adapter, VFRXDCTL); 2912 2913 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIMS)); 2914 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAM)); 2915 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAC)); 2916 } /* ixv_print_debug_info */ 2917 2918 /************************************************************************ 2919 * ixv_sysctl_debug 2920 ************************************************************************/ 2921 static int 2922 ixv_sysctl_debug(SYSCTLFN_ARGS) 2923 { 2924 struct sysctlnode node = *rnode; 2925 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2926 int error, result = 0; 2927 2928 node.sysctl_data = &result; 2929 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2930 2931 if (error || newp == NULL) 2932 return error; 2933 2934 if (result == 1) 2935 ixv_print_debug_info(adapter); 2936 2937 return 0; 2938 } /* ixv_sysctl_debug */ 2939 2940 /************************************************************************ 2941 * ixv_sysctl_rx_copy_len 2942 ************************************************************************/ 2943 static int 2944 ixv_sysctl_rx_copy_len(SYSCTLFN_ARGS) 2945 { 2946 struct sysctlnode node = *rnode; 2947 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2948 int error; 2949 int result = adapter->rx_copy_len; 2950 2951 node.sysctl_data = &result; 2952 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2953 2954 if (error || newp == NULL) 2955 return error; 2956 2957 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 2958 return EINVAL; 2959 2960 adapter->rx_copy_len = result; 2961 2962 return 0; 2963 } /* ixgbe_sysctl_rx_copy_len */ 2964 2965 /************************************************************************ 2966 * ixv_init_device_features 2967 ************************************************************************/ 2968 static void 2969 ixv_init_device_features(struct adapter *adapter) 2970 { 2971 adapter->feat_cap = IXGBE_FEATURE_NETMAP 2972 | IXGBE_FEATURE_VF 2973 | IXGBE_FEATURE_RSS 2974 | IXGBE_FEATURE_LEGACY_TX; 2975 2976 /* A tad short on feature flags for VFs, atm. */ 2977 switch (adapter->hw.mac.type) { 2978 case ixgbe_mac_82599_vf: 2979 break; 2980 case ixgbe_mac_X540_vf: 2981 break; 2982 case ixgbe_mac_X550_vf: 2983 case ixgbe_mac_X550EM_x_vf: 2984 case ixgbe_mac_X550EM_a_vf: 2985 adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 2986 break; 2987 default: 2988 break; 2989 } 2990 2991 /* Enabled by default... */ 2992 /* Is a virtual function (VF) */ 2993 if (adapter->feat_cap & IXGBE_FEATURE_VF) 2994 adapter->feat_en |= IXGBE_FEATURE_VF; 2995 /* Netmap */ 2996 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 2997 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 2998 /* Receive-Side Scaling (RSS) */ 2999 if (adapter->feat_cap & IXGBE_FEATURE_RSS) 3000 adapter->feat_en |= IXGBE_FEATURE_RSS; 3001 /* Needs advanced context descriptor regardless of offloads req'd */ 3002 if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 3003 adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 3004 3005 /* Enabled via sysctl... */ 3006 /* Legacy (single queue) transmit */ 3007 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 3008 ixv_enable_legacy_tx) 3009 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 3010 } /* ixv_init_device_features */ 3011 3012 /************************************************************************ 3013 * ixv_shutdown - Shutdown entry point 3014 ************************************************************************/ 3015 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3016 static int 3017 ixv_shutdown(device_t dev) 3018 { 3019 struct adapter *adapter = device_private(dev); 3020 IXGBE_CORE_LOCK(adapter); 3021 ixv_stop_locked(adapter); 3022 IXGBE_CORE_UNLOCK(adapter); 3023 3024 return (0); 3025 } /* ixv_shutdown */ 3026 #endif 3027 3028 static int 3029 ixv_ifflags_cb(struct ethercom *ec) 3030 { 3031 struct ifnet *ifp = &ec->ec_if; 3032 struct adapter *adapter = ifp->if_softc; 3033 u_short saved_flags; 3034 u_short change; 3035 int rv = 0; 3036 3037 IXGBE_CORE_LOCK(adapter); 3038 3039 saved_flags = adapter->if_flags; 3040 change = ifp->if_flags ^ adapter->if_flags; 3041 if (change != 0) 3042 adapter->if_flags = ifp->if_flags; 3043 3044 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 3045 rv = ENETRESET; 3046 goto out; 3047 } else if ((change & IFF_PROMISC) != 0) { 3048 rv = ixv_set_rxfilter(adapter); 3049 if (rv != 0) { 3050 /* Restore previous */ 3051 adapter->if_flags = saved_flags; 3052 goto out; 3053 } 3054 } 3055 3056 /* Check for ec_capenable. */ 3057 change = ec->ec_capenable ^ adapter->ec_capenable; 3058 adapter->ec_capenable = ec->ec_capenable; 3059 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 3060 | ETHERCAP_VLAN_HWFILTER)) != 0) { 3061 rv = ENETRESET; 3062 goto out; 3063 } 3064 3065 /* 3066 * Special handling is not required for ETHERCAP_VLAN_MTU. 3067 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 3068 */ 3069 3070 /* Set up VLAN support and filter */ 3071 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 3072 rv = ixv_setup_vlan_support(adapter); 3073 3074 out: 3075 IXGBE_CORE_UNLOCK(adapter); 3076 3077 return rv; 3078 } 3079 3080 3081 /************************************************************************ 3082 * ixv_ioctl - Ioctl entry point 3083 * 3084 * Called when the user wants to configure the interface. 3085 * 3086 * return 0 on success, positive on failure 3087 ************************************************************************/ 3088 static int 3089 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 3090 { 3091 struct adapter *adapter = ifp->if_softc; 3092 struct ixgbe_hw *hw = &adapter->hw; 3093 struct ifcapreq *ifcr = data; 3094 int error; 3095 int l4csum_en; 3096 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3097 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 3098 3099 switch (command) { 3100 case SIOCSIFFLAGS: 3101 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 3102 break; 3103 case SIOCADDMULTI: { 3104 struct ether_multi *enm; 3105 struct ether_multistep step; 3106 struct ethercom *ec = &adapter->osdep.ec; 3107 bool overflow = false; 3108 int mcnt = 0; 3109 3110 /* 3111 * Check the number of multicast address. If it exceeds, 3112 * return ENOSPC. 3113 * Update this code when we support API 1.3. 3114 */ 3115 ETHER_LOCK(ec); 3116 ETHER_FIRST_MULTI(step, ec, enm); 3117 while (enm != NULL) { 3118 mcnt++; 3119 3120 /* 3121 * This code is before adding, so one room is required 3122 * at least. 3123 */ 3124 if (mcnt > (IXGBE_MAX_VF_MC - 1)) { 3125 overflow = true; 3126 break; 3127 } 3128 ETHER_NEXT_MULTI(step, enm); 3129 } 3130 ETHER_UNLOCK(ec); 3131 error = 0; 3132 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) { 3133 error = hw->mac.ops.update_xcast_mode(hw, 3134 IXGBEVF_XCAST_MODE_ALLMULTI); 3135 if (error == IXGBE_ERR_NOT_TRUSTED) { 3136 device_printf(adapter->dev, 3137 "this interface is not trusted\n"); 3138 error = EPERM; 3139 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 3140 device_printf(adapter->dev, 3141 "the PF doesn't support allmulti mode\n"); 3142 error = EOPNOTSUPP; 3143 } else if (error) { 3144 device_printf(adapter->dev, 3145 "number of Ethernet multicast addresses " 3146 "exceeds the limit (%d). error = %d\n", 3147 IXGBE_MAX_VF_MC, error); 3148 error = ENOSPC; 3149 } else 3150 ec->ec_flags |= ETHER_F_ALLMULTI; 3151 } 3152 if (error) 3153 return error; 3154 } 3155 /*FALLTHROUGH*/ 3156 case SIOCDELMULTI: 3157 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 3158 break; 3159 case SIOCSIFMEDIA: 3160 case SIOCGIFMEDIA: 3161 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 3162 break; 3163 case SIOCSIFCAP: 3164 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 3165 break; 3166 case SIOCSIFMTU: 3167 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 3168 break; 3169 case SIOCZIFDATA: 3170 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 3171 ixv_update_stats(adapter); 3172 ixv_clear_evcnt(adapter); 3173 break; 3174 default: 3175 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 3176 break; 3177 } 3178 3179 switch (command) { 3180 case SIOCSIFCAP: 3181 /* Layer-4 Rx checksum offload has to be turned on and 3182 * off as a unit. 3183 */ 3184 l4csum_en = ifcr->ifcr_capenable & l4csum; 3185 if (l4csum_en != l4csum && l4csum_en != 0) 3186 return EINVAL; 3187 /*FALLTHROUGH*/ 3188 case SIOCADDMULTI: 3189 case SIOCDELMULTI: 3190 case SIOCSIFFLAGS: 3191 case SIOCSIFMTU: 3192 default: 3193 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 3194 return error; 3195 if ((ifp->if_flags & IFF_RUNNING) == 0) 3196 ; 3197 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 3198 IXGBE_CORE_LOCK(adapter); 3199 ixv_init_locked(adapter); 3200 IXGBE_CORE_UNLOCK(adapter); 3201 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 3202 /* 3203 * Multicast list has changed; set the hardware filter 3204 * accordingly. 3205 */ 3206 IXGBE_CORE_LOCK(adapter); 3207 ixv_disable_intr(adapter); 3208 ixv_set_rxfilter(adapter); 3209 ixv_enable_intr(adapter); 3210 IXGBE_CORE_UNLOCK(adapter); 3211 } 3212 return 0; 3213 } 3214 } /* ixv_ioctl */ 3215 3216 /************************************************************************ 3217 * ixv_init 3218 ************************************************************************/ 3219 static int 3220 ixv_init(struct ifnet *ifp) 3221 { 3222 struct adapter *adapter = ifp->if_softc; 3223 3224 IXGBE_CORE_LOCK(adapter); 3225 ixv_init_locked(adapter); 3226 IXGBE_CORE_UNLOCK(adapter); 3227 3228 return 0; 3229 } /* ixv_init */ 3230 3231 /************************************************************************ 3232 * ixv_handle_que 3233 ************************************************************************/ 3234 static void 3235 ixv_handle_que(void *context) 3236 { 3237 struct ix_queue *que = context; 3238 struct adapter *adapter = que->adapter; 3239 struct tx_ring *txr = que->txr; 3240 struct ifnet *ifp = adapter->ifp; 3241 bool more; 3242 3243 IXGBE_EVC_ADD(&que->handleq, 1); 3244 3245 if (ifp->if_flags & IFF_RUNNING) { 3246 more = ixgbe_rxeof(que); 3247 IXGBE_TX_LOCK(txr); 3248 more |= ixgbe_txeof(txr); 3249 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 3250 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 3251 ixgbe_mq_start_locked(ifp, txr); 3252 /* Only for queue 0 */ 3253 /* NetBSD still needs this for CBQ */ 3254 if ((&adapter->queues[0] == que) 3255 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 3256 ixgbe_legacy_start_locked(ifp, txr); 3257 IXGBE_TX_UNLOCK(txr); 3258 if (more) { 3259 IXGBE_EVC_ADD(&que->req, 1); 3260 if (adapter->txrx_use_workqueue) { 3261 /* 3262 * "enqueued flag" is not required here 3263 * the same as ixg(4). See ixgbe_msix_que(). 3264 */ 3265 workqueue_enqueue(adapter->que_wq, 3266 &que->wq_cookie, curcpu()); 3267 } else 3268 softint_schedule(que->que_si); 3269 return; 3270 } 3271 } 3272 3273 /* Re-enable this interrupt */ 3274 ixv_enable_queue(adapter, que->msix); 3275 3276 return; 3277 } /* ixv_handle_que */ 3278 3279 /************************************************************************ 3280 * ixv_handle_que_work 3281 ************************************************************************/ 3282 static void 3283 ixv_handle_que_work(struct work *wk, void *context) 3284 { 3285 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 3286 3287 /* 3288 * "enqueued flag" is not required here the same as ixg(4). 3289 * See ixgbe_msix_que(). 3290 */ 3291 ixv_handle_que(que); 3292 } 3293 3294 /************************************************************************ 3295 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 3296 ************************************************************************/ 3297 static int 3298 ixv_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 3299 { 3300 device_t dev = adapter->dev; 3301 struct ix_queue *que = adapter->queues; 3302 struct tx_ring *txr = adapter->tx_rings; 3303 int error, msix_ctrl, rid, vector = 0; 3304 pci_chipset_tag_t pc; 3305 pcitag_t tag; 3306 char intrbuf[PCI_INTRSTR_LEN]; 3307 char wqname[MAXCOMLEN]; 3308 char intr_xname[32]; 3309 const char *intrstr = NULL; 3310 kcpuset_t *affinity; 3311 int cpu_id = 0; 3312 3313 pc = adapter->osdep.pc; 3314 tag = adapter->osdep.tag; 3315 3316 adapter->osdep.nintrs = adapter->num_queues + 1; 3317 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 3318 adapter->osdep.nintrs) != 0) { 3319 aprint_error_dev(dev, 3320 "failed to allocate MSI-X interrupt\n"); 3321 return (ENXIO); 3322 } 3323 3324 kcpuset_create(&affinity, false); 3325 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 3326 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 3327 device_xname(dev), i); 3328 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 3329 sizeof(intrbuf)); 3330 #ifdef IXGBE_MPSAFE 3331 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 3332 true); 3333 #endif 3334 /* Set the handler function */ 3335 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 3336 adapter->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 3337 intr_xname); 3338 if (que->res == NULL) { 3339 pci_intr_release(pc, adapter->osdep.intrs, 3340 adapter->osdep.nintrs); 3341 aprint_error_dev(dev, 3342 "Failed to register QUE handler\n"); 3343 kcpuset_destroy(affinity); 3344 return (ENXIO); 3345 } 3346 que->msix = vector; 3347 adapter->active_queues |= (u64)(1 << que->msix); 3348 3349 cpu_id = i; 3350 /* Round-robin affinity */ 3351 kcpuset_zero(affinity); 3352 kcpuset_set(affinity, cpu_id % ncpu); 3353 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 3354 NULL); 3355 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 3356 intrstr); 3357 if (error == 0) 3358 aprint_normal(", bound queue %d to cpu %d\n", 3359 i, cpu_id % ncpu); 3360 else 3361 aprint_normal("\n"); 3362 3363 #ifndef IXGBE_LEGACY_TX 3364 txr->txr_si 3365 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3366 ixgbe_deferred_mq_start, txr); 3367 #endif 3368 que->que_si 3369 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3370 ixv_handle_que, que); 3371 if (que->que_si == NULL) { 3372 aprint_error_dev(dev, 3373 "could not establish software interrupt\n"); 3374 } 3375 } 3376 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 3377 error = workqueue_create(&adapter->txr_wq, wqname, 3378 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3379 IXGBE_WORKQUEUE_FLAGS); 3380 if (error) { 3381 aprint_error_dev(dev, 3382 "couldn't create workqueue for deferred Tx\n"); 3383 } 3384 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 3385 3386 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 3387 error = workqueue_create(&adapter->que_wq, wqname, 3388 ixv_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3389 IXGBE_WORKQUEUE_FLAGS); 3390 if (error) { 3391 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 3392 } 3393 3394 /* and Mailbox */ 3395 cpu_id++; 3396 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 3397 adapter->vector = vector; 3398 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 3399 sizeof(intrbuf)); 3400 #ifdef IXGBE_MPSAFE 3401 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 3402 true); 3403 #endif 3404 /* Set the mbx handler function */ 3405 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 3406 adapter->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, adapter, 3407 intr_xname); 3408 if (adapter->osdep.ihs[vector] == NULL) { 3409 aprint_error_dev(dev, "Failed to register LINK handler\n"); 3410 kcpuset_destroy(affinity); 3411 return (ENXIO); 3412 } 3413 /* Round-robin affinity */ 3414 kcpuset_zero(affinity); 3415 kcpuset_set(affinity, cpu_id % ncpu); 3416 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity, 3417 NULL); 3418 3419 aprint_normal_dev(dev, 3420 "for link, interrupting at %s", intrstr); 3421 if (error == 0) 3422 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 3423 else 3424 aprint_normal("\n"); 3425 3426 /* Tasklets for Mailbox */ 3427 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 3428 error = workqueue_create(&adapter->admin_wq, wqname, 3429 ixv_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3430 IXGBE_TASKLET_WQ_FLAGS); 3431 if (error) { 3432 aprint_error_dev(dev, 3433 "could not create admin workqueue (%d)\n", error); 3434 goto err_out; 3435 } 3436 3437 /* 3438 * Due to a broken design QEMU will fail to properly 3439 * enable the guest for MSI-X unless the vectors in 3440 * the table are all set up, so we must rewrite the 3441 * ENABLE in the MSI-X control register again at this 3442 * point to cause it to successfully initialize us. 3443 */ 3444 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 3445 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3446 rid += PCI_MSIX_CTL; 3447 msix_ctrl = pci_conf_read(pc, tag, rid); 3448 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3449 pci_conf_write(pc, tag, rid, msix_ctrl); 3450 } 3451 3452 kcpuset_destroy(affinity); 3453 return (0); 3454 err_out: 3455 kcpuset_destroy(affinity); 3456 ixv_free_deferred_handlers(adapter); 3457 ixv_free_pci_resources(adapter); 3458 return (error); 3459 } /* ixv_allocate_msix */ 3460 3461 /************************************************************************ 3462 * ixv_configure_interrupts - Setup MSI-X resources 3463 * 3464 * Note: The VF device MUST use MSI-X, there is no fallback. 3465 ************************************************************************/ 3466 static int 3467 ixv_configure_interrupts(struct adapter *adapter) 3468 { 3469 device_t dev = adapter->dev; 3470 int want, queues, msgs; 3471 3472 /* Must have at least 2 MSI-X vectors */ 3473 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 3474 if (msgs < 2) { 3475 aprint_error_dev(dev, "MSIX config error\n"); 3476 return (ENXIO); 3477 } 3478 msgs = MIN(msgs, IXG_MAX_NINTR); 3479 3480 /* Figure out a reasonable auto config value */ 3481 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3482 3483 if (ixv_num_queues != 0) 3484 queues = ixv_num_queues; 3485 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3486 queues = IXGBE_VF_MAX_TX_QUEUES; 3487 3488 /* 3489 * Want vectors for the queues, 3490 * plus an additional for mailbox. 3491 */ 3492 want = queues + 1; 3493 if (msgs >= want) 3494 msgs = want; 3495 else { 3496 aprint_error_dev(dev, 3497 "MSI-X Configuration Problem, " 3498 "%d vectors but %d queues wanted!\n", 3499 msgs, want); 3500 return -1; 3501 } 3502 3503 adapter->msix_mem = (void *)1; /* XXX */ 3504 aprint_normal_dev(dev, 3505 "Using MSI-X interrupts with %d vectors\n", msgs); 3506 adapter->num_queues = queues; 3507 3508 return (0); 3509 } /* ixv_configure_interrupts */ 3510 3511 3512 /************************************************************************ 3513 * ixv_handle_admin - Tasklet handler for MSI-X MBX interrupts 3514 * 3515 * Done outside of interrupt context since the driver might sleep 3516 ************************************************************************/ 3517 static void 3518 ixv_handle_admin(struct work *wk, void *context) 3519 { 3520 struct adapter *adapter = context; 3521 struct ixgbe_hw *hw = &adapter->hw; 3522 3523 IXGBE_CORE_LOCK(adapter); 3524 3525 IXGBE_EVC_ADD(&adapter->link_workev, 1); 3526 adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, 3527 &adapter->link_up, FALSE); 3528 ixv_update_link_status(adapter); 3529 3530 adapter->task_requests = 0; 3531 atomic_store_relaxed(&adapter->admin_pending, 0); 3532 3533 /* Re-enable interrupts */ 3534 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 3535 3536 IXGBE_CORE_UNLOCK(adapter); 3537 } /* ixv_handle_admin */ 3538 3539 /************************************************************************ 3540 * ixv_check_link - Used in the local timer to poll for link changes 3541 ************************************************************************/ 3542 static s32 3543 ixv_check_link(struct adapter *adapter) 3544 { 3545 s32 error; 3546 3547 KASSERT(mutex_owned(&adapter->core_mtx)); 3548 3549 adapter->hw.mac.get_link_status = TRUE; 3550 3551 error = adapter->hw.mac.ops.check_link(&adapter->hw, 3552 &adapter->link_speed, &adapter->link_up, FALSE); 3553 ixv_update_link_status(adapter); 3554 3555 return error; 3556 } /* ixv_check_link */ 3557
Indexes created Tue Sep 30 20:09:53 GMT 2025