1 /* $NetBSD: if_ixl.c,v 1.105 2026/09/19 13:34:11 tls Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 __KERNEL_RCSID(0, "$NetBSD: if_ixl.c,v 1.105 2026/09/19 13:34:11 tls Exp $"); 78 79 #ifdef _KERNEL_OPT 80 #include "opt_if_ixl.h" 81 #endif 82 83 #include <sys/param.h> 84 #include <sys/types.h> 85 86 #include <sys/bitops.h> 87 #include <sys/cpu.h> 88 #include <sys/device.h> 89 #include <sys/evcnt.h> 90 #include <sys/interrupt.h> 91 #include <sys/kmem.h> 92 #include <sys/module.h> 93 #include <sys/mutex.h> 94 #include <sys/pcq.h> 95 #include <sys/syslog.h> 96 #include <sys/workqueue.h> 97 #include <sys/xcall.h> 98 99 #include <sys/bus.h> 100 101 #include <net/bpf.h> 102 #include <net/if.h> 103 #include <net/if_dl.h> 104 #include <net/if_media.h> 105 #include <net/if_ether.h> 106 #include <net/rss_config.h> 107 108 #include <netinet/tcp.h> /* for struct tcphdr */ 109 #include <netinet/udp.h> /* for struct udphdr */ 110 111 #include <dev/pci/pcivar.h> 112 #include <dev/pci/pcidevs.h> 113 114 #include <dev/pci/if_ixlreg.h> 115 #include <dev/pci/if_ixlvar.h> 116 117 #include <prop/proplib.h> 118 119 struct ixl_softc; /* defined */ 120 121 #define I40E_PF_RESET_WAIT_COUNT 200 122 #define I40E_AQ_LARGE_BUF 512 123 124 /* bitfields for Tx queue mapping in QTX_CTL */ 125 #define I40E_QTX_CTL_VF_QUEUE 0x0 126 #define I40E_QTX_CTL_VM_QUEUE 0x1 127 #define I40E_QTX_CTL_PF_QUEUE 0x2 128 129 #define I40E_QUEUE_TYPE_EOL 0x7ff 130 #define I40E_INTR_NOTX_QUEUE 0 131 132 #define I40E_QUEUE_TYPE_RX 0x0 133 #define I40E_QUEUE_TYPE_TX 0x1 134 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 135 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 136 137 #define I40E_ITR_INDEX_RX 0x0 138 #define I40E_ITR_INDEX_TX 0x1 139 #define I40E_ITR_INDEX_OTHER 0x2 140 #define I40E_ITR_INDEX_NONE 0x3 141 #define IXL_ITR_RX 0x7a /* 4K intrs/sec */ 142 #define IXL_ITR_TX 0x7a /* 4K intrs/sec */ 143 144 #define I40E_INTR_NOTX_QUEUE 0 145 #define I40E_INTR_NOTX_INTR 0 146 #define I40E_INTR_NOTX_RX_QUEUE 0 147 #define I40E_INTR_NOTX_TX_QUEUE 1 148 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 149 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 150 151 #define I40E_HASH_LUT_SIZE_128 0 152 153 #define IXL_ICR0_CRIT_ERR_MASK \ 154 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 155 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 156 I40E_PFINT_ICR0_PE_CRITERR_MASK) 157 158 #define IXL_QUEUE_MAX_XL710 64 159 #define IXL_QUEUE_MAX_X722 128 160 161 #define IXL_TX_PKT_DESCS 8 162 #define IXL_TX_PKT_MAXSIZE (MCLBYTES * IXL_TX_PKT_DESCS) 163 #define IXL_TX_QUEUE_ALIGN 128 164 #define IXL_RX_QUEUE_ALIGN 128 165 166 #define IXL_MCLBYTES (MCLBYTES - ETHER_ALIGN) 167 #define IXL_MTU_ETHERLEN ETHER_HDR_LEN \ 168 + ETHER_CRC_LEN \ 169 + ETHER_VLAN_ENCAP_LEN 170 #if 0 171 #define IXL_MAX_MTU (9728 - IXL_MTU_ETHERLEN) 172 #else 173 /* (dbuff * 5) - ETHER_HDR_LEN - ETHER_CRC_LEN */ 174 #define IXL_MAX_MTU (9600 - IXL_MTU_ETHERLEN) 175 #endif 176 #define IXL_MIN_MTU (ETHER_MIN_LEN - ETHER_CRC_LEN) 177 178 #define IXL_PCIREG PCI_MAPREG_START 179 180 #define IXL_ITR0 0x0 181 #define IXL_ITR1 0x1 182 #define IXL_ITR2 0x2 183 #define IXL_NOITR 0x3 184 185 #define IXL_AQ_NUM 256 186 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 187 #define IXL_AQ_ALIGN 64 /* lol */ 188 #define IXL_AQ_BUFLEN 4096 189 190 #define IXL_HMC_ROUNDUP 512 191 #define IXL_HMC_PGSIZE 4096 192 #define IXL_HMC_DVASZ sizeof(uint64_t) 193 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 194 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 195 #define IXL_HMC_PDVALID 1ULL 196 197 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 198 199 #define IXL_SRRD_SRCTL_ATTEMPTS 100000 200 201 struct ixl_aq_regs { 202 bus_size_t atq_tail; 203 bus_size_t atq_head; 204 bus_size_t atq_len; 205 bus_size_t atq_bal; 206 bus_size_t atq_bah; 207 208 bus_size_t arq_tail; 209 bus_size_t arq_head; 210 bus_size_t arq_len; 211 bus_size_t arq_bal; 212 bus_size_t arq_bah; 213 214 uint32_t atq_len_enable; 215 uint32_t atq_tail_mask; 216 uint32_t atq_head_mask; 217 218 uint32_t arq_len_enable; 219 uint32_t arq_tail_mask; 220 uint32_t arq_head_mask; 221 }; 222 223 struct ixl_phy_type { 224 uint64_t phy_type; 225 uint64_t ifm_type; 226 }; 227 228 struct ixl_speed_type { 229 uint8_t dev_speed; 230 uint64_t net_speed; 231 }; 232 233 struct ixl_hmc_entry { 234 uint64_t hmc_base; 235 uint32_t hmc_count; 236 uint64_t hmc_size; 237 }; 238 239 enum ixl_hmc_types { 240 IXL_HMC_LAN_TX = 0, 241 IXL_HMC_LAN_RX, 242 IXL_HMC_FCOE_CTX, 243 IXL_HMC_FCOE_FILTER, 244 IXL_HMC_COUNT 245 }; 246 247 struct ixl_hmc_pack { 248 uint16_t offset; 249 uint16_t width; 250 uint16_t lsb; 251 }; 252 253 /* 254 * these hmc objects have weird sizes and alignments, so these are abstract 255 * representations of them that are nice for c to populate. 256 * 257 * the packing code relies on little-endian values being stored in the fields, 258 * no high bits in the fields being set, and the fields must be packed in the 259 * same order as they are in the ctx structure. 260 */ 261 262 struct ixl_hmc_rxq { 263 uint16_t head; 264 uint8_t cpuid; 265 uint64_t base; 266 #define IXL_HMC_RXQ_BASE_UNIT 128 267 uint16_t qlen; 268 uint16_t dbuff; 269 #define IXL_HMC_RXQ_DBUFF_UNIT 128 270 uint8_t hbuff; 271 #define IXL_HMC_RXQ_HBUFF_UNIT 64 272 uint8_t dtype; 273 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 274 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 275 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 276 uint8_t dsize; 277 #define IXL_HMC_RXQ_DSIZE_16 0 278 #define IXL_HMC_RXQ_DSIZE_32 1 279 uint8_t crcstrip; 280 uint8_t fc_ena; 281 uint8_t l2sel; 282 uint8_t hsplit_0; 283 uint8_t hsplit_1; 284 uint8_t showiv; 285 uint16_t rxmax; 286 uint8_t tphrdesc_ena; 287 uint8_t tphwdesc_ena; 288 uint8_t tphdata_ena; 289 uint8_t tphhead_ena; 290 uint8_t lrxqthresh; 291 uint8_t prefena; 292 }; 293 294 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 295 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 296 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 297 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 298 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 299 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 300 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 301 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 302 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 303 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 304 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 305 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 306 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 307 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 308 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 309 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 310 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 311 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 312 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 313 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 314 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 315 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 316 }; 317 318 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 319 320 struct ixl_hmc_txq { 321 uint16_t head; 322 uint8_t new_context; 323 uint64_t base; 324 #define IXL_HMC_TXQ_BASE_UNIT 128 325 uint8_t fc_ena; 326 uint8_t timesync_ena; 327 uint8_t fd_ena; 328 uint8_t alt_vlan_ena; 329 uint8_t cpuid; 330 uint16_t thead_wb; 331 uint8_t head_wb_ena; 332 #define IXL_HMC_TXQ_DESC_WB 0 333 #define IXL_HMC_TXQ_HEAD_WB 1 334 uint16_t qlen; 335 uint8_t tphrdesc_ena; 336 uint8_t tphrpacket_ena; 337 uint8_t tphwdesc_ena; 338 uint64_t head_wb_addr; 339 uint32_t crc; 340 uint16_t rdylist; 341 uint8_t rdylist_act; 342 }; 343 344 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 345 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 346 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 347 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 348 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 349 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 350 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 351 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 352 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 353 /* line 1 */ 354 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 355 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 356 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 357 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 358 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 359 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 360 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 361 /* line 7 */ 362 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 363 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 364 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 365 }; 366 367 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 368 369 struct ixl_work { 370 struct work ixw_cookie; 371 void (*ixw_func)(void *); 372 void *ixw_arg; 373 unsigned int ixw_added; 374 }; 375 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 376 377 struct ixl_tx_map { 378 struct mbuf *txm_m; 379 bus_dmamap_t txm_map; 380 unsigned int txm_eop; 381 }; 382 383 struct ixl_tx_ring { 384 kmutex_t txr_lock; 385 struct ixl_softc *txr_sc; 386 387 unsigned int txr_prod; 388 unsigned int txr_cons; 389 390 struct ixl_tx_map *txr_maps; 391 struct ixl_dmamem txr_mem; 392 393 bus_size_t txr_tail; 394 unsigned int txr_qid; 395 pcq_t *txr_intrq; 396 void *txr_si; 397 398 struct evcnt txr_defragged; 399 struct evcnt txr_defrag_failed; 400 struct evcnt txr_pcqdrop; 401 struct evcnt txr_transmitdef; 402 struct evcnt txr_intr; 403 struct evcnt txr_defer; 404 }; 405 406 struct ixl_rx_map { 407 struct mbuf *rxm_m; 408 bus_dmamap_t rxm_map; 409 }; 410 411 struct ixl_rx_ring { 412 kmutex_t rxr_lock; 413 414 unsigned int rxr_prod; 415 unsigned int rxr_cons; 416 417 struct ixl_rx_map *rxr_maps; 418 struct ixl_dmamem rxr_mem; 419 420 struct mbuf *rxr_m_head; 421 struct mbuf **rxr_m_tail; 422 423 bus_size_t rxr_tail; 424 unsigned int rxr_qid; 425 426 struct evcnt rxr_mgethdr_failed; 427 struct evcnt rxr_mgetcl_failed; 428 struct evcnt rxr_mbuf_load_failed; 429 struct evcnt rxr_intr; 430 struct evcnt rxr_defer; 431 }; 432 433 struct ixl_queue_pair { 434 struct ixl_softc *qp_sc; 435 struct ixl_tx_ring *qp_txr; 436 struct ixl_rx_ring *qp_rxr; 437 438 char qp_name[16]; 439 440 void *qp_si; 441 struct work qp_work; 442 bool qp_workqueue; 443 }; 444 445 struct ixl_atq { 446 struct ixl_aq_desc iatq_desc; 447 void (*iatq_fn)(struct ixl_softc *, 448 const struct ixl_aq_desc *); 449 bool iatq_inuse; 450 }; 451 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 452 453 struct ixl_product { 454 unsigned int vendor_id; 455 unsigned int product_id; 456 }; 457 458 struct ixl_stats_counters { 459 bool isc_has_offset; 460 struct evcnt isc_crc_errors; 461 uint64_t isc_crc_errors_offset; 462 struct evcnt isc_illegal_bytes; 463 uint64_t isc_illegal_bytes_offset; 464 struct evcnt isc_rx_bytes; 465 uint64_t isc_rx_bytes_offset; 466 struct evcnt isc_rx_discards; 467 uint64_t isc_rx_discards_offset; 468 struct evcnt isc_rx_unicast; 469 uint64_t isc_rx_unicast_offset; 470 struct evcnt isc_rx_multicast; 471 uint64_t isc_rx_multicast_offset; 472 struct evcnt isc_rx_broadcast; 473 uint64_t isc_rx_broadcast_offset; 474 struct evcnt isc_rx_size_64; 475 uint64_t isc_rx_size_64_offset; 476 struct evcnt isc_rx_size_127; 477 uint64_t isc_rx_size_127_offset; 478 struct evcnt isc_rx_size_255; 479 uint64_t isc_rx_size_255_offset; 480 struct evcnt isc_rx_size_511; 481 uint64_t isc_rx_size_511_offset; 482 struct evcnt isc_rx_size_1023; 483 uint64_t isc_rx_size_1023_offset; 484 struct evcnt isc_rx_size_1522; 485 uint64_t isc_rx_size_1522_offset; 486 struct evcnt isc_rx_size_big; 487 uint64_t isc_rx_size_big_offset; 488 struct evcnt isc_rx_undersize; 489 uint64_t isc_rx_undersize_offset; 490 struct evcnt isc_rx_oversize; 491 uint64_t isc_rx_oversize_offset; 492 struct evcnt isc_rx_fragments; 493 uint64_t isc_rx_fragments_offset; 494 struct evcnt isc_rx_jabber; 495 uint64_t isc_rx_jabber_offset; 496 struct evcnt isc_tx_bytes; 497 uint64_t isc_tx_bytes_offset; 498 struct evcnt isc_tx_dropped_link_down; 499 uint64_t isc_tx_dropped_link_down_offset; 500 struct evcnt isc_tx_unicast; 501 uint64_t isc_tx_unicast_offset; 502 struct evcnt isc_tx_multicast; 503 uint64_t isc_tx_multicast_offset; 504 struct evcnt isc_tx_broadcast; 505 uint64_t isc_tx_broadcast_offset; 506 struct evcnt isc_tx_size_64; 507 uint64_t isc_tx_size_64_offset; 508 struct evcnt isc_tx_size_127; 509 uint64_t isc_tx_size_127_offset; 510 struct evcnt isc_tx_size_255; 511 uint64_t isc_tx_size_255_offset; 512 struct evcnt isc_tx_size_511; 513 uint64_t isc_tx_size_511_offset; 514 struct evcnt isc_tx_size_1023; 515 uint64_t isc_tx_size_1023_offset; 516 struct evcnt isc_tx_size_1522; 517 uint64_t isc_tx_size_1522_offset; 518 struct evcnt isc_tx_size_big; 519 uint64_t isc_tx_size_big_offset; 520 struct evcnt isc_mac_local_faults; 521 uint64_t isc_mac_local_faults_offset; 522 struct evcnt isc_mac_remote_faults; 523 uint64_t isc_mac_remote_faults_offset; 524 struct evcnt isc_link_xon_rx; 525 uint64_t isc_link_xon_rx_offset; 526 struct evcnt isc_link_xon_tx; 527 uint64_t isc_link_xon_tx_offset; 528 struct evcnt isc_link_xoff_rx; 529 uint64_t isc_link_xoff_rx_offset; 530 struct evcnt isc_link_xoff_tx; 531 uint64_t isc_link_xoff_tx_offset; 532 struct evcnt isc_vsi_rx_discards; 533 uint64_t isc_vsi_rx_discards_offset; 534 struct evcnt isc_vsi_rx_bytes; 535 uint64_t isc_vsi_rx_bytes_offset; 536 struct evcnt isc_vsi_rx_unicast; 537 uint64_t isc_vsi_rx_unicast_offset; 538 struct evcnt isc_vsi_rx_multicast; 539 uint64_t isc_vsi_rx_multicast_offset; 540 struct evcnt isc_vsi_rx_broadcast; 541 uint64_t isc_vsi_rx_broadcast_offset; 542 struct evcnt isc_vsi_tx_errors; 543 uint64_t isc_vsi_tx_errors_offset; 544 struct evcnt isc_vsi_tx_bytes; 545 uint64_t isc_vsi_tx_bytes_offset; 546 struct evcnt isc_vsi_tx_unicast; 547 uint64_t isc_vsi_tx_unicast_offset; 548 struct evcnt isc_vsi_tx_multicast; 549 uint64_t isc_vsi_tx_multicast_offset; 550 struct evcnt isc_vsi_tx_broadcast; 551 uint64_t isc_vsi_tx_broadcast_offset; 552 }; 553 554 /* 555 * Locking notes: 556 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 557 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 558 * - more than one lock of them cannot be held at once. 559 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 560 * (a spin mutex). 561 * - the lock cannot held with txr_lock or rxr_lock. 562 * + a field named sc_arq_* is not protected by any lock. 563 * - operations for sc_arq_* is done in one context related to 564 * sc_arq_task. 565 * + other fields in ixl_softc is protected by sc_cfg_lock 566 * (an adaptive mutex) 567 * - It must be held before another lock is held, and It can be 568 * released after the other lock is released. 569 * */ 570 571 struct ixl_softc { 572 device_t sc_dev; 573 struct ethercom sc_ec; 574 bool sc_attached; 575 bool sc_dead; 576 uint32_t sc_port; 577 struct sysctllog *sc_sysctllog; 578 struct workqueue *sc_workq; 579 struct workqueue *sc_workq_txrx; 580 int sc_stats_intval; 581 callout_t sc_stats_callout; 582 struct ixl_work sc_stats_task; 583 struct ixl_stats_counters 584 sc_stats_counters; 585 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 586 struct ifmedia sc_media; 587 uint64_t sc_media_status; 588 uint64_t sc_media_active; 589 uint64_t sc_phy_types; 590 uint8_t sc_phy_abilities; 591 uint8_t sc_phy_linkspeed; 592 uint8_t sc_phy_fec_cfg; 593 uint16_t sc_eee_cap; 594 uint32_t sc_eeer_val; 595 uint8_t sc_d3_lpan; 596 kmutex_t sc_cfg_lock; 597 enum i40e_mac_type sc_mac_type; 598 uint32_t sc_rss_table_size; 599 uint32_t sc_rss_table_entry_width; 600 bool sc_txrx_workqueue; 601 u_int sc_tx_process_limit; 602 u_int sc_rx_process_limit; 603 u_int sc_tx_intr_process_limit; 604 u_int sc_rx_intr_process_limit; 605 606 int sc_cur_ec_capenable; 607 608 struct pci_attach_args sc_pa; 609 pci_intr_handle_t *sc_ihp; 610 void **sc_ihs; 611 unsigned int sc_nintrs; 612 613 bus_dma_tag_t sc_dmat; 614 bus_space_tag_t sc_memt; 615 bus_space_handle_t sc_memh; 616 bus_size_t sc_mems; 617 618 uint8_t sc_pf_id; 619 uint16_t sc_uplink_seid; /* le */ 620 uint16_t sc_downlink_seid; /* le */ 621 uint16_t sc_vsi_number; 622 uint16_t sc_vsi_stat_counter_idx; 623 uint16_t sc_seid; 624 unsigned int sc_base_queue; 625 626 pci_intr_type_t sc_intrtype; 627 unsigned int sc_msix_vector_queue; 628 629 struct ixl_dmamem sc_scratch; 630 struct ixl_dmamem sc_aqbuf; 631 632 const struct ixl_aq_regs * 633 sc_aq_regs; 634 uint32_t sc_aq_flags; 635 #define IXL_SC_AQ_FLAG_RXCTL __BIT(0) 636 #define IXL_SC_AQ_FLAG_NVMLOCK __BIT(1) 637 #define IXL_SC_AQ_FLAG_NVMREAD __BIT(2) 638 #define IXL_SC_AQ_FLAG_RSS __BIT(3) 639 640 kmutex_t sc_atq_lock; 641 kcondvar_t sc_atq_cv; 642 struct ixl_dmamem sc_atq; 643 unsigned int sc_atq_prod; 644 unsigned int sc_atq_cons; 645 struct ixl_atq sc_atq_poll; 646 647 struct ixl_dmamem sc_arq; 648 struct ixl_work sc_arq_task; 649 struct ixl_aq_bufs sc_arq_idle; 650 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 651 unsigned int sc_arq_prod; 652 unsigned int sc_arq_cons; 653 654 struct ixl_work sc_link_state_task; 655 struct ixl_work sc_link_state_done_task; 656 struct ixl_atq sc_link_state_atq; 657 658 struct ixl_dmamem sc_hmc_sd; 659 struct ixl_dmamem sc_hmc_pd; 660 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 661 662 struct if_percpuq *sc_ipq; 663 unsigned int sc_tx_ring_ndescs; 664 unsigned int sc_rx_ring_ndescs; 665 unsigned int sc_nqueue_pairs; 666 unsigned int sc_nqueue_pairs_max; 667 unsigned int sc_nqueue_pairs_device; 668 struct ixl_queue_pair *sc_qps; 669 uint32_t sc_itr_rx; 670 uint32_t sc_itr_tx; 671 672 struct evcnt sc_event_atq; 673 struct evcnt sc_event_link; 674 struct evcnt sc_event_ecc_err; 675 struct evcnt sc_event_pci_exception; 676 struct evcnt sc_event_crit_err; 677 }; 678 679 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 680 #define IXL_TX_PROCESS_LIMIT 256 681 #define IXL_RX_PROCESS_LIMIT 256 682 #define IXL_TX_INTR_PROCESS_LIMIT 256 683 #define IXL_RX_INTR_PROCESS_LIMIT 0U 684 685 #define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx | \ 686 IFCAP_CSUM_TCPv4_Rx | \ 687 IFCAP_CSUM_UDPv4_Rx | \ 688 IFCAP_CSUM_TCPv6_Rx | \ 689 IFCAP_CSUM_UDPv6_Rx) 690 #define IXL_IFCAP_TXCSUM (IFCAP_CSUM_IPv4_Tx | \ 691 IFCAP_CSUM_TCPv4_Tx | \ 692 IFCAP_CSUM_UDPv4_Tx | \ 693 IFCAP_CSUM_TCPv6_Tx | \ 694 IFCAP_CSUM_UDPv6_Tx) 695 #define IXL_CSUM_ALL_OFFLOAD (M_CSUM_IPv4 | \ 696 M_CSUM_TCPv4 | M_CSUM_TCPv6 | \ 697 M_CSUM_UDPv4 | M_CSUM_UDPv6) 698 699 #define delaymsec(_x) DELAY(1000 * (_x)) 700 #ifdef IXL_DEBUG 701 #define DDPRINTF(sc, fmt, args...) \ 702 do { \ 703 if ((sc) != NULL) { \ 704 device_printf( \ 705 ((struct ixl_softc *)(sc))->sc_dev, \ 706 ""); \ 707 } \ 708 printf("%s:\t" fmt, __func__, ##args); \ 709 } while (0) 710 #else 711 #define DDPRINTF(sc, fmt, args...) __nothing 712 #endif 713 #ifndef IXL_STATS_INTERVAL_MSEC 714 #define IXL_STATS_INTERVAL_MSEC 10000 715 #endif 716 #ifndef IXL_QUEUE_NUM 717 #define IXL_QUEUE_NUM 0 718 #endif 719 720 static bool ixl_param_nomsix = false; 721 static int ixl_param_stats_interval = IXL_STATS_INTERVAL_MSEC; 722 static int ixl_param_nqps_limit = IXL_QUEUE_NUM; 723 static unsigned int ixl_param_tx_ndescs = 512; 724 static unsigned int ixl_param_rx_ndescs = 512; 725 726 static enum i40e_mac_type 727 ixl_mactype(pci_product_id_t); 728 static void ixl_pci_csr_setup(pci_chipset_tag_t, pcitag_t); 729 static void ixl_clear_hw(struct ixl_softc *); 730 static int ixl_pf_reset(struct ixl_softc *); 731 732 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 733 bus_size_t, bus_size_t); 734 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 735 736 static int ixl_arq_fill(struct ixl_softc *); 737 static void ixl_arq_unfill(struct ixl_softc *); 738 739 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 740 unsigned int); 741 static void ixl_atq_set(struct ixl_atq *, 742 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 743 static void ixl_wakeup(struct ixl_softc *, const struct ixl_aq_desc *); 744 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 745 static void ixl_atq_done(struct ixl_softc *); 746 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 747 static int ixl_atq_exec_locked(struct ixl_softc *, struct ixl_atq *); 748 static int ixl_get_version(struct ixl_softc *); 749 static int ixl_get_nvm_version(struct ixl_softc *); 750 static int ixl_get_hw_capabilities(struct ixl_softc *); 751 static int ixl_pxe_clear(struct ixl_softc *); 752 static int ixl_lldp_shut(struct ixl_softc *); 753 static int ixl_get_mac(struct ixl_softc *); 754 static int ixl_get_switch_config(struct ixl_softc *); 755 static int ixl_phy_mask_ints(struct ixl_softc *); 756 static int ixl_get_phy_info(struct ixl_softc *); 757 static int ixl_set_phy_config(struct ixl_softc *, uint8_t, uint8_t, bool); 758 static int ixl_set_phy_autoselect(struct ixl_softc *); 759 static int ixl_restart_an(struct ixl_softc *); 760 static int ixl_hmc(struct ixl_softc *); 761 static void ixl_hmc_free(struct ixl_softc *); 762 static int ixl_get_vsi(struct ixl_softc *); 763 static int ixl_set_vsi(struct ixl_softc *); 764 static void ixl_set_filter_control(struct ixl_softc *); 765 static int ixl_get_link_status(struct ixl_softc *); 766 static void ixl_get_link_status_work(void *); 767 static int ixl_get_link_status_poll(struct ixl_softc *, int *); 768 static void ixl_get_link_status_done(struct ixl_softc *, 769 const struct ixl_aq_desc *); 770 static void ixl_get_link_status_done_work(void *); 771 static int ixl_set_link_status_locked(struct ixl_softc *, 772 const struct ixl_aq_desc *); 773 static uint64_t ixl_search_link_speed(uint8_t); 774 static uint8_t ixl_search_baudrate(uint64_t); 775 static void ixl_config_rss(struct ixl_softc *); 776 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 777 uint16_t, uint16_t); 778 static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *, 779 uint16_t, uint16_t); 780 static void ixl_arq(void *); 781 static void ixl_hmc_pack(void *, const void *, 782 const struct ixl_hmc_pack *, unsigned int); 783 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 784 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 785 static int ixl_rd16_nvm(struct ixl_softc *, uint16_t, uint16_t *); 786 787 static int ixl_match(device_t, cfdata_t, void *); 788 static void ixl_attach(device_t, device_t, void *); 789 static int ixl_detach(device_t, int); 790 791 static void ixl_media_add(struct ixl_softc *); 792 static int ixl_media_change(struct ifnet *); 793 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 794 static int ixl_ioctl(struct ifnet *, u_long, void *); 795 static void ixl_start(struct ifnet *); 796 static int ixl_transmit(struct ifnet *, struct mbuf *); 797 static void ixl_deferred_transmit(void *); 798 static int ixl_intr(void *); 799 static int ixl_queue_intr(void *); 800 static int ixl_other_intr(void *); 801 static void ixl_handle_queue(void *); 802 static void ixl_handle_queue_wk(struct work *, void *); 803 static void ixl_sched_handle_queue(struct ixl_softc *, 804 struct ixl_queue_pair *); 805 static int ixl_init(struct ifnet *); 806 static int ixl_init_locked(struct ixl_softc *); 807 static void ixl_stop(struct ifnet *, int); 808 static void ixl_stop_locked(struct ixl_softc *); 809 static int ixl_iff(struct ixl_softc *); 810 static int ixl_ifflags_cb(struct ethercom *); 811 static int ixl_setup_interrupts(struct ixl_softc *); 812 static int ixl_establish_intx(struct ixl_softc *); 813 static int ixl_establish_msix(struct ixl_softc *); 814 static void ixl_enable_queue_intr(struct ixl_softc *, 815 struct ixl_queue_pair *); 816 static void ixl_disable_queue_intr(struct ixl_softc *, 817 struct ixl_queue_pair *); 818 static void ixl_enable_other_intr(struct ixl_softc *); 819 static void ixl_disable_other_intr(struct ixl_softc *); 820 static void ixl_config_queue_intr(struct ixl_softc *); 821 static void ixl_config_other_intr(struct ixl_softc *); 822 823 static struct ixl_tx_ring * 824 ixl_txr_alloc(struct ixl_softc *, unsigned int); 825 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 826 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 827 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 828 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 829 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 830 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 831 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 832 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 833 834 static struct ixl_rx_ring * 835 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 836 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 837 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 838 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 839 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 840 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 841 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 842 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 843 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 844 845 static struct workqueue * 846 ixl_workq_create(const char *, pri_t, int, int); 847 static void ixl_workq_destroy(struct workqueue *); 848 static int ixl_workqs_teardown(device_t); 849 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 850 static void ixl_work_add(struct workqueue *, struct ixl_work *); 851 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 852 static void ixl_workq_work(struct work *, void *); 853 static const struct ixl_product * 854 ixl_lookup(const struct pci_attach_args *pa); 855 static void ixl_link_state_update(struct ixl_softc *, 856 const struct ixl_aq_desc *); 857 static int ixl_vlan_cb(struct ethercom *, uint16_t, bool); 858 static int ixl_setup_vlan_hwfilter(struct ixl_softc *); 859 static void ixl_teardown_vlan_hwfilter(struct ixl_softc *); 860 static int ixl_update_macvlan(struct ixl_softc *); 861 static int ixl_setup_interrupts(struct ixl_softc *); 862 static void ixl_teardown_interrupts(struct ixl_softc *); 863 static int ixl_setup_stats(struct ixl_softc *); 864 static void ixl_teardown_stats(struct ixl_softc *); 865 static void ixl_stats_callout(void *); 866 static void ixl_stats_update(void *); 867 static int ixl_setup_sysctls(struct ixl_softc *); 868 static void ixl_teardown_sysctls(struct ixl_softc *); 869 static int ixl_sysctl_itr_handler(SYSCTLFN_PROTO); 870 static int ixl_queue_pairs_alloc(struct ixl_softc *); 871 static void ixl_queue_pairs_free(struct ixl_softc *); 872 873 static const struct ixl_phy_type ixl_phy_type_map[] = { 874 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 875 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 876 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 877 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 878 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 879 { 1ULL << IXL_PHY_TYPE_XAUI | 880 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 881 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 882 { 1ULL << IXL_PHY_TYPE_XLAUI | 883 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 884 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 885 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 886 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 887 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 888 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 889 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 890 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 891 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 892 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 893 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 894 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 895 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 896 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 897 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 898 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 899 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 900 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 901 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 902 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 903 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 904 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 905 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 906 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 907 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_ACC }, 908 { 1ULL << IXL_PHY_TYPE_2500BASE_T_1, IFM_2500_T }, 909 { 1ULL << IXL_PHY_TYPE_5000BASE_T_1, IFM_5000_T }, 910 { 1ULL << IXL_PHY_TYPE_2500BASE_T_2, IFM_2500_T }, 911 { 1ULL << IXL_PHY_TYPE_5000BASE_T_2, IFM_5000_T }, 912 }; 913 914 static const struct ixl_speed_type ixl_speed_type_map[] = { 915 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 916 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 917 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 918 { IXL_AQ_LINK_SPEED_5000MB, IF_Mbps(5000) }, 919 { IXL_AQ_LINK_SPEED_2500MB, IF_Mbps(2500) }, 920 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 921 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 922 }; 923 924 static const struct ixl_aq_regs ixl_pf_aq_regs = { 925 .atq_tail = I40E_PF_ATQT, 926 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 927 .atq_head = I40E_PF_ATQH, 928 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 929 .atq_len = I40E_PF_ATQLEN, 930 .atq_bal = I40E_PF_ATQBAL, 931 .atq_bah = I40E_PF_ATQBAH, 932 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 933 934 .arq_tail = I40E_PF_ARQT, 935 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 936 .arq_head = I40E_PF_ARQH, 937 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 938 .arq_len = I40E_PF_ARQLEN, 939 .arq_bal = I40E_PF_ARQBAL, 940 .arq_bah = I40E_PF_ARQBAH, 941 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 942 }; 943 944 #define ixl_rd(_s, _r) \ 945 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 946 #define ixl_wr(_s, _r, _v) \ 947 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 948 #define ixl_barrier(_s, _r, _l, _o) \ 949 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 950 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 951 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 952 953 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 954 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 955 DVF_DETACH_SHUTDOWN); 956 957 static const struct ixl_product ixl_products[] = { 958 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 959 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 960 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 961 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 962 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 963 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 964 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_1 }, 965 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_2 }, 966 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 967 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 968 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 969 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 970 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 971 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 972 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 973 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 974 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 975 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 976 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 977 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_SFP }, 978 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_BP }, 979 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_V710_5G_T}, 980 /* required last entry */ 981 {0, 0} 982 }; 983 984 static const struct ixl_product * 985 ixl_lookup(const struct pci_attach_args *pa) 986 { 987 const struct ixl_product *ixlp; 988 989 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 990 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 991 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 992 return ixlp; 993 } 994 995 return NULL; 996 } 997 998 static void 999 ixl_intr_barrier(void) 1000 { 1001 1002 /* wait for finish of all handler */ 1003 xc_barrier(0); 1004 } 1005 1006 static int 1007 ixl_match(device_t parent, cfdata_t match, void *aux) 1008 { 1009 const struct pci_attach_args *pa = aux; 1010 1011 return (ixl_lookup(pa) != NULL) ? 1 : 0; 1012 } 1013 1014 static void 1015 ixl_attach(device_t parent, device_t self, void *aux) 1016 { 1017 struct ixl_softc *sc; 1018 struct pci_attach_args *pa = aux; 1019 struct ifnet *ifp; 1020 pcireg_t memtype; 1021 uint32_t firstq, port, ari, func; 1022 char xnamebuf[32]; 1023 int tries, rv, link; 1024 1025 sc = device_private(self); 1026 sc->sc_dev = self; 1027 ifp = &sc->sc_ec.ec_if; 1028 1029 sc->sc_pa = *pa; 1030 sc->sc_dmat = (pci_dma64_available(pa)) ? 1031 pa->pa_dmat64 : pa->pa_dmat; 1032 sc->sc_aq_regs = &ixl_pf_aq_regs; 1033 1034 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(pa->pa_id)); 1035 1036 ixl_pci_csr_setup(pa->pa_pc, pa->pa_tag); 1037 1038 pci_aprint_devinfo(pa, "Ethernet controller"); 1039 1040 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 1041 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 1042 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 1043 aprint_error(": unable to map registers\n"); 1044 return; 1045 } 1046 1047 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 1048 1049 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 1050 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 1051 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 1052 sc->sc_base_queue = firstq; 1053 1054 ixl_clear_hw(sc); 1055 if (ixl_pf_reset(sc) == -1) { 1056 /* error printed by ixl pf_reset */ 1057 goto unmap; 1058 } 1059 1060 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 1061 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 1062 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 1063 sc->sc_port = port; 1064 aprint_normal_dev(self, "port %u", sc->sc_port); 1065 1066 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 1067 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 1068 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 1069 1070 func = ixl_rd(sc, I40E_PF_FUNC_RID); 1071 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 1072 1073 /* initialise the adminq */ 1074 1075 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 1076 1077 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 1078 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1079 aprint_error("\n" "%s: unable to allocate atq\n", 1080 device_xname(self)); 1081 goto unmap; 1082 } 1083 1084 SIMPLEQ_INIT(&sc->sc_arq_idle); 1085 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 1086 sc->sc_arq_cons = 0; 1087 sc->sc_arq_prod = 0; 1088 1089 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 1090 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1091 aprint_error("\n" "%s: unable to allocate arq\n", 1092 device_xname(self)); 1093 goto free_atq; 1094 } 1095 1096 if (!ixl_arq_fill(sc)) { 1097 aprint_error("\n" "%s: unable to fill arq descriptors\n", 1098 device_xname(self)); 1099 goto free_arq; 1100 } 1101 1102 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1103 0, IXL_DMA_LEN(&sc->sc_atq), 1104 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1105 1106 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1107 0, IXL_DMA_LEN(&sc->sc_arq), 1108 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1109 1110 for (tries = 0; tries < 10; tries++) { 1111 sc->sc_atq_cons = 0; 1112 sc->sc_atq_prod = 0; 1113 1114 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1115 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1116 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1117 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1118 1119 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1120 1121 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1122 ixl_dmamem_lo(&sc->sc_atq)); 1123 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1124 ixl_dmamem_hi(&sc->sc_atq)); 1125 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1126 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1127 1128 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1129 ixl_dmamem_lo(&sc->sc_arq)); 1130 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1131 ixl_dmamem_hi(&sc->sc_arq)); 1132 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1133 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1134 1135 rv = ixl_get_version(sc); 1136 if (rv == 0) 1137 break; 1138 if (rv != ETIMEDOUT) { 1139 aprint_error(", unable to get firmware version\n"); 1140 goto shutdown; 1141 } 1142 1143 delaymsec(100); 1144 } 1145 1146 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1147 1148 if (ixl_dmamem_alloc(sc, &sc->sc_aqbuf, IXL_AQ_BUFLEN, 0) != 0) { 1149 aprint_error_dev(self, ", unable to allocate nvm buffer\n"); 1150 goto shutdown; 1151 } 1152 1153 ixl_get_nvm_version(sc); 1154 1155 if (sc->sc_mac_type == I40E_MAC_X722) 1156 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_X722; 1157 else 1158 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_XL710; 1159 1160 rv = ixl_get_hw_capabilities(sc); 1161 if (rv != 0) { 1162 aprint_error(", GET HW CAPABILITIES %s\n", 1163 rv == ETIMEDOUT ? "timeout" : "error"); 1164 goto free_aqbuf; 1165 } 1166 1167 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1168 if (ixl_param_nqps_limit > 0) { 1169 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_max, 1170 ixl_param_nqps_limit); 1171 } 1172 1173 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1174 sc->sc_tx_ring_ndescs = ixl_param_tx_ndescs; 1175 sc->sc_rx_ring_ndescs = ixl_param_rx_ndescs; 1176 1177 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1178 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1179 KASSERT(sc->sc_rx_ring_ndescs == 1180 (1U << (fls32(sc->sc_rx_ring_ndescs) - 1))); 1181 KASSERT(sc->sc_tx_ring_ndescs == 1182 (1U << (fls32(sc->sc_tx_ring_ndescs) - 1))); 1183 1184 if (ixl_get_mac(sc) != 0) { 1185 /* error printed by ixl_get_mac */ 1186 goto free_aqbuf; 1187 } 1188 1189 aprint_normal("\n"); 1190 aprint_naive("\n"); 1191 1192 aprint_normal_dev(self, "Ethernet address %s\n", 1193 ether_sprintf(sc->sc_enaddr)); 1194 1195 rv = ixl_pxe_clear(sc); 1196 if (rv != 0) { 1197 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1198 rv == ETIMEDOUT ? "timeout" : "error"); 1199 } 1200 1201 ixl_set_filter_control(sc); 1202 1203 if (ixl_hmc(sc) != 0) { 1204 /* error printed by ixl_hmc */ 1205 goto free_aqbuf; 1206 } 1207 1208 if (ixl_lldp_shut(sc) != 0) { 1209 /* error printed by ixl_lldp_shut */ 1210 goto free_hmc; 1211 } 1212 1213 if (ixl_phy_mask_ints(sc) != 0) { 1214 /* error printed by ixl_phy_mask_ints */ 1215 goto free_hmc; 1216 } 1217 1218 if (ixl_restart_an(sc) != 0) { 1219 /* error printed by ixl_restart_an */ 1220 goto free_hmc; 1221 } 1222 1223 if (ixl_get_switch_config(sc) != 0) { 1224 /* error printed by ixl_get_switch_config */ 1225 goto free_hmc; 1226 } 1227 1228 rv = ixl_get_link_status_poll(sc, NULL); 1229 if (rv != 0) { 1230 aprint_error_dev(self, "GET LINK STATUS %s\n", 1231 rv == ETIMEDOUT ? "timeout" : "error"); 1232 goto free_hmc; 1233 } 1234 1235 /* 1236 * The FW often returns EIO in "Get PHY Abilities" command 1237 * if there is no delay 1238 */ 1239 DELAY(500); 1240 if (ixl_get_phy_info(sc) != 0) { 1241 /* error printed by ixl_get_phy_info */ 1242 goto free_hmc; 1243 } 1244 1245 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1246 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1247 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1248 goto free_hmc; 1249 } 1250 1251 rv = ixl_get_vsi(sc); 1252 if (rv != 0) { 1253 aprint_error_dev(self, "GET VSI %s %d\n", 1254 rv == ETIMEDOUT ? "timeout" : "error", rv); 1255 goto free_scratch; 1256 } 1257 1258 rv = ixl_set_vsi(sc); 1259 if (rv != 0) { 1260 aprint_error_dev(self, "UPDATE VSI error %s %d\n", 1261 rv == ETIMEDOUT ? "timeout" : "error", rv); 1262 goto free_scratch; 1263 } 1264 1265 if (ixl_queue_pairs_alloc(sc) != 0) { 1266 /* error printed by ixl_queue_pairs_alloc */ 1267 goto free_scratch; 1268 } 1269 1270 if (ixl_setup_interrupts(sc) != 0) { 1271 /* error printed by ixl_setup_interrupts */ 1272 goto free_queue_pairs; 1273 } 1274 1275 if (ixl_setup_stats(sc) != 0) { 1276 aprint_error_dev(self, "failed to setup event counters\n"); 1277 goto teardown_intrs; 1278 } 1279 1280 if (ixl_setup_sysctls(sc) != 0) { 1281 /* error printed by ixl_setup_sysctls */ 1282 goto teardown_stats; 1283 } 1284 1285 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1286 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1287 IPL_NET, WQ_MPSAFE); 1288 if (sc->sc_workq == NULL) 1289 goto teardown_sysctls; 1290 1291 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1292 rv = workqueue_create(&sc->sc_workq_txrx, xnamebuf, ixl_handle_queue_wk, 1293 sc, IXL_WORKQUEUE_PRI, IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1294 if (rv != 0) { 1295 sc->sc_workq_txrx = NULL; 1296 goto teardown_wqs; 1297 } 1298 1299 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1300 cv_init(&sc->sc_atq_cv, xnamebuf); 1301 1302 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1303 1304 ifp->if_softc = sc; 1305 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1306 ifp->if_extflags = IFEF_MPSAFE; 1307 ifp->if_ioctl = ixl_ioctl; 1308 ifp->if_start = ixl_start; 1309 ifp->if_transmit = ixl_transmit; 1310 ifp->if_init = ixl_init; 1311 ifp->if_stop = ixl_stop; 1312 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1313 IFQ_SET_READY(&ifp->if_snd); 1314 ifp->if_capabilities |= IXL_IFCAP_RXCSUM; 1315 ifp->if_capabilities |= IXL_IFCAP_TXCSUM; 1316 #if 0 1317 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1318 #endif 1319 ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb); 1320 sc->sc_ec.ec_capabilities |= ETHERCAP_JUMBO_MTU; 1321 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_MTU; 1322 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1323 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1324 1325 sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities; 1326 /* Disable VLAN_HWFILTER by default */ 1327 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1328 1329 sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable; 1330 1331 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1332 ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, ixl_media_change, 1333 ixl_media_status, &sc->sc_cfg_lock); 1334 1335 ixl_media_add(sc); 1336 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1337 if (ISSET(sc->sc_phy_abilities, 1338 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1339 ifmedia_add(&sc->sc_media, 1340 IFM_ETHER | IFM_AUTO | IFM_FLOW, 0, NULL); 1341 } 1342 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_NONE, 0, NULL); 1343 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1344 1345 if_initialize(ifp); 1346 1347 sc->sc_ipq = if_percpuq_create(ifp); 1348 if_deferred_start_init(ifp, NULL); 1349 ether_ifattach(ifp, sc->sc_enaddr); 1350 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1351 1352 rv = ixl_get_link_status_poll(sc, &link); 1353 if (rv != 0) 1354 link = LINK_STATE_UNKNOWN; 1355 if_link_state_change(ifp, link); 1356 1357 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 1358 ixl_work_set(&sc->sc_link_state_task, 1359 ixl_get_link_status_work, sc); 1360 ixl_work_set(&sc->sc_link_state_done_task, 1361 ixl_get_link_status_done_work, sc); 1362 1363 ixl_config_other_intr(sc); 1364 ixl_enable_other_intr(sc); 1365 1366 ixl_set_phy_autoselect(sc); 1367 1368 /* remove default mac filter and replace it so we can see vlans */ 1369 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 1370 if (rv != ENOENT) { 1371 aprint_debug_dev(self, 1372 "unable to remove macvlan %u\n", rv); 1373 } 1374 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1375 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1376 if (rv != ENOENT) { 1377 aprint_debug_dev(self, 1378 "unable to remove macvlan, ignore vlan %u\n", rv); 1379 } 1380 1381 if (ixl_update_macvlan(sc) != 0) { 1382 aprint_debug_dev(self, 1383 "couldn't enable vlan hardware filter\n"); 1384 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1385 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1386 } 1387 1388 sc->sc_txrx_workqueue = true; 1389 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1390 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1391 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1392 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1393 1394 ixl_stats_update(sc); 1395 sc->sc_stats_counters.isc_has_offset = true; 1396 1397 if (pmf_device_register(self, NULL, NULL) != true) 1398 aprint_debug_dev(self, "couldn't establish power handler\n"); 1399 sc->sc_itr_rx = IXL_ITR_RX; 1400 sc->sc_itr_tx = IXL_ITR_TX; 1401 sc->sc_attached = true; 1402 if_register(ifp); 1403 1404 return; 1405 1406 teardown_wqs: 1407 config_finalize_register(self, ixl_workqs_teardown); 1408 teardown_sysctls: 1409 ixl_teardown_sysctls(sc); 1410 teardown_stats: 1411 ixl_teardown_stats(sc); 1412 teardown_intrs: 1413 ixl_teardown_interrupts(sc); 1414 free_queue_pairs: 1415 ixl_queue_pairs_free(sc); 1416 free_scratch: 1417 ixl_dmamem_free(sc, &sc->sc_scratch); 1418 free_hmc: 1419 ixl_hmc_free(sc); 1420 free_aqbuf: 1421 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1422 shutdown: 1423 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1424 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1425 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1426 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1427 1428 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1429 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1430 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1431 1432 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1433 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1434 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1435 1436 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1437 0, IXL_DMA_LEN(&sc->sc_arq), 1438 BUS_DMASYNC_POSTREAD); 1439 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1440 0, IXL_DMA_LEN(&sc->sc_atq), 1441 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1442 1443 ixl_arq_unfill(sc); 1444 free_arq: 1445 ixl_dmamem_free(sc, &sc->sc_arq); 1446 free_atq: 1447 ixl_dmamem_free(sc, &sc->sc_atq); 1448 unmap: 1449 mutex_destroy(&sc->sc_atq_lock); 1450 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1451 mutex_destroy(&sc->sc_cfg_lock); 1452 sc->sc_mems = 0; 1453 1454 sc->sc_attached = false; 1455 } 1456 1457 static int 1458 ixl_detach(device_t self, int flags) 1459 { 1460 struct ixl_softc *sc = device_private(self); 1461 struct ifnet *ifp = &sc->sc_ec.ec_if; 1462 1463 if (!sc->sc_attached) 1464 return 0; 1465 1466 ixl_stop(ifp, 1); 1467 1468 callout_halt(&sc->sc_stats_callout, NULL); 1469 ixl_work_wait(sc->sc_workq, &sc->sc_stats_task); 1470 1471 /* detach the I/F before stop adminq due to callbacks */ 1472 ether_ifdetach(ifp); 1473 if_detach(ifp); 1474 ifmedia_fini(&sc->sc_media); 1475 if_percpuq_destroy(sc->sc_ipq); 1476 1477 ixl_disable_other_intr(sc); 1478 ixl_intr_barrier(); 1479 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1480 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1481 1482 if (sc->sc_workq != NULL) { 1483 ixl_workq_destroy(sc->sc_workq); 1484 sc->sc_workq = NULL; 1485 } 1486 1487 if (sc->sc_workq_txrx != NULL) { 1488 workqueue_destroy(sc->sc_workq_txrx); 1489 sc->sc_workq_txrx = NULL; 1490 } 1491 1492 ixl_teardown_interrupts(sc); 1493 ixl_teardown_stats(sc); 1494 ixl_teardown_sysctls(sc); 1495 1496 ixl_queue_pairs_free(sc); 1497 1498 ixl_dmamem_free(sc, &sc->sc_scratch); 1499 ixl_hmc_free(sc); 1500 1501 /* shutdown */ 1502 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1503 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1504 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1505 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1506 1507 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1508 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1509 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1510 1511 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1512 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1513 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1514 1515 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1516 0, IXL_DMA_LEN(&sc->sc_arq), 1517 BUS_DMASYNC_POSTREAD); 1518 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1519 0, IXL_DMA_LEN(&sc->sc_atq), 1520 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1521 1522 ixl_arq_unfill(sc); 1523 1524 ixl_dmamem_free(sc, &sc->sc_arq); 1525 ixl_dmamem_free(sc, &sc->sc_atq); 1526 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1527 1528 cv_destroy(&sc->sc_atq_cv); 1529 mutex_destroy(&sc->sc_atq_lock); 1530 1531 if (sc->sc_mems != 0) { 1532 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1533 sc->sc_mems = 0; 1534 } 1535 1536 mutex_destroy(&sc->sc_cfg_lock); 1537 1538 return 0; 1539 } 1540 1541 static int 1542 ixl_workqs_teardown(device_t self) 1543 { 1544 struct ixl_softc *sc = device_private(self); 1545 1546 if (sc->sc_workq != NULL) { 1547 ixl_workq_destroy(sc->sc_workq); 1548 sc->sc_workq = NULL; 1549 } 1550 1551 if (sc->sc_workq_txrx != NULL) { 1552 workqueue_destroy(sc->sc_workq_txrx); 1553 sc->sc_workq_txrx = NULL; 1554 } 1555 1556 return 0; 1557 } 1558 1559 static int 1560 ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 1561 { 1562 struct ifnet *ifp = &ec->ec_if; 1563 struct ixl_softc *sc = ifp->if_softc; 1564 int rv; 1565 1566 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1567 return 0; 1568 } 1569 1570 if (set) { 1571 rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid, 1572 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1573 if (rv == 0) { 1574 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 1575 vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1576 } 1577 } else { 1578 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid, 1579 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1580 (void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid, 1581 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1582 } 1583 1584 return rv; 1585 } 1586 1587 static void 1588 ixl_media_add(struct ixl_softc *sc) 1589 { 1590 struct ifmedia *ifm = &sc->sc_media; 1591 const struct ixl_phy_type *itype; 1592 unsigned int i; 1593 bool flow; 1594 1595 if (ISSET(sc->sc_phy_abilities, 1596 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1597 flow = true; 1598 } else { 1599 flow = false; 1600 } 1601 1602 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1603 itype = &ixl_phy_type_map[i]; 1604 1605 if (ISSET(sc->sc_phy_types, itype->phy_type)) { 1606 ifmedia_add(ifm, 1607 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1608 1609 if (flow) { 1610 ifmedia_add(ifm, 1611 IFM_ETHER | IFM_FDX | IFM_FLOW | 1612 itype->ifm_type, 0, NULL); 1613 } 1614 1615 if (itype->ifm_type != IFM_100_TX) 1616 continue; 1617 1618 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1619 0, NULL); 1620 if (flow) { 1621 ifmedia_add(ifm, 1622 IFM_ETHER | IFM_FLOW | itype->ifm_type, 1623 0, NULL); 1624 } 1625 } 1626 } 1627 } 1628 1629 static void 1630 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1631 { 1632 struct ixl_softc *sc = ifp->if_softc; 1633 1634 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1635 1636 ifmr->ifm_status = sc->sc_media_status; 1637 ifmr->ifm_active = sc->sc_media_active; 1638 } 1639 1640 static int 1641 ixl_media_change(struct ifnet *ifp) 1642 { 1643 struct ixl_softc *sc = ifp->if_softc; 1644 struct ifmedia *ifm = &sc->sc_media; 1645 uint64_t ifm_active = sc->sc_media_active; 1646 uint8_t link_speed, abilities; 1647 1648 switch (IFM_SUBTYPE(ifm_active)) { 1649 case IFM_1000_SGMII: 1650 case IFM_1000_KX: 1651 case IFM_10G_KX4: 1652 case IFM_10G_KR: 1653 case IFM_40G_KR4: 1654 case IFM_20G_KR2: 1655 case IFM_25G_KR: 1656 /* backplanes */ 1657 return EINVAL; 1658 } 1659 1660 abilities = IXL_PHY_ABILITY_AUTONEGO | IXL_PHY_ABILITY_LINKUP; 1661 1662 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1663 case IFM_AUTO: 1664 link_speed = sc->sc_phy_linkspeed; 1665 break; 1666 case IFM_NONE: 1667 link_speed = 0; 1668 CLR(abilities, IXL_PHY_ABILITY_LINKUP); 1669 break; 1670 default: 1671 link_speed = ixl_search_baudrate( 1672 ifmedia_baudrate(ifm->ifm_media)); 1673 } 1674 1675 if (ISSET(abilities, IXL_PHY_ABILITY_LINKUP)) { 1676 if (ISSET(link_speed, sc->sc_phy_linkspeed) == 0) 1677 return EINVAL; 1678 } 1679 1680 if (ifm->ifm_media & IFM_FLOW) { 1681 abilities |= sc->sc_phy_abilities & 1682 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX); 1683 } 1684 1685 return ixl_set_phy_config(sc, link_speed, abilities, false); 1686 } 1687 1688 1689 static void 1690 ixl_del_all_multiaddr(struct ixl_softc *sc) 1691 { 1692 struct ethercom *ec = &sc->sc_ec; 1693 struct ether_multi *enm; 1694 struct ether_multistep step; 1695 1696 ETHER_LOCK(ec); 1697 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1698 ETHER_NEXT_MULTI(step, enm)) { 1699 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1700 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1701 } 1702 ETHER_UNLOCK(ec); 1703 } 1704 1705 static int 1706 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1707 { 1708 struct ifnet *ifp = &sc->sc_ec.ec_if; 1709 int rv; 1710 1711 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1712 return 0; 1713 1714 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1715 ixl_del_all_multiaddr(sc); 1716 SET(ifp->if_flags, IFF_ALLMULTI); 1717 return ENETRESET; 1718 } 1719 1720 /* multicast address can not use VLAN HWFILTER */ 1721 rv = ixl_add_macvlan(sc, addrlo, 0, 1722 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1723 1724 if (rv == ENOSPC) { 1725 ixl_del_all_multiaddr(sc); 1726 SET(ifp->if_flags, IFF_ALLMULTI); 1727 return ENETRESET; 1728 } 1729 1730 return rv; 1731 } 1732 1733 static int 1734 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1735 { 1736 struct ifnet *ifp = &sc->sc_ec.ec_if; 1737 struct ethercom *ec = &sc->sc_ec; 1738 struct ether_multi *enm, *enm_last; 1739 struct ether_multistep step; 1740 int error, rv = 0; 1741 1742 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1743 ixl_remove_macvlan(sc, addrlo, 0, 1744 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1745 return 0; 1746 } 1747 1748 ETHER_LOCK(ec); 1749 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1750 ETHER_NEXT_MULTI(step, enm)) { 1751 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1752 ETHER_ADDR_LEN) != 0) { 1753 goto out; 1754 } 1755 } 1756 1757 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1758 ETHER_NEXT_MULTI(step, enm)) { 1759 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1760 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1761 if (error != 0) 1762 break; 1763 } 1764 1765 if (enm != NULL) { 1766 enm_last = enm; 1767 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1768 ETHER_NEXT_MULTI(step, enm)) { 1769 if (enm == enm_last) 1770 break; 1771 1772 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1773 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1774 } 1775 } else { 1776 CLR(ifp->if_flags, IFF_ALLMULTI); 1777 rv = ENETRESET; 1778 } 1779 1780 out: 1781 ETHER_UNLOCK(ec); 1782 return rv; 1783 } 1784 1785 static int 1786 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1787 { 1788 struct ifreq *ifr = (struct ifreq *)data; 1789 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1790 const struct sockaddr *sa; 1791 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1792 int s, error = 0; 1793 unsigned int nmtu; 1794 1795 switch (cmd) { 1796 case SIOCSIFMTU: 1797 nmtu = ifr->ifr_mtu; 1798 1799 if (nmtu < IXL_MIN_MTU || nmtu > IXL_MAX_MTU) { 1800 error = EINVAL; 1801 break; 1802 } 1803 if (ifp->if_mtu != nmtu) { 1804 s = splnet(); 1805 error = ether_ioctl(ifp, cmd, data); 1806 splx(s); 1807 if (error == ENETRESET) 1808 error = ixl_init(ifp); 1809 } 1810 break; 1811 case SIOCADDMULTI: 1812 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1813 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1814 error = ether_multiaddr(sa, addrlo, addrhi); 1815 if (error != 0) 1816 return error; 1817 1818 error = ixl_add_multi(sc, addrlo, addrhi); 1819 if (error != 0 && error != ENETRESET) { 1820 ether_delmulti(sa, &sc->sc_ec); 1821 error = EIO; 1822 } 1823 } 1824 break; 1825 1826 case SIOCDELMULTI: 1827 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1828 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1829 error = ether_multiaddr(sa, addrlo, addrhi); 1830 if (error != 0) 1831 return error; 1832 1833 error = ixl_del_multi(sc, addrlo, addrhi); 1834 } 1835 break; 1836 1837 default: 1838 s = splnet(); 1839 error = ether_ioctl(ifp, cmd, data); 1840 splx(s); 1841 } 1842 1843 if (error == ENETRESET) 1844 error = ixl_iff(sc); 1845 1846 return error; 1847 } 1848 1849 static enum i40e_mac_type 1850 ixl_mactype(pci_product_id_t id) 1851 { 1852 1853 switch (id) { 1854 case PCI_PRODUCT_INTEL_XL710_SFP: 1855 case PCI_PRODUCT_INTEL_XL710_KX_B: 1856 case PCI_PRODUCT_INTEL_XL710_KX_C: 1857 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1858 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1859 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1860 case PCI_PRODUCT_INTEL_X710_10G_T_1: 1861 case PCI_PRODUCT_INTEL_X710_10G_T_2: 1862 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1863 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1864 case PCI_PRODUCT_INTEL_X710_T4_10G: 1865 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1866 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1867 case PCI_PRODUCT_INTEL_X710_10G_SFP: 1868 case PCI_PRODUCT_INTEL_X710_10G_BP: 1869 return I40E_MAC_XL710; 1870 1871 case PCI_PRODUCT_INTEL_X722_KX: 1872 case PCI_PRODUCT_INTEL_X722_QSFP: 1873 case PCI_PRODUCT_INTEL_X722_SFP: 1874 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1875 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1876 case PCI_PRODUCT_INTEL_X722_I_SFP: 1877 return I40E_MAC_X722; 1878 } 1879 1880 return I40E_MAC_GENERIC; 1881 } 1882 1883 static void 1884 ixl_pci_csr_setup(pci_chipset_tag_t pc, pcitag_t tag) 1885 { 1886 pcireg_t csr; 1887 1888 csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 1889 csr |= (PCI_COMMAND_MASTER_ENABLE | 1890 PCI_COMMAND_MEM_ENABLE); 1891 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr); 1892 } 1893 1894 static inline void * 1895 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1896 { 1897 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1898 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1899 1900 if (i >= e->hmc_count) 1901 return NULL; 1902 1903 kva += e->hmc_base; 1904 kva += i * e->hmc_size; 1905 1906 return kva; 1907 } 1908 1909 static inline size_t 1910 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1911 { 1912 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1913 1914 return e->hmc_size; 1915 } 1916 1917 static void 1918 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1919 { 1920 struct ixl_rx_ring *rxr = qp->qp_rxr; 1921 1922 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1923 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1924 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1925 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1926 ixl_flush(sc); 1927 } 1928 1929 static void 1930 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1931 { 1932 struct ixl_rx_ring *rxr = qp->qp_rxr; 1933 1934 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1935 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1936 ixl_flush(sc); 1937 } 1938 1939 static void 1940 ixl_enable_other_intr(struct ixl_softc *sc) 1941 { 1942 1943 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1944 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1945 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1946 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1947 ixl_flush(sc); 1948 } 1949 1950 static void 1951 ixl_disable_other_intr(struct ixl_softc *sc) 1952 { 1953 1954 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1955 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1956 ixl_flush(sc); 1957 } 1958 1959 static int 1960 ixl_reinit(struct ixl_softc *sc) 1961 { 1962 struct ixl_rx_ring *rxr; 1963 struct ixl_tx_ring *txr; 1964 unsigned int i; 1965 uint32_t reg; 1966 1967 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1968 1969 if (ixl_get_vsi(sc) != 0) 1970 return EIO; 1971 1972 if (ixl_set_vsi(sc) != 0) 1973 return EIO; 1974 1975 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1976 txr = sc->sc_qps[i].qp_txr; 1977 rxr = sc->sc_qps[i].qp_rxr; 1978 1979 ixl_txr_config(sc, txr); 1980 ixl_rxr_config(sc, rxr); 1981 } 1982 1983 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1984 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 1985 1986 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1987 txr = sc->sc_qps[i].qp_txr; 1988 rxr = sc->sc_qps[i].qp_rxr; 1989 1990 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 1991 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 1992 ixl_flush(sc); 1993 1994 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 1995 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 1996 1997 /* ixl_rxfill() needs lock held */ 1998 mutex_enter(&rxr->rxr_lock); 1999 ixl_rxfill(sc, rxr); 2000 mutex_exit(&rxr->rxr_lock); 2001 2002 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2003 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2004 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2005 if (ixl_rxr_enabled(sc, rxr) != 0) 2006 goto stop; 2007 2008 ixl_txr_qdis(sc, txr, 1); 2009 2010 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2011 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2012 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2013 2014 if (ixl_txr_enabled(sc, txr) != 0) 2015 goto stop; 2016 } 2017 2018 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2019 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2020 2021 return 0; 2022 2023 stop: 2024 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2025 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2026 2027 return ETIMEDOUT; 2028 } 2029 2030 static int 2031 ixl_init_locked(struct ixl_softc *sc) 2032 { 2033 struct ifnet *ifp = &sc->sc_ec.ec_if; 2034 unsigned int i; 2035 int error, eccap_change; 2036 2037 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2038 2039 if (ISSET(ifp->if_flags, IFF_RUNNING)) 2040 ixl_stop_locked(sc); 2041 2042 if (sc->sc_dead) { 2043 return ENXIO; 2044 } 2045 2046 eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable; 2047 if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING)) 2048 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 2049 2050 if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) { 2051 if (ixl_update_macvlan(sc) == 0) { 2052 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 2053 } else { 2054 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 2055 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 2056 } 2057 } 2058 2059 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 2060 sc->sc_nqueue_pairs = 1; 2061 else 2062 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 2063 2064 error = ixl_reinit(sc); 2065 if (error) { 2066 ixl_stop_locked(sc); 2067 return error; 2068 } 2069 2070 SET(ifp->if_flags, IFF_RUNNING); 2071 CLR(ifp->if_flags, IFF_OACTIVE); 2072 2073 ixl_config_rss(sc); 2074 ixl_config_queue_intr(sc); 2075 2076 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2077 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 2078 } 2079 ixl_enable_other_intr(sc); 2080 2081 error = ixl_iff(sc); 2082 if (error) { 2083 ixl_stop_locked(sc); 2084 return error; 2085 } 2086 2087 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 2088 2089 return 0; 2090 } 2091 2092 static int 2093 ixl_init(struct ifnet *ifp) 2094 { 2095 struct ixl_softc *sc = ifp->if_softc; 2096 int error; 2097 2098 mutex_enter(&sc->sc_cfg_lock); 2099 error = ixl_init_locked(sc); 2100 mutex_exit(&sc->sc_cfg_lock); 2101 2102 if (error == 0) { 2103 (void)ixl_get_link_status(sc); 2104 } 2105 2106 return error; 2107 } 2108 2109 static int 2110 ixl_iff(struct ixl_softc *sc) 2111 { 2112 struct ifnet *ifp = &sc->sc_ec.ec_if; 2113 struct ixl_atq iatq; 2114 struct ixl_aq_desc *iaq; 2115 struct ixl_aq_vsi_promisc_param *param; 2116 uint16_t flag_add, flag_del; 2117 int error; 2118 2119 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2120 return 0; 2121 2122 memset(&iatq, 0, sizeof(iatq)); 2123 2124 iaq = &iatq.iatq_desc; 2125 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 2126 2127 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 2128 param->flags = htole16(0); 2129 2130 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER) 2131 || ISSET(ifp->if_flags, IFF_PROMISC)) { 2132 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2133 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2134 } 2135 2136 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 2137 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2138 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2139 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 2140 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2141 } 2142 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2143 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2144 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2145 param->seid = sc->sc_seid; 2146 2147 error = ixl_atq_exec(sc, &iatq); 2148 if (error) 2149 return error; 2150 2151 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 2152 return EIO; 2153 2154 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 2155 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 2156 flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH; 2157 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH; 2158 } else { 2159 flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN; 2160 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN; 2161 } 2162 2163 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del); 2164 2165 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2166 ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add); 2167 } 2168 return 0; 2169 } 2170 2171 static void 2172 ixl_stop_locked(struct ixl_softc *sc) 2173 { 2174 struct ifnet *ifp = &sc->sc_ec.ec_if; 2175 struct ixl_rx_ring *rxr; 2176 struct ixl_tx_ring *txr; 2177 unsigned int i; 2178 uint32_t reg; 2179 2180 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2181 2182 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 2183 callout_stop(&sc->sc_stats_callout); 2184 2185 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2186 txr = sc->sc_qps[i].qp_txr; 2187 rxr = sc->sc_qps[i].qp_rxr; 2188 2189 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 2190 2191 mutex_enter(&txr->txr_lock); 2192 ixl_txr_qdis(sc, txr, 0); 2193 mutex_exit(&txr->txr_lock); 2194 } 2195 2196 /* XXX wait at least 400 usec for all tx queues in one go */ 2197 ixl_flush(sc); 2198 DELAY(500); 2199 2200 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2201 txr = sc->sc_qps[i].qp_txr; 2202 rxr = sc->sc_qps[i].qp_rxr; 2203 2204 mutex_enter(&txr->txr_lock); 2205 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2206 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2207 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2208 mutex_exit(&txr->txr_lock); 2209 2210 mutex_enter(&rxr->rxr_lock); 2211 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2212 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2213 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2214 mutex_exit(&rxr->rxr_lock); 2215 } 2216 2217 /* XXX short wait for all queue disables to settle */ 2218 ixl_flush(sc); 2219 DELAY(50); 2220 2221 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2222 txr = sc->sc_qps[i].qp_txr; 2223 rxr = sc->sc_qps[i].qp_rxr; 2224 2225 mutex_enter(&txr->txr_lock); 2226 if (ixl_txr_disabled(sc, txr) != 0) { 2227 mutex_exit(&txr->txr_lock); 2228 goto die; 2229 } 2230 mutex_exit(&txr->txr_lock); 2231 2232 mutex_enter(&rxr->rxr_lock); 2233 if (ixl_rxr_disabled(sc, rxr) != 0) { 2234 mutex_exit(&rxr->rxr_lock); 2235 goto die; 2236 } 2237 mutex_exit(&rxr->rxr_lock); 2238 } 2239 2240 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2241 sc->sc_qps[i].qp_workqueue = false; 2242 workqueue_wait(sc->sc_workq_txrx, 2243 &sc->sc_qps[i].qp_work); 2244 } 2245 2246 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2247 txr = sc->sc_qps[i].qp_txr; 2248 rxr = sc->sc_qps[i].qp_rxr; 2249 2250 mutex_enter(&txr->txr_lock); 2251 ixl_txr_unconfig(sc, txr); 2252 mutex_exit(&txr->txr_lock); 2253 2254 mutex_enter(&rxr->rxr_lock); 2255 ixl_rxr_unconfig(sc, rxr); 2256 mutex_exit(&rxr->rxr_lock); 2257 2258 ixl_txr_clean(sc, txr); 2259 ixl_rxr_clean(sc, rxr); 2260 } 2261 2262 return; 2263 die: 2264 sc->sc_dead = true; 2265 log(LOG_CRIT, "%s: failed to shut down rings", 2266 device_xname(sc->sc_dev)); 2267 return; 2268 } 2269 2270 static void 2271 ixl_stop(struct ifnet *ifp, int disable) 2272 { 2273 struct ixl_softc *sc = ifp->if_softc; 2274 2275 mutex_enter(&sc->sc_cfg_lock); 2276 ixl_stop_locked(sc); 2277 mutex_exit(&sc->sc_cfg_lock); 2278 } 2279 2280 static int 2281 ixl_queue_pairs_alloc(struct ixl_softc *sc) 2282 { 2283 struct ixl_queue_pair *qp; 2284 unsigned int i; 2285 size_t sz; 2286 2287 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2288 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 2289 2290 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2291 qp = &sc->sc_qps[i]; 2292 2293 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2294 ixl_handle_queue, qp); 2295 if (qp->qp_si == NULL) 2296 goto free; 2297 2298 qp->qp_txr = ixl_txr_alloc(sc, i); 2299 if (qp->qp_txr == NULL) 2300 goto free; 2301 2302 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2303 if (qp->qp_rxr == NULL) 2304 goto free; 2305 2306 qp->qp_sc = sc; 2307 snprintf(qp->qp_name, sizeof(qp->qp_name), 2308 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2309 } 2310 2311 return 0; 2312 free: 2313 if (sc->sc_qps != NULL) { 2314 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2315 qp = &sc->sc_qps[i]; 2316 2317 if (qp->qp_txr != NULL) 2318 ixl_txr_free(sc, qp->qp_txr); 2319 if (qp->qp_rxr != NULL) 2320 ixl_rxr_free(sc, qp->qp_rxr); 2321 if (qp->qp_si != NULL) 2322 softint_disestablish(qp->qp_si); 2323 } 2324 2325 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2326 kmem_free(sc->sc_qps, sz); 2327 sc->sc_qps = NULL; 2328 } 2329 2330 return -1; 2331 } 2332 2333 static void 2334 ixl_queue_pairs_free(struct ixl_softc *sc) 2335 { 2336 struct ixl_queue_pair *qp; 2337 unsigned int i; 2338 size_t sz; 2339 2340 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2341 qp = &sc->sc_qps[i]; 2342 ixl_txr_free(sc, qp->qp_txr); 2343 ixl_rxr_free(sc, qp->qp_rxr); 2344 softint_disestablish(qp->qp_si); 2345 } 2346 2347 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2348 kmem_free(sc->sc_qps, sz); 2349 sc->sc_qps = NULL; 2350 } 2351 2352 static struct ixl_tx_ring * 2353 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2354 { 2355 struct ixl_tx_ring *txr = NULL; 2356 struct ixl_tx_map *maps = NULL, *txm; 2357 unsigned int i; 2358 2359 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2360 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2361 KM_SLEEP); 2362 2363 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2364 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2365 IXL_TX_QUEUE_ALIGN) != 0) 2366 goto free; 2367 2368 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2369 txm = &maps[i]; 2370 2371 if (bus_dmamap_create(sc->sc_dmat, IXL_TX_PKT_MAXSIZE, 2372 IXL_TX_PKT_DESCS, IXL_TX_PKT_MAXSIZE, 0, 2373 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2374 goto uncreate; 2375 2376 txm->txm_eop = -1; 2377 txm->txm_m = NULL; 2378 } 2379 2380 txr->txr_cons = txr->txr_prod = 0; 2381 txr->txr_maps = maps; 2382 2383 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2384 if (txr->txr_intrq == NULL) 2385 goto uncreate; 2386 2387 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2388 ixl_deferred_transmit, txr); 2389 if (txr->txr_si == NULL) 2390 goto destroy_pcq; 2391 2392 txr->txr_tail = I40E_QTX_TAIL(qid); 2393 txr->txr_qid = qid; 2394 txr->txr_sc = sc; 2395 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2396 2397 return txr; 2398 2399 destroy_pcq: 2400 pcq_destroy(txr->txr_intrq); 2401 uncreate: 2402 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2403 txm = &maps[i]; 2404 2405 if (txm->txm_map == NULL) 2406 continue; 2407 2408 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2409 } 2410 2411 ixl_dmamem_free(sc, &txr->txr_mem); 2412 free: 2413 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2414 kmem_free(txr, sizeof(*txr)); 2415 2416 return NULL; 2417 } 2418 2419 static void 2420 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2421 { 2422 unsigned int qid; 2423 bus_size_t reg; 2424 uint32_t r; 2425 2426 qid = txr->txr_qid + sc->sc_base_queue; 2427 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2428 qid %= 128; 2429 2430 r = ixl_rd(sc, reg); 2431 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2432 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2433 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2434 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2435 ixl_wr(sc, reg, r); 2436 } 2437 2438 static void 2439 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2440 { 2441 struct ixl_hmc_txq txq; 2442 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2443 void *hmc; 2444 2445 memset(&txq, 0, sizeof(txq)); 2446 txq.head = htole16(txr->txr_cons); 2447 txq.new_context = 1; 2448 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2449 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2450 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2451 txq.tphrdesc_ena = 0; 2452 txq.tphrpacket_ena = 0; 2453 txq.tphwdesc_ena = 0; 2454 txq.rdylist = data->qs_handle[0]; 2455 2456 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2457 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2458 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2459 __arraycount(ixl_hmc_pack_txq)); 2460 } 2461 2462 static void 2463 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2464 { 2465 void *hmc; 2466 2467 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2468 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2469 txr->txr_cons = txr->txr_prod = 0; 2470 } 2471 2472 static void 2473 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2474 { 2475 struct ixl_tx_map *maps, *txm; 2476 bus_dmamap_t map; 2477 unsigned int i; 2478 2479 maps = txr->txr_maps; 2480 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2481 txm = &maps[i]; 2482 2483 if (txm->txm_m == NULL) 2484 continue; 2485 2486 map = txm->txm_map; 2487 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2488 BUS_DMASYNC_POSTWRITE); 2489 bus_dmamap_unload(sc->sc_dmat, map); 2490 2491 m_freem(txm->txm_m); 2492 txm->txm_m = NULL; 2493 } 2494 } 2495 2496 static int 2497 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2498 { 2499 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2500 uint32_t reg; 2501 int i; 2502 2503 for (i = 0; i < 10; i++) { 2504 reg = ixl_rd(sc, ena); 2505 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2506 return 0; 2507 2508 delaymsec(10); 2509 } 2510 2511 return ETIMEDOUT; 2512 } 2513 2514 static int 2515 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2516 { 2517 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2518 uint32_t reg; 2519 int i; 2520 2521 KASSERT(mutex_owned(&txr->txr_lock)); 2522 2523 for (i = 0; i < 10; i++) { 2524 reg = ixl_rd(sc, ena); 2525 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2526 return 0; 2527 2528 delaymsec(10); 2529 } 2530 2531 return ETIMEDOUT; 2532 } 2533 2534 static void 2535 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2536 { 2537 struct ixl_tx_map *maps, *txm; 2538 struct mbuf *m; 2539 unsigned int i; 2540 2541 softint_disestablish(txr->txr_si); 2542 2543 maps = txr->txr_maps; 2544 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2545 txm = &maps[i]; 2546 2547 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2548 } 2549 2550 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2551 m_freem(m); 2552 pcq_destroy(txr->txr_intrq); 2553 2554 ixl_dmamem_free(sc, &txr->txr_mem); 2555 mutex_destroy(&txr->txr_lock); 2556 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2557 kmem_free(txr, sizeof(*txr)); 2558 } 2559 2560 static inline int 2561 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2562 struct ixl_tx_ring *txr) 2563 { 2564 struct mbuf *m; 2565 int error; 2566 2567 KASSERT(mutex_owned(&txr->txr_lock)); 2568 2569 m = *m0; 2570 2571 error = bus_dmamap_load_mbuf(dmat, map, m, 2572 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2573 if (error != EFBIG) 2574 return error; 2575 2576 m = m_defrag(m, M_DONTWAIT); 2577 if (m != NULL) { 2578 *m0 = m; 2579 txr->txr_defragged.ev_count++; 2580 2581 error = bus_dmamap_load_mbuf(dmat, map, m, 2582 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2583 } else { 2584 txr->txr_defrag_failed.ev_count++; 2585 error = ENOBUFS; 2586 } 2587 2588 return error; 2589 } 2590 2591 static inline int 2592 ixl_tx_setup_offloads(struct mbuf *m, uint64_t *cmd_txd) 2593 { 2594 struct ether_header *eh; 2595 size_t len; 2596 uint64_t cmd; 2597 2598 cmd = 0; 2599 2600 eh = mtod(m, struct ether_header *); 2601 switch (htons(eh->ether_type)) { 2602 case ETHERTYPE_IP: 2603 case ETHERTYPE_IPV6: 2604 len = ETHER_HDR_LEN; 2605 break; 2606 case ETHERTYPE_VLAN: 2607 len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2608 break; 2609 default: 2610 len = 0; 2611 } 2612 cmd |= ((len >> 1) << IXL_TX_DESC_MACLEN_SHIFT); 2613 2614 if (m->m_pkthdr.csum_flags & 2615 (M_CSUM_TSOv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) { 2616 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4; 2617 } 2618 if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) { 2619 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4_CSUM; 2620 } 2621 2622 if (m->m_pkthdr.csum_flags & 2623 (M_CSUM_TSOv6 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) { 2624 cmd |= IXL_TX_DESC_CMD_IIPT_IPV6; 2625 } 2626 2627 switch (cmd & IXL_TX_DESC_CMD_IIPT_MASK) { 2628 case IXL_TX_DESC_CMD_IIPT_IPV4: 2629 case IXL_TX_DESC_CMD_IIPT_IPV4_CSUM: 2630 len = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); 2631 break; 2632 case IXL_TX_DESC_CMD_IIPT_IPV6: 2633 len = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); 2634 break; 2635 default: 2636 len = 0; 2637 } 2638 cmd |= ((len >> 2) << IXL_TX_DESC_IPLEN_SHIFT); 2639 2640 if (m->m_pkthdr.csum_flags & 2641 (M_CSUM_TSOv4 | M_CSUM_TSOv6 | M_CSUM_TCPv4 | M_CSUM_TCPv6)) { 2642 len = sizeof(struct tcphdr); 2643 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_TCP; 2644 } else if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) { 2645 len = sizeof(struct udphdr); 2646 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_UDP; 2647 } else { 2648 len = 0; 2649 } 2650 cmd |= ((len >> 2) << IXL_TX_DESC_L4LEN_SHIFT); 2651 2652 *cmd_txd |= cmd; 2653 return 0; 2654 } 2655 2656 static void 2657 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2658 bool is_transmit) 2659 { 2660 struct ixl_softc *sc = ifp->if_softc; 2661 struct ixl_tx_desc *ring, *txd; 2662 struct ixl_tx_map *txm; 2663 bus_dmamap_t map; 2664 struct mbuf *m; 2665 uint64_t cmd, cmd_txd; 2666 unsigned int prod, free, last, i; 2667 unsigned int mask; 2668 int post = 0; 2669 2670 KASSERT(mutex_owned(&txr->txr_lock)); 2671 2672 if (!ISSET(ifp->if_flags, IFF_RUNNING) 2673 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2674 if (!is_transmit) 2675 IFQ_PURGE(&ifp->if_snd); 2676 return; 2677 } 2678 2679 prod = txr->txr_prod; 2680 free = txr->txr_cons; 2681 if (free <= prod) 2682 free += sc->sc_tx_ring_ndescs; 2683 free -= prod; 2684 2685 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2686 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2687 2688 ring = IXL_DMA_KVA(&txr->txr_mem); 2689 mask = sc->sc_tx_ring_ndescs - 1; 2690 last = prod; 2691 cmd = 0; 2692 txd = NULL; 2693 2694 for (;;) { 2695 if (free <= IXL_TX_PKT_DESCS) { 2696 if (!is_transmit) 2697 SET(ifp->if_flags, IFF_OACTIVE); 2698 break; 2699 } 2700 2701 if (is_transmit) 2702 m = pcq_get(txr->txr_intrq); 2703 else 2704 IFQ_DEQUEUE(&ifp->if_snd, m); 2705 2706 if (m == NULL) 2707 break; 2708 2709 txm = &txr->txr_maps[prod]; 2710 map = txm->txm_map; 2711 2712 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2713 if_statinc(ifp, if_oerrors); 2714 m_freem(m); 2715 continue; 2716 } 2717 2718 cmd_txd = 0; 2719 if (m->m_pkthdr.csum_flags & IXL_CSUM_ALL_OFFLOAD) { 2720 ixl_tx_setup_offloads(m, &cmd_txd); 2721 } 2722 2723 if (vlan_has_tag(m)) { 2724 uint16_t vtag; 2725 vtag = htole16(vlan_get_tag(m)); 2726 cmd_txd |= (uint64_t)vtag << 2727 IXL_TX_DESC_L2TAG1_SHIFT; 2728 cmd_txd |= IXL_TX_DESC_CMD_IL2TAG1; 2729 } 2730 2731 bus_dmamap_sync(sc->sc_dmat, map, 0, 2732 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2733 2734 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2735 txd = &ring[prod]; 2736 2737 cmd = (uint64_t)map->dm_segs[i].ds_len << 2738 IXL_TX_DESC_BSIZE_SHIFT; 2739 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2740 cmd |= cmd_txd; 2741 2742 txd->addr = htole64(map->dm_segs[i].ds_addr); 2743 txd->cmd = htole64(cmd); 2744 2745 last = prod; 2746 2747 prod++; 2748 prod &= mask; 2749 } 2750 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2751 txd->cmd = htole64(cmd); 2752 2753 txm->txm_m = m; 2754 txm->txm_eop = last; 2755 2756 bpf_mtap(ifp, m, BPF_D_OUT); 2757 2758 free -= i; 2759 post = 1; 2760 } 2761 2762 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2763 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2764 2765 if (post) { 2766 txr->txr_prod = prod; 2767 ixl_wr(sc, txr->txr_tail, prod); 2768 } 2769 } 2770 2771 static int 2772 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2773 { 2774 struct ifnet *ifp = &sc->sc_ec.ec_if; 2775 struct ixl_tx_desc *ring, *txd; 2776 struct ixl_tx_map *txm; 2777 struct mbuf *m; 2778 bus_dmamap_t map; 2779 unsigned int cons, prod, last; 2780 unsigned int mask; 2781 uint64_t dtype; 2782 int done = 0, more = 0; 2783 2784 KASSERT(mutex_owned(&txr->txr_lock)); 2785 2786 prod = txr->txr_prod; 2787 cons = txr->txr_cons; 2788 2789 if (cons == prod) 2790 return 0; 2791 2792 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2793 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2794 2795 ring = IXL_DMA_KVA(&txr->txr_mem); 2796 mask = sc->sc_tx_ring_ndescs - 1; 2797 2798 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2799 2800 do { 2801 if (txlimit-- <= 0) { 2802 more = 1; 2803 break; 2804 } 2805 2806 txm = &txr->txr_maps[cons]; 2807 last = txm->txm_eop; 2808 txd = &ring[last]; 2809 2810 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2811 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2812 break; 2813 2814 map = txm->txm_map; 2815 2816 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2817 BUS_DMASYNC_POSTWRITE); 2818 bus_dmamap_unload(sc->sc_dmat, map); 2819 2820 m = txm->txm_m; 2821 if (m != NULL) { 2822 if_statinc_ref(ifp, nsr, if_opackets); 2823 if_statadd_ref(ifp, nsr, if_obytes, m->m_pkthdr.len); 2824 if (ISSET(m->m_flags, M_MCAST)) 2825 if_statinc_ref(ifp, nsr, if_omcasts); 2826 m_freem(m); 2827 } 2828 2829 txm->txm_m = NULL; 2830 txm->txm_eop = -1; 2831 2832 cons = last + 1; 2833 cons &= mask; 2834 done = 1; 2835 } while (cons != prod); 2836 2837 IF_STAT_PUTREF(ifp); 2838 2839 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2840 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2841 2842 txr->txr_cons = cons; 2843 2844 if (done) { 2845 softint_schedule(txr->txr_si); 2846 if (txr->txr_qid == 0) { 2847 CLR(ifp->if_flags, IFF_OACTIVE); 2848 if_schedule_deferred_start(ifp); 2849 } 2850 } 2851 2852 return more; 2853 } 2854 2855 static void 2856 ixl_start(struct ifnet *ifp) 2857 { 2858 struct ixl_softc *sc; 2859 struct ixl_tx_ring *txr; 2860 2861 sc = ifp->if_softc; 2862 txr = sc->sc_qps[0].qp_txr; 2863 2864 mutex_enter(&txr->txr_lock); 2865 ixl_tx_common_locked(ifp, txr, false); 2866 mutex_exit(&txr->txr_lock); 2867 } 2868 2869 static inline unsigned int 2870 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2871 { 2872 u_int cpuid; 2873 2874 cpuid = cpu_index(curcpu()); 2875 2876 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2877 } 2878 2879 static int 2880 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2881 { 2882 struct ixl_softc *sc; 2883 struct ixl_tx_ring *txr; 2884 unsigned int qid; 2885 2886 sc = ifp->if_softc; 2887 qid = ixl_select_txqueue(sc, m); 2888 2889 txr = sc->sc_qps[qid].qp_txr; 2890 2891 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2892 mutex_enter(&txr->txr_lock); 2893 txr->txr_pcqdrop.ev_count++; 2894 mutex_exit(&txr->txr_lock); 2895 2896 m_freem(m); 2897 return ENOBUFS; 2898 } 2899 2900 #ifdef IXL_ALWAYS_TXDEFER 2901 kpreempt_disable(); 2902 softint_schedule(txr->txr_si); 2903 kpreempt_enable(); 2904 #else 2905 if (mutex_tryenter(&txr->txr_lock)) { 2906 ixl_tx_common_locked(ifp, txr, true); 2907 mutex_exit(&txr->txr_lock); 2908 } else { 2909 kpreempt_disable(); 2910 softint_schedule(txr->txr_si); 2911 kpreempt_enable(); 2912 } 2913 #endif 2914 2915 return 0; 2916 } 2917 2918 static void 2919 ixl_deferred_transmit(void *xtxr) 2920 { 2921 struct ixl_tx_ring *txr = xtxr; 2922 struct ixl_softc *sc = txr->txr_sc; 2923 struct ifnet *ifp = &sc->sc_ec.ec_if; 2924 2925 mutex_enter(&txr->txr_lock); 2926 txr->txr_transmitdef.ev_count++; 2927 if (pcq_peek(txr->txr_intrq) != NULL) 2928 ixl_tx_common_locked(ifp, txr, true); 2929 mutex_exit(&txr->txr_lock); 2930 } 2931 2932 static struct ixl_rx_ring * 2933 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2934 { 2935 struct ixl_rx_ring *rxr = NULL; 2936 struct ixl_rx_map *maps = NULL, *rxm; 2937 unsigned int i; 2938 2939 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2940 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2941 KM_SLEEP); 2942 2943 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2944 sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs, 2945 IXL_RX_QUEUE_ALIGN) != 0) 2946 goto free; 2947 2948 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2949 rxm = &maps[i]; 2950 2951 if (bus_dmamap_create(sc->sc_dmat, 2952 IXL_MCLBYTES, 1, IXL_MCLBYTES, 0, 2953 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2954 goto uncreate; 2955 2956 rxm->rxm_m = NULL; 2957 } 2958 2959 rxr->rxr_cons = rxr->rxr_prod = 0; 2960 rxr->rxr_m_head = NULL; 2961 rxr->rxr_m_tail = &rxr->rxr_m_head; 2962 rxr->rxr_maps = maps; 2963 2964 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2965 rxr->rxr_qid = qid; 2966 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2967 2968 return rxr; 2969 2970 uncreate: 2971 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2972 rxm = &maps[i]; 2973 2974 if (rxm->rxm_map == NULL) 2975 continue; 2976 2977 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2978 } 2979 2980 ixl_dmamem_free(sc, &rxr->rxr_mem); 2981 free: 2982 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2983 kmem_free(rxr, sizeof(*rxr)); 2984 2985 return NULL; 2986 } 2987 2988 static void 2989 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2990 { 2991 struct ixl_rx_map *maps, *rxm; 2992 bus_dmamap_t map; 2993 unsigned int i; 2994 2995 maps = rxr->rxr_maps; 2996 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2997 rxm = &maps[i]; 2998 2999 if (rxm->rxm_m == NULL) 3000 continue; 3001 3002 map = rxm->rxm_map; 3003 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3004 BUS_DMASYNC_POSTWRITE); 3005 bus_dmamap_unload(sc->sc_dmat, map); 3006 3007 m_freem(rxm->rxm_m); 3008 rxm->rxm_m = NULL; 3009 } 3010 3011 m_freem(rxr->rxr_m_head); 3012 rxr->rxr_m_head = NULL; 3013 rxr->rxr_m_tail = &rxr->rxr_m_head; 3014 3015 rxr->rxr_prod = rxr->rxr_cons = 0; 3016 } 3017 3018 static int 3019 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3020 { 3021 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3022 uint32_t reg; 3023 int i; 3024 3025 for (i = 0; i < 10; i++) { 3026 reg = ixl_rd(sc, ena); 3027 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 3028 return 0; 3029 3030 delaymsec(10); 3031 } 3032 3033 return ETIMEDOUT; 3034 } 3035 3036 static int 3037 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3038 { 3039 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3040 uint32_t reg; 3041 int i; 3042 3043 KASSERT(mutex_owned(&rxr->rxr_lock)); 3044 3045 for (i = 0; i < 10; i++) { 3046 reg = ixl_rd(sc, ena); 3047 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 3048 return 0; 3049 3050 delaymsec(10); 3051 } 3052 3053 return ETIMEDOUT; 3054 } 3055 3056 static void 3057 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3058 { 3059 struct ixl_hmc_rxq rxq; 3060 struct ifnet *ifp = &sc->sc_ec.ec_if; 3061 uint16_t rxmax; 3062 void *hmc; 3063 3064 memset(&rxq, 0, sizeof(rxq)); 3065 rxmax = ifp->if_mtu + IXL_MTU_ETHERLEN; 3066 if (!ISSET(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_MTU)) 3067 rxmax -= ETHER_VLAN_ENCAP_LEN; 3068 3069 rxq.head = htole16(rxr->rxr_cons); 3070 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 3071 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 3072 rxq.dbuff = htole16(IXL_MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 3073 rxq.hbuff = 0; 3074 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 3075 rxq.dsize = IXL_HMC_RXQ_DSIZE_32; 3076 rxq.crcstrip = 1; 3077 rxq.l2sel = 1; 3078 rxq.showiv = 1; 3079 rxq.rxmax = htole16(rxmax); 3080 rxq.tphrdesc_ena = 0; 3081 rxq.tphwdesc_ena = 0; 3082 rxq.tphdata_ena = 0; 3083 rxq.tphhead_ena = 0; 3084 rxq.lrxqthresh = 0; 3085 rxq.prefena = 1; 3086 3087 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3088 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3089 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 3090 __arraycount(ixl_hmc_pack_rxq)); 3091 } 3092 3093 static void 3094 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3095 { 3096 void *hmc; 3097 3098 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3099 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3100 rxr->rxr_cons = rxr->rxr_prod = 0; 3101 } 3102 3103 static void 3104 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3105 { 3106 struct ixl_rx_map *maps, *rxm; 3107 unsigned int i; 3108 3109 maps = rxr->rxr_maps; 3110 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3111 rxm = &maps[i]; 3112 3113 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 3114 } 3115 3116 ixl_dmamem_free(sc, &rxr->rxr_mem); 3117 mutex_destroy(&rxr->rxr_lock); 3118 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3119 kmem_free(rxr, sizeof(*rxr)); 3120 } 3121 3122 static inline void 3123 ixl_rx_csum(struct mbuf *m, uint64_t qword) 3124 { 3125 int flags_mask; 3126 3127 if (!ISSET(qword, IXL_RX_DESC_L3L4P)) { 3128 /* No L3 or L4 checksum was calculated */ 3129 return; 3130 } 3131 3132 switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) { 3133 case IXL_RX_DESC_PTYPE_IPV4FRAG: 3134 case IXL_RX_DESC_PTYPE_IPV4: 3135 case IXL_RX_DESC_PTYPE_SCTPV4: 3136 case IXL_RX_DESC_PTYPE_ICMPV4: 3137 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3138 break; 3139 case IXL_RX_DESC_PTYPE_TCPV4: 3140 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3141 flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD; 3142 break; 3143 case IXL_RX_DESC_PTYPE_UDPV4: 3144 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3145 flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD; 3146 break; 3147 case IXL_RX_DESC_PTYPE_TCPV6: 3148 flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD; 3149 break; 3150 case IXL_RX_DESC_PTYPE_UDPV6: 3151 flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD; 3152 break; 3153 default: 3154 flags_mask = 0; 3155 } 3156 3157 m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 | 3158 M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6)); 3159 3160 if (ISSET(qword, IXL_RX_DESC_IPE)) { 3161 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD); 3162 } 3163 3164 if (ISSET(qword, IXL_RX_DESC_L4E)) { 3165 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD); 3166 } 3167 } 3168 3169 static int 3170 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 3171 { 3172 struct ifnet *ifp = &sc->sc_ec.ec_if; 3173 struct ixl_rx_wb_desc_32 *ring, *rxd; 3174 struct ixl_rx_map *rxm; 3175 bus_dmamap_t map; 3176 unsigned int cons, prod; 3177 struct mbuf *m; 3178 uint64_t word, word0; 3179 unsigned int len; 3180 unsigned int mask; 3181 int done = 0, more = 0; 3182 3183 KASSERT(mutex_owned(&rxr->rxr_lock)); 3184 3185 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3186 return 0; 3187 3188 prod = rxr->rxr_prod; 3189 cons = rxr->rxr_cons; 3190 3191 if (cons == prod) 3192 return 0; 3193 3194 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3195 0, IXL_DMA_LEN(&rxr->rxr_mem), 3196 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3197 3198 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3199 mask = sc->sc_rx_ring_ndescs - 1; 3200 3201 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 3202 3203 do { 3204 if (rxlimit-- <= 0) { 3205 more = 1; 3206 break; 3207 } 3208 3209 rxd = &ring[cons]; 3210 3211 word = le64toh(rxd->qword1); 3212 3213 if (!ISSET(word, IXL_RX_DESC_DD)) 3214 break; 3215 3216 rxm = &rxr->rxr_maps[cons]; 3217 3218 map = rxm->rxm_map; 3219 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3220 BUS_DMASYNC_POSTREAD); 3221 bus_dmamap_unload(sc->sc_dmat, map); 3222 3223 m = rxm->rxm_m; 3224 rxm->rxm_m = NULL; 3225 3226 KASSERT(m != NULL); 3227 3228 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 3229 m->m_len = len; 3230 m->m_pkthdr.len = 0; 3231 3232 m->m_next = NULL; 3233 *rxr->rxr_m_tail = m; 3234 rxr->rxr_m_tail = &m->m_next; 3235 3236 m = rxr->rxr_m_head; 3237 m->m_pkthdr.len += len; 3238 3239 if (ISSET(word, IXL_RX_DESC_EOP)) { 3240 word0 = le64toh(rxd->qword0); 3241 3242 if (ISSET(word, IXL_RX_DESC_L2TAG1P)) { 3243 uint16_t vtag; 3244 vtag = __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK); 3245 vlan_set_tag(m, le16toh(vtag)); 3246 } 3247 3248 if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0) 3249 ixl_rx_csum(m, word); 3250 3251 if (!ISSET(word, 3252 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 3253 m_set_rcvif(m, ifp); 3254 if_statinc_ref(ifp, nsr, if_ipackets); 3255 if_statadd_ref(ifp, nsr, if_ibytes, 3256 m->m_pkthdr.len); 3257 if_percpuq_enqueue(sc->sc_ipq, m); 3258 } else { 3259 if_statinc_ref(ifp, nsr, if_ierrors); 3260 m_freem(m); 3261 } 3262 3263 rxr->rxr_m_head = NULL; 3264 rxr->rxr_m_tail = &rxr->rxr_m_head; 3265 } 3266 3267 cons++; 3268 cons &= mask; 3269 3270 done = 1; 3271 } while (cons != prod); 3272 3273 if (done) { 3274 rxr->rxr_cons = cons; 3275 if (ixl_rxfill(sc, rxr) == -1) 3276 if_statinc_ref(ifp, nsr, if_iqdrops); 3277 } 3278 3279 IF_STAT_PUTREF(ifp); 3280 3281 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3282 0, IXL_DMA_LEN(&rxr->rxr_mem), 3283 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3284 3285 return more; 3286 } 3287 3288 static int 3289 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3290 { 3291 struct ixl_rx_rd_desc_32 *ring, *rxd; 3292 struct ixl_rx_map *rxm; 3293 bus_dmamap_t map; 3294 struct mbuf *m; 3295 unsigned int prod; 3296 unsigned int slots; 3297 unsigned int mask; 3298 int post = 0, error = 0; 3299 3300 KASSERT(mutex_owned(&rxr->rxr_lock)); 3301 3302 prod = rxr->rxr_prod; 3303 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 3304 sc->sc_rx_ring_ndescs); 3305 3306 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3307 mask = sc->sc_rx_ring_ndescs - 1; 3308 3309 if (__predict_false(slots <= 0)) 3310 return -1; 3311 3312 do { 3313 rxm = &rxr->rxr_maps[prod]; 3314 3315 MGETHDR(m, M_DONTWAIT, MT_DATA); 3316 if (m == NULL) { 3317 rxr->rxr_mgethdr_failed.ev_count++; 3318 error = -1; 3319 break; 3320 } 3321 3322 MCLGET(m, M_DONTWAIT); 3323 if (!ISSET(m->m_flags, M_EXT)) { 3324 rxr->rxr_mgetcl_failed.ev_count++; 3325 error = -1; 3326 m_freem(m); 3327 break; 3328 } 3329 3330 m->m_len = m->m_pkthdr.len = MCLBYTES; 3331 m_adj(m, ETHER_ALIGN); 3332 3333 map = rxm->rxm_map; 3334 3335 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 3336 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 3337 rxr->rxr_mbuf_load_failed.ev_count++; 3338 error = -1; 3339 m_freem(m); 3340 break; 3341 } 3342 3343 rxm->rxm_m = m; 3344 3345 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3346 BUS_DMASYNC_PREREAD); 3347 3348 rxd = &ring[prod]; 3349 3350 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 3351 rxd->haddr = htole64(0); 3352 3353 prod++; 3354 prod &= mask; 3355 3356 post = 1; 3357 3358 } while (--slots); 3359 3360 if (post) { 3361 rxr->rxr_prod = prod; 3362 ixl_wr(sc, rxr->rxr_tail, prod); 3363 } 3364 3365 return error; 3366 } 3367 3368 static inline int 3369 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 3370 u_int txlimit, struct evcnt *txevcnt, 3371 u_int rxlimit, struct evcnt *rxevcnt) 3372 { 3373 struct ixl_tx_ring *txr = qp->qp_txr; 3374 struct ixl_rx_ring *rxr = qp->qp_rxr; 3375 int txmore, rxmore; 3376 int rv; 3377 3378 mutex_enter(&txr->txr_lock); 3379 txevcnt->ev_count++; 3380 txmore = ixl_txeof(sc, txr, txlimit); 3381 mutex_exit(&txr->txr_lock); 3382 3383 mutex_enter(&rxr->rxr_lock); 3384 rxevcnt->ev_count++; 3385 rxmore = ixl_rxeof(sc, rxr, rxlimit); 3386 mutex_exit(&rxr->rxr_lock); 3387 3388 rv = txmore | (rxmore << 1); 3389 3390 return rv; 3391 } 3392 3393 static void 3394 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 3395 { 3396 3397 if (qp->qp_workqueue) 3398 workqueue_enqueue(sc->sc_workq_txrx, &qp->qp_work, NULL); 3399 else 3400 softint_schedule(qp->qp_si); 3401 } 3402 3403 static int 3404 ixl_intr(void *xsc) 3405 { 3406 struct ixl_softc *sc = xsc; 3407 struct ixl_tx_ring *txr; 3408 struct ixl_rx_ring *rxr; 3409 uint32_t icr, rxintr, txintr; 3410 int rv = 0; 3411 unsigned int i; 3412 3413 KASSERT(sc != NULL); 3414 3415 ixl_enable_other_intr(sc); 3416 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3417 3418 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3419 atomic_inc_64(&sc->sc_event_atq.ev_count); 3420 ixl_atq_done(sc); 3421 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3422 rv = 1; 3423 } 3424 3425 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3426 atomic_inc_64(&sc->sc_event_link.ev_count); 3427 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3428 rv = 1; 3429 } 3430 3431 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 3432 txintr = icr & I40E_INTR_NOTX_TX_MASK; 3433 3434 if (txintr || rxintr) { 3435 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 3436 txr = sc->sc_qps[i].qp_txr; 3437 rxr = sc->sc_qps[i].qp_rxr; 3438 3439 ixl_handle_queue_common(sc, &sc->sc_qps[i], 3440 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 3441 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 3442 } 3443 rv = 1; 3444 } 3445 3446 return rv; 3447 } 3448 3449 static int 3450 ixl_queue_intr(void *xqp) 3451 { 3452 struct ixl_queue_pair *qp = xqp; 3453 struct ixl_tx_ring *txr = qp->qp_txr; 3454 struct ixl_rx_ring *rxr = qp->qp_rxr; 3455 struct ixl_softc *sc = qp->qp_sc; 3456 u_int txlimit, rxlimit; 3457 int more; 3458 3459 txlimit = sc->sc_tx_intr_process_limit; 3460 rxlimit = sc->sc_rx_intr_process_limit; 3461 qp->qp_workqueue = sc->sc_txrx_workqueue; 3462 3463 more = ixl_handle_queue_common(sc, qp, 3464 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3465 3466 if (more != 0) { 3467 ixl_sched_handle_queue(sc, qp); 3468 } else { 3469 /* for ALTQ */ 3470 if (txr->txr_qid == 0) 3471 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3472 softint_schedule(txr->txr_si); 3473 3474 ixl_enable_queue_intr(sc, qp); 3475 } 3476 3477 return 1; 3478 } 3479 3480 static void 3481 ixl_handle_queue_wk(struct work *wk, void *xsc) 3482 { 3483 struct ixl_queue_pair *qp; 3484 3485 qp = container_of(wk, struct ixl_queue_pair, qp_work); 3486 ixl_handle_queue(qp); 3487 } 3488 3489 static void 3490 ixl_handle_queue(void *xqp) 3491 { 3492 struct ixl_queue_pair *qp = xqp; 3493 struct ixl_softc *sc = qp->qp_sc; 3494 struct ixl_tx_ring *txr = qp->qp_txr; 3495 struct ixl_rx_ring *rxr = qp->qp_rxr; 3496 u_int txlimit, rxlimit; 3497 int more; 3498 3499 txlimit = sc->sc_tx_process_limit; 3500 rxlimit = sc->sc_rx_process_limit; 3501 3502 more = ixl_handle_queue_common(sc, qp, 3503 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3504 3505 if (more != 0) 3506 ixl_sched_handle_queue(sc, qp); 3507 else 3508 ixl_enable_queue_intr(sc, qp); 3509 } 3510 3511 static inline void 3512 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3513 { 3514 uint32_t hmc_idx, hmc_isvf; 3515 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3516 3517 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3518 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3519 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3520 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3521 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3522 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3523 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3524 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3525 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3526 3527 device_printf(sc->sc_dev, 3528 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3529 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3530 } 3531 3532 static int 3533 ixl_other_intr(void *xsc) 3534 { 3535 struct ixl_softc *sc = xsc; 3536 uint32_t icr, mask, reg; 3537 int rv = 0; 3538 3539 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3540 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3541 3542 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3543 atomic_inc_64(&sc->sc_event_atq.ev_count); 3544 ixl_atq_done(sc); 3545 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3546 rv = 1; 3547 } 3548 3549 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3550 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3551 device_printf(sc->sc_dev, "link stat changed\n"); 3552 3553 atomic_inc_64(&sc->sc_event_link.ev_count); 3554 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3555 rv = 1; 3556 } 3557 3558 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3559 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3560 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3561 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3562 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3563 3564 device_printf(sc->sc_dev, "GRST: %s\n", 3565 reg == I40E_RESET_CORER ? "CORER" : 3566 reg == I40E_RESET_GLOBR ? "GLOBR" : 3567 reg == I40E_RESET_EMPR ? "EMPR" : 3568 "POR"); 3569 } 3570 3571 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3572 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3573 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3574 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3575 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3576 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3577 3578 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3579 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3580 device_printf(sc->sc_dev, "critical error\n"); 3581 } 3582 3583 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3584 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3585 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3586 ixl_print_hmc_error(sc, reg); 3587 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3588 } 3589 3590 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3591 ixl_flush(sc); 3592 ixl_enable_other_intr(sc); 3593 return rv; 3594 } 3595 3596 static void 3597 ixl_get_link_status_done_work(void *xsc) 3598 { 3599 struct ixl_softc *sc = xsc; 3600 struct ixl_aq_desc *iaq, iaq_buf; 3601 3602 mutex_enter(&sc->sc_atq_lock); 3603 iaq = &sc->sc_link_state_atq.iatq_desc; 3604 iaq_buf = *iaq; 3605 mutex_exit(&sc->sc_atq_lock); 3606 3607 ixl_link_state_update(sc, &iaq_buf); 3608 3609 mutex_enter(&sc->sc_atq_lock); 3610 CLR(iaq->iaq_flags, htole16(IXL_AQ_DD)); 3611 ixl_wakeup(sc, iaq); 3612 mutex_exit(&sc->sc_atq_lock); 3613 } 3614 3615 static void 3616 ixl_get_link_status_done(struct ixl_softc *sc, 3617 const struct ixl_aq_desc *iaq) 3618 { 3619 3620 ixl_work_add(sc->sc_workq, &sc->sc_link_state_done_task); 3621 } 3622 3623 static int 3624 ixl_get_link_status(struct ixl_softc *sc) 3625 { 3626 struct ixl_atq *iatq; 3627 struct ixl_aq_desc *iaq; 3628 struct ixl_aq_link_param *param; 3629 int error; 3630 3631 mutex_enter(&sc->sc_atq_lock); 3632 3633 iatq = &sc->sc_link_state_atq; 3634 iaq = &iatq->iatq_desc; 3635 3636 if (!sc->sc_link_state_atq.iatq_inuse && 3637 !ISSET(iaq->iaq_flags, htole16(IXL_AQ_DD))) { 3638 memset(iaq, 0, sizeof(*iaq)); 3639 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3640 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3641 param->notify = IXL_AQ_LINK_NOTIFY; 3642 3643 KASSERT(iatq->iatq_fn == ixl_get_link_status_done); 3644 error = ixl_atq_post_locked(sc, iatq); 3645 } else { 3646 /* the previous command is not completed */ 3647 error = EBUSY; 3648 } 3649 3650 mutex_exit(&sc->sc_atq_lock); 3651 3652 return error; 3653 } 3654 3655 static void 3656 ixl_get_link_status_work(void *xsc) 3657 { 3658 struct ixl_softc *sc = xsc; 3659 3660 (void)ixl_get_link_status(sc); 3661 } 3662 3663 static void 3664 ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3665 { 3666 struct ifnet *ifp = &sc->sc_ec.ec_if; 3667 int link_state; 3668 3669 mutex_enter(&sc->sc_cfg_lock); 3670 link_state = ixl_set_link_status_locked(sc, iaq); 3671 mutex_exit(&sc->sc_cfg_lock); 3672 3673 if (ifp->if_link_state != link_state) 3674 if_link_state_change(ifp, link_state); 3675 3676 if (link_state != LINK_STATE_DOWN) { 3677 kpreempt_disable(); 3678 if_schedule_deferred_start(ifp); 3679 kpreempt_enable(); 3680 } 3681 } 3682 3683 static void 3684 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3685 const char *msg) 3686 { 3687 char buf[512]; 3688 size_t len; 3689 3690 len = sizeof(buf); 3691 buf[--len] = '\0'; 3692 3693 device_printf(sc->sc_dev, "%s\n", msg); 3694 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3695 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3696 buf, le16toh(iaq->iaq_opcode)); 3697 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3698 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3699 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3700 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3701 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3702 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3703 } 3704 3705 static void 3706 ixl_arq(void *xsc) 3707 { 3708 struct ixl_softc *sc = xsc; 3709 struct ixl_aq_desc *arq, *iaq; 3710 struct ixl_aq_buf *aqb; 3711 unsigned int cons = sc->sc_arq_cons; 3712 unsigned int prod; 3713 int done = 0; 3714 3715 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3716 sc->sc_aq_regs->arq_head_mask; 3717 3718 if (cons == prod) 3719 goto done; 3720 3721 arq = IXL_DMA_KVA(&sc->sc_arq); 3722 3723 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3724 0, IXL_DMA_LEN(&sc->sc_arq), 3725 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3726 3727 do { 3728 iaq = &arq[cons]; 3729 aqb = sc->sc_arq_live[cons]; 3730 3731 KASSERT(aqb != NULL); 3732 3733 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3734 BUS_DMASYNC_POSTREAD); 3735 3736 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3737 ixl_aq_dump(sc, iaq, "arq event"); 3738 3739 switch (iaq->iaq_opcode) { 3740 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3741 ixl_link_state_update(sc, iaq); 3742 break; 3743 } 3744 3745 memset(iaq, 0, sizeof(*iaq)); 3746 sc->sc_arq_live[cons] = NULL; 3747 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3748 3749 cons++; 3750 cons &= IXL_AQ_MASK; 3751 3752 done = 1; 3753 } while (cons != prod); 3754 3755 if (done) { 3756 sc->sc_arq_cons = cons; 3757 ixl_arq_fill(sc); 3758 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3759 0, IXL_DMA_LEN(&sc->sc_arq), 3760 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3761 } 3762 3763 done: 3764 ixl_enable_other_intr(sc); 3765 } 3766 3767 static void 3768 ixl_atq_set(struct ixl_atq *iatq, 3769 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3770 { 3771 3772 iatq->iatq_fn = fn; 3773 } 3774 3775 static int 3776 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3777 { 3778 struct ixl_aq_desc *atq, *slot; 3779 unsigned int prod, cons, prod_next; 3780 3781 /* assert locked */ 3782 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3783 3784 atq = IXL_DMA_KVA(&sc->sc_atq); 3785 prod = sc->sc_atq_prod; 3786 cons = sc->sc_atq_cons; 3787 prod_next = (prod +1) & IXL_AQ_MASK; 3788 3789 if (cons == prod_next) 3790 return ENOMEM; 3791 3792 slot = &atq[prod]; 3793 3794 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3795 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD| 3796 BUS_DMASYNC_POSTWRITE); 3797 3798 *slot = iatq->iatq_desc; 3799 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3800 3801 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3802 ixl_aq_dump(sc, slot, "atq command"); 3803 3804 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3805 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD| 3806 BUS_DMASYNC_PREWRITE); 3807 3808 sc->sc_atq_prod = prod_next; 3809 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3810 iatq->iatq_inuse = true; 3811 3812 return 0; 3813 } 3814 3815 static void 3816 ixl_atq_done_locked(struct ixl_softc *sc) 3817 { 3818 struct ixl_aq_desc *atq, *slot; 3819 struct ixl_atq *iatq; 3820 unsigned int cons; 3821 unsigned int prod; 3822 3823 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3824 3825 prod = sc->sc_atq_prod; 3826 cons = sc->sc_atq_cons; 3827 3828 if (prod == cons) 3829 return; 3830 3831 atq = IXL_DMA_KVA(&sc->sc_atq); 3832 3833 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3834 0, IXL_DMA_LEN(&sc->sc_atq), 3835 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3836 3837 do { 3838 slot = &atq[cons]; 3839 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3840 break; 3841 3842 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3843 iatq->iatq_desc = *slot; 3844 iatq->iatq_inuse = false; 3845 3846 memset(slot, 0, sizeof(*slot)); 3847 3848 if (iatq != NULL) { 3849 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3850 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3851 3852 if (iatq->iatq_fn != NULL) 3853 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3854 } 3855 3856 cons++; 3857 cons &= IXL_AQ_MASK; 3858 } while (cons != prod); 3859 3860 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3861 0, IXL_DMA_LEN(&sc->sc_atq), 3862 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3863 3864 sc->sc_atq_cons = cons; 3865 } 3866 3867 static void 3868 ixl_atq_done(struct ixl_softc *sc) 3869 { 3870 3871 mutex_enter(&sc->sc_atq_lock); 3872 ixl_atq_done_locked(sc); 3873 mutex_exit(&sc->sc_atq_lock); 3874 } 3875 3876 static void 3877 ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3878 { 3879 3880 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3881 3882 cv_broadcast(&sc->sc_atq_cv); 3883 } 3884 3885 static int 3886 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3887 { 3888 int error; 3889 3890 mutex_enter(&sc->sc_atq_lock); 3891 error = ixl_atq_exec_locked(sc, iatq); 3892 mutex_exit(&sc->sc_atq_lock); 3893 3894 return error; 3895 } 3896 3897 static int 3898 ixl_atq_exec_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3899 { 3900 const unsigned deadline = getticks() + IXL_ATQ_EXEC_TIMEOUT; 3901 int error; 3902 3903 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3904 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3905 3906 ixl_atq_set(iatq, ixl_wakeup); 3907 3908 error = ixl_atq_post_locked(sc, iatq); 3909 if (error) 3910 return error; 3911 3912 while (iatq->iatq_inuse) { 3913 const int left = deadline - getticks(); 3914 3915 if (left <= 0) { 3916 error = EWOULDBLOCK; 3917 break; 3918 } 3919 (void)cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, left); 3920 } 3921 3922 return error; 3923 } 3924 3925 static int 3926 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3927 { 3928 struct ixl_atq *iatq = &sc->sc_atq_poll; 3929 unsigned int t = 0; 3930 int error; 3931 3932 mutex_enter(&sc->sc_atq_lock); 3933 3934 /* Drain any descriptors that finished earlier */ 3935 ixl_atq_done_locked(sc); 3936 3937 if (iatq->iatq_inuse) { 3938 mutex_exit(&sc->sc_atq_lock); 3939 return EBUSY; 3940 } 3941 3942 memset(iatq, 0, sizeof(*iatq)); 3943 iatq->iatq_desc = *iaq; 3944 iatq->iatq_desc.iaq_flags |= htole16(IXL_AQ_SI); 3945 3946 error = ixl_atq_post_locked(sc, iatq); 3947 if (error != 0) { 3948 mutex_exit(&sc->sc_atq_lock); 3949 return error; 3950 } 3951 3952 while (iatq->iatq_inuse) { 3953 ixl_atq_done_locked(sc); 3954 if (!iatq->iatq_inuse) 3955 break; 3956 3957 delaymsec(1); 3958 3959 if (t++ > tm) { 3960 mutex_exit(&sc->sc_atq_lock); 3961 return ETIMEDOUT; 3962 } 3963 } 3964 3965 *iaq = iatq->iatq_desc; 3966 3967 mutex_exit(&sc->sc_atq_lock); 3968 3969 return 0; 3970 } 3971 3972 static int 3973 ixl_get_version(struct ixl_softc *sc) 3974 { 3975 struct ixl_aq_desc iaq; 3976 uint32_t fwbuild, fwver, apiver; 3977 uint16_t api_maj_ver, api_min_ver; 3978 3979 memset(&iaq, 0, sizeof(iaq)); 3980 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3981 3982 iaq.iaq_retval = le16toh(23); 3983 3984 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3985 return ETIMEDOUT; 3986 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3987 return EIO; 3988 3989 fwbuild = le32toh(iaq.iaq_param[1]); 3990 fwver = le32toh(iaq.iaq_param[2]); 3991 apiver = le32toh(iaq.iaq_param[3]); 3992 3993 api_maj_ver = (uint16_t)apiver; 3994 api_min_ver = (uint16_t)(apiver >> 16); 3995 3996 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 3997 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 3998 3999 if (sc->sc_mac_type == I40E_MAC_X722) { 4000 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK | 4001 IXL_SC_AQ_FLAG_NVMREAD); 4002 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 4003 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS); 4004 } 4005 4006 #define IXL_API_VER(maj, min) (((uint32_t)(maj) << 16) | (min)) 4007 if (IXL_API_VER(api_maj_ver, api_min_ver) >= IXL_API_VER(1, 5)) { 4008 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 4009 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK); 4010 } 4011 #undef IXL_API_VER 4012 4013 return 0; 4014 } 4015 4016 static int 4017 ixl_get_nvm_version(struct ixl_softc *sc) 4018 { 4019 uint16_t nvmver, cfg_ptr, eetrack_hi, eetrack_lo, oem_hi, oem_lo; 4020 uint32_t eetrack, oem; 4021 uint16_t nvm_maj_ver, nvm_min_ver, oem_build; 4022 uint8_t oem_ver, oem_patch; 4023 4024 nvmver = cfg_ptr = eetrack_hi = eetrack_lo = oem_hi = oem_lo = 0; 4025 ixl_rd16_nvm(sc, I40E_SR_NVM_DEV_STARTER_VERSION, &nvmver); 4026 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_HI, &eetrack_hi); 4027 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_LO, &eetrack_lo); 4028 ixl_rd16_nvm(sc, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr); 4029 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF, &oem_hi); 4030 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF + 1, &oem_lo); 4031 4032 nvm_maj_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_HI_MASK); 4033 nvm_min_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_LO_MASK); 4034 eetrack = ((uint32_t)eetrack_hi << 16) | eetrack_lo; 4035 oem = ((uint32_t)oem_hi << 16) | oem_lo; 4036 oem_ver = __SHIFTOUT(oem, IXL_NVM_OEMVERSION_MASK); 4037 oem_build = __SHIFTOUT(oem, IXL_NVM_OEMBUILD_MASK); 4038 oem_patch = __SHIFTOUT(oem, IXL_NVM_OEMPATCH_MASK); 4039 4040 aprint_normal(" nvm %x.%02x etid %08x oem %d.%d.%d", 4041 nvm_maj_ver, nvm_min_ver, eetrack, 4042 oem_ver, oem_build, oem_patch); 4043 4044 return 0; 4045 } 4046 4047 static int 4048 ixl_pxe_clear(struct ixl_softc *sc) 4049 { 4050 struct ixl_aq_desc iaq; 4051 int rv; 4052 4053 memset(&iaq, 0, sizeof(iaq)); 4054 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 4055 iaq.iaq_param[0] = htole32(0x2); 4056 4057 rv = ixl_atq_poll(sc, &iaq, 250); 4058 4059 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 4060 4061 if (rv != 0) 4062 return ETIMEDOUT; 4063 4064 switch (iaq.iaq_retval) { 4065 case htole16(IXL_AQ_RC_OK): 4066 case htole16(IXL_AQ_RC_EEXIST): 4067 break; 4068 default: 4069 return EIO; 4070 } 4071 4072 return 0; 4073 } 4074 4075 static int 4076 ixl_lldp_shut(struct ixl_softc *sc) 4077 { 4078 struct ixl_aq_desc iaq; 4079 4080 memset(&iaq, 0, sizeof(iaq)); 4081 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 4082 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 4083 4084 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4085 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 4086 return -1; 4087 } 4088 4089 switch (iaq.iaq_retval) { 4090 case htole16(IXL_AQ_RC_EMODE): 4091 case htole16(IXL_AQ_RC_EPERM): 4092 /* ignore silently */ 4093 default: 4094 break; 4095 } 4096 4097 return 0; 4098 } 4099 4100 static void 4101 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 4102 { 4103 uint16_t id; 4104 uint32_t number, logical_id; 4105 4106 id = le16toh(cap->cap_id); 4107 number = le32toh(cap->number); 4108 logical_id = le32toh(cap->logical_id); 4109 4110 switch (id) { 4111 case IXL_AQ_CAP_RSS: 4112 sc->sc_rss_table_size = number; 4113 sc->sc_rss_table_entry_width = logical_id; 4114 break; 4115 case IXL_AQ_CAP_RXQ: 4116 case IXL_AQ_CAP_TXQ: 4117 sc->sc_nqueue_pairs_device = MIN(number, 4118 sc->sc_nqueue_pairs_device); 4119 break; 4120 } 4121 } 4122 4123 static int 4124 ixl_get_hw_capabilities(struct ixl_softc *sc) 4125 { 4126 struct ixl_dmamem idm; 4127 struct ixl_aq_desc iaq; 4128 struct ixl_aq_capability *caps; 4129 size_t i, ncaps; 4130 bus_size_t caps_size; 4131 uint16_t status; 4132 int rv; 4133 4134 caps_size = sizeof(caps[0]) * 40; 4135 memset(&iaq, 0, sizeof(iaq)); 4136 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 4137 4138 do { 4139 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 4140 return -1; 4141 } 4142 4143 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4144 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4145 iaq.iaq_datalen = htole16(caps_size); 4146 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4147 4148 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4149 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 4150 4151 rv = ixl_atq_poll(sc, &iaq, 250); 4152 4153 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4154 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 4155 4156 if (rv != 0) { 4157 aprint_error(", HW capabilities timeout\n"); 4158 goto done; 4159 } 4160 4161 status = le16toh(iaq.iaq_retval); 4162 4163 if (status == IXL_AQ_RC_ENOMEM) { 4164 caps_size = le16toh(iaq.iaq_datalen); 4165 ixl_dmamem_free(sc, &idm); 4166 } 4167 } while (status == IXL_AQ_RC_ENOMEM); 4168 4169 if (status != IXL_AQ_RC_OK) { 4170 aprint_error(", HW capabilities error\n"); 4171 goto done; 4172 } 4173 4174 caps = IXL_DMA_KVA(&idm); 4175 ncaps = le16toh(iaq.iaq_param[1]); 4176 4177 for (i = 0; i < ncaps; i++) { 4178 ixl_parse_hw_capability(sc, &caps[i]); 4179 } 4180 4181 done: 4182 ixl_dmamem_free(sc, &idm); 4183 return rv; 4184 } 4185 4186 static int 4187 ixl_get_mac(struct ixl_softc *sc) 4188 { 4189 struct ixl_dmamem idm; 4190 struct ixl_aq_desc iaq; 4191 struct ixl_aq_mac_addresses *addrs; 4192 int rv; 4193 4194 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 4195 aprint_error(", unable to allocate mac addresses\n"); 4196 return -1; 4197 } 4198 4199 memset(&iaq, 0, sizeof(iaq)); 4200 iaq.iaq_flags = htole16(IXL_AQ_BUF); 4201 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 4202 iaq.iaq_datalen = htole16(sizeof(*addrs)); 4203 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4204 4205 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4206 BUS_DMASYNC_PREREAD); 4207 4208 rv = ixl_atq_poll(sc, &iaq, 250); 4209 4210 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4211 BUS_DMASYNC_POSTREAD); 4212 4213 if (rv != 0) { 4214 aprint_error(", MAC ADDRESS READ timeout\n"); 4215 rv = -1; 4216 goto done; 4217 } 4218 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4219 aprint_error(", MAC ADDRESS READ error\n"); 4220 rv = -1; 4221 goto done; 4222 } 4223 4224 addrs = IXL_DMA_KVA(&idm); 4225 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 4226 printf(", port address is not valid\n"); 4227 goto done; 4228 } 4229 4230 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 4231 rv = 0; 4232 4233 done: 4234 ixl_dmamem_free(sc, &idm); 4235 return rv; 4236 } 4237 4238 static int 4239 ixl_get_switch_config(struct ixl_softc *sc) 4240 { 4241 struct ixl_dmamem idm; 4242 struct ixl_aq_desc iaq; 4243 struct ixl_aq_switch_config *hdr; 4244 struct ixl_aq_switch_config_element *elms, *elm; 4245 unsigned int nelm, i; 4246 int rv; 4247 4248 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4249 aprint_error_dev(sc->sc_dev, 4250 "unable to allocate switch config buffer\n"); 4251 return -1; 4252 } 4253 4254 memset(&iaq, 0, sizeof(iaq)); 4255 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4256 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4257 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 4258 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 4259 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4260 4261 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4262 BUS_DMASYNC_PREREAD); 4263 4264 rv = ixl_atq_poll(sc, &iaq, 250); 4265 4266 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4267 BUS_DMASYNC_POSTREAD); 4268 4269 if (rv != 0) { 4270 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 4271 rv = -1; 4272 goto done; 4273 } 4274 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4275 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 4276 rv = -1; 4277 goto done; 4278 } 4279 4280 hdr = IXL_DMA_KVA(&idm); 4281 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 4282 4283 nelm = le16toh(hdr->num_reported); 4284 if (nelm < 1) { 4285 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 4286 rv = -1; 4287 goto done; 4288 } 4289 4290 for (i = 0; i < nelm; i++) { 4291 elm = &elms[i]; 4292 4293 aprint_debug_dev(sc->sc_dev, 4294 "type %x revision %u seid %04x\n", 4295 elm->type, elm->revision, le16toh(elm->seid)); 4296 aprint_debug_dev(sc->sc_dev, 4297 "uplink %04x downlink %04x\n", 4298 le16toh(elm->uplink_seid), 4299 le16toh(elm->downlink_seid)); 4300 aprint_debug_dev(sc->sc_dev, 4301 "conntype %x scheduler %04x extra %04x\n", 4302 elm->connection_type, 4303 le16toh(elm->scheduler_id), 4304 le16toh(elm->element_info)); 4305 } 4306 4307 elm = &elms[0]; 4308 4309 sc->sc_uplink_seid = elm->uplink_seid; 4310 sc->sc_downlink_seid = elm->downlink_seid; 4311 sc->sc_seid = elm->seid; 4312 4313 if ((sc->sc_uplink_seid == htole16(0)) != 4314 (sc->sc_downlink_seid == htole16(0))) { 4315 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 4316 rv = -1; 4317 goto done; 4318 } 4319 4320 done: 4321 ixl_dmamem_free(sc, &idm); 4322 return rv; 4323 } 4324 4325 static int 4326 ixl_phy_mask_ints(struct ixl_softc *sc) 4327 { 4328 struct ixl_aq_desc iaq; 4329 4330 memset(&iaq, 0, sizeof(iaq)); 4331 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 4332 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 4333 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 4334 IXL_AQ_PHY_EV_MEDIA_NA)); 4335 4336 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4337 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 4338 return -1; 4339 } 4340 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4341 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 4342 return -1; 4343 } 4344 4345 return 0; 4346 } 4347 4348 static int 4349 ixl_get_phy_abilities(struct ixl_softc *sc, struct ixl_dmamem *idm) 4350 { 4351 struct ixl_aq_desc iaq; 4352 int rv; 4353 4354 memset(&iaq, 0, sizeof(iaq)); 4355 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4356 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4357 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 4358 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 4359 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 4360 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 4361 4362 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4363 BUS_DMASYNC_PREREAD); 4364 4365 rv = ixl_atq_poll(sc, &iaq, 250); 4366 4367 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4368 BUS_DMASYNC_POSTREAD); 4369 4370 if (rv != 0) 4371 return -1; 4372 4373 return le16toh(iaq.iaq_retval); 4374 } 4375 4376 static int 4377 ixl_get_phy_info(struct ixl_softc *sc) 4378 { 4379 struct ixl_dmamem idm; 4380 struct ixl_aq_phy_abilities *phy; 4381 int rv; 4382 4383 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4384 aprint_error_dev(sc->sc_dev, 4385 "unable to allocate phy abilities buffer\n"); 4386 return -1; 4387 } 4388 4389 rv = ixl_get_phy_abilities(sc, &idm); 4390 switch (rv) { 4391 case -1: 4392 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 4393 goto done; 4394 case IXL_AQ_RC_OK: 4395 break; 4396 case IXL_AQ_RC_EIO: 4397 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 4398 goto done; 4399 default: 4400 aprint_error_dev(sc->sc_dev, 4401 "GET PHY ABILITIES error %u\n", rv); 4402 goto done; 4403 } 4404 4405 phy = IXL_DMA_KVA(&idm); 4406 4407 sc->sc_phy_types = le32toh(phy->phy_type); 4408 sc->sc_phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 4409 4410 sc->sc_phy_abilities = phy->abilities; 4411 sc->sc_phy_linkspeed = phy->link_speed; 4412 sc->sc_phy_fec_cfg = phy->fec_cfg_curr_mod_ext_info & 4413 (IXL_AQ_ENABLE_FEC_KR | IXL_AQ_ENABLE_FEC_RS | 4414 IXL_AQ_REQUEST_FEC_KR | IXL_AQ_REQUEST_FEC_RS); 4415 sc->sc_eee_cap = phy->eee_capability; 4416 sc->sc_eeer_val = phy->eeer_val; 4417 sc->sc_d3_lpan = phy->d3_lpan; 4418 4419 rv = 0; 4420 4421 done: 4422 ixl_dmamem_free(sc, &idm); 4423 return rv; 4424 } 4425 4426 static int 4427 ixl_set_phy_config(struct ixl_softc *sc, 4428 uint8_t link_speed, uint8_t abilities, bool polling) 4429 { 4430 struct ixl_aq_phy_param *param; 4431 struct ixl_atq iatq; 4432 struct ixl_aq_desc *iaq; 4433 int error; 4434 4435 memset(&iatq, 0, sizeof(iatq)); 4436 4437 iaq = &iatq.iatq_desc; 4438 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_CONFIG); 4439 param = (struct ixl_aq_phy_param *)&iaq->iaq_param; 4440 param->phy_types = htole32((uint32_t)sc->sc_phy_types); 4441 param->phy_type_ext = (uint8_t)(sc->sc_phy_types >> 32); 4442 param->link_speed = link_speed; 4443 param->abilities = abilities | IXL_AQ_PHY_ABILITY_AUTO_LINK; 4444 param->fec_cfg = sc->sc_phy_fec_cfg; 4445 param->eee_capability = sc->sc_eee_cap; 4446 param->eeer_val = sc->sc_eeer_val; 4447 param->d3_lpan = sc->sc_d3_lpan; 4448 4449 if (polling) 4450 error = ixl_atq_poll(sc, iaq, 250); 4451 else 4452 error = ixl_atq_exec(sc, &iatq); 4453 4454 if (error != 0) 4455 return error; 4456 4457 switch (le16toh(iaq->iaq_retval)) { 4458 case IXL_AQ_RC_OK: 4459 break; 4460 case IXL_AQ_RC_EPERM: 4461 return EPERM; 4462 default: 4463 return EIO; 4464 } 4465 4466 return 0; 4467 } 4468 4469 static int 4470 ixl_set_phy_autoselect(struct ixl_softc *sc) 4471 { 4472 uint8_t link_speed, abilities; 4473 4474 link_speed = sc->sc_phy_linkspeed; 4475 abilities = IXL_PHY_ABILITY_LINKUP | IXL_PHY_ABILITY_AUTONEGO; 4476 4477 return ixl_set_phy_config(sc, link_speed, abilities, true); 4478 } 4479 4480 static int 4481 ixl_get_link_status_poll(struct ixl_softc *sc, int *l) 4482 { 4483 struct ixl_aq_desc iaq; 4484 struct ixl_aq_link_param *param; 4485 int link; 4486 4487 memset(&iaq, 0, sizeof(iaq)); 4488 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 4489 param = (struct ixl_aq_link_param *)iaq.iaq_param; 4490 param->notify = IXL_AQ_LINK_NOTIFY; 4491 4492 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4493 return ETIMEDOUT; 4494 } 4495 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4496 return EIO; 4497 } 4498 4499 /* It is unnecessary to hold lock */ 4500 link = ixl_set_link_status_locked(sc, &iaq); 4501 4502 if (l != NULL) 4503 *l = link; 4504 4505 return 0; 4506 } 4507 4508 static int 4509 ixl_get_vsi(struct ixl_softc *sc) 4510 { 4511 struct ixl_dmamem *vsi = &sc->sc_scratch; 4512 struct ixl_aq_desc iaq; 4513 struct ixl_aq_vsi_param *param; 4514 struct ixl_aq_vsi_reply *reply; 4515 struct ixl_aq_vsi_data *data; 4516 int rv; 4517 4518 /* grumble, vsi info isn't "known" at compile time */ 4519 4520 memset(&iaq, 0, sizeof(iaq)); 4521 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4522 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4523 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 4524 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4525 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4526 4527 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4528 param->uplink_seid = sc->sc_seid; 4529 4530 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4531 BUS_DMASYNC_PREREAD); 4532 4533 rv = ixl_atq_poll(sc, &iaq, 250); 4534 4535 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4536 BUS_DMASYNC_POSTREAD); 4537 4538 if (rv != 0) { 4539 return ETIMEDOUT; 4540 } 4541 4542 switch (le16toh(iaq.iaq_retval)) { 4543 case IXL_AQ_RC_OK: 4544 break; 4545 case IXL_AQ_RC_ENOENT: 4546 return ENOENT; 4547 case IXL_AQ_RC_EACCES: 4548 return EACCES; 4549 default: 4550 return EIO; 4551 } 4552 4553 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 4554 sc->sc_vsi_number = le16toh(reply->vsi_number); 4555 data = IXL_DMA_KVA(vsi); 4556 sc->sc_vsi_stat_counter_idx = le16toh(data->stat_counter_idx); 4557 4558 return 0; 4559 } 4560 4561 static int 4562 ixl_set_vsi(struct ixl_softc *sc) 4563 { 4564 struct ixl_dmamem *vsi = &sc->sc_scratch; 4565 struct ixl_aq_desc iaq; 4566 struct ixl_aq_vsi_param *param; 4567 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 4568 unsigned int qnum; 4569 uint16_t val; 4570 int rv; 4571 4572 qnum = sc->sc_nqueue_pairs - 1; 4573 4574 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 4575 IXL_AQ_VSI_VALID_VLAN); 4576 4577 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 4578 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 4579 data->queue_mapping[0] = htole16(0); 4580 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 4581 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 4582 4583 val = le16toh(data->port_vlan_flags); 4584 CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK); 4585 SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL); 4586 4587 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { 4588 SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH); 4589 } else { 4590 SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING); 4591 } 4592 4593 data->port_vlan_flags = htole16(val); 4594 4595 /* grumble, vsi info isn't "known" at compile time */ 4596 4597 memset(&iaq, 0, sizeof(iaq)); 4598 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4599 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4600 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 4601 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4602 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4603 4604 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4605 param->uplink_seid = sc->sc_seid; 4606 4607 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4608 BUS_DMASYNC_PREWRITE); 4609 4610 rv = ixl_atq_poll(sc, &iaq, 250); 4611 4612 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4613 BUS_DMASYNC_POSTWRITE); 4614 4615 if (rv != 0) { 4616 return ETIMEDOUT; 4617 } 4618 4619 switch (le16toh(iaq.iaq_retval)) { 4620 case IXL_AQ_RC_OK: 4621 break; 4622 case IXL_AQ_RC_ENOENT: 4623 return ENOENT; 4624 case IXL_AQ_RC_EACCES: 4625 return EACCES; 4626 default: 4627 return EIO; 4628 } 4629 4630 return 0; 4631 } 4632 4633 static void 4634 ixl_set_filter_control(struct ixl_softc *sc) 4635 { 4636 uint32_t reg; 4637 4638 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4639 4640 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4641 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4642 4643 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4644 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4645 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4646 4647 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4648 } 4649 4650 static inline void 4651 ixl_get_default_rss_key(uint32_t *buf, size_t len) 4652 { 4653 size_t cplen; 4654 uint8_t rss_seed[RSS_KEYSIZE]; 4655 4656 rss_getkey(rss_seed); 4657 memset(buf, 0, len); 4658 4659 cplen = MIN(len, sizeof(rss_seed)); 4660 memcpy(buf, rss_seed, cplen); 4661 } 4662 4663 static int 4664 ixl_set_rss_key(struct ixl_softc *sc, uint8_t *key, size_t keylen) 4665 { 4666 struct ixl_dmamem *idm; 4667 struct ixl_atq iatq; 4668 struct ixl_aq_desc *iaq; 4669 struct ixl_aq_rss_key_param *param; 4670 struct ixl_aq_rss_key_data *data; 4671 size_t len, datalen, stdlen, extlen; 4672 uint16_t vsi_id; 4673 int rv; 4674 4675 memset(&iatq, 0, sizeof(iatq)); 4676 iaq = &iatq.iatq_desc; 4677 idm = &sc->sc_aqbuf; 4678 4679 datalen = sizeof(*data); 4680 4681 /*XXX The buf size has to be less than the size of the register */ 4682 datalen = MIN(IXL_RSS_KEY_SIZE_REG * sizeof(uint32_t), datalen); 4683 4684 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4685 (datalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4686 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_KEY); 4687 iaq->iaq_datalen = htole16(datalen); 4688 4689 param = (struct ixl_aq_rss_key_param *)iaq->iaq_param; 4690 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSKEY_VSI_ID_SHIFT) | 4691 IXL_AQ_RSSKEY_VSI_VALID; 4692 param->vsi_id = htole16(vsi_id); 4693 4694 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4695 data = IXL_DMA_KVA(idm); 4696 4697 len = MIN(keylen, datalen); 4698 stdlen = MIN(sizeof(data->standard_rss_key), len); 4699 memcpy(data->standard_rss_key, key, stdlen); 4700 len = (len > stdlen) ? (len - stdlen) : 0; 4701 4702 extlen = MIN(sizeof(data->extended_hash_key), len); 4703 extlen = (stdlen < keylen) ? 0 : keylen - stdlen; 4704 memcpy(data->extended_hash_key, key + stdlen, extlen); 4705 4706 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4707 4708 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4709 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4710 4711 rv = ixl_atq_exec(sc, &iatq); 4712 4713 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4714 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4715 4716 if (rv != 0) { 4717 return ETIMEDOUT; 4718 } 4719 4720 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4721 return EIO; 4722 } 4723 4724 return 0; 4725 } 4726 4727 static int 4728 ixl_set_rss_lut(struct ixl_softc *sc, uint8_t *lut, size_t lutlen) 4729 { 4730 struct ixl_dmamem *idm; 4731 struct ixl_atq iatq; 4732 struct ixl_aq_desc *iaq; 4733 struct ixl_aq_rss_lut_param *param; 4734 uint16_t vsi_id; 4735 uint8_t *data; 4736 size_t dmalen; 4737 int rv; 4738 4739 memset(&iatq, 0, sizeof(iatq)); 4740 iaq = &iatq.iatq_desc; 4741 idm = &sc->sc_aqbuf; 4742 4743 dmalen = MIN(lutlen, IXL_DMA_LEN(idm)); 4744 4745 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4746 (dmalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4747 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_LUT); 4748 iaq->iaq_datalen = htole16(dmalen); 4749 4750 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4751 data = IXL_DMA_KVA(idm); 4752 memcpy(data, lut, dmalen); 4753 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4754 4755 param = (struct ixl_aq_rss_lut_param *)iaq->iaq_param; 4756 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSLUT_VSI_ID_SHIFT) | 4757 IXL_AQ_RSSLUT_VSI_VALID; 4758 param->vsi_id = htole16(vsi_id); 4759 param->flags = htole16(IXL_AQ_RSSLUT_TABLE_TYPE_PF << 4760 IXL_AQ_RSSLUT_TABLE_TYPE_SHIFT); 4761 4762 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4763 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4764 4765 rv = ixl_atq_exec(sc, &iatq); 4766 4767 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4768 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4769 4770 if (rv != 0) { 4771 return ETIMEDOUT; 4772 } 4773 4774 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4775 return EIO; 4776 } 4777 4778 return 0; 4779 } 4780 4781 static int 4782 ixl_register_rss_key(struct ixl_softc *sc) 4783 { 4784 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4785 int rv; 4786 size_t i; 4787 4788 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4789 4790 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4791 rv = ixl_set_rss_key(sc, (uint8_t*)rss_seed, 4792 sizeof(rss_seed)); 4793 } else { 4794 rv = 0; 4795 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4796 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4797 } 4798 } 4799 4800 return rv; 4801 } 4802 4803 static void 4804 ixl_register_rss_pctype(struct ixl_softc *sc) 4805 { 4806 uint64_t set_hena = 0; 4807 uint32_t hena0, hena1; 4808 4809 /* 4810 * We use TCP/UDP with IPv4/IPv6 by default. 4811 * Note: the device can not use just IP header in each 4812 * TCP/UDP packets for the RSS hash calculation. 4813 */ 4814 if (sc->sc_mac_type == I40E_MAC_X722) 4815 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4816 else 4817 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4818 4819 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4820 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4821 4822 SET(hena0, set_hena); 4823 SET(hena1, set_hena >> 32); 4824 4825 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4826 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4827 } 4828 4829 static int 4830 ixl_register_rss_hlut(struct ixl_softc *sc) 4831 { 4832 unsigned int qid; 4833 uint8_t hlut_buf[512], lut_mask; 4834 uint32_t *hluts; 4835 size_t i, hluts_num; 4836 int rv; 4837 4838 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4839 4840 for (i = 0; i < sc->sc_rss_table_size; i++) { 4841 qid = i % sc->sc_nqueue_pairs; 4842 hlut_buf[i] = qid & lut_mask; 4843 } 4844 4845 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4846 rv = ixl_set_rss_lut(sc, hlut_buf, sizeof(hlut_buf)); 4847 } else { 4848 rv = 0; 4849 hluts = (uint32_t *)hlut_buf; 4850 hluts_num = sc->sc_rss_table_size >> 2; 4851 for (i = 0; i < hluts_num; i++) { 4852 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4853 } 4854 ixl_flush(sc); 4855 } 4856 4857 return rv; 4858 } 4859 4860 static void 4861 ixl_config_rss(struct ixl_softc *sc) 4862 { 4863 4864 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4865 4866 ixl_register_rss_key(sc); 4867 ixl_register_rss_pctype(sc); 4868 ixl_register_rss_hlut(sc); 4869 } 4870 4871 static const struct ixl_phy_type * 4872 ixl_search_phy_type(uint8_t phy_type) 4873 { 4874 const struct ixl_phy_type *itype; 4875 uint64_t mask; 4876 unsigned int i; 4877 4878 if (phy_type >= 64) 4879 return NULL; 4880 4881 mask = 1ULL << phy_type; 4882 4883 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4884 itype = &ixl_phy_type_map[i]; 4885 4886 if (ISSET(itype->phy_type, mask)) 4887 return itype; 4888 } 4889 4890 return NULL; 4891 } 4892 4893 static uint64_t 4894 ixl_search_link_speed(uint8_t link_speed) 4895 { 4896 const struct ixl_speed_type *type; 4897 unsigned int i; 4898 4899 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4900 type = &ixl_speed_type_map[i]; 4901 4902 if (ISSET(type->dev_speed, link_speed)) 4903 return type->net_speed; 4904 } 4905 4906 return 0; 4907 } 4908 4909 static uint8_t 4910 ixl_search_baudrate(uint64_t baudrate) 4911 { 4912 const struct ixl_speed_type *type; 4913 unsigned int i; 4914 4915 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4916 type = &ixl_speed_type_map[i]; 4917 4918 if (type->net_speed == baudrate) { 4919 return type->dev_speed; 4920 } 4921 } 4922 4923 return 0; 4924 } 4925 4926 static int 4927 ixl_restart_an(struct ixl_softc *sc) 4928 { 4929 struct ixl_aq_desc iaq; 4930 4931 memset(&iaq, 0, sizeof(iaq)); 4932 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4933 iaq.iaq_param[0] = 4934 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4935 4936 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4937 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4938 return -1; 4939 } 4940 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4941 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4942 return -1; 4943 } 4944 4945 return 0; 4946 } 4947 4948 static int 4949 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4950 uint16_t vlan, uint16_t flags) 4951 { 4952 struct ixl_aq_desc iaq; 4953 struct ixl_aq_add_macvlan *param; 4954 struct ixl_aq_add_macvlan_elem *elem; 4955 4956 memset(&iaq, 0, sizeof(iaq)); 4957 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4958 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4959 iaq.iaq_datalen = htole16(sizeof(*elem)); 4960 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4961 4962 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4963 param->num_addrs = htole16(1); 4964 param->seid0 = htole16(0x8000) | sc->sc_seid; 4965 param->seid1 = 0; 4966 param->seid2 = 0; 4967 4968 elem = IXL_DMA_KVA(&sc->sc_scratch); 4969 memset(elem, 0, sizeof(*elem)); 4970 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4971 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4972 elem->vlan = htole16(vlan); 4973 4974 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4975 return IXL_AQ_RC_EINVAL; 4976 } 4977 4978 switch (le16toh(iaq.iaq_retval)) { 4979 case IXL_AQ_RC_OK: 4980 break; 4981 case IXL_AQ_RC_ENOSPC: 4982 return ENOSPC; 4983 case IXL_AQ_RC_ENOENT: 4984 return ENOENT; 4985 case IXL_AQ_RC_EACCES: 4986 return EACCES; 4987 case IXL_AQ_RC_EEXIST: 4988 return EEXIST; 4989 case IXL_AQ_RC_EINVAL: 4990 return EINVAL; 4991 default: 4992 return EIO; 4993 } 4994 4995 return 0; 4996 } 4997 4998 static int 4999 ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 5000 uint16_t vlan, uint16_t flags) 5001 { 5002 struct ixl_aq_desc iaq; 5003 struct ixl_aq_remove_macvlan *param; 5004 struct ixl_aq_remove_macvlan_elem *elem; 5005 5006 memset(&iaq, 0, sizeof(iaq)); 5007 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 5008 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 5009 iaq.iaq_datalen = htole16(sizeof(*elem)); 5010 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 5011 5012 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 5013 param->num_addrs = htole16(1); 5014 param->seid0 = htole16(0x8000) | sc->sc_seid; 5015 param->seid1 = 0; 5016 param->seid2 = 0; 5017 5018 elem = IXL_DMA_KVA(&sc->sc_scratch); 5019 memset(elem, 0, sizeof(*elem)); 5020 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 5021 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 5022 elem->vlan = htole16(vlan); 5023 5024 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 5025 return EINVAL; 5026 } 5027 5028 switch (le16toh(iaq.iaq_retval)) { 5029 case IXL_AQ_RC_OK: 5030 break; 5031 case IXL_AQ_RC_ENOENT: 5032 return ENOENT; 5033 case IXL_AQ_RC_EACCES: 5034 return EACCES; 5035 case IXL_AQ_RC_EINVAL: 5036 return EINVAL; 5037 default: 5038 return EIO; 5039 } 5040 5041 return 0; 5042 } 5043 5044 static int 5045 ixl_hmc(struct ixl_softc *sc) 5046 { 5047 struct { 5048 uint32_t count; 5049 uint32_t minsize; 5050 bus_size_t objsiz; 5051 bus_size_t setoff; 5052 bus_size_t setcnt; 5053 } regs[] = { 5054 { 5055 0, 5056 IXL_HMC_TXQ_MINSIZE, 5057 I40E_GLHMC_LANTXOBJSZ, 5058 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 5059 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 5060 }, 5061 { 5062 0, 5063 IXL_HMC_RXQ_MINSIZE, 5064 I40E_GLHMC_LANRXOBJSZ, 5065 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 5066 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 5067 }, 5068 { 5069 0, 5070 0, 5071 I40E_GLHMC_FCOEDDPOBJSZ, 5072 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 5073 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 5074 }, 5075 { 5076 0, 5077 0, 5078 I40E_GLHMC_FCOEFOBJSZ, 5079 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 5080 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 5081 }, 5082 }; 5083 struct ixl_hmc_entry *e; 5084 uint64_t size, dva; 5085 uint8_t *kva; 5086 uint64_t *sdpage; 5087 unsigned int i; 5088 int npages, tables; 5089 uint32_t reg; 5090 5091 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 5092 5093 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 5094 ixl_rd(sc, I40E_GLHMC_LANQMAX); 5095 5096 size = 0; 5097 for (i = 0; i < __arraycount(regs); i++) { 5098 e = &sc->sc_hmc_entries[i]; 5099 5100 e->hmc_count = regs[i].count; 5101 reg = ixl_rd(sc, regs[i].objsiz); 5102 e->hmc_size = IXL_BIT_ULL(0x3F & reg); 5103 e->hmc_base = size; 5104 5105 if ((e->hmc_size * 8) < regs[i].minsize) { 5106 aprint_error_dev(sc->sc_dev, 5107 "kernel hmc entry is too big\n"); 5108 return -1; 5109 } 5110 5111 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 5112 } 5113 size = roundup(size, IXL_HMC_PGSIZE); 5114 npages = size / IXL_HMC_PGSIZE; 5115 5116 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 5117 5118 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 5119 aprint_error_dev(sc->sc_dev, 5120 "unable to allocate hmc pd memory\n"); 5121 return -1; 5122 } 5123 5124 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 5125 IXL_HMC_PGSIZE) != 0) { 5126 aprint_error_dev(sc->sc_dev, 5127 "unable to allocate hmc sd memory\n"); 5128 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5129 return -1; 5130 } 5131 5132 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 5133 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 5134 5135 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 5136 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 5137 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5138 5139 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 5140 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 5141 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 5142 5143 for (i = 0; (int)i < npages; i++) { 5144 *sdpage = htole64(dva | IXL_HMC_PDVALID); 5145 sdpage++; 5146 5147 dva += IXL_HMC_PGSIZE; 5148 } 5149 5150 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 5151 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 5152 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5153 5154 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 5155 for (i = 0; (int)i < tables; i++) { 5156 uint32_t count; 5157 5158 KASSERT(npages >= 0); 5159 5160 count = ((unsigned int)npages > IXL_HMC_PGS) ? 5161 IXL_HMC_PGS : (unsigned int)npages; 5162 5163 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 5164 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 5165 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 5166 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 5167 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 5168 ixl_wr(sc, I40E_PFHMC_SDCMD, 5169 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 5170 5171 npages -= IXL_HMC_PGS; 5172 dva += IXL_HMC_PGSIZE; 5173 } 5174 5175 for (i = 0; i < __arraycount(regs); i++) { 5176 e = &sc->sc_hmc_entries[i]; 5177 5178 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 5179 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 5180 } 5181 5182 return 0; 5183 } 5184 5185 static void 5186 ixl_hmc_free(struct ixl_softc *sc) 5187 { 5188 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 5189 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5190 } 5191 5192 static void 5193 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 5194 unsigned int npacking) 5195 { 5196 uint8_t *dst = d; 5197 const uint8_t *src = s; 5198 unsigned int i; 5199 5200 for (i = 0; i < npacking; i++) { 5201 const struct ixl_hmc_pack *pack = &packing[i]; 5202 unsigned int offset = pack->lsb / 8; 5203 unsigned int align = pack->lsb % 8; 5204 const uint8_t *in = src + pack->offset; 5205 uint8_t *out = dst + offset; 5206 int width = pack->width; 5207 unsigned int inbits = 0; 5208 5209 if (align) { 5210 inbits = (*in++) << align; 5211 *out++ |= (inbits & 0xff); 5212 inbits >>= 8; 5213 5214 width -= 8 - align; 5215 } 5216 5217 while (width >= 8) { 5218 inbits |= (*in++) << align; 5219 *out++ = (inbits & 0xff); 5220 inbits >>= 8; 5221 5222 width -= 8; 5223 } 5224 5225 if (width > 0) { 5226 inbits |= (*in) << align; 5227 *out |= (inbits & ((1 << width) - 1)); 5228 } 5229 } 5230 } 5231 5232 static struct ixl_aq_buf * 5233 ixl_aqb_alloc(struct ixl_softc *sc) 5234 { 5235 struct ixl_aq_buf *aqb; 5236 5237 aqb = kmem_alloc(sizeof(*aqb), KM_SLEEP); 5238 5239 aqb->aqb_size = IXL_AQ_BUFLEN; 5240 5241 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 5242 aqb->aqb_size, 0, 5243 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 5244 goto free; 5245 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 5246 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 5247 BUS_DMA_WAITOK) != 0) 5248 goto destroy; 5249 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 5250 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 5251 goto dma_free; 5252 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 5253 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 5254 goto unmap; 5255 5256 return aqb; 5257 unmap: 5258 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5259 dma_free: 5260 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5261 destroy: 5262 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5263 free: 5264 kmem_free(aqb, sizeof(*aqb)); 5265 5266 return NULL; 5267 } 5268 5269 static void 5270 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 5271 { 5272 5273 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 5274 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5275 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5276 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5277 kmem_free(aqb, sizeof(*aqb)); 5278 } 5279 5280 static int 5281 ixl_arq_fill(struct ixl_softc *sc) 5282 { 5283 struct ixl_aq_buf *aqb; 5284 struct ixl_aq_desc *arq, *iaq; 5285 unsigned int prod = sc->sc_arq_prod; 5286 unsigned int n; 5287 int post = 0; 5288 5289 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 5290 IXL_AQ_NUM); 5291 arq = IXL_DMA_KVA(&sc->sc_arq); 5292 5293 if (__predict_false(n <= 0)) 5294 return 0; 5295 5296 do { 5297 aqb = sc->sc_arq_live[prod]; 5298 iaq = &arq[prod]; 5299 5300 if (aqb == NULL) { 5301 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 5302 if (aqb != NULL) { 5303 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5304 ixl_aq_buf, aqb_entry); 5305 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 5306 break; 5307 } 5308 5309 sc->sc_arq_live[prod] = aqb; 5310 memset(aqb->aqb_data, 0, aqb->aqb_size); 5311 5312 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 5313 aqb->aqb_size, BUS_DMASYNC_PREREAD); 5314 5315 iaq->iaq_flags = htole16(IXL_AQ_BUF | 5316 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 5317 IXL_AQ_LB : 0)); 5318 iaq->iaq_opcode = 0; 5319 iaq->iaq_datalen = htole16(aqb->aqb_size); 5320 iaq->iaq_retval = 0; 5321 iaq->iaq_cookie = 0; 5322 iaq->iaq_param[0] = 0; 5323 iaq->iaq_param[1] = 0; 5324 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 5325 } 5326 5327 prod++; 5328 prod &= IXL_AQ_MASK; 5329 5330 post = 1; 5331 5332 } while (--n); 5333 5334 if (post) { 5335 sc->sc_arq_prod = prod; 5336 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 5337 } 5338 5339 return post; 5340 } 5341 5342 static void 5343 ixl_arq_unfill(struct ixl_softc *sc) 5344 { 5345 struct ixl_aq_buf *aqb; 5346 unsigned int i; 5347 5348 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 5349 aqb = sc->sc_arq_live[i]; 5350 if (aqb == NULL) 5351 continue; 5352 5353 sc->sc_arq_live[i] = NULL; 5354 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 5355 BUS_DMASYNC_POSTREAD); 5356 ixl_aqb_free(sc, aqb); 5357 } 5358 5359 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 5360 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5361 ixl_aq_buf, aqb_entry); 5362 ixl_aqb_free(sc, aqb); 5363 } 5364 } 5365 5366 static void 5367 ixl_clear_hw(struct ixl_softc *sc) 5368 { 5369 uint32_t num_queues, base_queue; 5370 uint32_t num_pf_int; 5371 uint32_t num_vf_int; 5372 uint32_t num_vfs; 5373 uint32_t i, j; 5374 uint32_t val; 5375 uint32_t eol = 0x7ff; 5376 5377 /* get number of interrupts, queues, and vfs */ 5378 val = ixl_rd(sc, I40E_GLPCI_CNF2); 5379 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 5380 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 5381 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 5382 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 5383 5384 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 5385 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 5386 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 5387 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 5388 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 5389 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 5390 num_queues = (j - base_queue) + 1; 5391 else 5392 num_queues = 0; 5393 5394 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 5395 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 5396 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 5397 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 5398 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 5399 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 5400 num_vfs = (j - i) + 1; 5401 else 5402 num_vfs = 0; 5403 5404 /* stop all the interrupts */ 5405 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5406 ixl_flush(sc); 5407 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 5408 for (i = 0; i < num_pf_int - 2; i++) 5409 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 5410 ixl_flush(sc); 5411 5412 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 5413 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5414 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 5415 for (i = 0; i < num_pf_int - 2; i++) 5416 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 5417 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5418 for (i = 0; i < num_vfs; i++) 5419 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 5420 for (i = 0; i < num_vf_int - 2; i++) 5421 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 5422 5423 /* warn the HW of the coming Tx disables */ 5424 for (i = 0; i < num_queues; i++) { 5425 uint32_t abs_queue_idx = base_queue + i; 5426 uint32_t reg_block = 0; 5427 5428 if (abs_queue_idx >= 128) { 5429 reg_block = abs_queue_idx / 128; 5430 abs_queue_idx %= 128; 5431 } 5432 5433 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 5434 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 5435 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 5436 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 5437 5438 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 5439 } 5440 delaymsec(400); 5441 5442 /* stop all the queues */ 5443 for (i = 0; i < num_queues; i++) { 5444 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 5445 ixl_wr(sc, I40E_QTX_ENA(i), 0); 5446 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 5447 ixl_wr(sc, I40E_QRX_ENA(i), 0); 5448 } 5449 5450 /* short wait for all queue disables to settle */ 5451 delaymsec(50); 5452 } 5453 5454 static int 5455 ixl_pf_reset(struct ixl_softc *sc) 5456 { 5457 uint32_t cnt = 0; 5458 uint32_t cnt1 = 0; 5459 uint32_t reg = 0, reg0 = 0; 5460 uint32_t grst_del; 5461 5462 /* 5463 * Poll for Global Reset steady state in case of recent GRST. 5464 * The grst delay value is in 100ms units, and we'll wait a 5465 * couple counts longer to be sure we don't just miss the end. 5466 */ 5467 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 5468 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 5469 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 5470 5471 grst_del = grst_del * 20; 5472 5473 for (cnt = 0; cnt < grst_del; cnt++) { 5474 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 5475 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 5476 break; 5477 delaymsec(100); 5478 } 5479 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5480 aprint_error(", Global reset polling failed to complete\n"); 5481 return -1; 5482 } 5483 5484 /* Now Wait for the FW to be ready */ 5485 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 5486 reg = ixl_rd(sc, I40E_GLNVM_ULD); 5487 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5488 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 5489 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5490 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 5491 break; 5492 5493 delaymsec(10); 5494 } 5495 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5496 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 5497 aprint_error(", wait for FW Reset complete timed out " 5498 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 5499 return -1; 5500 } 5501 5502 /* 5503 * If there was a Global Reset in progress when we got here, 5504 * we don't need to do the PF Reset 5505 */ 5506 if (cnt == 0) { 5507 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5508 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 5509 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 5510 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5511 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 5512 break; 5513 delaymsec(1); 5514 5515 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 5516 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5517 aprint_error(", Core reset upcoming." 5518 " Skipping PF reset reset request\n"); 5519 return -1; 5520 } 5521 } 5522 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 5523 aprint_error(", PF reset polling failed to complete" 5524 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 5525 return -1; 5526 } 5527 } 5528 5529 return 0; 5530 } 5531 5532 static int 5533 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 5534 bus_size_t size, bus_size_t align) 5535 { 5536 ixm->ixm_size = size; 5537 5538 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 5539 ixm->ixm_size, 0, 5540 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 5541 &ixm->ixm_map) != 0) 5542 return 1; 5543 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 5544 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 5545 BUS_DMA_WAITOK) != 0) 5546 goto destroy; 5547 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 5548 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 5549 goto free; 5550 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 5551 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 5552 goto unmap; 5553 5554 memset(ixm->ixm_kva, 0, ixm->ixm_size); 5555 5556 return 0; 5557 unmap: 5558 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5559 free: 5560 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5561 destroy: 5562 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5563 return 1; 5564 } 5565 5566 static void 5567 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 5568 { 5569 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 5570 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5571 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5572 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5573 } 5574 5575 static int 5576 ixl_setup_vlan_hwfilter(struct ixl_softc *sc) 5577 { 5578 struct ethercom *ec = &sc->sc_ec; 5579 struct vlanid_list *vlanidp; 5580 int rv; 5581 5582 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5583 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5584 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5585 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5586 5587 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5588 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5589 if (rv != 0) 5590 return rv; 5591 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5592 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5593 if (rv != 0) 5594 return rv; 5595 5596 ETHER_LOCK(ec); 5597 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5598 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 5599 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5600 if (rv != 0) 5601 break; 5602 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 5603 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5604 if (rv != 0) 5605 break; 5606 } 5607 ETHER_UNLOCK(ec); 5608 5609 return rv; 5610 } 5611 5612 static void 5613 ixl_teardown_vlan_hwfilter(struct ixl_softc *sc) 5614 { 5615 struct vlanid_list *vlanidp; 5616 struct ethercom *ec = &sc->sc_ec; 5617 5618 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5619 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5620 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5621 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5622 5623 ETHER_LOCK(ec); 5624 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5625 ixl_remove_macvlan(sc, sc->sc_enaddr, 5626 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5627 ixl_remove_macvlan(sc, etherbroadcastaddr, 5628 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5629 } 5630 ETHER_UNLOCK(ec); 5631 5632 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5633 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5634 ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5635 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5636 } 5637 5638 static int 5639 ixl_update_macvlan(struct ixl_softc *sc) 5640 { 5641 int rv = 0; 5642 int next_ec_capenable = sc->sc_ec.ec_capenable; 5643 5644 if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 5645 rv = ixl_setup_vlan_hwfilter(sc); 5646 if (rv != 0) 5647 ixl_teardown_vlan_hwfilter(sc); 5648 } else { 5649 ixl_teardown_vlan_hwfilter(sc); 5650 } 5651 5652 return rv; 5653 } 5654 5655 static int 5656 ixl_ifflags_cb(struct ethercom *ec) 5657 { 5658 struct ifnet *ifp = &ec->ec_if; 5659 struct ixl_softc *sc = ifp->if_softc; 5660 int rv, change, reset_bits; 5661 5662 mutex_enter(&sc->sc_cfg_lock); 5663 5664 change = ec->ec_capenable ^ sc->sc_cur_ec_capenable; 5665 reset_bits = change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_MTU); 5666 if (reset_bits != 0) { 5667 sc->sc_cur_ec_capenable ^= reset_bits; 5668 rv = ENETRESET; 5669 goto out; 5670 } 5671 5672 if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) { 5673 rv = ixl_update_macvlan(sc); 5674 if (rv == 0) { 5675 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 5676 } else { 5677 CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER); 5678 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 5679 } 5680 } 5681 5682 rv = ixl_iff(sc); 5683 out: 5684 mutex_exit(&sc->sc_cfg_lock); 5685 5686 return rv; 5687 } 5688 5689 static int 5690 ixl_set_link_status_locked(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 5691 { 5692 const struct ixl_aq_link_status *status; 5693 const struct ixl_phy_type *itype; 5694 5695 uint64_t ifm_active = IFM_ETHER; 5696 uint64_t ifm_status = IFM_AVALID; 5697 int link_state = LINK_STATE_DOWN; 5698 uint64_t baudrate = 0; 5699 5700 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 5701 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) { 5702 ifm_active |= IFM_NONE; 5703 goto done; 5704 } 5705 5706 ifm_active |= IFM_FDX; 5707 ifm_status |= IFM_ACTIVE; 5708 link_state = LINK_STATE_UP; 5709 5710 itype = ixl_search_phy_type(status->phy_type); 5711 if (itype != NULL) 5712 ifm_active |= itype->ifm_type; 5713 5714 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 5715 ifm_active |= IFM_ETH_TXPAUSE; 5716 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 5717 ifm_active |= IFM_ETH_RXPAUSE; 5718 5719 baudrate = ixl_search_link_speed(status->link_speed); 5720 5721 done: 5722 /* sc->sc_cfg_lock held expect during attach */ 5723 sc->sc_media_active = ifm_active; 5724 sc->sc_media_status = ifm_status; 5725 5726 sc->sc_ec.ec_if.if_baudrate = baudrate; 5727 5728 return link_state; 5729 } 5730 5731 static int 5732 ixl_establish_intx(struct ixl_softc *sc) 5733 { 5734 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5735 pci_intr_handle_t *intr; 5736 char xnamebuf[32]; 5737 char intrbuf[PCI_INTRSTR_LEN]; 5738 char const *intrstr; 5739 5740 KASSERT(sc->sc_nintrs == 1); 5741 5742 intr = &sc->sc_ihp[0]; 5743 5744 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 5745 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 5746 device_xname(sc->sc_dev)); 5747 5748 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 5749 sc, xnamebuf); 5750 5751 if (sc->sc_ihs[0] == NULL) { 5752 aprint_error_dev(sc->sc_dev, 5753 "unable to establish interrupt at %s\n", intrstr); 5754 return -1; 5755 } 5756 5757 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 5758 return 0; 5759 } 5760 5761 static int 5762 ixl_establish_msix(struct ixl_softc *sc) 5763 { 5764 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5765 kcpuset_t *affinity; 5766 unsigned int vector = 0; 5767 unsigned int i; 5768 int affinity_to, r; 5769 char xnamebuf[32]; 5770 char intrbuf[PCI_INTRSTR_LEN]; 5771 char const *intrstr; 5772 5773 kcpuset_create(&affinity, false); 5774 5775 /* the "other" intr is mapped to vector 0 */ 5776 vector = 0; 5777 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5778 intrbuf, sizeof(intrbuf)); 5779 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 5780 device_xname(sc->sc_dev)); 5781 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5782 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 5783 sc, xnamebuf); 5784 if (sc->sc_ihs[vector] == NULL) { 5785 aprint_error_dev(sc->sc_dev, 5786 "unable to establish interrupt at %s\n", intrstr); 5787 goto fail; 5788 } 5789 5790 aprint_normal_dev(sc->sc_dev, "other interrupt at %s", intrstr); 5791 5792 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5793 affinity_to = (affinity_to + sc->sc_nqueue_pairs_max) % ncpu; 5794 5795 kcpuset_zero(affinity); 5796 kcpuset_set(affinity, affinity_to); 5797 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5798 if (r == 0) { 5799 aprint_normal(", affinity to %u", affinity_to); 5800 } 5801 aprint_normal("\n"); 5802 vector++; 5803 5804 sc->sc_msix_vector_queue = vector; 5805 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5806 5807 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5808 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5809 intrbuf, sizeof(intrbuf)); 5810 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 5811 device_xname(sc->sc_dev), i); 5812 5813 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5814 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 5815 (void *)&sc->sc_qps[i], xnamebuf); 5816 5817 if (sc->sc_ihs[vector] == NULL) { 5818 aprint_error_dev(sc->sc_dev, 5819 "unable to establish interrupt at %s\n", intrstr); 5820 goto fail; 5821 } 5822 5823 aprint_normal_dev(sc->sc_dev, 5824 "for TXRX%d interrupt at %s", i, intrstr); 5825 5826 kcpuset_zero(affinity); 5827 kcpuset_set(affinity, affinity_to); 5828 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5829 if (r == 0) { 5830 aprint_normal(", affinity to %u", affinity_to); 5831 affinity_to = (affinity_to + 1) % ncpu; 5832 } 5833 aprint_normal("\n"); 5834 vector++; 5835 } 5836 5837 kcpuset_destroy(affinity); 5838 5839 return 0; 5840 fail: 5841 for (i = 0; i < vector; i++) { 5842 pci_intr_disestablish(pc, sc->sc_ihs[i]); 5843 } 5844 5845 sc->sc_msix_vector_queue = 0; 5846 sc->sc_msix_vector_queue = 0; 5847 kcpuset_destroy(affinity); 5848 5849 return -1; 5850 } 5851 5852 static void 5853 ixl_config_queue_intr(struct ixl_softc *sc) 5854 { 5855 unsigned int i, vector; 5856 5857 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5858 vector = sc->sc_msix_vector_queue; 5859 } else { 5860 vector = I40E_INTR_NOTX_INTR; 5861 5862 ixl_wr(sc, I40E_PFINT_LNKLST0, 5863 (I40E_INTR_NOTX_QUEUE << 5864 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5865 (I40E_QUEUE_TYPE_RX << 5866 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5867 } 5868 5869 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5870 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5871 ixl_flush(sc); 5872 5873 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5874 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5875 (I40E_QUEUE_TYPE_RX << 5876 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5877 5878 ixl_wr(sc, I40E_QINT_RQCTL(i), 5879 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5880 (I40E_ITR_INDEX_RX << 5881 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5882 (I40E_INTR_NOTX_RX_QUEUE << 5883 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5884 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5885 (I40E_QUEUE_TYPE_TX << 5886 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5887 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5888 5889 ixl_wr(sc, I40E_QINT_TQCTL(i), 5890 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5891 (I40E_ITR_INDEX_TX << 5892 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5893 (I40E_INTR_NOTX_TX_QUEUE << 5894 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5895 (I40E_QUEUE_TYPE_EOL << 5896 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5897 (I40E_QUEUE_TYPE_RX << 5898 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5899 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5900 5901 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5902 ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_RX, i), 5903 sc->sc_itr_rx); 5904 ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_TX, i), 5905 sc->sc_itr_tx); 5906 vector++; 5907 } 5908 } 5909 ixl_flush(sc); 5910 5911 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), sc->sc_itr_rx); 5912 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), sc->sc_itr_tx); 5913 ixl_flush(sc); 5914 } 5915 5916 static void 5917 ixl_config_other_intr(struct ixl_softc *sc) 5918 { 5919 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5920 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5921 5922 /* 5923 * We use the hardware LINK_STAT_CHANGE event to trigger receipt 5924 * of Link Status Events on the firmware queue, unlike Intel's 5925 * drivers which poll. 5926 */ 5927 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5928 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5929 I40E_PFINT_ICR0_ENA_GRST_MASK | 5930 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5931 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5932 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5933 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5934 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5935 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5936 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5937 5938 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5939 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5940 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5941 (I40E_ITR_INDEX_OTHER << 5942 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5943 ixl_flush(sc); 5944 } 5945 5946 static int 5947 ixl_setup_interrupts(struct ixl_softc *sc) 5948 { 5949 struct pci_attach_args *pa = &sc->sc_pa; 5950 pci_intr_type_t max_type, intr_type; 5951 int counts[PCI_INTR_TYPE_SIZE]; 5952 int error; 5953 unsigned int i; 5954 bool retry; 5955 5956 memset(counts, 0, sizeof(counts)); 5957 max_type = PCI_INTR_TYPE_MSIX; 5958 /* QPs + other interrupt */ 5959 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5960 counts[PCI_INTR_TYPE_INTX] = 1; 5961 5962 if (ixl_param_nomsix) 5963 counts[PCI_INTR_TYPE_MSIX] = 0; 5964 5965 do { 5966 retry = false; 5967 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5968 if (error != 0) { 5969 aprint_error_dev(sc->sc_dev, 5970 "couldn't map interrupt\n"); 5971 break; 5972 } 5973 5974 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5975 sc->sc_nintrs = counts[intr_type]; 5976 KASSERT(sc->sc_nintrs > 0); 5977 5978 for (i = 0; i < sc->sc_nintrs; i++) { 5979 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5980 PCI_INTR_MPSAFE, true); 5981 } 5982 5983 sc->sc_ihs = kmem_zalloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5984 KM_SLEEP); 5985 5986 if (intr_type == PCI_INTR_TYPE_MSIX) { 5987 error = ixl_establish_msix(sc); 5988 if (error) { 5989 counts[PCI_INTR_TYPE_MSIX] = 0; 5990 retry = true; 5991 } 5992 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5993 error = ixl_establish_intx(sc); 5994 } else { 5995 error = -1; 5996 } 5997 5998 if (error) { 5999 kmem_free(sc->sc_ihs, 6000 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 6001 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 6002 } else { 6003 sc->sc_intrtype = intr_type; 6004 } 6005 } while (retry); 6006 6007 return error; 6008 } 6009 6010 static void 6011 ixl_teardown_interrupts(struct ixl_softc *sc) 6012 { 6013 struct pci_attach_args *pa = &sc->sc_pa; 6014 unsigned int i; 6015 6016 for (i = 0; i < sc->sc_nintrs; i++) { 6017 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 6018 } 6019 6020 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 6021 6022 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 6023 sc->sc_ihs = NULL; 6024 sc->sc_nintrs = 0; 6025 } 6026 6027 static int 6028 ixl_setup_stats(struct ixl_softc *sc) 6029 { 6030 struct ixl_queue_pair *qp; 6031 struct ixl_tx_ring *txr; 6032 struct ixl_rx_ring *rxr; 6033 struct ixl_stats_counters *isc; 6034 unsigned int i; 6035 6036 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6037 qp = &sc->sc_qps[i]; 6038 txr = qp->qp_txr; 6039 rxr = qp->qp_rxr; 6040 6041 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 6042 NULL, qp->qp_name, "m_defrag succeeded"); 6043 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 6044 NULL, qp->qp_name, "m_defrag failed"); 6045 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 6046 NULL, qp->qp_name, "Dropped in pcq"); 6047 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 6048 NULL, qp->qp_name, "Deferred transmit"); 6049 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 6050 NULL, qp->qp_name, "Interrupt on queue"); 6051 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 6052 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6053 6054 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 6055 NULL, qp->qp_name, "MGETHDR failed"); 6056 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 6057 NULL, qp->qp_name, "MCLGET failed"); 6058 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 6059 EVCNT_TYPE_MISC, NULL, qp->qp_name, 6060 "bus_dmamap_load_mbuf failed"); 6061 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 6062 NULL, qp->qp_name, "Interrupt on queue"); 6063 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 6064 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6065 } 6066 6067 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 6068 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 6069 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 6070 NULL, device_xname(sc->sc_dev), "Link status event"); 6071 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 6072 NULL, device_xname(sc->sc_dev), "ECC error"); 6073 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 6074 NULL, device_xname(sc->sc_dev), "PCI exception"); 6075 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 6076 NULL, device_xname(sc->sc_dev), "Critical error"); 6077 6078 isc = &sc->sc_stats_counters; 6079 evcnt_attach_dynamic(&isc->isc_crc_errors, EVCNT_TYPE_MISC, 6080 NULL, device_xname(sc->sc_dev), "CRC errors"); 6081 evcnt_attach_dynamic(&isc->isc_illegal_bytes, EVCNT_TYPE_MISC, 6082 NULL, device_xname(sc->sc_dev), "Illegal bytes"); 6083 evcnt_attach_dynamic(&isc->isc_mac_local_faults, EVCNT_TYPE_MISC, 6084 NULL, device_xname(sc->sc_dev), "Mac local faults"); 6085 evcnt_attach_dynamic(&isc->isc_mac_remote_faults, EVCNT_TYPE_MISC, 6086 NULL, device_xname(sc->sc_dev), "Mac remote faults"); 6087 evcnt_attach_dynamic(&isc->isc_link_xon_rx, EVCNT_TYPE_MISC, 6088 NULL, device_xname(sc->sc_dev), "Rx xon"); 6089 evcnt_attach_dynamic(&isc->isc_link_xon_tx, EVCNT_TYPE_MISC, 6090 NULL, device_xname(sc->sc_dev), "Tx xon"); 6091 evcnt_attach_dynamic(&isc->isc_link_xoff_rx, EVCNT_TYPE_MISC, 6092 NULL, device_xname(sc->sc_dev), "Rx xoff"); 6093 evcnt_attach_dynamic(&isc->isc_link_xoff_tx, EVCNT_TYPE_MISC, 6094 NULL, device_xname(sc->sc_dev), "Tx xoff"); 6095 evcnt_attach_dynamic(&isc->isc_rx_fragments, EVCNT_TYPE_MISC, 6096 NULL, device_xname(sc->sc_dev), "Rx fragments"); 6097 evcnt_attach_dynamic(&isc->isc_rx_jabber, EVCNT_TYPE_MISC, 6098 NULL, device_xname(sc->sc_dev), "Rx jabber"); 6099 6100 evcnt_attach_dynamic(&isc->isc_rx_size_64, EVCNT_TYPE_MISC, 6101 NULL, device_xname(sc->sc_dev), "Rx size 64"); 6102 evcnt_attach_dynamic(&isc->isc_rx_size_127, EVCNT_TYPE_MISC, 6103 NULL, device_xname(sc->sc_dev), "Rx size 127"); 6104 evcnt_attach_dynamic(&isc->isc_rx_size_255, EVCNT_TYPE_MISC, 6105 NULL, device_xname(sc->sc_dev), "Rx size 255"); 6106 evcnt_attach_dynamic(&isc->isc_rx_size_511, EVCNT_TYPE_MISC, 6107 NULL, device_xname(sc->sc_dev), "Rx size 511"); 6108 evcnt_attach_dynamic(&isc->isc_rx_size_1023, EVCNT_TYPE_MISC, 6109 NULL, device_xname(sc->sc_dev), "Rx size 1023"); 6110 evcnt_attach_dynamic(&isc->isc_rx_size_1522, EVCNT_TYPE_MISC, 6111 NULL, device_xname(sc->sc_dev), "Rx size 1522"); 6112 evcnt_attach_dynamic(&isc->isc_rx_size_big, EVCNT_TYPE_MISC, 6113 NULL, device_xname(sc->sc_dev), "Rx jumbo packets"); 6114 evcnt_attach_dynamic(&isc->isc_rx_undersize, EVCNT_TYPE_MISC, 6115 NULL, device_xname(sc->sc_dev), "Rx under size"); 6116 evcnt_attach_dynamic(&isc->isc_rx_oversize, EVCNT_TYPE_MISC, 6117 NULL, device_xname(sc->sc_dev), "Rx over size"); 6118 6119 evcnt_attach_dynamic(&isc->isc_rx_bytes, EVCNT_TYPE_MISC, 6120 NULL, device_xname(sc->sc_dev), "Rx bytes / port"); 6121 evcnt_attach_dynamic(&isc->isc_rx_discards, EVCNT_TYPE_MISC, 6122 NULL, device_xname(sc->sc_dev), "Rx discards / port"); 6123 evcnt_attach_dynamic(&isc->isc_rx_unicast, EVCNT_TYPE_MISC, 6124 NULL, device_xname(sc->sc_dev), "Rx unicast / port"); 6125 evcnt_attach_dynamic(&isc->isc_rx_multicast, EVCNT_TYPE_MISC, 6126 NULL, device_xname(sc->sc_dev), "Rx multicast / port"); 6127 evcnt_attach_dynamic(&isc->isc_rx_broadcast, EVCNT_TYPE_MISC, 6128 NULL, device_xname(sc->sc_dev), "Rx broadcast / port"); 6129 6130 evcnt_attach_dynamic(&isc->isc_vsi_rx_bytes, EVCNT_TYPE_MISC, 6131 NULL, device_xname(sc->sc_dev), "Rx bytes / vsi"); 6132 evcnt_attach_dynamic(&isc->isc_vsi_rx_discards, EVCNT_TYPE_MISC, 6133 NULL, device_xname(sc->sc_dev), "Rx discards / vsi"); 6134 evcnt_attach_dynamic(&isc->isc_vsi_rx_unicast, EVCNT_TYPE_MISC, 6135 NULL, device_xname(sc->sc_dev), "Rx unicast / vsi"); 6136 evcnt_attach_dynamic(&isc->isc_vsi_rx_multicast, EVCNT_TYPE_MISC, 6137 NULL, device_xname(sc->sc_dev), "Rx multicast / vsi"); 6138 evcnt_attach_dynamic(&isc->isc_vsi_rx_broadcast, EVCNT_TYPE_MISC, 6139 NULL, device_xname(sc->sc_dev), "Rx broadcast / vsi"); 6140 6141 evcnt_attach_dynamic(&isc->isc_tx_size_64, EVCNT_TYPE_MISC, 6142 NULL, device_xname(sc->sc_dev), "Tx size 64"); 6143 evcnt_attach_dynamic(&isc->isc_tx_size_127, EVCNT_TYPE_MISC, 6144 NULL, device_xname(sc->sc_dev), "Tx size 127"); 6145 evcnt_attach_dynamic(&isc->isc_tx_size_255, EVCNT_TYPE_MISC, 6146 NULL, device_xname(sc->sc_dev), "Tx size 255"); 6147 evcnt_attach_dynamic(&isc->isc_tx_size_511, EVCNT_TYPE_MISC, 6148 NULL, device_xname(sc->sc_dev), "Tx size 511"); 6149 evcnt_attach_dynamic(&isc->isc_tx_size_1023, EVCNT_TYPE_MISC, 6150 NULL, device_xname(sc->sc_dev), "Tx size 1023"); 6151 evcnt_attach_dynamic(&isc->isc_tx_size_1522, EVCNT_TYPE_MISC, 6152 NULL, device_xname(sc->sc_dev), "Tx size 1522"); 6153 evcnt_attach_dynamic(&isc->isc_tx_size_big, EVCNT_TYPE_MISC, 6154 NULL, device_xname(sc->sc_dev), "Tx jumbo packets"); 6155 6156 evcnt_attach_dynamic(&isc->isc_tx_bytes, EVCNT_TYPE_MISC, 6157 NULL, device_xname(sc->sc_dev), "Tx bytes / port"); 6158 evcnt_attach_dynamic(&isc->isc_tx_dropped_link_down, EVCNT_TYPE_MISC, 6159 NULL, device_xname(sc->sc_dev), 6160 "Tx dropped due to link down / port"); 6161 evcnt_attach_dynamic(&isc->isc_tx_unicast, EVCNT_TYPE_MISC, 6162 NULL, device_xname(sc->sc_dev), "Tx unicast / port"); 6163 evcnt_attach_dynamic(&isc->isc_tx_multicast, EVCNT_TYPE_MISC, 6164 NULL, device_xname(sc->sc_dev), "Tx multicast / port"); 6165 evcnt_attach_dynamic(&isc->isc_tx_broadcast, EVCNT_TYPE_MISC, 6166 NULL, device_xname(sc->sc_dev), "Tx broadcast / port"); 6167 6168 evcnt_attach_dynamic(&isc->isc_vsi_tx_bytes, EVCNT_TYPE_MISC, 6169 NULL, device_xname(sc->sc_dev), "Tx bytes / vsi"); 6170 evcnt_attach_dynamic(&isc->isc_vsi_tx_errors, EVCNT_TYPE_MISC, 6171 NULL, device_xname(sc->sc_dev), "Tx errors / vsi"); 6172 evcnt_attach_dynamic(&isc->isc_vsi_tx_unicast, EVCNT_TYPE_MISC, 6173 NULL, device_xname(sc->sc_dev), "Tx unicast / vsi"); 6174 evcnt_attach_dynamic(&isc->isc_vsi_tx_multicast, EVCNT_TYPE_MISC, 6175 NULL, device_xname(sc->sc_dev), "Tx multicast / vsi"); 6176 evcnt_attach_dynamic(&isc->isc_vsi_tx_broadcast, EVCNT_TYPE_MISC, 6177 NULL, device_xname(sc->sc_dev), "Tx broadcast / vsi"); 6178 6179 sc->sc_stats_intval = ixl_param_stats_interval; 6180 callout_init(&sc->sc_stats_callout, CALLOUT_MPSAFE); 6181 callout_setfunc(&sc->sc_stats_callout, ixl_stats_callout, sc); 6182 ixl_work_set(&sc->sc_stats_task, ixl_stats_update, sc); 6183 6184 return 0; 6185 } 6186 6187 static void 6188 ixl_teardown_stats(struct ixl_softc *sc) 6189 { 6190 struct ixl_tx_ring *txr; 6191 struct ixl_rx_ring *rxr; 6192 struct ixl_stats_counters *isc; 6193 unsigned int i; 6194 6195 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6196 txr = sc->sc_qps[i].qp_txr; 6197 rxr = sc->sc_qps[i].qp_rxr; 6198 6199 evcnt_detach(&txr->txr_defragged); 6200 evcnt_detach(&txr->txr_defrag_failed); 6201 evcnt_detach(&txr->txr_pcqdrop); 6202 evcnt_detach(&txr->txr_transmitdef); 6203 evcnt_detach(&txr->txr_intr); 6204 evcnt_detach(&txr->txr_defer); 6205 6206 evcnt_detach(&rxr->rxr_mgethdr_failed); 6207 evcnt_detach(&rxr->rxr_mgetcl_failed); 6208 evcnt_detach(&rxr->rxr_mbuf_load_failed); 6209 evcnt_detach(&rxr->rxr_intr); 6210 evcnt_detach(&rxr->rxr_defer); 6211 } 6212 6213 isc = &sc->sc_stats_counters; 6214 evcnt_detach(&isc->isc_crc_errors); 6215 evcnt_detach(&isc->isc_illegal_bytes); 6216 evcnt_detach(&isc->isc_mac_local_faults); 6217 evcnt_detach(&isc->isc_mac_remote_faults); 6218 evcnt_detach(&isc->isc_link_xon_rx); 6219 evcnt_detach(&isc->isc_link_xon_tx); 6220 evcnt_detach(&isc->isc_link_xoff_rx); 6221 evcnt_detach(&isc->isc_link_xoff_tx); 6222 evcnt_detach(&isc->isc_rx_fragments); 6223 evcnt_detach(&isc->isc_rx_jabber); 6224 evcnt_detach(&isc->isc_rx_bytes); 6225 evcnt_detach(&isc->isc_rx_discards); 6226 evcnt_detach(&isc->isc_rx_unicast); 6227 evcnt_detach(&isc->isc_rx_multicast); 6228 evcnt_detach(&isc->isc_rx_broadcast); 6229 evcnt_detach(&isc->isc_rx_size_64); 6230 evcnt_detach(&isc->isc_rx_size_127); 6231 evcnt_detach(&isc->isc_rx_size_255); 6232 evcnt_detach(&isc->isc_rx_size_511); 6233 evcnt_detach(&isc->isc_rx_size_1023); 6234 evcnt_detach(&isc->isc_rx_size_1522); 6235 evcnt_detach(&isc->isc_rx_size_big); 6236 evcnt_detach(&isc->isc_rx_undersize); 6237 evcnt_detach(&isc->isc_rx_oversize); 6238 evcnt_detach(&isc->isc_tx_bytes); 6239 evcnt_detach(&isc->isc_tx_dropped_link_down); 6240 evcnt_detach(&isc->isc_tx_unicast); 6241 evcnt_detach(&isc->isc_tx_multicast); 6242 evcnt_detach(&isc->isc_tx_broadcast); 6243 evcnt_detach(&isc->isc_tx_size_64); 6244 evcnt_detach(&isc->isc_tx_size_127); 6245 evcnt_detach(&isc->isc_tx_size_255); 6246 evcnt_detach(&isc->isc_tx_size_511); 6247 evcnt_detach(&isc->isc_tx_size_1023); 6248 evcnt_detach(&isc->isc_tx_size_1522); 6249 evcnt_detach(&isc->isc_tx_size_big); 6250 evcnt_detach(&isc->isc_vsi_rx_discards); 6251 evcnt_detach(&isc->isc_vsi_rx_bytes); 6252 evcnt_detach(&isc->isc_vsi_rx_unicast); 6253 evcnt_detach(&isc->isc_vsi_rx_multicast); 6254 evcnt_detach(&isc->isc_vsi_rx_broadcast); 6255 evcnt_detach(&isc->isc_vsi_tx_errors); 6256 evcnt_detach(&isc->isc_vsi_tx_bytes); 6257 evcnt_detach(&isc->isc_vsi_tx_unicast); 6258 evcnt_detach(&isc->isc_vsi_tx_multicast); 6259 evcnt_detach(&isc->isc_vsi_tx_broadcast); 6260 6261 evcnt_detach(&sc->sc_event_atq); 6262 evcnt_detach(&sc->sc_event_link); 6263 evcnt_detach(&sc->sc_event_ecc_err); 6264 evcnt_detach(&sc->sc_event_pci_exception); 6265 evcnt_detach(&sc->sc_event_crit_err); 6266 6267 callout_destroy(&sc->sc_stats_callout); 6268 } 6269 6270 static void 6271 ixl_stats_callout(void *xsc) 6272 { 6273 struct ixl_softc *sc = xsc; 6274 6275 ixl_work_add(sc->sc_workq, &sc->sc_stats_task); 6276 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 6277 } 6278 6279 static uint64_t 6280 ixl_stat_delta(struct ixl_softc *sc, uint32_t reg_hi, uint32_t reg_lo, 6281 uint64_t *offset, bool has_offset) 6282 { 6283 uint64_t value, delta; 6284 int bitwidth; 6285 6286 bitwidth = reg_hi == 0 ? 32 : 48; 6287 6288 value = ixl_rd(sc, reg_lo); 6289 6290 if (bitwidth > 32) { 6291 value |= ((uint64_t)ixl_rd(sc, reg_hi) << 32); 6292 } 6293 6294 if (__predict_true(has_offset)) { 6295 delta = value; 6296 if (value < *offset) 6297 delta += ((uint64_t)1 << bitwidth); 6298 delta -= *offset; 6299 } else { 6300 delta = 0; 6301 } 6302 atomic_swap_64(offset, value); 6303 6304 return delta; 6305 } 6306 6307 static void 6308 ixl_stats_update(void *xsc) 6309 { 6310 struct ixl_softc *sc = xsc; 6311 struct ixl_stats_counters *isc; 6312 uint64_t delta; 6313 6314 isc = &sc->sc_stats_counters; 6315 6316 /* errors */ 6317 delta = ixl_stat_delta(sc, 6318 0, I40E_GLPRT_CRCERRS(sc->sc_port), 6319 &isc->isc_crc_errors_offset, isc->isc_has_offset); 6320 atomic_add_64(&isc->isc_crc_errors.ev_count, delta); 6321 6322 delta = ixl_stat_delta(sc, 6323 0, I40E_GLPRT_ILLERRC(sc->sc_port), 6324 &isc->isc_illegal_bytes_offset, isc->isc_has_offset); 6325 atomic_add_64(&isc->isc_illegal_bytes.ev_count, delta); 6326 6327 /* rx */ 6328 delta = ixl_stat_delta(sc, 6329 I40E_GLPRT_GORCH(sc->sc_port), I40E_GLPRT_GORCL(sc->sc_port), 6330 &isc->isc_rx_bytes_offset, isc->isc_has_offset); 6331 atomic_add_64(&isc->isc_rx_bytes.ev_count, delta); 6332 6333 delta = ixl_stat_delta(sc, 6334 0, I40E_GLPRT_RDPC(sc->sc_port), 6335 &isc->isc_rx_discards_offset, isc->isc_has_offset); 6336 atomic_add_64(&isc->isc_rx_discards.ev_count, delta); 6337 6338 delta = ixl_stat_delta(sc, 6339 I40E_GLPRT_UPRCH(sc->sc_port), I40E_GLPRT_UPRCL(sc->sc_port), 6340 &isc->isc_rx_unicast_offset, isc->isc_has_offset); 6341 atomic_add_64(&isc->isc_rx_unicast.ev_count, delta); 6342 6343 delta = ixl_stat_delta(sc, 6344 I40E_GLPRT_MPRCH(sc->sc_port), I40E_GLPRT_MPRCL(sc->sc_port), 6345 &isc->isc_rx_multicast_offset, isc->isc_has_offset); 6346 atomic_add_64(&isc->isc_rx_multicast.ev_count, delta); 6347 6348 delta = ixl_stat_delta(sc, 6349 I40E_GLPRT_BPRCH(sc->sc_port), I40E_GLPRT_BPRCL(sc->sc_port), 6350 &isc->isc_rx_broadcast_offset, isc->isc_has_offset); 6351 atomic_add_64(&isc->isc_rx_broadcast.ev_count, delta); 6352 6353 /* Packet size stats rx */ 6354 delta = ixl_stat_delta(sc, 6355 I40E_GLPRT_PRC64H(sc->sc_port), I40E_GLPRT_PRC64L(sc->sc_port), 6356 &isc->isc_rx_size_64_offset, isc->isc_has_offset); 6357 atomic_add_64(&isc->isc_rx_size_64.ev_count, delta); 6358 6359 delta = ixl_stat_delta(sc, 6360 I40E_GLPRT_PRC127H(sc->sc_port), I40E_GLPRT_PRC127L(sc->sc_port), 6361 &isc->isc_rx_size_127_offset, isc->isc_has_offset); 6362 atomic_add_64(&isc->isc_rx_size_127.ev_count, delta); 6363 6364 delta = ixl_stat_delta(sc, 6365 I40E_GLPRT_PRC255H(sc->sc_port), I40E_GLPRT_PRC255L(sc->sc_port), 6366 &isc->isc_rx_size_255_offset, isc->isc_has_offset); 6367 atomic_add_64(&isc->isc_rx_size_255.ev_count, delta); 6368 6369 delta = ixl_stat_delta(sc, 6370 I40E_GLPRT_PRC511H(sc->sc_port), I40E_GLPRT_PRC511L(sc->sc_port), 6371 &isc->isc_rx_size_511_offset, isc->isc_has_offset); 6372 atomic_add_64(&isc->isc_rx_size_511.ev_count, delta); 6373 6374 delta = ixl_stat_delta(sc, 6375 I40E_GLPRT_PRC1023H(sc->sc_port), I40E_GLPRT_PRC1023L(sc->sc_port), 6376 &isc->isc_rx_size_1023_offset, isc->isc_has_offset); 6377 atomic_add_64(&isc->isc_rx_size_1023.ev_count, delta); 6378 6379 delta = ixl_stat_delta(sc, 6380 I40E_GLPRT_PRC1522H(sc->sc_port), I40E_GLPRT_PRC1522L(sc->sc_port), 6381 &isc->isc_rx_size_1522_offset, isc->isc_has_offset); 6382 atomic_add_64(&isc->isc_rx_size_1522.ev_count, delta); 6383 6384 delta = ixl_stat_delta(sc, 6385 I40E_GLPRT_PRC9522H(sc->sc_port), I40E_GLPRT_PRC9522L(sc->sc_port), 6386 &isc->isc_rx_size_big_offset, isc->isc_has_offset); 6387 atomic_add_64(&isc->isc_rx_size_big.ev_count, delta); 6388 6389 delta = ixl_stat_delta(sc, 6390 0, I40E_GLPRT_RUC(sc->sc_port), 6391 &isc->isc_rx_undersize_offset, isc->isc_has_offset); 6392 atomic_add_64(&isc->isc_rx_undersize.ev_count, delta); 6393 6394 delta = ixl_stat_delta(sc, 6395 0, I40E_GLPRT_ROC(sc->sc_port), 6396 &isc->isc_rx_oversize_offset, isc->isc_has_offset); 6397 atomic_add_64(&isc->isc_rx_oversize.ev_count, delta); 6398 6399 /* tx */ 6400 delta = ixl_stat_delta(sc, 6401 I40E_GLPRT_GOTCH(sc->sc_port), I40E_GLPRT_GOTCL(sc->sc_port), 6402 &isc->isc_tx_bytes_offset, isc->isc_has_offset); 6403 atomic_add_64(&isc->isc_tx_bytes.ev_count, delta); 6404 6405 delta = ixl_stat_delta(sc, 6406 0, I40E_GLPRT_TDOLD(sc->sc_port), 6407 &isc->isc_tx_dropped_link_down_offset, isc->isc_has_offset); 6408 atomic_add_64(&isc->isc_tx_dropped_link_down.ev_count, delta); 6409 6410 delta = ixl_stat_delta(sc, 6411 I40E_GLPRT_UPTCH(sc->sc_port), I40E_GLPRT_UPTCL(sc->sc_port), 6412 &isc->isc_tx_unicast_offset, isc->isc_has_offset); 6413 atomic_add_64(&isc->isc_tx_unicast.ev_count, delta); 6414 6415 delta = ixl_stat_delta(sc, 6416 I40E_GLPRT_MPTCH(sc->sc_port), I40E_GLPRT_MPTCL(sc->sc_port), 6417 &isc->isc_tx_multicast_offset, isc->isc_has_offset); 6418 atomic_add_64(&isc->isc_tx_multicast.ev_count, delta); 6419 6420 delta = ixl_stat_delta(sc, 6421 I40E_GLPRT_BPTCH(sc->sc_port), I40E_GLPRT_BPTCL(sc->sc_port), 6422 &isc->isc_tx_broadcast_offset, isc->isc_has_offset); 6423 atomic_add_64(&isc->isc_tx_broadcast.ev_count, delta); 6424 6425 /* Packet size stats tx */ 6426 delta = ixl_stat_delta(sc, 6427 I40E_GLPRT_PTC64L(sc->sc_port), I40E_GLPRT_PTC64L(sc->sc_port), 6428 &isc->isc_tx_size_64_offset, isc->isc_has_offset); 6429 atomic_add_64(&isc->isc_tx_size_64.ev_count, delta); 6430 6431 delta = ixl_stat_delta(sc, 6432 I40E_GLPRT_PTC127H(sc->sc_port), I40E_GLPRT_PTC127L(sc->sc_port), 6433 &isc->isc_tx_size_127_offset, isc->isc_has_offset); 6434 atomic_add_64(&isc->isc_tx_size_127.ev_count, delta); 6435 6436 delta = ixl_stat_delta(sc, 6437 I40E_GLPRT_PTC255H(sc->sc_port), I40E_GLPRT_PTC255L(sc->sc_port), 6438 &isc->isc_tx_size_255_offset, isc->isc_has_offset); 6439 atomic_add_64(&isc->isc_tx_size_255.ev_count, delta); 6440 6441 delta = ixl_stat_delta(sc, 6442 I40E_GLPRT_PTC511H(sc->sc_port), I40E_GLPRT_PTC511L(sc->sc_port), 6443 &isc->isc_tx_size_511_offset, isc->isc_has_offset); 6444 atomic_add_64(&isc->isc_tx_size_511.ev_count, delta); 6445 6446 delta = ixl_stat_delta(sc, 6447 I40E_GLPRT_PTC1023H(sc->sc_port), I40E_GLPRT_PTC1023L(sc->sc_port), 6448 &isc->isc_tx_size_1023_offset, isc->isc_has_offset); 6449 atomic_add_64(&isc->isc_tx_size_1023.ev_count, delta); 6450 6451 delta = ixl_stat_delta(sc, 6452 I40E_GLPRT_PTC1522H(sc->sc_port), I40E_GLPRT_PTC1522L(sc->sc_port), 6453 &isc->isc_tx_size_1522_offset, isc->isc_has_offset); 6454 atomic_add_64(&isc->isc_tx_size_1522.ev_count, delta); 6455 6456 delta = ixl_stat_delta(sc, 6457 I40E_GLPRT_PTC9522H(sc->sc_port), I40E_GLPRT_PTC9522L(sc->sc_port), 6458 &isc->isc_tx_size_big_offset, isc->isc_has_offset); 6459 atomic_add_64(&isc->isc_tx_size_big.ev_count, delta); 6460 6461 /* mac faults */ 6462 delta = ixl_stat_delta(sc, 6463 0, I40E_GLPRT_MLFC(sc->sc_port), 6464 &isc->isc_mac_local_faults_offset, isc->isc_has_offset); 6465 atomic_add_64(&isc->isc_mac_local_faults.ev_count, delta); 6466 6467 delta = ixl_stat_delta(sc, 6468 0, I40E_GLPRT_MRFC(sc->sc_port), 6469 &isc->isc_mac_remote_faults_offset, isc->isc_has_offset); 6470 atomic_add_64(&isc->isc_mac_remote_faults.ev_count, delta); 6471 6472 /* Flow control (LFC) stats */ 6473 delta = ixl_stat_delta(sc, 6474 0, I40E_GLPRT_LXONRXC(sc->sc_port), 6475 &isc->isc_link_xon_rx_offset, isc->isc_has_offset); 6476 atomic_add_64(&isc->isc_link_xon_rx.ev_count, delta); 6477 6478 delta = ixl_stat_delta(sc, 6479 0, I40E_GLPRT_LXONTXC(sc->sc_port), 6480 &isc->isc_link_xon_tx_offset, isc->isc_has_offset); 6481 atomic_add_64(&isc->isc_link_xon_tx.ev_count, delta); 6482 6483 delta = ixl_stat_delta(sc, 6484 0, I40E_GLPRT_LXOFFRXC(sc->sc_port), 6485 &isc->isc_link_xoff_rx_offset, isc->isc_has_offset); 6486 atomic_add_64(&isc->isc_link_xoff_rx.ev_count, delta); 6487 6488 delta = ixl_stat_delta(sc, 6489 0, I40E_GLPRT_LXOFFTXC(sc->sc_port), 6490 &isc->isc_link_xoff_tx_offset, isc->isc_has_offset); 6491 atomic_add_64(&isc->isc_link_xoff_tx.ev_count, delta); 6492 6493 /* fragments */ 6494 delta = ixl_stat_delta(sc, 6495 0, I40E_GLPRT_RFC(sc->sc_port), 6496 &isc->isc_rx_fragments_offset, isc->isc_has_offset); 6497 atomic_add_64(&isc->isc_rx_fragments.ev_count, delta); 6498 6499 delta = ixl_stat_delta(sc, 6500 0, I40E_GLPRT_RJC(sc->sc_port), 6501 &isc->isc_rx_jabber_offset, isc->isc_has_offset); 6502 atomic_add_64(&isc->isc_rx_jabber.ev_count, delta); 6503 6504 /* VSI rx counters */ 6505 delta = ixl_stat_delta(sc, 6506 0, I40E_GLV_RDPC(sc->sc_vsi_stat_counter_idx), 6507 &isc->isc_vsi_rx_discards_offset, isc->isc_has_offset); 6508 atomic_add_64(&isc->isc_vsi_rx_discards.ev_count, delta); 6509 6510 delta = ixl_stat_delta(sc, 6511 I40E_GLV_GORCH(sc->sc_vsi_stat_counter_idx), 6512 I40E_GLV_GORCL(sc->sc_vsi_stat_counter_idx), 6513 &isc->isc_vsi_rx_bytes_offset, isc->isc_has_offset); 6514 atomic_add_64(&isc->isc_vsi_rx_bytes.ev_count, delta); 6515 6516 delta = ixl_stat_delta(sc, 6517 I40E_GLV_UPRCH(sc->sc_vsi_stat_counter_idx), 6518 I40E_GLV_UPRCL(sc->sc_vsi_stat_counter_idx), 6519 &isc->isc_vsi_rx_unicast_offset, isc->isc_has_offset); 6520 atomic_add_64(&isc->isc_vsi_rx_unicast.ev_count, delta); 6521 6522 delta = ixl_stat_delta(sc, 6523 I40E_GLV_MPRCH(sc->sc_vsi_stat_counter_idx), 6524 I40E_GLV_MPRCL(sc->sc_vsi_stat_counter_idx), 6525 &isc->isc_vsi_rx_multicast_offset, isc->isc_has_offset); 6526 atomic_add_64(&isc->isc_vsi_rx_multicast.ev_count, delta); 6527 6528 delta = ixl_stat_delta(sc, 6529 I40E_GLV_BPRCH(sc->sc_vsi_stat_counter_idx), 6530 I40E_GLV_BPRCL(sc->sc_vsi_stat_counter_idx), 6531 &isc->isc_vsi_rx_broadcast_offset, isc->isc_has_offset); 6532 atomic_add_64(&isc->isc_vsi_rx_broadcast.ev_count, delta); 6533 6534 /* VSI tx counters */ 6535 delta = ixl_stat_delta(sc, 6536 0, I40E_GLV_TEPC(sc->sc_vsi_stat_counter_idx), 6537 &isc->isc_vsi_tx_errors_offset, isc->isc_has_offset); 6538 atomic_add_64(&isc->isc_vsi_tx_errors.ev_count, delta); 6539 6540 delta = ixl_stat_delta(sc, 6541 I40E_GLV_GOTCH(sc->sc_vsi_stat_counter_idx), 6542 I40E_GLV_GOTCL(sc->sc_vsi_stat_counter_idx), 6543 &isc->isc_vsi_tx_bytes_offset, isc->isc_has_offset); 6544 atomic_add_64(&isc->isc_vsi_tx_bytes.ev_count, delta); 6545 6546 delta = ixl_stat_delta(sc, 6547 I40E_GLV_UPTCH(sc->sc_vsi_stat_counter_idx), 6548 I40E_GLV_UPTCL(sc->sc_vsi_stat_counter_idx), 6549 &isc->isc_vsi_tx_unicast_offset, isc->isc_has_offset); 6550 atomic_add_64(&isc->isc_vsi_tx_unicast.ev_count, delta); 6551 6552 delta = ixl_stat_delta(sc, 6553 I40E_GLV_MPTCH(sc->sc_vsi_stat_counter_idx), 6554 I40E_GLV_MPTCL(sc->sc_vsi_stat_counter_idx), 6555 &isc->isc_vsi_tx_multicast_offset, isc->isc_has_offset); 6556 atomic_add_64(&isc->isc_vsi_tx_multicast.ev_count, delta); 6557 6558 delta = ixl_stat_delta(sc, 6559 I40E_GLV_BPTCH(sc->sc_vsi_stat_counter_idx), 6560 I40E_GLV_BPTCL(sc->sc_vsi_stat_counter_idx), 6561 &isc->isc_vsi_tx_broadcast_offset, isc->isc_has_offset); 6562 atomic_add_64(&isc->isc_vsi_tx_broadcast.ev_count, delta); 6563 } 6564 6565 static int 6566 ixl_setup_sysctls(struct ixl_softc *sc) 6567 { 6568 const char *devname; 6569 struct sysctllog **log; 6570 const struct sysctlnode *rnode, *rxnode, *txnode; 6571 int error; 6572 6573 log = &sc->sc_sysctllog; 6574 devname = device_xname(sc->sc_dev); 6575 6576 error = sysctl_createv(log, 0, NULL, &rnode, 6577 0, CTLTYPE_NODE, devname, 6578 SYSCTL_DESCR("ixl information and settings"), 6579 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 6580 if (error) 6581 goto out; 6582 6583 error = sysctl_createv(log, 0, &rnode, NULL, 6584 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 6585 SYSCTL_DESCR("Use workqueue for packet processing"), 6586 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 6587 if (error) 6588 goto out; 6589 6590 error = sysctl_createv(log, 0, &rnode, NULL, 6591 CTLFLAG_READONLY, CTLTYPE_INT, "stats_interval", 6592 SYSCTL_DESCR("Statistics collection interval in milliseconds"), 6593 NULL, 0, &sc->sc_stats_intval, 0, CTL_CREATE, CTL_EOL); 6594 6595 error = sysctl_createv(log, 0, &rnode, &rxnode, 6596 0, CTLTYPE_NODE, "rx", 6597 SYSCTL_DESCR("ixl information and settings for Rx"), 6598 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6599 if (error) 6600 goto out; 6601 6602 error = sysctl_createv(log, 0, &rxnode, NULL, 6603 CTLFLAG_READWRITE, CTLTYPE_INT, "itr", 6604 SYSCTL_DESCR("Interrupt Throttling"), 6605 ixl_sysctl_itr_handler, 0, 6606 (void *)sc, 0, CTL_CREATE, CTL_EOL); 6607 if (error) 6608 goto out; 6609 6610 error = sysctl_createv(log, 0, &rxnode, NULL, 6611 CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num", 6612 SYSCTL_DESCR("the number of rx descriptors"), 6613 NULL, 0, &sc->sc_rx_ring_ndescs, 0, CTL_CREATE, CTL_EOL); 6614 if (error) 6615 goto out; 6616 6617 error = sysctl_createv(log, 0, &rxnode, NULL, 6618 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6619 SYSCTL_DESCR("max number of Rx packets" 6620 " to process for interrupt processing"), 6621 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6622 if (error) 6623 goto out; 6624 6625 error = sysctl_createv(log, 0, &rxnode, NULL, 6626 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6627 SYSCTL_DESCR("max number of Rx packets" 6628 " to process for deferred processing"), 6629 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 6630 if (error) 6631 goto out; 6632 6633 error = sysctl_createv(log, 0, &rnode, &txnode, 6634 0, CTLTYPE_NODE, "tx", 6635 SYSCTL_DESCR("ixl information and settings for Tx"), 6636 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6637 if (error) 6638 goto out; 6639 6640 error = sysctl_createv(log, 0, &txnode, NULL, 6641 CTLFLAG_READWRITE, CTLTYPE_INT, "itr", 6642 SYSCTL_DESCR("Interrupt Throttling"), 6643 ixl_sysctl_itr_handler, 0, 6644 (void *)sc, 0, CTL_CREATE, CTL_EOL); 6645 if (error) 6646 goto out; 6647 6648 error = sysctl_createv(log, 0, &txnode, NULL, 6649 CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num", 6650 SYSCTL_DESCR("the number of tx descriptors"), 6651 NULL, 0, &sc->sc_tx_ring_ndescs, 0, CTL_CREATE, CTL_EOL); 6652 if (error) 6653 goto out; 6654 6655 error = sysctl_createv(log, 0, &txnode, NULL, 6656 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6657 SYSCTL_DESCR("max number of Tx packets" 6658 " to process for interrupt processing"), 6659 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6660 if (error) 6661 goto out; 6662 6663 error = sysctl_createv(log, 0, &txnode, NULL, 6664 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6665 SYSCTL_DESCR("max number of Tx packets" 6666 " to process for deferred processing"), 6667 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 6668 if (error) 6669 goto out; 6670 6671 out: 6672 if (error) { 6673 aprint_error_dev(sc->sc_dev, 6674 "unable to create sysctl node\n"); 6675 sysctl_teardown(log); 6676 } 6677 6678 return error; 6679 } 6680 6681 static void 6682 ixl_teardown_sysctls(struct ixl_softc *sc) 6683 { 6684 6685 sysctl_teardown(&sc->sc_sysctllog); 6686 } 6687 6688 static bool 6689 ixl_sysctlnode_is_rx(struct sysctlnode *node) 6690 { 6691 6692 if (strstr(node->sysctl_parent->sysctl_name, "rx") != NULL) 6693 return true; 6694 6695 return false; 6696 } 6697 6698 static int 6699 ixl_sysctl_itr_handler(SYSCTLFN_ARGS) 6700 { 6701 struct sysctlnode node = *rnode; 6702 struct ixl_softc *sc = (struct ixl_softc *)node.sysctl_data; 6703 struct ifnet *ifp = &sc->sc_ec.ec_if; 6704 uint32_t newitr, *itrptr; 6705 int error; 6706 6707 if (ixl_sysctlnode_is_rx(&node)) { 6708 itrptr = &sc->sc_itr_rx; 6709 } else { 6710 itrptr = &sc->sc_itr_tx; 6711 } 6712 6713 newitr = *itrptr; 6714 node.sysctl_data = &newitr; 6715 node.sysctl_size = sizeof(newitr); 6716 6717 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6718 6719 if (error || newp == NULL) 6720 return error; 6721 6722 /* ITRs are applied in ixl_init() for simple implementation */ 6723 if (ISSET(ifp->if_flags, IFF_RUNNING)) 6724 return EBUSY; 6725 6726 if (newitr > 0x07ff) 6727 return EINVAL; 6728 6729 *itrptr = newitr; 6730 6731 return 0; 6732 } 6733 6734 static struct workqueue * 6735 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 6736 { 6737 struct workqueue *wq; 6738 int error; 6739 6740 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 6741 prio, ipl, flags); 6742 6743 if (error) 6744 return NULL; 6745 6746 return wq; 6747 } 6748 6749 static void 6750 ixl_workq_destroy(struct workqueue *wq) 6751 { 6752 6753 workqueue_destroy(wq); 6754 } 6755 6756 static void 6757 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 6758 { 6759 6760 memset(work, 0, sizeof(*work)); 6761 work->ixw_func = func; 6762 work->ixw_arg = arg; 6763 } 6764 6765 static void 6766 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 6767 { 6768 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 6769 return; 6770 6771 kpreempt_disable(); 6772 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 6773 kpreempt_enable(); 6774 } 6775 6776 static void 6777 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 6778 { 6779 6780 workqueue_wait(wq, &work->ixw_cookie); 6781 } 6782 6783 static void 6784 ixl_workq_work(struct work *wk, void *context) 6785 { 6786 struct ixl_work *work; 6787 6788 work = container_of(wk, struct ixl_work, ixw_cookie); 6789 6790 atomic_swap_uint(&work->ixw_added, 0); 6791 work->ixw_func(work->ixw_arg); 6792 } 6793 6794 static int 6795 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 6796 { 6797 struct ixl_aq_desc iaq; 6798 6799 memset(&iaq, 0, sizeof(iaq)); 6800 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 6801 iaq.iaq_param[1] = htole32(reg); 6802 6803 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6804 return ETIMEDOUT; 6805 6806 switch (htole16(iaq.iaq_retval)) { 6807 case IXL_AQ_RC_OK: 6808 /* success */ 6809 break; 6810 case IXL_AQ_RC_EACCES: 6811 return EPERM; 6812 case IXL_AQ_RC_EAGAIN: 6813 return EAGAIN; 6814 default: 6815 return EIO; 6816 } 6817 6818 *rv = htole32(iaq.iaq_param[3]); 6819 return 0; 6820 } 6821 6822 static uint32_t 6823 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 6824 { 6825 uint32_t val; 6826 int rv, retry, retry_limit; 6827 6828 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6829 retry_limit = 5; 6830 } else { 6831 retry_limit = 0; 6832 } 6833 6834 for (retry = 0; retry < retry_limit; retry++) { 6835 rv = ixl_rx_ctl_read(sc, reg, &val); 6836 if (rv == 0) 6837 return val; 6838 else if (rv == EAGAIN) 6839 delaymsec(1); 6840 else 6841 break; 6842 } 6843 6844 val = ixl_rd(sc, reg); 6845 6846 return val; 6847 } 6848 6849 static int 6850 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6851 { 6852 struct ixl_aq_desc iaq; 6853 6854 memset(&iaq, 0, sizeof(iaq)); 6855 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 6856 iaq.iaq_param[1] = htole32(reg); 6857 iaq.iaq_param[3] = htole32(value); 6858 6859 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6860 return ETIMEDOUT; 6861 6862 switch (htole16(iaq.iaq_retval)) { 6863 case IXL_AQ_RC_OK: 6864 /* success */ 6865 break; 6866 case IXL_AQ_RC_EACCES: 6867 return EPERM; 6868 case IXL_AQ_RC_EAGAIN: 6869 return EAGAIN; 6870 default: 6871 return EIO; 6872 } 6873 6874 return 0; 6875 } 6876 6877 static void 6878 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6879 { 6880 int rv, retry, retry_limit; 6881 6882 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6883 retry_limit = 5; 6884 } else { 6885 retry_limit = 0; 6886 } 6887 6888 for (retry = 0; retry < retry_limit; retry++) { 6889 rv = ixl_rx_ctl_write(sc, reg, value); 6890 if (rv == 0) 6891 return; 6892 else if (rv == EAGAIN) 6893 delaymsec(1); 6894 else 6895 break; 6896 } 6897 6898 ixl_wr(sc, reg, value); 6899 } 6900 6901 static int 6902 ixl_nvm_lock(struct ixl_softc *sc, char rw) 6903 { 6904 struct ixl_aq_desc iaq; 6905 struct ixl_aq_req_resource_param *param; 6906 int rv; 6907 6908 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6909 return 0; 6910 6911 memset(&iaq, 0, sizeof(iaq)); 6912 iaq.iaq_opcode = htole16(IXL_AQ_OP_REQUEST_RESOURCE); 6913 6914 param = (struct ixl_aq_req_resource_param *)&iaq.iaq_param; 6915 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6916 if (rw == 'R') { 6917 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_READ); 6918 } else { 6919 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_WRITE); 6920 } 6921 6922 rv = ixl_atq_poll(sc, &iaq, 250); 6923 6924 if (rv != 0) 6925 return ETIMEDOUT; 6926 6927 switch (le16toh(iaq.iaq_retval)) { 6928 case IXL_AQ_RC_OK: 6929 break; 6930 case IXL_AQ_RC_EACCES: 6931 return EACCES; 6932 case IXL_AQ_RC_EBUSY: 6933 return EBUSY; 6934 case IXL_AQ_RC_EPERM: 6935 return EPERM; 6936 } 6937 6938 return 0; 6939 } 6940 6941 static int 6942 ixl_nvm_unlock(struct ixl_softc *sc) 6943 { 6944 struct ixl_aq_desc iaq; 6945 struct ixl_aq_rel_resource_param *param; 6946 int rv; 6947 6948 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6949 return 0; 6950 6951 memset(&iaq, 0, sizeof(iaq)); 6952 iaq.iaq_opcode = htole16(IXL_AQ_OP_RELEASE_RESOURCE); 6953 6954 param = (struct ixl_aq_rel_resource_param *)&iaq.iaq_param; 6955 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6956 6957 rv = ixl_atq_poll(sc, &iaq, 250); 6958 6959 if (rv != 0) 6960 return ETIMEDOUT; 6961 6962 switch (le16toh(iaq.iaq_retval)) { 6963 case IXL_AQ_RC_OK: 6964 break; 6965 default: 6966 return EIO; 6967 } 6968 return 0; 6969 } 6970 6971 static int 6972 ixl_srdone_poll(struct ixl_softc *sc) 6973 { 6974 int wait_count; 6975 uint32_t reg; 6976 6977 for (wait_count = 0; wait_count < IXL_SRRD_SRCTL_ATTEMPTS; 6978 wait_count++) { 6979 reg = ixl_rd(sc, I40E_GLNVM_SRCTL); 6980 if (ISSET(reg, I40E_GLNVM_SRCTL_DONE_MASK)) 6981 break; 6982 6983 delaymsec(5); 6984 } 6985 6986 if (wait_count == IXL_SRRD_SRCTL_ATTEMPTS) 6987 return -1; 6988 6989 return 0; 6990 } 6991 6992 static int 6993 ixl_nvm_read_srctl(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 6994 { 6995 uint32_t reg; 6996 6997 if (ixl_srdone_poll(sc) != 0) 6998 return ETIMEDOUT; 6999 7000 reg = ((uint32_t)offset << I40E_GLNVM_SRCTL_ADDR_SHIFT) | 7001 __BIT(I40E_GLNVM_SRCTL_START_SHIFT); 7002 ixl_wr(sc, I40E_GLNVM_SRCTL, reg); 7003 7004 if (ixl_srdone_poll(sc) != 0) { 7005 aprint_debug("NVM read error: couldn't access " 7006 "Shadow RAM address: 0x%x\n", offset); 7007 return ETIMEDOUT; 7008 } 7009 7010 reg = ixl_rd(sc, I40E_GLNVM_SRDATA); 7011 *data = (uint16_t)__SHIFTOUT(reg, I40E_GLNVM_SRDATA_RDDATA_MASK); 7012 7013 return 0; 7014 } 7015 7016 static int 7017 ixl_nvm_read_aq(struct ixl_softc *sc, uint16_t offset_word, 7018 void *data, size_t len) 7019 { 7020 struct ixl_dmamem *idm; 7021 struct ixl_aq_desc iaq; 7022 struct ixl_aq_nvm_param *param; 7023 uint32_t offset_bytes; 7024 int rv; 7025 7026 idm = &sc->sc_aqbuf; 7027 if (len > IXL_DMA_LEN(idm)) 7028 return ENOMEM; 7029 7030 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 7031 memset(&iaq, 0, sizeof(iaq)); 7032 iaq.iaq_opcode = htole16(IXL_AQ_OP_NVM_READ); 7033 iaq.iaq_flags = htole16(IXL_AQ_BUF | 7034 ((len > I40E_AQ_LARGE_BUF) ? IXL_AQ_LB : 0)); 7035 iaq.iaq_datalen = htole16(len); 7036 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 7037 7038 param = (struct ixl_aq_nvm_param *)iaq.iaq_param; 7039 param->command_flags = IXL_AQ_NVM_LAST_CMD; 7040 param->module_pointer = 0; 7041 param->length = htole16(len); 7042 offset_bytes = (uint32_t)offset_word * 2; 7043 offset_bytes &= 0x00FFFFFF; 7044 param->offset = htole32(offset_bytes); 7045 7046 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 7047 BUS_DMASYNC_PREREAD); 7048 7049 rv = ixl_atq_poll(sc, &iaq, 250); 7050 7051 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 7052 BUS_DMASYNC_POSTREAD); 7053 7054 if (rv != 0) { 7055 return ETIMEDOUT; 7056 } 7057 7058 switch (le16toh(iaq.iaq_retval)) { 7059 case IXL_AQ_RC_OK: 7060 break; 7061 case IXL_AQ_RC_EPERM: 7062 return EPERM; 7063 case IXL_AQ_RC_EINVAL: 7064 return EINVAL; 7065 case IXL_AQ_RC_EBUSY: 7066 return EBUSY; 7067 case IXL_AQ_RC_EIO: 7068 default: 7069 return EIO; 7070 } 7071 7072 memcpy(data, IXL_DMA_KVA(idm), len); 7073 7074 return 0; 7075 } 7076 7077 static int 7078 ixl_rd16_nvm(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 7079 { 7080 int error; 7081 uint16_t buf; 7082 7083 error = ixl_nvm_lock(sc, 'R'); 7084 if (error) 7085 return error; 7086 7087 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMREAD)) { 7088 error = ixl_nvm_read_aq(sc, offset, 7089 &buf, sizeof(buf)); 7090 if (error == 0) 7091 *data = le16toh(buf); 7092 } else { 7093 error = ixl_nvm_read_srctl(sc, offset, &buf); 7094 if (error == 0) 7095 *data = buf; 7096 } 7097 7098 ixl_nvm_unlock(sc); 7099 7100 return error; 7101 } 7102 7103 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 7104 7105 #ifdef _MODULE 7106 #include "ioconf.c" 7107 #endif 7108 7109 #ifdef _MODULE 7110 static void 7111 ixl_parse_modprop(prop_dictionary_t dict) 7112 { 7113 prop_object_t obj; 7114 int64_t val; 7115 uint64_t uval; 7116 7117 if (dict == NULL) 7118 return; 7119 7120 obj = prop_dictionary_get(dict, "nomsix"); 7121 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_BOOL) { 7122 ixl_param_nomsix = prop_bool_true((prop_bool_t)obj); 7123 } 7124 7125 obj = prop_dictionary_get(dict, "stats_interval"); 7126 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7127 val = prop_number_signed_value((prop_number_t)obj); 7128 7129 /* the range has no reason */ 7130 if (100 < val && val < 180000) { 7131 ixl_param_stats_interval = val; 7132 } 7133 } 7134 7135 obj = prop_dictionary_get(dict, "nqps_limit"); 7136 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7137 val = prop_number_signed_value((prop_number_t)obj); 7138 7139 if (val <= INT32_MAX) 7140 ixl_param_nqps_limit = val; 7141 } 7142 7143 obj = prop_dictionary_get(dict, "rx_ndescs"); 7144 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7145 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7146 7147 if (uval > 8) 7148 ixl_param_rx_ndescs = uval; 7149 } 7150 7151 obj = prop_dictionary_get(dict, "tx_ndescs"); 7152 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7153 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7154 7155 if (uval > IXL_TX_PKT_DESCS) 7156 ixl_param_tx_ndescs = uval; 7157 } 7158 7159 } 7160 #endif 7161 7162 static int 7163 if_ixl_modcmd(modcmd_t cmd, void *opaque) 7164 { 7165 int error = 0; 7166 7167 #ifdef _MODULE 7168 switch (cmd) { 7169 case MODULE_CMD_INIT: 7170 ixl_parse_modprop((prop_dictionary_t)opaque); 7171 error = config_init_component(cfdriver_ioconf_if_ixl, 7172 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7173 break; 7174 case MODULE_CMD_FINI: 7175 error = config_fini_component(cfdriver_ioconf_if_ixl, 7176 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7177 break; 7178 default: 7179 error = ENOTTY; 7180 break; 7181 } 7182 #endif 7183 7184 return error; 7185 } 7186
Indexes created Sun Oct 25 00:26:45 UTC 2026