if_ixl.c revision 1.93
1 /* $NetBSD: if_ixl.c,v 1.93 2023/10/11 07:44:53 rin Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 __KERNEL_RCSID(0, "$NetBSD: if_ixl.c,v 1.93 2023/10/11 07:44:53 rin Exp $"); 78 79 #ifdef _KERNEL_OPT 80 #include "opt_net_mpsafe.h" 81 #include "opt_if_ixl.h" 82 #endif 83 84 #include <sys/param.h> 85 #include <sys/types.h> 86 87 #include <sys/bitops.h> 88 #include <sys/cpu.h> 89 #include <sys/device.h> 90 #include <sys/evcnt.h> 91 #include <sys/interrupt.h> 92 #include <sys/kmem.h> 93 #include <sys/module.h> 94 #include <sys/mutex.h> 95 #include <sys/pcq.h> 96 #include <sys/syslog.h> 97 #include <sys/workqueue.h> 98 #include <sys/xcall.h> 99 100 #include <sys/bus.h> 101 102 #include <net/bpf.h> 103 #include <net/if.h> 104 #include <net/if_dl.h> 105 #include <net/if_media.h> 106 #include <net/if_ether.h> 107 #include <net/rss_config.h> 108 109 #include <netinet/tcp.h> /* for struct tcphdr */ 110 #include <netinet/udp.h> /* for struct udphdr */ 111 112 #include <dev/pci/pcivar.h> 113 #include <dev/pci/pcidevs.h> 114 115 #include <dev/pci/if_ixlreg.h> 116 #include <dev/pci/if_ixlvar.h> 117 118 #include <prop/proplib.h> 119 120 struct ixl_softc; /* defined */ 121 122 #define I40E_PF_RESET_WAIT_COUNT 200 123 #define I40E_AQ_LARGE_BUF 512 124 125 /* bitfields for Tx queue mapping in QTX_CTL */ 126 #define I40E_QTX_CTL_VF_QUEUE 0x0 127 #define I40E_QTX_CTL_VM_QUEUE 0x1 128 #define I40E_QTX_CTL_PF_QUEUE 0x2 129 130 #define I40E_QUEUE_TYPE_EOL 0x7ff 131 #define I40E_INTR_NOTX_QUEUE 0 132 133 #define I40E_QUEUE_TYPE_RX 0x0 134 #define I40E_QUEUE_TYPE_TX 0x1 135 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 136 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 137 138 #define I40E_ITR_INDEX_RX 0x0 139 #define I40E_ITR_INDEX_TX 0x1 140 #define I40E_ITR_INDEX_OTHER 0x2 141 #define I40E_ITR_INDEX_NONE 0x3 142 #define IXL_ITR_RX 0x7a /* 4K intrs/sec */ 143 #define IXL_ITR_TX 0x7a /* 4K intrs/sec */ 144 145 #define I40E_INTR_NOTX_QUEUE 0 146 #define I40E_INTR_NOTX_INTR 0 147 #define I40E_INTR_NOTX_RX_QUEUE 0 148 #define I40E_INTR_NOTX_TX_QUEUE 1 149 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 150 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 151 152 #define I40E_HASH_LUT_SIZE_128 0 153 154 #define IXL_ICR0_CRIT_ERR_MASK \ 155 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 156 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 157 I40E_PFINT_ICR0_PE_CRITERR_MASK) 158 159 #define IXL_QUEUE_MAX_XL710 64 160 #define IXL_QUEUE_MAX_X722 128 161 162 #define IXL_TX_PKT_DESCS 8 163 #define IXL_TX_PKT_MAXSIZE (MCLBYTES * IXL_TX_PKT_DESCS) 164 #define IXL_TX_QUEUE_ALIGN 128 165 #define IXL_RX_QUEUE_ALIGN 128 166 167 #define IXL_MCLBYTES (MCLBYTES - ETHER_ALIGN) 168 #define IXL_MTU_ETHERLEN ETHER_HDR_LEN \ 169 + ETHER_CRC_LEN 170 #if 0 171 #define IXL_MAX_MTU (9728 - IXL_MTU_ETHERLEN) 172 #else 173 /* (dbuff * 5) - ETHER_HDR_LEN - ETHER_CRC_LEN */ 174 #define IXL_MAX_MTU (9600 - IXL_MTU_ETHERLEN) 175 #endif 176 #define IXL_MIN_MTU (ETHER_MIN_LEN - ETHER_CRC_LEN) 177 178 #define IXL_PCIREG PCI_MAPREG_START 179 180 #define IXL_ITR0 0x0 181 #define IXL_ITR1 0x1 182 #define IXL_ITR2 0x2 183 #define IXL_NOITR 0x3 184 185 #define IXL_AQ_NUM 256 186 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 187 #define IXL_AQ_ALIGN 64 /* lol */ 188 #define IXL_AQ_BUFLEN 4096 189 190 #define IXL_HMC_ROUNDUP 512 191 #define IXL_HMC_PGSIZE 4096 192 #define IXL_HMC_DVASZ sizeof(uint64_t) 193 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 194 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 195 #define IXL_HMC_PDVALID 1ULL 196 197 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 198 199 #define IXL_SRRD_SRCTL_ATTEMPTS 100000 200 201 struct ixl_aq_regs { 202 bus_size_t atq_tail; 203 bus_size_t atq_head; 204 bus_size_t atq_len; 205 bus_size_t atq_bal; 206 bus_size_t atq_bah; 207 208 bus_size_t arq_tail; 209 bus_size_t arq_head; 210 bus_size_t arq_len; 211 bus_size_t arq_bal; 212 bus_size_t arq_bah; 213 214 uint32_t atq_len_enable; 215 uint32_t atq_tail_mask; 216 uint32_t atq_head_mask; 217 218 uint32_t arq_len_enable; 219 uint32_t arq_tail_mask; 220 uint32_t arq_head_mask; 221 }; 222 223 struct ixl_phy_type { 224 uint64_t phy_type; 225 uint64_t ifm_type; 226 }; 227 228 struct ixl_speed_type { 229 uint8_t dev_speed; 230 uint64_t net_speed; 231 }; 232 233 struct ixl_hmc_entry { 234 uint64_t hmc_base; 235 uint32_t hmc_count; 236 uint64_t hmc_size; 237 }; 238 239 enum ixl_hmc_types { 240 IXL_HMC_LAN_TX = 0, 241 IXL_HMC_LAN_RX, 242 IXL_HMC_FCOE_CTX, 243 IXL_HMC_FCOE_FILTER, 244 IXL_HMC_COUNT 245 }; 246 247 struct ixl_hmc_pack { 248 uint16_t offset; 249 uint16_t width; 250 uint16_t lsb; 251 }; 252 253 /* 254 * these hmc objects have weird sizes and alignments, so these are abstract 255 * representations of them that are nice for c to populate. 256 * 257 * the packing code relies on little-endian values being stored in the fields, 258 * no high bits in the fields being set, and the fields must be packed in the 259 * same order as they are in the ctx structure. 260 */ 261 262 struct ixl_hmc_rxq { 263 uint16_t head; 264 uint8_t cpuid; 265 uint64_t base; 266 #define IXL_HMC_RXQ_BASE_UNIT 128 267 uint16_t qlen; 268 uint16_t dbuff; 269 #define IXL_HMC_RXQ_DBUFF_UNIT 128 270 uint8_t hbuff; 271 #define IXL_HMC_RXQ_HBUFF_UNIT 64 272 uint8_t dtype; 273 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 274 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 275 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 276 uint8_t dsize; 277 #define IXL_HMC_RXQ_DSIZE_16 0 278 #define IXL_HMC_RXQ_DSIZE_32 1 279 uint8_t crcstrip; 280 uint8_t fc_ena; 281 uint8_t l2sel; 282 uint8_t hsplit_0; 283 uint8_t hsplit_1; 284 uint8_t showiv; 285 uint16_t rxmax; 286 uint8_t tphrdesc_ena; 287 uint8_t tphwdesc_ena; 288 uint8_t tphdata_ena; 289 uint8_t tphhead_ena; 290 uint8_t lrxqthresh; 291 uint8_t prefena; 292 }; 293 294 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 295 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 296 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 297 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 298 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 299 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 300 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 301 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 302 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 303 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 304 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 305 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 306 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 307 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 308 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 309 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 310 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 311 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 312 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 313 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 314 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 315 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 316 }; 317 318 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 319 320 struct ixl_hmc_txq { 321 uint16_t head; 322 uint8_t new_context; 323 uint64_t base; 324 #define IXL_HMC_TXQ_BASE_UNIT 128 325 uint8_t fc_ena; 326 uint8_t timesync_ena; 327 uint8_t fd_ena; 328 uint8_t alt_vlan_ena; 329 uint8_t cpuid; 330 uint16_t thead_wb; 331 uint8_t head_wb_ena; 332 #define IXL_HMC_TXQ_DESC_WB 0 333 #define IXL_HMC_TXQ_HEAD_WB 1 334 uint16_t qlen; 335 uint8_t tphrdesc_ena; 336 uint8_t tphrpacket_ena; 337 uint8_t tphwdesc_ena; 338 uint64_t head_wb_addr; 339 uint32_t crc; 340 uint16_t rdylist; 341 uint8_t rdylist_act; 342 }; 343 344 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 345 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 346 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 347 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 348 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 349 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 350 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 351 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 352 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 353 /* line 1 */ 354 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 355 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 356 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 357 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 358 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 359 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 360 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 361 /* line 7 */ 362 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 363 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 364 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 365 }; 366 367 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 368 369 struct ixl_work { 370 struct work ixw_cookie; 371 void (*ixw_func)(void *); 372 void *ixw_arg; 373 unsigned int ixw_added; 374 }; 375 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 376 377 struct ixl_tx_map { 378 struct mbuf *txm_m; 379 bus_dmamap_t txm_map; 380 unsigned int txm_eop; 381 }; 382 383 struct ixl_tx_ring { 384 kmutex_t txr_lock; 385 struct ixl_softc *txr_sc; 386 387 unsigned int txr_prod; 388 unsigned int txr_cons; 389 390 struct ixl_tx_map *txr_maps; 391 struct ixl_dmamem txr_mem; 392 393 bus_size_t txr_tail; 394 unsigned int txr_qid; 395 pcq_t *txr_intrq; 396 void *txr_si; 397 398 struct evcnt txr_defragged; 399 struct evcnt txr_defrag_failed; 400 struct evcnt txr_pcqdrop; 401 struct evcnt txr_transmitdef; 402 struct evcnt txr_intr; 403 struct evcnt txr_defer; 404 }; 405 406 struct ixl_rx_map { 407 struct mbuf *rxm_m; 408 bus_dmamap_t rxm_map; 409 }; 410 411 struct ixl_rx_ring { 412 kmutex_t rxr_lock; 413 414 unsigned int rxr_prod; 415 unsigned int rxr_cons; 416 417 struct ixl_rx_map *rxr_maps; 418 struct ixl_dmamem rxr_mem; 419 420 struct mbuf *rxr_m_head; 421 struct mbuf **rxr_m_tail; 422 423 bus_size_t rxr_tail; 424 unsigned int rxr_qid; 425 426 struct evcnt rxr_mgethdr_failed; 427 struct evcnt rxr_mgetcl_failed; 428 struct evcnt rxr_mbuf_load_failed; 429 struct evcnt rxr_intr; 430 struct evcnt rxr_defer; 431 }; 432 433 struct ixl_queue_pair { 434 struct ixl_softc *qp_sc; 435 struct ixl_tx_ring *qp_txr; 436 struct ixl_rx_ring *qp_rxr; 437 438 char qp_name[16]; 439 440 void *qp_si; 441 struct work qp_work; 442 bool qp_workqueue; 443 }; 444 445 struct ixl_atq { 446 struct ixl_aq_desc iatq_desc; 447 void (*iatq_fn)(struct ixl_softc *, 448 const struct ixl_aq_desc *); 449 bool iatq_inuse; 450 }; 451 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 452 453 struct ixl_product { 454 unsigned int vendor_id; 455 unsigned int product_id; 456 }; 457 458 struct ixl_stats_counters { 459 bool isc_has_offset; 460 struct evcnt isc_crc_errors; 461 uint64_t isc_crc_errors_offset; 462 struct evcnt isc_illegal_bytes; 463 uint64_t isc_illegal_bytes_offset; 464 struct evcnt isc_rx_bytes; 465 uint64_t isc_rx_bytes_offset; 466 struct evcnt isc_rx_discards; 467 uint64_t isc_rx_discards_offset; 468 struct evcnt isc_rx_unicast; 469 uint64_t isc_rx_unicast_offset; 470 struct evcnt isc_rx_multicast; 471 uint64_t isc_rx_multicast_offset; 472 struct evcnt isc_rx_broadcast; 473 uint64_t isc_rx_broadcast_offset; 474 struct evcnt isc_rx_size_64; 475 uint64_t isc_rx_size_64_offset; 476 struct evcnt isc_rx_size_127; 477 uint64_t isc_rx_size_127_offset; 478 struct evcnt isc_rx_size_255; 479 uint64_t isc_rx_size_255_offset; 480 struct evcnt isc_rx_size_511; 481 uint64_t isc_rx_size_511_offset; 482 struct evcnt isc_rx_size_1023; 483 uint64_t isc_rx_size_1023_offset; 484 struct evcnt isc_rx_size_1522; 485 uint64_t isc_rx_size_1522_offset; 486 struct evcnt isc_rx_size_big; 487 uint64_t isc_rx_size_big_offset; 488 struct evcnt isc_rx_undersize; 489 uint64_t isc_rx_undersize_offset; 490 struct evcnt isc_rx_oversize; 491 uint64_t isc_rx_oversize_offset; 492 struct evcnt isc_rx_fragments; 493 uint64_t isc_rx_fragments_offset; 494 struct evcnt isc_rx_jabber; 495 uint64_t isc_rx_jabber_offset; 496 struct evcnt isc_tx_bytes; 497 uint64_t isc_tx_bytes_offset; 498 struct evcnt isc_tx_dropped_link_down; 499 uint64_t isc_tx_dropped_link_down_offset; 500 struct evcnt isc_tx_unicast; 501 uint64_t isc_tx_unicast_offset; 502 struct evcnt isc_tx_multicast; 503 uint64_t isc_tx_multicast_offset; 504 struct evcnt isc_tx_broadcast; 505 uint64_t isc_tx_broadcast_offset; 506 struct evcnt isc_tx_size_64; 507 uint64_t isc_tx_size_64_offset; 508 struct evcnt isc_tx_size_127; 509 uint64_t isc_tx_size_127_offset; 510 struct evcnt isc_tx_size_255; 511 uint64_t isc_tx_size_255_offset; 512 struct evcnt isc_tx_size_511; 513 uint64_t isc_tx_size_511_offset; 514 struct evcnt isc_tx_size_1023; 515 uint64_t isc_tx_size_1023_offset; 516 struct evcnt isc_tx_size_1522; 517 uint64_t isc_tx_size_1522_offset; 518 struct evcnt isc_tx_size_big; 519 uint64_t isc_tx_size_big_offset; 520 struct evcnt isc_mac_local_faults; 521 uint64_t isc_mac_local_faults_offset; 522 struct evcnt isc_mac_remote_faults; 523 uint64_t isc_mac_remote_faults_offset; 524 struct evcnt isc_link_xon_rx; 525 uint64_t isc_link_xon_rx_offset; 526 struct evcnt isc_link_xon_tx; 527 uint64_t isc_link_xon_tx_offset; 528 struct evcnt isc_link_xoff_rx; 529 uint64_t isc_link_xoff_rx_offset; 530 struct evcnt isc_link_xoff_tx; 531 uint64_t isc_link_xoff_tx_offset; 532 struct evcnt isc_vsi_rx_discards; 533 uint64_t isc_vsi_rx_discards_offset; 534 struct evcnt isc_vsi_rx_bytes; 535 uint64_t isc_vsi_rx_bytes_offset; 536 struct evcnt isc_vsi_rx_unicast; 537 uint64_t isc_vsi_rx_unicast_offset; 538 struct evcnt isc_vsi_rx_multicast; 539 uint64_t isc_vsi_rx_multicast_offset; 540 struct evcnt isc_vsi_rx_broadcast; 541 uint64_t isc_vsi_rx_broadcast_offset; 542 struct evcnt isc_vsi_tx_errors; 543 uint64_t isc_vsi_tx_errors_offset; 544 struct evcnt isc_vsi_tx_bytes; 545 uint64_t isc_vsi_tx_bytes_offset; 546 struct evcnt isc_vsi_tx_unicast; 547 uint64_t isc_vsi_tx_unicast_offset; 548 struct evcnt isc_vsi_tx_multicast; 549 uint64_t isc_vsi_tx_multicast_offset; 550 struct evcnt isc_vsi_tx_broadcast; 551 uint64_t isc_vsi_tx_broadcast_offset; 552 }; 553 554 /* 555 * Locking notes: 556 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 557 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 558 * - more than one lock of them cannot be held at once. 559 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 560 * (a spin mutex). 561 * - the lock cannot held with txr_lock or rxr_lock. 562 * + a field named sc_arq_* is not protected by any lock. 563 * - operations for sc_arq_* is done in one context related to 564 * sc_arq_task. 565 * + other fields in ixl_softc is protected by sc_cfg_lock 566 * (an adaptive mutex) 567 * - It must be held before another lock is held, and It can be 568 * released after the other lock is released. 569 * */ 570 571 struct ixl_softc { 572 device_t sc_dev; 573 struct ethercom sc_ec; 574 bool sc_attached; 575 bool sc_dead; 576 uint32_t sc_port; 577 struct sysctllog *sc_sysctllog; 578 struct workqueue *sc_workq; 579 struct workqueue *sc_workq_txrx; 580 int sc_stats_intval; 581 callout_t sc_stats_callout; 582 struct ixl_work sc_stats_task; 583 struct ixl_stats_counters 584 sc_stats_counters; 585 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 586 struct ifmedia sc_media; 587 uint64_t sc_media_status; 588 uint64_t sc_media_active; 589 uint64_t sc_phy_types; 590 uint8_t sc_phy_abilities; 591 uint8_t sc_phy_linkspeed; 592 uint8_t sc_phy_fec_cfg; 593 uint16_t sc_eee_cap; 594 uint32_t sc_eeer_val; 595 uint8_t sc_d3_lpan; 596 kmutex_t sc_cfg_lock; 597 enum i40e_mac_type sc_mac_type; 598 uint32_t sc_rss_table_size; 599 uint32_t sc_rss_table_entry_width; 600 bool sc_txrx_workqueue; 601 u_int sc_tx_process_limit; 602 u_int sc_rx_process_limit; 603 u_int sc_tx_intr_process_limit; 604 u_int sc_rx_intr_process_limit; 605 606 int sc_cur_ec_capenable; 607 608 struct pci_attach_args sc_pa; 609 pci_intr_handle_t *sc_ihp; 610 void **sc_ihs; 611 unsigned int sc_nintrs; 612 613 bus_dma_tag_t sc_dmat; 614 bus_space_tag_t sc_memt; 615 bus_space_handle_t sc_memh; 616 bus_size_t sc_mems; 617 618 uint8_t sc_pf_id; 619 uint16_t sc_uplink_seid; /* le */ 620 uint16_t sc_downlink_seid; /* le */ 621 uint16_t sc_vsi_number; 622 uint16_t sc_vsi_stat_counter_idx; 623 uint16_t sc_seid; 624 unsigned int sc_base_queue; 625 626 pci_intr_type_t sc_intrtype; 627 unsigned int sc_msix_vector_queue; 628 629 struct ixl_dmamem sc_scratch; 630 struct ixl_dmamem sc_aqbuf; 631 632 const struct ixl_aq_regs * 633 sc_aq_regs; 634 uint32_t sc_aq_flags; 635 #define IXL_SC_AQ_FLAG_RXCTL __BIT(0) 636 #define IXL_SC_AQ_FLAG_NVMLOCK __BIT(1) 637 #define IXL_SC_AQ_FLAG_NVMREAD __BIT(2) 638 #define IXL_SC_AQ_FLAG_RSS __BIT(3) 639 640 kmutex_t sc_atq_lock; 641 kcondvar_t sc_atq_cv; 642 struct ixl_dmamem sc_atq; 643 unsigned int sc_atq_prod; 644 unsigned int sc_atq_cons; 645 646 struct ixl_dmamem sc_arq; 647 struct ixl_work sc_arq_task; 648 struct ixl_aq_bufs sc_arq_idle; 649 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 650 unsigned int sc_arq_prod; 651 unsigned int sc_arq_cons; 652 653 struct ixl_work sc_link_state_task; 654 struct ixl_work sc_link_state_done_task; 655 struct ixl_atq sc_link_state_atq; 656 657 struct ixl_dmamem sc_hmc_sd; 658 struct ixl_dmamem sc_hmc_pd; 659 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 660 661 struct if_percpuq *sc_ipq; 662 unsigned int sc_tx_ring_ndescs; 663 unsigned int sc_rx_ring_ndescs; 664 unsigned int sc_nqueue_pairs; 665 unsigned int sc_nqueue_pairs_max; 666 unsigned int sc_nqueue_pairs_device; 667 struct ixl_queue_pair *sc_qps; 668 uint32_t sc_itr_rx; 669 uint32_t sc_itr_tx; 670 671 struct evcnt sc_event_atq; 672 struct evcnt sc_event_link; 673 struct evcnt sc_event_ecc_err; 674 struct evcnt sc_event_pci_exception; 675 struct evcnt sc_event_crit_err; 676 }; 677 678 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 679 #define IXL_TX_PROCESS_LIMIT 256 680 #define IXL_RX_PROCESS_LIMIT 256 681 #define IXL_TX_INTR_PROCESS_LIMIT 256 682 #define IXL_RX_INTR_PROCESS_LIMIT 0U 683 684 #define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx | \ 685 IFCAP_CSUM_TCPv4_Rx | \ 686 IFCAP_CSUM_UDPv4_Rx | \ 687 IFCAP_CSUM_TCPv6_Rx | \ 688 IFCAP_CSUM_UDPv6_Rx) 689 #define IXL_IFCAP_TXCSUM (IFCAP_CSUM_IPv4_Tx | \ 690 IFCAP_CSUM_TCPv4_Tx | \ 691 IFCAP_CSUM_UDPv4_Tx | \ 692 IFCAP_CSUM_TCPv6_Tx | \ 693 IFCAP_CSUM_UDPv6_Tx) 694 #define IXL_CSUM_ALL_OFFLOAD (M_CSUM_IPv4 | \ 695 M_CSUM_TCPv4 | M_CSUM_TCPv6 | \ 696 M_CSUM_UDPv4 | M_CSUM_UDPv6) 697 698 #define delaymsec(_x) DELAY(1000 * (_x)) 699 #ifdef IXL_DEBUG 700 #define DDPRINTF(sc, fmt, args...) \ 701 do { \ 702 if ((sc) != NULL) { \ 703 device_printf( \ 704 ((struct ixl_softc *)(sc))->sc_dev, \ 705 ""); \ 706 } \ 707 printf("%s:\t" fmt, __func__, ##args); \ 708 } while (0) 709 #else 710 #define DDPRINTF(sc, fmt, args...) __nothing 711 #endif 712 #ifndef IXL_STATS_INTERVAL_MSEC 713 #define IXL_STATS_INTERVAL_MSEC 10000 714 #endif 715 #ifndef IXL_QUEUE_NUM 716 #define IXL_QUEUE_NUM 0 717 #endif 718 719 enum ixl_link_flags { 720 IXL_LINK_NOFLAGS = 0, 721 IXL_LINK_FLAG_WAITDONE = __BIT(0), 722 }; 723 724 static bool ixl_param_nomsix = false; 725 static int ixl_param_stats_interval = IXL_STATS_INTERVAL_MSEC; 726 static int ixl_param_nqps_limit = IXL_QUEUE_NUM; 727 static unsigned int ixl_param_tx_ndescs = 512; 728 static unsigned int ixl_param_rx_ndescs = 512; 729 730 static enum i40e_mac_type 731 ixl_mactype(pci_product_id_t); 732 static void ixl_pci_csr_setup(pci_chipset_tag_t, pcitag_t); 733 static void ixl_clear_hw(struct ixl_softc *); 734 static int ixl_pf_reset(struct ixl_softc *); 735 736 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 737 bus_size_t, bus_size_t); 738 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 739 740 static int ixl_arq_fill(struct ixl_softc *); 741 static void ixl_arq_unfill(struct ixl_softc *); 742 743 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 744 unsigned int); 745 static void ixl_atq_set(struct ixl_atq *, 746 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 747 static void ixl_wakeup(struct ixl_softc *, const struct ixl_aq_desc *); 748 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 749 static void ixl_atq_done(struct ixl_softc *); 750 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 751 static int ixl_atq_exec_locked(struct ixl_softc *, struct ixl_atq *); 752 static int ixl_get_version(struct ixl_softc *); 753 static int ixl_get_nvm_version(struct ixl_softc *); 754 static int ixl_get_hw_capabilities(struct ixl_softc *); 755 static int ixl_pxe_clear(struct ixl_softc *); 756 static int ixl_lldp_shut(struct ixl_softc *); 757 static int ixl_get_mac(struct ixl_softc *); 758 static int ixl_get_switch_config(struct ixl_softc *); 759 static int ixl_phy_mask_ints(struct ixl_softc *); 760 static int ixl_get_phy_info(struct ixl_softc *); 761 static int ixl_set_phy_config(struct ixl_softc *, uint8_t, uint8_t, bool); 762 static int ixl_set_phy_autoselect(struct ixl_softc *); 763 static int ixl_restart_an(struct ixl_softc *); 764 static int ixl_hmc(struct ixl_softc *); 765 static void ixl_hmc_free(struct ixl_softc *); 766 static int ixl_get_vsi(struct ixl_softc *); 767 static int ixl_set_vsi(struct ixl_softc *); 768 static void ixl_set_filter_control(struct ixl_softc *); 769 static int ixl_get_link_status(struct ixl_softc *, enum ixl_link_flags); 770 static void ixl_get_link_status_work(void *); 771 static int ixl_get_link_status_poll(struct ixl_softc *, int *); 772 static void ixl_get_link_status_done(struct ixl_softc *, 773 const struct ixl_aq_desc *); 774 static void ixl_get_link_status_done_work(void *); 775 static int ixl_set_link_status_locked(struct ixl_softc *, 776 const struct ixl_aq_desc *); 777 static uint64_t ixl_search_link_speed(uint8_t); 778 static uint8_t ixl_search_baudrate(uint64_t); 779 static void ixl_config_rss(struct ixl_softc *); 780 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 781 uint16_t, uint16_t); 782 static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *, 783 uint16_t, uint16_t); 784 static void ixl_arq(void *); 785 static void ixl_hmc_pack(void *, const void *, 786 const struct ixl_hmc_pack *, unsigned int); 787 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 788 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 789 static int ixl_rd16_nvm(struct ixl_softc *, uint16_t, uint16_t *); 790 791 static int ixl_match(device_t, cfdata_t, void *); 792 static void ixl_attach(device_t, device_t, void *); 793 static int ixl_detach(device_t, int); 794 795 static void ixl_media_add(struct ixl_softc *); 796 static int ixl_media_change(struct ifnet *); 797 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 798 static int ixl_ioctl(struct ifnet *, u_long, void *); 799 static void ixl_start(struct ifnet *); 800 static int ixl_transmit(struct ifnet *, struct mbuf *); 801 static void ixl_deferred_transmit(void *); 802 static int ixl_intr(void *); 803 static int ixl_queue_intr(void *); 804 static int ixl_other_intr(void *); 805 static void ixl_handle_queue(void *); 806 static void ixl_handle_queue_wk(struct work *, void *); 807 static void ixl_sched_handle_queue(struct ixl_softc *, 808 struct ixl_queue_pair *); 809 static int ixl_init(struct ifnet *); 810 static int ixl_init_locked(struct ixl_softc *); 811 static void ixl_stop(struct ifnet *, int); 812 static void ixl_stop_locked(struct ixl_softc *); 813 static int ixl_iff(struct ixl_softc *); 814 static int ixl_ifflags_cb(struct ethercom *); 815 static int ixl_setup_interrupts(struct ixl_softc *); 816 static int ixl_establish_intx(struct ixl_softc *); 817 static int ixl_establish_msix(struct ixl_softc *); 818 static void ixl_enable_queue_intr(struct ixl_softc *, 819 struct ixl_queue_pair *); 820 static void ixl_disable_queue_intr(struct ixl_softc *, 821 struct ixl_queue_pair *); 822 static void ixl_enable_other_intr(struct ixl_softc *); 823 static void ixl_disable_other_intr(struct ixl_softc *); 824 static void ixl_config_queue_intr(struct ixl_softc *); 825 static void ixl_config_other_intr(struct ixl_softc *); 826 827 static struct ixl_tx_ring * 828 ixl_txr_alloc(struct ixl_softc *, unsigned int); 829 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 830 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 831 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 832 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 833 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 834 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 835 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 836 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 837 838 static struct ixl_rx_ring * 839 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 840 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 841 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 842 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 843 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 844 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 845 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 846 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 847 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 848 849 static struct workqueue * 850 ixl_workq_create(const char *, pri_t, int, int); 851 static void ixl_workq_destroy(struct workqueue *); 852 static int ixl_workqs_teardown(device_t); 853 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 854 static void ixl_work_add(struct workqueue *, struct ixl_work *); 855 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 856 static void ixl_workq_work(struct work *, void *); 857 static const struct ixl_product * 858 ixl_lookup(const struct pci_attach_args *pa); 859 static void ixl_link_state_update(struct ixl_softc *, 860 const struct ixl_aq_desc *); 861 static int ixl_vlan_cb(struct ethercom *, uint16_t, bool); 862 static int ixl_setup_vlan_hwfilter(struct ixl_softc *); 863 static void ixl_teardown_vlan_hwfilter(struct ixl_softc *); 864 static int ixl_update_macvlan(struct ixl_softc *); 865 static int ixl_setup_interrupts(struct ixl_softc *); 866 static void ixl_teardown_interrupts(struct ixl_softc *); 867 static int ixl_setup_stats(struct ixl_softc *); 868 static void ixl_teardown_stats(struct ixl_softc *); 869 static void ixl_stats_callout(void *); 870 static void ixl_stats_update(void *); 871 static int ixl_setup_sysctls(struct ixl_softc *); 872 static void ixl_teardown_sysctls(struct ixl_softc *); 873 static int ixl_sysctl_itr_handler(SYSCTLFN_PROTO); 874 static int ixl_queue_pairs_alloc(struct ixl_softc *); 875 static void ixl_queue_pairs_free(struct ixl_softc *); 876 877 static const struct ixl_phy_type ixl_phy_type_map[] = { 878 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 879 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 880 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 881 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 882 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 883 { 1ULL << IXL_PHY_TYPE_XAUI | 884 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 885 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 886 { 1ULL << IXL_PHY_TYPE_XLAUI | 887 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 888 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 889 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 890 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 891 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 892 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 893 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 894 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 895 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 896 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 897 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 898 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 899 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 900 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 901 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 902 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 903 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 904 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 905 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 906 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 907 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 908 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 909 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 910 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 911 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_ACC }, 912 { 1ULL << IXL_PHY_TYPE_2500BASE_T_1, IFM_2500_T }, 913 { 1ULL << IXL_PHY_TYPE_5000BASE_T_1, IFM_5000_T }, 914 { 1ULL << IXL_PHY_TYPE_2500BASE_T_2, IFM_2500_T }, 915 { 1ULL << IXL_PHY_TYPE_5000BASE_T_2, IFM_5000_T }, 916 }; 917 918 static const struct ixl_speed_type ixl_speed_type_map[] = { 919 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 920 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 921 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 922 { IXL_AQ_LINK_SPEED_5000MB, IF_Mbps(5000) }, 923 { IXL_AQ_LINK_SPEED_2500MB, IF_Mbps(2500) }, 924 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 925 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 926 }; 927 928 static const struct ixl_aq_regs ixl_pf_aq_regs = { 929 .atq_tail = I40E_PF_ATQT, 930 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 931 .atq_head = I40E_PF_ATQH, 932 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 933 .atq_len = I40E_PF_ATQLEN, 934 .atq_bal = I40E_PF_ATQBAL, 935 .atq_bah = I40E_PF_ATQBAH, 936 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 937 938 .arq_tail = I40E_PF_ARQT, 939 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 940 .arq_head = I40E_PF_ARQH, 941 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 942 .arq_len = I40E_PF_ARQLEN, 943 .arq_bal = I40E_PF_ARQBAL, 944 .arq_bah = I40E_PF_ARQBAH, 945 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 946 }; 947 948 #define ixl_rd(_s, _r) \ 949 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 950 #define ixl_wr(_s, _r, _v) \ 951 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 952 #define ixl_barrier(_s, _r, _l, _o) \ 953 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 954 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 955 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 956 957 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 958 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 959 DVF_DETACH_SHUTDOWN); 960 961 static const struct ixl_product ixl_products[] = { 962 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 963 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 964 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 965 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 966 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 967 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 968 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_1 }, 969 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_2 }, 970 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 971 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 972 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 973 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 974 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 975 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 976 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 977 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 978 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 979 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 980 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 981 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_SFP }, 982 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_BP }, 983 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_V710_5G_T}, 984 /* required last entry */ 985 {0, 0} 986 }; 987 988 static const struct ixl_product * 989 ixl_lookup(const struct pci_attach_args *pa) 990 { 991 const struct ixl_product *ixlp; 992 993 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 994 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 995 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 996 return ixlp; 997 } 998 999 return NULL; 1000 } 1001 1002 static void 1003 ixl_intr_barrier(void) 1004 { 1005 1006 /* wait for finish of all handler */ 1007 xc_barrier(0); 1008 } 1009 1010 static int 1011 ixl_match(device_t parent, cfdata_t match, void *aux) 1012 { 1013 const struct pci_attach_args *pa = aux; 1014 1015 return (ixl_lookup(pa) != NULL) ? 1 : 0; 1016 } 1017 1018 static void 1019 ixl_attach(device_t parent, device_t self, void *aux) 1020 { 1021 struct ixl_softc *sc; 1022 struct pci_attach_args *pa = aux; 1023 struct ifnet *ifp; 1024 pcireg_t memtype; 1025 uint32_t firstq, port, ari, func; 1026 char xnamebuf[32]; 1027 int tries, rv, link; 1028 1029 sc = device_private(self); 1030 sc->sc_dev = self; 1031 ifp = &sc->sc_ec.ec_if; 1032 1033 sc->sc_pa = *pa; 1034 sc->sc_dmat = (pci_dma64_available(pa)) ? 1035 pa->pa_dmat64 : pa->pa_dmat; 1036 sc->sc_aq_regs = &ixl_pf_aq_regs; 1037 1038 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(pa->pa_id)); 1039 1040 ixl_pci_csr_setup(pa->pa_pc, pa->pa_tag); 1041 1042 pci_aprint_devinfo(pa, "Ethernet controller"); 1043 1044 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 1045 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 1046 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 1047 aprint_error(": unable to map registers\n"); 1048 return; 1049 } 1050 1051 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 1052 1053 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 1054 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 1055 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 1056 sc->sc_base_queue = firstq; 1057 1058 ixl_clear_hw(sc); 1059 if (ixl_pf_reset(sc) == -1) { 1060 /* error printed by ixl pf_reset */ 1061 goto unmap; 1062 } 1063 1064 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 1065 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 1066 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 1067 sc->sc_port = port; 1068 aprint_normal_dev(self, "port %u", sc->sc_port); 1069 1070 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 1071 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 1072 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 1073 1074 func = ixl_rd(sc, I40E_PF_FUNC_RID); 1075 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 1076 1077 /* initialise the adminq */ 1078 1079 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 1080 1081 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 1082 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1083 aprint_error("\n" "%s: unable to allocate atq\n", 1084 device_xname(self)); 1085 goto unmap; 1086 } 1087 1088 SIMPLEQ_INIT(&sc->sc_arq_idle); 1089 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 1090 sc->sc_arq_cons = 0; 1091 sc->sc_arq_prod = 0; 1092 1093 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 1094 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1095 aprint_error("\n" "%s: unable to allocate arq\n", 1096 device_xname(self)); 1097 goto free_atq; 1098 } 1099 1100 if (!ixl_arq_fill(sc)) { 1101 aprint_error("\n" "%s: unable to fill arq descriptors\n", 1102 device_xname(self)); 1103 goto free_arq; 1104 } 1105 1106 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1107 0, IXL_DMA_LEN(&sc->sc_atq), 1108 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1109 1110 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1111 0, IXL_DMA_LEN(&sc->sc_arq), 1112 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1113 1114 for (tries = 0; tries < 10; tries++) { 1115 sc->sc_atq_cons = 0; 1116 sc->sc_atq_prod = 0; 1117 1118 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1119 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1120 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1121 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1122 1123 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1124 1125 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1126 ixl_dmamem_lo(&sc->sc_atq)); 1127 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1128 ixl_dmamem_hi(&sc->sc_atq)); 1129 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1130 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1131 1132 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1133 ixl_dmamem_lo(&sc->sc_arq)); 1134 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1135 ixl_dmamem_hi(&sc->sc_arq)); 1136 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1137 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1138 1139 rv = ixl_get_version(sc); 1140 if (rv == 0) 1141 break; 1142 if (rv != ETIMEDOUT) { 1143 aprint_error(", unable to get firmware version\n"); 1144 goto shutdown; 1145 } 1146 1147 delaymsec(100); 1148 } 1149 1150 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1151 1152 if (ixl_dmamem_alloc(sc, &sc->sc_aqbuf, IXL_AQ_BUFLEN, 0) != 0) { 1153 aprint_error_dev(self, ", unable to allocate nvm buffer\n"); 1154 goto shutdown; 1155 } 1156 1157 ixl_get_nvm_version(sc); 1158 1159 if (sc->sc_mac_type == I40E_MAC_X722) 1160 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_X722; 1161 else 1162 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_XL710; 1163 1164 rv = ixl_get_hw_capabilities(sc); 1165 if (rv != 0) { 1166 aprint_error(", GET HW CAPABILITIES %s\n", 1167 rv == ETIMEDOUT ? "timeout" : "error"); 1168 goto free_aqbuf; 1169 } 1170 1171 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1172 if (ixl_param_nqps_limit > 0) { 1173 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_max, 1174 ixl_param_nqps_limit); 1175 } 1176 1177 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1178 sc->sc_tx_ring_ndescs = ixl_param_tx_ndescs; 1179 sc->sc_rx_ring_ndescs = ixl_param_rx_ndescs; 1180 1181 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1182 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1183 KASSERT(sc->sc_rx_ring_ndescs == 1184 (1U << (fls32(sc->sc_rx_ring_ndescs) - 1))); 1185 KASSERT(sc->sc_tx_ring_ndescs == 1186 (1U << (fls32(sc->sc_tx_ring_ndescs) - 1))); 1187 1188 if (ixl_get_mac(sc) != 0) { 1189 /* error printed by ixl_get_mac */ 1190 goto free_aqbuf; 1191 } 1192 1193 aprint_normal("\n"); 1194 aprint_naive("\n"); 1195 1196 aprint_normal_dev(self, "Ethernet address %s\n", 1197 ether_sprintf(sc->sc_enaddr)); 1198 1199 rv = ixl_pxe_clear(sc); 1200 if (rv != 0) { 1201 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1202 rv == ETIMEDOUT ? "timeout" : "error"); 1203 } 1204 1205 ixl_set_filter_control(sc); 1206 1207 if (ixl_hmc(sc) != 0) { 1208 /* error printed by ixl_hmc */ 1209 goto free_aqbuf; 1210 } 1211 1212 if (ixl_lldp_shut(sc) != 0) { 1213 /* error printed by ixl_lldp_shut */ 1214 goto free_hmc; 1215 } 1216 1217 if (ixl_phy_mask_ints(sc) != 0) { 1218 /* error printed by ixl_phy_mask_ints */ 1219 goto free_hmc; 1220 } 1221 1222 if (ixl_restart_an(sc) != 0) { 1223 /* error printed by ixl_restart_an */ 1224 goto free_hmc; 1225 } 1226 1227 if (ixl_get_switch_config(sc) != 0) { 1228 /* error printed by ixl_get_switch_config */ 1229 goto free_hmc; 1230 } 1231 1232 rv = ixl_get_link_status_poll(sc, NULL); 1233 if (rv != 0) { 1234 aprint_error_dev(self, "GET LINK STATUS %s\n", 1235 rv == ETIMEDOUT ? "timeout" : "error"); 1236 goto free_hmc; 1237 } 1238 1239 /* 1240 * The FW often returns EIO in "Get PHY Abilities" command 1241 * if there is no delay 1242 */ 1243 DELAY(500); 1244 if (ixl_get_phy_info(sc) != 0) { 1245 /* error printed by ixl_get_phy_info */ 1246 goto free_hmc; 1247 } 1248 1249 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1250 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1251 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1252 goto free_hmc; 1253 } 1254 1255 rv = ixl_get_vsi(sc); 1256 if (rv != 0) { 1257 aprint_error_dev(self, "GET VSI %s %d\n", 1258 rv == ETIMEDOUT ? "timeout" : "error", rv); 1259 goto free_scratch; 1260 } 1261 1262 rv = ixl_set_vsi(sc); 1263 if (rv != 0) { 1264 aprint_error_dev(self, "UPDATE VSI error %s %d\n", 1265 rv == ETIMEDOUT ? "timeout" : "error", rv); 1266 goto free_scratch; 1267 } 1268 1269 if (ixl_queue_pairs_alloc(sc) != 0) { 1270 /* error printed by ixl_queue_pairs_alloc */ 1271 goto free_scratch; 1272 } 1273 1274 if (ixl_setup_interrupts(sc) != 0) { 1275 /* error printed by ixl_setup_interrupts */ 1276 goto free_queue_pairs; 1277 } 1278 1279 if (ixl_setup_stats(sc) != 0) { 1280 aprint_error_dev(self, "failed to setup event counters\n"); 1281 goto teardown_intrs; 1282 } 1283 1284 if (ixl_setup_sysctls(sc) != 0) { 1285 /* error printed by ixl_setup_sysctls */ 1286 goto teardown_stats; 1287 } 1288 1289 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1290 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1291 IPL_NET, WQ_MPSAFE); 1292 if (sc->sc_workq == NULL) 1293 goto teardown_sysctls; 1294 1295 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1296 rv = workqueue_create(&sc->sc_workq_txrx, xnamebuf, ixl_handle_queue_wk, 1297 sc, IXL_WORKQUEUE_PRI, IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1298 if (rv != 0) { 1299 sc->sc_workq_txrx = NULL; 1300 goto teardown_wqs; 1301 } 1302 1303 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1304 cv_init(&sc->sc_atq_cv, xnamebuf); 1305 1306 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1307 1308 ifp->if_softc = sc; 1309 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1310 ifp->if_extflags = IFEF_MPSAFE; 1311 ifp->if_ioctl = ixl_ioctl; 1312 ifp->if_start = ixl_start; 1313 ifp->if_transmit = ixl_transmit; 1314 ifp->if_init = ixl_init; 1315 ifp->if_stop = ixl_stop; 1316 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1317 IFQ_SET_READY(&ifp->if_snd); 1318 ifp->if_capabilities |= IXL_IFCAP_RXCSUM; 1319 ifp->if_capabilities |= IXL_IFCAP_TXCSUM; 1320 #if 0 1321 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1322 #endif 1323 ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb); 1324 sc->sc_ec.ec_capabilities |= ETHERCAP_JUMBO_MTU; 1325 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1326 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1327 1328 sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities; 1329 /* Disable VLAN_HWFILTER by default */ 1330 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1331 1332 sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable; 1333 1334 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1335 ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, ixl_media_change, 1336 ixl_media_status, &sc->sc_cfg_lock); 1337 1338 ixl_media_add(sc); 1339 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1340 if (ISSET(sc->sc_phy_abilities, 1341 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1342 ifmedia_add(&sc->sc_media, 1343 IFM_ETHER | IFM_AUTO | IFM_FLOW, 0, NULL); 1344 } 1345 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_NONE, 0, NULL); 1346 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1347 1348 if_initialize(ifp); 1349 1350 sc->sc_ipq = if_percpuq_create(ifp); 1351 if_deferred_start_init(ifp, NULL); 1352 ether_ifattach(ifp, sc->sc_enaddr); 1353 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1354 1355 rv = ixl_get_link_status_poll(sc, &link); 1356 if (rv != 0) 1357 link = LINK_STATE_UNKNOWN; 1358 if_link_state_change(ifp, link); 1359 1360 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 1361 ixl_work_set(&sc->sc_link_state_task, 1362 ixl_get_link_status_work, sc); 1363 ixl_work_set(&sc->sc_link_state_done_task, 1364 ixl_get_link_status_done_work, sc); 1365 1366 ixl_config_other_intr(sc); 1367 ixl_enable_other_intr(sc); 1368 1369 ixl_set_phy_autoselect(sc); 1370 1371 /* remove default mac filter and replace it so we can see vlans */ 1372 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 1373 if (rv != ENOENT) { 1374 aprint_debug_dev(self, 1375 "unable to remove macvlan %u\n", rv); 1376 } 1377 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1378 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1379 if (rv != ENOENT) { 1380 aprint_debug_dev(self, 1381 "unable to remove macvlan, ignore vlan %u\n", rv); 1382 } 1383 1384 if (ixl_update_macvlan(sc) != 0) { 1385 aprint_debug_dev(self, 1386 "couldn't enable vlan hardware filter\n"); 1387 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1388 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1389 } 1390 1391 sc->sc_txrx_workqueue = true; 1392 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1393 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1394 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1395 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1396 1397 ixl_stats_update(sc); 1398 sc->sc_stats_counters.isc_has_offset = true; 1399 1400 if (pmf_device_register(self, NULL, NULL) != true) 1401 aprint_debug_dev(self, "couldn't establish power handler\n"); 1402 sc->sc_itr_rx = IXL_ITR_RX; 1403 sc->sc_itr_tx = IXL_ITR_TX; 1404 sc->sc_attached = true; 1405 if_register(ifp); 1406 1407 return; 1408 1409 teardown_wqs: 1410 config_finalize_register(self, ixl_workqs_teardown); 1411 teardown_sysctls: 1412 ixl_teardown_sysctls(sc); 1413 teardown_stats: 1414 ixl_teardown_stats(sc); 1415 teardown_intrs: 1416 ixl_teardown_interrupts(sc); 1417 free_queue_pairs: 1418 ixl_queue_pairs_free(sc); 1419 free_scratch: 1420 ixl_dmamem_free(sc, &sc->sc_scratch); 1421 free_hmc: 1422 ixl_hmc_free(sc); 1423 free_aqbuf: 1424 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1425 shutdown: 1426 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1427 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1428 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1429 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1430 1431 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1432 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1433 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1434 1435 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1436 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1437 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1438 1439 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1440 0, IXL_DMA_LEN(&sc->sc_arq), 1441 BUS_DMASYNC_POSTREAD); 1442 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1443 0, IXL_DMA_LEN(&sc->sc_atq), 1444 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1445 1446 ixl_arq_unfill(sc); 1447 free_arq: 1448 ixl_dmamem_free(sc, &sc->sc_arq); 1449 free_atq: 1450 ixl_dmamem_free(sc, &sc->sc_atq); 1451 unmap: 1452 mutex_destroy(&sc->sc_atq_lock); 1453 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1454 mutex_destroy(&sc->sc_cfg_lock); 1455 sc->sc_mems = 0; 1456 1457 sc->sc_attached = false; 1458 } 1459 1460 static int 1461 ixl_detach(device_t self, int flags) 1462 { 1463 struct ixl_softc *sc = device_private(self); 1464 struct ifnet *ifp = &sc->sc_ec.ec_if; 1465 1466 if (!sc->sc_attached) 1467 return 0; 1468 1469 ixl_stop(ifp, 1); 1470 1471 callout_halt(&sc->sc_stats_callout, NULL); 1472 ixl_work_wait(sc->sc_workq, &sc->sc_stats_task); 1473 1474 /* detach the I/F before stop adminq due to callbacks */ 1475 ether_ifdetach(ifp); 1476 if_detach(ifp); 1477 ifmedia_fini(&sc->sc_media); 1478 if_percpuq_destroy(sc->sc_ipq); 1479 1480 ixl_disable_other_intr(sc); 1481 ixl_intr_barrier(); 1482 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1483 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1484 1485 if (sc->sc_workq != NULL) { 1486 ixl_workq_destroy(sc->sc_workq); 1487 sc->sc_workq = NULL; 1488 } 1489 1490 if (sc->sc_workq_txrx != NULL) { 1491 workqueue_destroy(sc->sc_workq_txrx); 1492 sc->sc_workq_txrx = NULL; 1493 } 1494 1495 ixl_teardown_interrupts(sc); 1496 ixl_teardown_stats(sc); 1497 ixl_teardown_sysctls(sc); 1498 1499 ixl_queue_pairs_free(sc); 1500 1501 ixl_dmamem_free(sc, &sc->sc_scratch); 1502 ixl_hmc_free(sc); 1503 1504 /* shutdown */ 1505 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1506 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1507 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1508 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1509 1510 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1511 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1512 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1513 1514 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1515 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1516 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1517 1518 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1519 0, IXL_DMA_LEN(&sc->sc_arq), 1520 BUS_DMASYNC_POSTREAD); 1521 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1522 0, IXL_DMA_LEN(&sc->sc_atq), 1523 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1524 1525 ixl_arq_unfill(sc); 1526 1527 ixl_dmamem_free(sc, &sc->sc_arq); 1528 ixl_dmamem_free(sc, &sc->sc_atq); 1529 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1530 1531 cv_destroy(&sc->sc_atq_cv); 1532 mutex_destroy(&sc->sc_atq_lock); 1533 1534 if (sc->sc_mems != 0) { 1535 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1536 sc->sc_mems = 0; 1537 } 1538 1539 mutex_destroy(&sc->sc_cfg_lock); 1540 1541 return 0; 1542 } 1543 1544 static int 1545 ixl_workqs_teardown(device_t self) 1546 { 1547 struct ixl_softc *sc = device_private(self); 1548 1549 if (sc->sc_workq != NULL) { 1550 ixl_workq_destroy(sc->sc_workq); 1551 sc->sc_workq = NULL; 1552 } 1553 1554 if (sc->sc_workq_txrx != NULL) { 1555 workqueue_destroy(sc->sc_workq_txrx); 1556 sc->sc_workq_txrx = NULL; 1557 } 1558 1559 return 0; 1560 } 1561 1562 static int 1563 ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 1564 { 1565 struct ifnet *ifp = &ec->ec_if; 1566 struct ixl_softc *sc = ifp->if_softc; 1567 int rv; 1568 1569 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1570 return 0; 1571 } 1572 1573 if (set) { 1574 rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid, 1575 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1576 if (rv == 0) { 1577 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 1578 vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1579 } 1580 } else { 1581 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid, 1582 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1583 (void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid, 1584 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1585 } 1586 1587 return rv; 1588 } 1589 1590 static void 1591 ixl_media_add(struct ixl_softc *sc) 1592 { 1593 struct ifmedia *ifm = &sc->sc_media; 1594 const struct ixl_phy_type *itype; 1595 unsigned int i; 1596 bool flow; 1597 1598 if (ISSET(sc->sc_phy_abilities, 1599 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1600 flow = true; 1601 } else { 1602 flow = false; 1603 } 1604 1605 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1606 itype = &ixl_phy_type_map[i]; 1607 1608 if (ISSET(sc->sc_phy_types, itype->phy_type)) { 1609 ifmedia_add(ifm, 1610 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1611 1612 if (flow) { 1613 ifmedia_add(ifm, 1614 IFM_ETHER | IFM_FDX | IFM_FLOW | 1615 itype->ifm_type, 0, NULL); 1616 } 1617 1618 if (itype->ifm_type != IFM_100_TX) 1619 continue; 1620 1621 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1622 0, NULL); 1623 if (flow) { 1624 ifmedia_add(ifm, 1625 IFM_ETHER | IFM_FLOW | itype->ifm_type, 1626 0, NULL); 1627 } 1628 } 1629 } 1630 } 1631 1632 static void 1633 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1634 { 1635 struct ixl_softc *sc = ifp->if_softc; 1636 1637 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1638 1639 ifmr->ifm_status = sc->sc_media_status; 1640 ifmr->ifm_active = sc->sc_media_active; 1641 } 1642 1643 static int 1644 ixl_media_change(struct ifnet *ifp) 1645 { 1646 struct ixl_softc *sc = ifp->if_softc; 1647 struct ifmedia *ifm = &sc->sc_media; 1648 uint64_t ifm_active = sc->sc_media_active; 1649 uint8_t link_speed, abilities; 1650 1651 switch (IFM_SUBTYPE(ifm_active)) { 1652 case IFM_1000_SGMII: 1653 case IFM_1000_KX: 1654 case IFM_10G_KX4: 1655 case IFM_10G_KR: 1656 case IFM_40G_KR4: 1657 case IFM_20G_KR2: 1658 case IFM_25G_KR: 1659 /* backplanes */ 1660 return EINVAL; 1661 } 1662 1663 abilities = IXL_PHY_ABILITY_AUTONEGO | IXL_PHY_ABILITY_LINKUP; 1664 1665 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1666 case IFM_AUTO: 1667 link_speed = sc->sc_phy_linkspeed; 1668 break; 1669 case IFM_NONE: 1670 link_speed = 0; 1671 CLR(abilities, IXL_PHY_ABILITY_LINKUP); 1672 break; 1673 default: 1674 link_speed = ixl_search_baudrate( 1675 ifmedia_baudrate(ifm->ifm_media)); 1676 } 1677 1678 if (ISSET(abilities, IXL_PHY_ABILITY_LINKUP)) { 1679 if (ISSET(link_speed, sc->sc_phy_linkspeed) == 0) 1680 return EINVAL; 1681 } 1682 1683 if (ifm->ifm_media & IFM_FLOW) { 1684 abilities |= sc->sc_phy_abilities & 1685 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX); 1686 } 1687 1688 return ixl_set_phy_config(sc, link_speed, abilities, false); 1689 } 1690 1691 1692 static void 1693 ixl_del_all_multiaddr(struct ixl_softc *sc) 1694 { 1695 struct ethercom *ec = &sc->sc_ec; 1696 struct ether_multi *enm; 1697 struct ether_multistep step; 1698 1699 ETHER_LOCK(ec); 1700 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1701 ETHER_NEXT_MULTI(step, enm)) { 1702 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1703 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1704 } 1705 ETHER_UNLOCK(ec); 1706 } 1707 1708 static int 1709 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1710 { 1711 struct ifnet *ifp = &sc->sc_ec.ec_if; 1712 int rv; 1713 1714 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1715 return 0; 1716 1717 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1718 ixl_del_all_multiaddr(sc); 1719 SET(ifp->if_flags, IFF_ALLMULTI); 1720 return ENETRESET; 1721 } 1722 1723 /* multicast address can not use VLAN HWFILTER */ 1724 rv = ixl_add_macvlan(sc, addrlo, 0, 1725 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1726 1727 if (rv == ENOSPC) { 1728 ixl_del_all_multiaddr(sc); 1729 SET(ifp->if_flags, IFF_ALLMULTI); 1730 return ENETRESET; 1731 } 1732 1733 return rv; 1734 } 1735 1736 static int 1737 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1738 { 1739 struct ifnet *ifp = &sc->sc_ec.ec_if; 1740 struct ethercom *ec = &sc->sc_ec; 1741 struct ether_multi *enm, *enm_last; 1742 struct ether_multistep step; 1743 int error, rv = 0; 1744 1745 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1746 ixl_remove_macvlan(sc, addrlo, 0, 1747 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1748 return 0; 1749 } 1750 1751 ETHER_LOCK(ec); 1752 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1753 ETHER_NEXT_MULTI(step, enm)) { 1754 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1755 ETHER_ADDR_LEN) != 0) { 1756 goto out; 1757 } 1758 } 1759 1760 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1761 ETHER_NEXT_MULTI(step, enm)) { 1762 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1763 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1764 if (error != 0) 1765 break; 1766 } 1767 1768 if (enm != NULL) { 1769 enm_last = enm; 1770 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1771 ETHER_NEXT_MULTI(step, enm)) { 1772 if (enm == enm_last) 1773 break; 1774 1775 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1776 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1777 } 1778 } else { 1779 CLR(ifp->if_flags, IFF_ALLMULTI); 1780 rv = ENETRESET; 1781 } 1782 1783 out: 1784 ETHER_UNLOCK(ec); 1785 return rv; 1786 } 1787 1788 static int 1789 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1790 { 1791 struct ifreq *ifr = (struct ifreq *)data; 1792 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1793 const struct sockaddr *sa; 1794 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1795 int s, error = 0; 1796 unsigned int nmtu; 1797 1798 switch (cmd) { 1799 case SIOCSIFMTU: 1800 nmtu = ifr->ifr_mtu; 1801 1802 if (nmtu < IXL_MIN_MTU || nmtu > IXL_MAX_MTU) { 1803 error = EINVAL; 1804 break; 1805 } 1806 if (ifp->if_mtu != nmtu) { 1807 s = splnet(); 1808 error = ether_ioctl(ifp, cmd, data); 1809 splx(s); 1810 if (error == ENETRESET) 1811 error = ixl_init(ifp); 1812 } 1813 break; 1814 case SIOCADDMULTI: 1815 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1816 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1817 error = ether_multiaddr(sa, addrlo, addrhi); 1818 if (error != 0) 1819 return error; 1820 1821 error = ixl_add_multi(sc, addrlo, addrhi); 1822 if (error != 0 && error != ENETRESET) { 1823 ether_delmulti(sa, &sc->sc_ec); 1824 error = EIO; 1825 } 1826 } 1827 break; 1828 1829 case SIOCDELMULTI: 1830 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1831 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1832 error = ether_multiaddr(sa, addrlo, addrhi); 1833 if (error != 0) 1834 return error; 1835 1836 error = ixl_del_multi(sc, addrlo, addrhi); 1837 } 1838 break; 1839 1840 default: 1841 s = splnet(); 1842 error = ether_ioctl(ifp, cmd, data); 1843 splx(s); 1844 } 1845 1846 if (error == ENETRESET) 1847 error = ixl_iff(sc); 1848 1849 return error; 1850 } 1851 1852 static enum i40e_mac_type 1853 ixl_mactype(pci_product_id_t id) 1854 { 1855 1856 switch (id) { 1857 case PCI_PRODUCT_INTEL_XL710_SFP: 1858 case PCI_PRODUCT_INTEL_XL710_KX_B: 1859 case PCI_PRODUCT_INTEL_XL710_KX_C: 1860 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1861 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1862 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1863 case PCI_PRODUCT_INTEL_X710_10G_T_1: 1864 case PCI_PRODUCT_INTEL_X710_10G_T_2: 1865 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1866 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1867 case PCI_PRODUCT_INTEL_X710_T4_10G: 1868 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1869 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1870 case PCI_PRODUCT_INTEL_X710_10G_SFP: 1871 case PCI_PRODUCT_INTEL_X710_10G_BP: 1872 return I40E_MAC_XL710; 1873 1874 case PCI_PRODUCT_INTEL_X722_KX: 1875 case PCI_PRODUCT_INTEL_X722_QSFP: 1876 case PCI_PRODUCT_INTEL_X722_SFP: 1877 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1878 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1879 case PCI_PRODUCT_INTEL_X722_I_SFP: 1880 return I40E_MAC_X722; 1881 } 1882 1883 return I40E_MAC_GENERIC; 1884 } 1885 1886 static void 1887 ixl_pci_csr_setup(pci_chipset_tag_t pc, pcitag_t tag) 1888 { 1889 pcireg_t csr; 1890 1891 csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 1892 csr |= (PCI_COMMAND_MASTER_ENABLE | 1893 PCI_COMMAND_MEM_ENABLE); 1894 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr); 1895 } 1896 1897 static inline void * 1898 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1899 { 1900 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1901 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1902 1903 if (i >= e->hmc_count) 1904 return NULL; 1905 1906 kva += e->hmc_base; 1907 kva += i * e->hmc_size; 1908 1909 return kva; 1910 } 1911 1912 static inline size_t 1913 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1914 { 1915 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1916 1917 return e->hmc_size; 1918 } 1919 1920 static void 1921 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1922 { 1923 struct ixl_rx_ring *rxr = qp->qp_rxr; 1924 1925 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1926 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1927 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1928 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1929 ixl_flush(sc); 1930 } 1931 1932 static void 1933 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1934 { 1935 struct ixl_rx_ring *rxr = qp->qp_rxr; 1936 1937 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1938 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1939 ixl_flush(sc); 1940 } 1941 1942 static void 1943 ixl_enable_other_intr(struct ixl_softc *sc) 1944 { 1945 1946 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1947 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1948 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1949 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1950 ixl_flush(sc); 1951 } 1952 1953 static void 1954 ixl_disable_other_intr(struct ixl_softc *sc) 1955 { 1956 1957 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1958 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1959 ixl_flush(sc); 1960 } 1961 1962 static int 1963 ixl_reinit(struct ixl_softc *sc) 1964 { 1965 struct ixl_rx_ring *rxr; 1966 struct ixl_tx_ring *txr; 1967 unsigned int i; 1968 uint32_t reg; 1969 1970 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1971 1972 if (ixl_get_vsi(sc) != 0) 1973 return EIO; 1974 1975 if (ixl_set_vsi(sc) != 0) 1976 return EIO; 1977 1978 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1979 txr = sc->sc_qps[i].qp_txr; 1980 rxr = sc->sc_qps[i].qp_rxr; 1981 1982 ixl_txr_config(sc, txr); 1983 ixl_rxr_config(sc, rxr); 1984 } 1985 1986 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1987 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 1988 1989 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1990 txr = sc->sc_qps[i].qp_txr; 1991 rxr = sc->sc_qps[i].qp_rxr; 1992 1993 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 1994 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 1995 ixl_flush(sc); 1996 1997 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 1998 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 1999 2000 /* ixl_rxfill() needs lock held */ 2001 mutex_enter(&rxr->rxr_lock); 2002 ixl_rxfill(sc, rxr); 2003 mutex_exit(&rxr->rxr_lock); 2004 2005 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2006 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2007 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2008 if (ixl_rxr_enabled(sc, rxr) != 0) 2009 goto stop; 2010 2011 ixl_txr_qdis(sc, txr, 1); 2012 2013 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2014 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2015 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2016 2017 if (ixl_txr_enabled(sc, txr) != 0) 2018 goto stop; 2019 } 2020 2021 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2022 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2023 2024 return 0; 2025 2026 stop: 2027 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2028 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2029 2030 return ETIMEDOUT; 2031 } 2032 2033 static int 2034 ixl_init_locked(struct ixl_softc *sc) 2035 { 2036 struct ifnet *ifp = &sc->sc_ec.ec_if; 2037 unsigned int i; 2038 int error, eccap_change; 2039 2040 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2041 2042 if (ISSET(ifp->if_flags, IFF_RUNNING)) 2043 ixl_stop_locked(sc); 2044 2045 if (sc->sc_dead) { 2046 return ENXIO; 2047 } 2048 2049 eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable; 2050 if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING)) 2051 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 2052 2053 if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) { 2054 if (ixl_update_macvlan(sc) == 0) { 2055 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 2056 } else { 2057 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 2058 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 2059 } 2060 } 2061 2062 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 2063 sc->sc_nqueue_pairs = 1; 2064 else 2065 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 2066 2067 error = ixl_reinit(sc); 2068 if (error) { 2069 ixl_stop_locked(sc); 2070 return error; 2071 } 2072 2073 SET(ifp->if_flags, IFF_RUNNING); 2074 CLR(ifp->if_flags, IFF_OACTIVE); 2075 2076 ixl_config_rss(sc); 2077 ixl_config_queue_intr(sc); 2078 2079 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2080 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 2081 } 2082 2083 error = ixl_iff(sc); 2084 if (error) { 2085 ixl_stop_locked(sc); 2086 return error; 2087 } 2088 2089 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 2090 2091 return 0; 2092 } 2093 2094 static int 2095 ixl_init(struct ifnet *ifp) 2096 { 2097 struct ixl_softc *sc = ifp->if_softc; 2098 int error; 2099 2100 mutex_enter(&sc->sc_cfg_lock); 2101 error = ixl_init_locked(sc); 2102 mutex_exit(&sc->sc_cfg_lock); 2103 2104 if (error == 0) { 2105 error = ixl_get_link_status(sc, 2106 IXL_LINK_FLAG_WAITDONE); 2107 } 2108 2109 return error; 2110 } 2111 2112 static int 2113 ixl_iff(struct ixl_softc *sc) 2114 { 2115 struct ifnet *ifp = &sc->sc_ec.ec_if; 2116 struct ixl_atq iatq; 2117 struct ixl_aq_desc *iaq; 2118 struct ixl_aq_vsi_promisc_param *param; 2119 uint16_t flag_add, flag_del; 2120 int error; 2121 2122 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2123 return 0; 2124 2125 memset(&iatq, 0, sizeof(iatq)); 2126 2127 iaq = &iatq.iatq_desc; 2128 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 2129 2130 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 2131 param->flags = htole16(0); 2132 2133 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER) 2134 || ISSET(ifp->if_flags, IFF_PROMISC)) { 2135 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2136 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2137 } 2138 2139 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 2140 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2141 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2142 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 2143 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2144 } 2145 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2146 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2147 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2148 param->seid = sc->sc_seid; 2149 2150 error = ixl_atq_exec(sc, &iatq); 2151 if (error) 2152 return error; 2153 2154 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 2155 return EIO; 2156 2157 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 2158 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 2159 flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH; 2160 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH; 2161 } else { 2162 flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN; 2163 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN; 2164 } 2165 2166 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del); 2167 2168 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2169 ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add); 2170 } 2171 return 0; 2172 } 2173 2174 static void 2175 ixl_stop_locked(struct ixl_softc *sc) 2176 { 2177 struct ifnet *ifp = &sc->sc_ec.ec_if; 2178 struct ixl_rx_ring *rxr; 2179 struct ixl_tx_ring *txr; 2180 unsigned int i; 2181 uint32_t reg; 2182 2183 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2184 2185 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 2186 callout_stop(&sc->sc_stats_callout); 2187 2188 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2189 txr = sc->sc_qps[i].qp_txr; 2190 rxr = sc->sc_qps[i].qp_rxr; 2191 2192 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 2193 2194 mutex_enter(&txr->txr_lock); 2195 ixl_txr_qdis(sc, txr, 0); 2196 mutex_exit(&txr->txr_lock); 2197 } 2198 2199 /* XXX wait at least 400 usec for all tx queues in one go */ 2200 ixl_flush(sc); 2201 DELAY(500); 2202 2203 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2204 txr = sc->sc_qps[i].qp_txr; 2205 rxr = sc->sc_qps[i].qp_rxr; 2206 2207 mutex_enter(&txr->txr_lock); 2208 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2209 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2210 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2211 mutex_exit(&txr->txr_lock); 2212 2213 mutex_enter(&rxr->rxr_lock); 2214 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2215 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2216 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2217 mutex_exit(&rxr->rxr_lock); 2218 } 2219 2220 /* XXX short wait for all queue disables to settle */ 2221 ixl_flush(sc); 2222 DELAY(50); 2223 2224 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2225 txr = sc->sc_qps[i].qp_txr; 2226 rxr = sc->sc_qps[i].qp_rxr; 2227 2228 mutex_enter(&txr->txr_lock); 2229 if (ixl_txr_disabled(sc, txr) != 0) { 2230 mutex_exit(&txr->txr_lock); 2231 goto die; 2232 } 2233 mutex_exit(&txr->txr_lock); 2234 2235 mutex_enter(&rxr->rxr_lock); 2236 if (ixl_rxr_disabled(sc, rxr) != 0) { 2237 mutex_exit(&rxr->rxr_lock); 2238 goto die; 2239 } 2240 mutex_exit(&rxr->rxr_lock); 2241 } 2242 2243 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2244 sc->sc_qps[i].qp_workqueue = false; 2245 workqueue_wait(sc->sc_workq_txrx, 2246 &sc->sc_qps[i].qp_work); 2247 } 2248 2249 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2250 txr = sc->sc_qps[i].qp_txr; 2251 rxr = sc->sc_qps[i].qp_rxr; 2252 2253 mutex_enter(&txr->txr_lock); 2254 ixl_txr_unconfig(sc, txr); 2255 mutex_exit(&txr->txr_lock); 2256 2257 mutex_enter(&rxr->rxr_lock); 2258 ixl_rxr_unconfig(sc, rxr); 2259 mutex_exit(&rxr->rxr_lock); 2260 2261 ixl_txr_clean(sc, txr); 2262 ixl_rxr_clean(sc, rxr); 2263 } 2264 2265 return; 2266 die: 2267 sc->sc_dead = true; 2268 log(LOG_CRIT, "%s: failed to shut down rings", 2269 device_xname(sc->sc_dev)); 2270 return; 2271 } 2272 2273 static void 2274 ixl_stop(struct ifnet *ifp, int disable) 2275 { 2276 struct ixl_softc *sc = ifp->if_softc; 2277 2278 mutex_enter(&sc->sc_cfg_lock); 2279 ixl_stop_locked(sc); 2280 mutex_exit(&sc->sc_cfg_lock); 2281 } 2282 2283 static int 2284 ixl_queue_pairs_alloc(struct ixl_softc *sc) 2285 { 2286 struct ixl_queue_pair *qp; 2287 unsigned int i; 2288 size_t sz; 2289 2290 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2291 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 2292 2293 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2294 qp = &sc->sc_qps[i]; 2295 2296 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2297 ixl_handle_queue, qp); 2298 if (qp->qp_si == NULL) 2299 goto free; 2300 2301 qp->qp_txr = ixl_txr_alloc(sc, i); 2302 if (qp->qp_txr == NULL) 2303 goto free; 2304 2305 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2306 if (qp->qp_rxr == NULL) 2307 goto free; 2308 2309 qp->qp_sc = sc; 2310 snprintf(qp->qp_name, sizeof(qp->qp_name), 2311 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2312 } 2313 2314 return 0; 2315 free: 2316 if (sc->sc_qps != NULL) { 2317 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2318 qp = &sc->sc_qps[i]; 2319 2320 if (qp->qp_txr != NULL) 2321 ixl_txr_free(sc, qp->qp_txr); 2322 if (qp->qp_rxr != NULL) 2323 ixl_rxr_free(sc, qp->qp_rxr); 2324 if (qp->qp_si != NULL) 2325 softint_disestablish(qp->qp_si); 2326 } 2327 2328 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2329 kmem_free(sc->sc_qps, sz); 2330 sc->sc_qps = NULL; 2331 } 2332 2333 return -1; 2334 } 2335 2336 static void 2337 ixl_queue_pairs_free(struct ixl_softc *sc) 2338 { 2339 struct ixl_queue_pair *qp; 2340 unsigned int i; 2341 size_t sz; 2342 2343 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2344 qp = &sc->sc_qps[i]; 2345 ixl_txr_free(sc, qp->qp_txr); 2346 ixl_rxr_free(sc, qp->qp_rxr); 2347 softint_disestablish(qp->qp_si); 2348 } 2349 2350 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2351 kmem_free(sc->sc_qps, sz); 2352 sc->sc_qps = NULL; 2353 } 2354 2355 static struct ixl_tx_ring * 2356 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2357 { 2358 struct ixl_tx_ring *txr = NULL; 2359 struct ixl_tx_map *maps = NULL, *txm; 2360 unsigned int i; 2361 2362 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2363 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2364 KM_SLEEP); 2365 2366 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2367 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2368 IXL_TX_QUEUE_ALIGN) != 0) 2369 goto free; 2370 2371 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2372 txm = &maps[i]; 2373 2374 if (bus_dmamap_create(sc->sc_dmat, IXL_TX_PKT_MAXSIZE, 2375 IXL_TX_PKT_DESCS, IXL_TX_PKT_MAXSIZE, 0, 2376 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2377 goto uncreate; 2378 2379 txm->txm_eop = -1; 2380 txm->txm_m = NULL; 2381 } 2382 2383 txr->txr_cons = txr->txr_prod = 0; 2384 txr->txr_maps = maps; 2385 2386 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2387 if (txr->txr_intrq == NULL) 2388 goto uncreate; 2389 2390 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2391 ixl_deferred_transmit, txr); 2392 if (txr->txr_si == NULL) 2393 goto destroy_pcq; 2394 2395 txr->txr_tail = I40E_QTX_TAIL(qid); 2396 txr->txr_qid = qid; 2397 txr->txr_sc = sc; 2398 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2399 2400 return txr; 2401 2402 destroy_pcq: 2403 pcq_destroy(txr->txr_intrq); 2404 uncreate: 2405 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2406 txm = &maps[i]; 2407 2408 if (txm->txm_map == NULL) 2409 continue; 2410 2411 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2412 } 2413 2414 ixl_dmamem_free(sc, &txr->txr_mem); 2415 free: 2416 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2417 kmem_free(txr, sizeof(*txr)); 2418 2419 return NULL; 2420 } 2421 2422 static void 2423 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2424 { 2425 unsigned int qid; 2426 bus_size_t reg; 2427 uint32_t r; 2428 2429 qid = txr->txr_qid + sc->sc_base_queue; 2430 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2431 qid %= 128; 2432 2433 r = ixl_rd(sc, reg); 2434 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2435 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2436 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2437 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2438 ixl_wr(sc, reg, r); 2439 } 2440 2441 static void 2442 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2443 { 2444 struct ixl_hmc_txq txq; 2445 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2446 void *hmc; 2447 2448 memset(&txq, 0, sizeof(txq)); 2449 txq.head = htole16(txr->txr_cons); 2450 txq.new_context = 1; 2451 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2452 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2453 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2454 txq.tphrdesc_ena = 0; 2455 txq.tphrpacket_ena = 0; 2456 txq.tphwdesc_ena = 0; 2457 txq.rdylist = data->qs_handle[0]; 2458 2459 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2460 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2461 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2462 __arraycount(ixl_hmc_pack_txq)); 2463 } 2464 2465 static void 2466 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2467 { 2468 void *hmc; 2469 2470 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2471 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2472 txr->txr_cons = txr->txr_prod = 0; 2473 } 2474 2475 static void 2476 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2477 { 2478 struct ixl_tx_map *maps, *txm; 2479 bus_dmamap_t map; 2480 unsigned int i; 2481 2482 maps = txr->txr_maps; 2483 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2484 txm = &maps[i]; 2485 2486 if (txm->txm_m == NULL) 2487 continue; 2488 2489 map = txm->txm_map; 2490 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2491 BUS_DMASYNC_POSTWRITE); 2492 bus_dmamap_unload(sc->sc_dmat, map); 2493 2494 m_freem(txm->txm_m); 2495 txm->txm_m = NULL; 2496 } 2497 } 2498 2499 static int 2500 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2501 { 2502 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2503 uint32_t reg; 2504 int i; 2505 2506 for (i = 0; i < 10; i++) { 2507 reg = ixl_rd(sc, ena); 2508 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2509 return 0; 2510 2511 delaymsec(10); 2512 } 2513 2514 return ETIMEDOUT; 2515 } 2516 2517 static int 2518 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2519 { 2520 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2521 uint32_t reg; 2522 int i; 2523 2524 KASSERT(mutex_owned(&txr->txr_lock)); 2525 2526 for (i = 0; i < 10; i++) { 2527 reg = ixl_rd(sc, ena); 2528 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2529 return 0; 2530 2531 delaymsec(10); 2532 } 2533 2534 return ETIMEDOUT; 2535 } 2536 2537 static void 2538 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2539 { 2540 struct ixl_tx_map *maps, *txm; 2541 struct mbuf *m; 2542 unsigned int i; 2543 2544 softint_disestablish(txr->txr_si); 2545 2546 maps = txr->txr_maps; 2547 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2548 txm = &maps[i]; 2549 2550 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2551 } 2552 2553 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2554 m_freem(m); 2555 pcq_destroy(txr->txr_intrq); 2556 2557 ixl_dmamem_free(sc, &txr->txr_mem); 2558 mutex_destroy(&txr->txr_lock); 2559 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2560 kmem_free(txr, sizeof(*txr)); 2561 } 2562 2563 static inline int 2564 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2565 struct ixl_tx_ring *txr) 2566 { 2567 struct mbuf *m; 2568 int error; 2569 2570 KASSERT(mutex_owned(&txr->txr_lock)); 2571 2572 m = *m0; 2573 2574 error = bus_dmamap_load_mbuf(dmat, map, m, 2575 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2576 if (error != EFBIG) 2577 return error; 2578 2579 m = m_defrag(m, M_DONTWAIT); 2580 if (m != NULL) { 2581 *m0 = m; 2582 txr->txr_defragged.ev_count++; 2583 2584 error = bus_dmamap_load_mbuf(dmat, map, m, 2585 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2586 } else { 2587 txr->txr_defrag_failed.ev_count++; 2588 error = ENOBUFS; 2589 } 2590 2591 return error; 2592 } 2593 2594 static inline int 2595 ixl_tx_setup_offloads(struct mbuf *m, uint64_t *cmd_txd) 2596 { 2597 struct ether_header *eh; 2598 size_t len; 2599 uint64_t cmd; 2600 2601 cmd = 0; 2602 2603 eh = mtod(m, struct ether_header *); 2604 switch (htons(eh->ether_type)) { 2605 case ETHERTYPE_IP: 2606 case ETHERTYPE_IPV6: 2607 len = ETHER_HDR_LEN; 2608 break; 2609 case ETHERTYPE_VLAN: 2610 len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2611 break; 2612 default: 2613 len = 0; 2614 } 2615 cmd |= ((len >> 1) << IXL_TX_DESC_MACLEN_SHIFT); 2616 2617 if (m->m_pkthdr.csum_flags & 2618 (M_CSUM_TSOv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) { 2619 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4; 2620 } 2621 if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) { 2622 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4_CSUM; 2623 } 2624 2625 if (m->m_pkthdr.csum_flags & 2626 (M_CSUM_TSOv6 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) { 2627 cmd |= IXL_TX_DESC_CMD_IIPT_IPV6; 2628 } 2629 2630 switch (cmd & IXL_TX_DESC_CMD_IIPT_MASK) { 2631 case IXL_TX_DESC_CMD_IIPT_IPV4: 2632 case IXL_TX_DESC_CMD_IIPT_IPV4_CSUM: 2633 len = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); 2634 break; 2635 case IXL_TX_DESC_CMD_IIPT_IPV6: 2636 len = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); 2637 break; 2638 default: 2639 len = 0; 2640 } 2641 cmd |= ((len >> 2) << IXL_TX_DESC_IPLEN_SHIFT); 2642 2643 if (m->m_pkthdr.csum_flags & 2644 (M_CSUM_TSOv4 | M_CSUM_TSOv6 | M_CSUM_TCPv4 | M_CSUM_TCPv6)) { 2645 len = sizeof(struct tcphdr); 2646 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_TCP; 2647 } else if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) { 2648 len = sizeof(struct udphdr); 2649 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_UDP; 2650 } else { 2651 len = 0; 2652 } 2653 cmd |= ((len >> 2) << IXL_TX_DESC_L4LEN_SHIFT); 2654 2655 *cmd_txd |= cmd; 2656 return 0; 2657 } 2658 2659 static void 2660 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2661 bool is_transmit) 2662 { 2663 struct ixl_softc *sc = ifp->if_softc; 2664 struct ixl_tx_desc *ring, *txd; 2665 struct ixl_tx_map *txm; 2666 bus_dmamap_t map; 2667 struct mbuf *m; 2668 uint64_t cmd, cmd_txd; 2669 unsigned int prod, free, last, i; 2670 unsigned int mask; 2671 int post = 0; 2672 2673 KASSERT(mutex_owned(&txr->txr_lock)); 2674 2675 if (!ISSET(ifp->if_flags, IFF_RUNNING) 2676 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2677 if (!is_transmit) 2678 IFQ_PURGE(&ifp->if_snd); 2679 return; 2680 } 2681 2682 prod = txr->txr_prod; 2683 free = txr->txr_cons; 2684 if (free <= prod) 2685 free += sc->sc_tx_ring_ndescs; 2686 free -= prod; 2687 2688 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2689 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2690 2691 ring = IXL_DMA_KVA(&txr->txr_mem); 2692 mask = sc->sc_tx_ring_ndescs - 1; 2693 last = prod; 2694 cmd = 0; 2695 txd = NULL; 2696 2697 for (;;) { 2698 if (free <= IXL_TX_PKT_DESCS) { 2699 if (!is_transmit) 2700 SET(ifp->if_flags, IFF_OACTIVE); 2701 break; 2702 } 2703 2704 if (is_transmit) 2705 m = pcq_get(txr->txr_intrq); 2706 else 2707 IFQ_DEQUEUE(&ifp->if_snd, m); 2708 2709 if (m == NULL) 2710 break; 2711 2712 txm = &txr->txr_maps[prod]; 2713 map = txm->txm_map; 2714 2715 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2716 if_statinc(ifp, if_oerrors); 2717 m_freem(m); 2718 continue; 2719 } 2720 2721 cmd_txd = 0; 2722 if (m->m_pkthdr.csum_flags & IXL_CSUM_ALL_OFFLOAD) { 2723 ixl_tx_setup_offloads(m, &cmd_txd); 2724 } 2725 2726 if (vlan_has_tag(m)) { 2727 uint16_t vtag; 2728 vtag = htole16(vlan_get_tag(m)); 2729 cmd_txd |= (uint64_t)vtag << 2730 IXL_TX_DESC_L2TAG1_SHIFT; 2731 cmd_txd |= IXL_TX_DESC_CMD_IL2TAG1; 2732 } 2733 2734 bus_dmamap_sync(sc->sc_dmat, map, 0, 2735 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2736 2737 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2738 txd = &ring[prod]; 2739 2740 cmd = (uint64_t)map->dm_segs[i].ds_len << 2741 IXL_TX_DESC_BSIZE_SHIFT; 2742 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2743 cmd |= cmd_txd; 2744 2745 txd->addr = htole64(map->dm_segs[i].ds_addr); 2746 txd->cmd = htole64(cmd); 2747 2748 last = prod; 2749 2750 prod++; 2751 prod &= mask; 2752 } 2753 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2754 txd->cmd = htole64(cmd); 2755 2756 txm->txm_m = m; 2757 txm->txm_eop = last; 2758 2759 bpf_mtap(ifp, m, BPF_D_OUT); 2760 2761 free -= i; 2762 post = 1; 2763 } 2764 2765 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2766 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2767 2768 if (post) { 2769 txr->txr_prod = prod; 2770 ixl_wr(sc, txr->txr_tail, prod); 2771 } 2772 } 2773 2774 static int 2775 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2776 { 2777 struct ifnet *ifp = &sc->sc_ec.ec_if; 2778 struct ixl_tx_desc *ring, *txd; 2779 struct ixl_tx_map *txm; 2780 struct mbuf *m; 2781 bus_dmamap_t map; 2782 unsigned int cons, prod, last; 2783 unsigned int mask; 2784 uint64_t dtype; 2785 int done = 0, more = 0; 2786 2787 KASSERT(mutex_owned(&txr->txr_lock)); 2788 2789 prod = txr->txr_prod; 2790 cons = txr->txr_cons; 2791 2792 if (cons == prod) 2793 return 0; 2794 2795 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2796 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2797 2798 ring = IXL_DMA_KVA(&txr->txr_mem); 2799 mask = sc->sc_tx_ring_ndescs - 1; 2800 2801 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2802 2803 do { 2804 if (txlimit-- <= 0) { 2805 more = 1; 2806 break; 2807 } 2808 2809 txm = &txr->txr_maps[cons]; 2810 last = txm->txm_eop; 2811 txd = &ring[last]; 2812 2813 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2814 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2815 break; 2816 2817 map = txm->txm_map; 2818 2819 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2820 BUS_DMASYNC_POSTWRITE); 2821 bus_dmamap_unload(sc->sc_dmat, map); 2822 2823 m = txm->txm_m; 2824 if (m != NULL) { 2825 if_statinc_ref(nsr, if_opackets); 2826 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 2827 if (ISSET(m->m_flags, M_MCAST)) 2828 if_statinc_ref(nsr, if_omcasts); 2829 m_freem(m); 2830 } 2831 2832 txm->txm_m = NULL; 2833 txm->txm_eop = -1; 2834 2835 cons = last + 1; 2836 cons &= mask; 2837 done = 1; 2838 } while (cons != prod); 2839 2840 IF_STAT_PUTREF(ifp); 2841 2842 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2843 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2844 2845 txr->txr_cons = cons; 2846 2847 if (done) { 2848 softint_schedule(txr->txr_si); 2849 if (txr->txr_qid == 0) { 2850 CLR(ifp->if_flags, IFF_OACTIVE); 2851 if_schedule_deferred_start(ifp); 2852 } 2853 } 2854 2855 return more; 2856 } 2857 2858 static void 2859 ixl_start(struct ifnet *ifp) 2860 { 2861 struct ixl_softc *sc; 2862 struct ixl_tx_ring *txr; 2863 2864 sc = ifp->if_softc; 2865 txr = sc->sc_qps[0].qp_txr; 2866 2867 mutex_enter(&txr->txr_lock); 2868 ixl_tx_common_locked(ifp, txr, false); 2869 mutex_exit(&txr->txr_lock); 2870 } 2871 2872 static inline unsigned int 2873 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2874 { 2875 u_int cpuid; 2876 2877 cpuid = cpu_index(curcpu()); 2878 2879 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2880 } 2881 2882 static int 2883 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2884 { 2885 struct ixl_softc *sc; 2886 struct ixl_tx_ring *txr; 2887 unsigned int qid; 2888 2889 sc = ifp->if_softc; 2890 qid = ixl_select_txqueue(sc, m); 2891 2892 txr = sc->sc_qps[qid].qp_txr; 2893 2894 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2895 mutex_enter(&txr->txr_lock); 2896 txr->txr_pcqdrop.ev_count++; 2897 mutex_exit(&txr->txr_lock); 2898 2899 m_freem(m); 2900 return ENOBUFS; 2901 } 2902 2903 #ifdef IXL_ALWAYS_TXDEFER 2904 kpreempt_disable(); 2905 softint_schedule(txr->txr_si); 2906 kpreempt_enable(); 2907 #else 2908 if (mutex_tryenter(&txr->txr_lock)) { 2909 ixl_tx_common_locked(ifp, txr, true); 2910 mutex_exit(&txr->txr_lock); 2911 } else { 2912 kpreempt_disable(); 2913 softint_schedule(txr->txr_si); 2914 kpreempt_enable(); 2915 } 2916 #endif 2917 2918 return 0; 2919 } 2920 2921 static void 2922 ixl_deferred_transmit(void *xtxr) 2923 { 2924 struct ixl_tx_ring *txr = xtxr; 2925 struct ixl_softc *sc = txr->txr_sc; 2926 struct ifnet *ifp = &sc->sc_ec.ec_if; 2927 2928 mutex_enter(&txr->txr_lock); 2929 txr->txr_transmitdef.ev_count++; 2930 if (pcq_peek(txr->txr_intrq) != NULL) 2931 ixl_tx_common_locked(ifp, txr, true); 2932 mutex_exit(&txr->txr_lock); 2933 } 2934 2935 static struct ixl_rx_ring * 2936 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2937 { 2938 struct ixl_rx_ring *rxr = NULL; 2939 struct ixl_rx_map *maps = NULL, *rxm; 2940 unsigned int i; 2941 2942 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2943 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2944 KM_SLEEP); 2945 2946 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2947 sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs, 2948 IXL_RX_QUEUE_ALIGN) != 0) 2949 goto free; 2950 2951 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2952 rxm = &maps[i]; 2953 2954 if (bus_dmamap_create(sc->sc_dmat, 2955 IXL_MCLBYTES, 1, IXL_MCLBYTES, 0, 2956 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2957 goto uncreate; 2958 2959 rxm->rxm_m = NULL; 2960 } 2961 2962 rxr->rxr_cons = rxr->rxr_prod = 0; 2963 rxr->rxr_m_head = NULL; 2964 rxr->rxr_m_tail = &rxr->rxr_m_head; 2965 rxr->rxr_maps = maps; 2966 2967 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2968 rxr->rxr_qid = qid; 2969 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2970 2971 return rxr; 2972 2973 uncreate: 2974 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2975 rxm = &maps[i]; 2976 2977 if (rxm->rxm_map == NULL) 2978 continue; 2979 2980 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2981 } 2982 2983 ixl_dmamem_free(sc, &rxr->rxr_mem); 2984 free: 2985 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2986 kmem_free(rxr, sizeof(*rxr)); 2987 2988 return NULL; 2989 } 2990 2991 static void 2992 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2993 { 2994 struct ixl_rx_map *maps, *rxm; 2995 bus_dmamap_t map; 2996 unsigned int i; 2997 2998 maps = rxr->rxr_maps; 2999 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3000 rxm = &maps[i]; 3001 3002 if (rxm->rxm_m == NULL) 3003 continue; 3004 3005 map = rxm->rxm_map; 3006 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3007 BUS_DMASYNC_POSTWRITE); 3008 bus_dmamap_unload(sc->sc_dmat, map); 3009 3010 m_freem(rxm->rxm_m); 3011 rxm->rxm_m = NULL; 3012 } 3013 3014 m_freem(rxr->rxr_m_head); 3015 rxr->rxr_m_head = NULL; 3016 rxr->rxr_m_tail = &rxr->rxr_m_head; 3017 3018 rxr->rxr_prod = rxr->rxr_cons = 0; 3019 } 3020 3021 static int 3022 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3023 { 3024 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3025 uint32_t reg; 3026 int i; 3027 3028 for (i = 0; i < 10; i++) { 3029 reg = ixl_rd(sc, ena); 3030 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 3031 return 0; 3032 3033 delaymsec(10); 3034 } 3035 3036 return ETIMEDOUT; 3037 } 3038 3039 static int 3040 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3041 { 3042 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3043 uint32_t reg; 3044 int i; 3045 3046 KASSERT(mutex_owned(&rxr->rxr_lock)); 3047 3048 for (i = 0; i < 10; i++) { 3049 reg = ixl_rd(sc, ena); 3050 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 3051 return 0; 3052 3053 delaymsec(10); 3054 } 3055 3056 return ETIMEDOUT; 3057 } 3058 3059 static void 3060 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3061 { 3062 struct ixl_hmc_rxq rxq; 3063 struct ifnet *ifp = &sc->sc_ec.ec_if; 3064 uint16_t rxmax; 3065 void *hmc; 3066 3067 memset(&rxq, 0, sizeof(rxq)); 3068 rxmax = ifp->if_mtu + IXL_MTU_ETHERLEN; 3069 3070 rxq.head = htole16(rxr->rxr_cons); 3071 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 3072 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 3073 rxq.dbuff = htole16(IXL_MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 3074 rxq.hbuff = 0; 3075 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 3076 rxq.dsize = IXL_HMC_RXQ_DSIZE_32; 3077 rxq.crcstrip = 1; 3078 rxq.l2sel = 1; 3079 rxq.showiv = 1; 3080 rxq.rxmax = htole16(rxmax); 3081 rxq.tphrdesc_ena = 0; 3082 rxq.tphwdesc_ena = 0; 3083 rxq.tphdata_ena = 0; 3084 rxq.tphhead_ena = 0; 3085 rxq.lrxqthresh = 0; 3086 rxq.prefena = 1; 3087 3088 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3089 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3090 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 3091 __arraycount(ixl_hmc_pack_rxq)); 3092 } 3093 3094 static void 3095 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3096 { 3097 void *hmc; 3098 3099 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3100 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3101 rxr->rxr_cons = rxr->rxr_prod = 0; 3102 } 3103 3104 static void 3105 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3106 { 3107 struct ixl_rx_map *maps, *rxm; 3108 unsigned int i; 3109 3110 maps = rxr->rxr_maps; 3111 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3112 rxm = &maps[i]; 3113 3114 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 3115 } 3116 3117 ixl_dmamem_free(sc, &rxr->rxr_mem); 3118 mutex_destroy(&rxr->rxr_lock); 3119 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3120 kmem_free(rxr, sizeof(*rxr)); 3121 } 3122 3123 static inline void 3124 ixl_rx_csum(struct mbuf *m, uint64_t qword) 3125 { 3126 int flags_mask; 3127 3128 if (!ISSET(qword, IXL_RX_DESC_L3L4P)) { 3129 /* No L3 or L4 checksum was calculated */ 3130 return; 3131 } 3132 3133 switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) { 3134 case IXL_RX_DESC_PTYPE_IPV4FRAG: 3135 case IXL_RX_DESC_PTYPE_IPV4: 3136 case IXL_RX_DESC_PTYPE_SCTPV4: 3137 case IXL_RX_DESC_PTYPE_ICMPV4: 3138 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3139 break; 3140 case IXL_RX_DESC_PTYPE_TCPV4: 3141 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3142 flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD; 3143 break; 3144 case IXL_RX_DESC_PTYPE_UDPV4: 3145 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3146 flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD; 3147 break; 3148 case IXL_RX_DESC_PTYPE_TCPV6: 3149 flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD; 3150 break; 3151 case IXL_RX_DESC_PTYPE_UDPV6: 3152 flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD; 3153 break; 3154 default: 3155 flags_mask = 0; 3156 } 3157 3158 m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 | 3159 M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6)); 3160 3161 if (ISSET(qword, IXL_RX_DESC_IPE)) { 3162 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD); 3163 } 3164 3165 if (ISSET(qword, IXL_RX_DESC_L4E)) { 3166 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD); 3167 } 3168 } 3169 3170 static int 3171 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 3172 { 3173 struct ifnet *ifp = &sc->sc_ec.ec_if; 3174 struct ixl_rx_wb_desc_32 *ring, *rxd; 3175 struct ixl_rx_map *rxm; 3176 bus_dmamap_t map; 3177 unsigned int cons, prod; 3178 struct mbuf *m; 3179 uint64_t word, word0; 3180 unsigned int len; 3181 unsigned int mask; 3182 int done = 0, more = 0; 3183 3184 KASSERT(mutex_owned(&rxr->rxr_lock)); 3185 3186 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3187 return 0; 3188 3189 prod = rxr->rxr_prod; 3190 cons = rxr->rxr_cons; 3191 3192 if (cons == prod) 3193 return 0; 3194 3195 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3196 0, IXL_DMA_LEN(&rxr->rxr_mem), 3197 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3198 3199 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3200 mask = sc->sc_rx_ring_ndescs - 1; 3201 3202 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 3203 3204 do { 3205 if (rxlimit-- <= 0) { 3206 more = 1; 3207 break; 3208 } 3209 3210 rxd = &ring[cons]; 3211 3212 word = le64toh(rxd->qword1); 3213 3214 if (!ISSET(word, IXL_RX_DESC_DD)) 3215 break; 3216 3217 rxm = &rxr->rxr_maps[cons]; 3218 3219 map = rxm->rxm_map; 3220 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3221 BUS_DMASYNC_POSTREAD); 3222 bus_dmamap_unload(sc->sc_dmat, map); 3223 3224 m = rxm->rxm_m; 3225 rxm->rxm_m = NULL; 3226 3227 KASSERT(m != NULL); 3228 3229 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 3230 m->m_len = len; 3231 m->m_pkthdr.len = 0; 3232 3233 m->m_next = NULL; 3234 *rxr->rxr_m_tail = m; 3235 rxr->rxr_m_tail = &m->m_next; 3236 3237 m = rxr->rxr_m_head; 3238 m->m_pkthdr.len += len; 3239 3240 if (ISSET(word, IXL_RX_DESC_EOP)) { 3241 word0 = le64toh(rxd->qword0); 3242 3243 if (ISSET(word, IXL_RX_DESC_L2TAG1P)) { 3244 uint16_t vtag; 3245 vtag = __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK); 3246 vlan_set_tag(m, le16toh(vtag)); 3247 } 3248 3249 if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0) 3250 ixl_rx_csum(m, word); 3251 3252 if (!ISSET(word, 3253 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 3254 m_set_rcvif(m, ifp); 3255 if_statinc_ref(nsr, if_ipackets); 3256 if_statadd_ref(nsr, if_ibytes, 3257 m->m_pkthdr.len); 3258 if_percpuq_enqueue(sc->sc_ipq, m); 3259 } else { 3260 if_statinc_ref(nsr, if_ierrors); 3261 m_freem(m); 3262 } 3263 3264 rxr->rxr_m_head = NULL; 3265 rxr->rxr_m_tail = &rxr->rxr_m_head; 3266 } 3267 3268 cons++; 3269 cons &= mask; 3270 3271 done = 1; 3272 } while (cons != prod); 3273 3274 if (done) { 3275 rxr->rxr_cons = cons; 3276 if (ixl_rxfill(sc, rxr) == -1) 3277 if_statinc_ref(nsr, if_iqdrops); 3278 } 3279 3280 IF_STAT_PUTREF(ifp); 3281 3282 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3283 0, IXL_DMA_LEN(&rxr->rxr_mem), 3284 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3285 3286 return more; 3287 } 3288 3289 static int 3290 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3291 { 3292 struct ixl_rx_rd_desc_32 *ring, *rxd; 3293 struct ixl_rx_map *rxm; 3294 bus_dmamap_t map; 3295 struct mbuf *m; 3296 unsigned int prod; 3297 unsigned int slots; 3298 unsigned int mask; 3299 int post = 0, error = 0; 3300 3301 KASSERT(mutex_owned(&rxr->rxr_lock)); 3302 3303 prod = rxr->rxr_prod; 3304 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 3305 sc->sc_rx_ring_ndescs); 3306 3307 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3308 mask = sc->sc_rx_ring_ndescs - 1; 3309 3310 if (__predict_false(slots <= 0)) 3311 return -1; 3312 3313 do { 3314 rxm = &rxr->rxr_maps[prod]; 3315 3316 MGETHDR(m, M_DONTWAIT, MT_DATA); 3317 if (m == NULL) { 3318 rxr->rxr_mgethdr_failed.ev_count++; 3319 error = -1; 3320 break; 3321 } 3322 3323 MCLGET(m, M_DONTWAIT); 3324 if (!ISSET(m->m_flags, M_EXT)) { 3325 rxr->rxr_mgetcl_failed.ev_count++; 3326 error = -1; 3327 m_freem(m); 3328 break; 3329 } 3330 3331 m->m_len = m->m_pkthdr.len = MCLBYTES; 3332 m_adj(m, ETHER_ALIGN); 3333 3334 map = rxm->rxm_map; 3335 3336 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 3337 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 3338 rxr->rxr_mbuf_load_failed.ev_count++; 3339 error = -1; 3340 m_freem(m); 3341 break; 3342 } 3343 3344 rxm->rxm_m = m; 3345 3346 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3347 BUS_DMASYNC_PREREAD); 3348 3349 rxd = &ring[prod]; 3350 3351 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 3352 rxd->haddr = htole64(0); 3353 3354 prod++; 3355 prod &= mask; 3356 3357 post = 1; 3358 3359 } while (--slots); 3360 3361 if (post) { 3362 rxr->rxr_prod = prod; 3363 ixl_wr(sc, rxr->rxr_tail, prod); 3364 } 3365 3366 return error; 3367 } 3368 3369 static inline int 3370 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 3371 u_int txlimit, struct evcnt *txevcnt, 3372 u_int rxlimit, struct evcnt *rxevcnt) 3373 { 3374 struct ixl_tx_ring *txr = qp->qp_txr; 3375 struct ixl_rx_ring *rxr = qp->qp_rxr; 3376 int txmore, rxmore; 3377 int rv; 3378 3379 mutex_enter(&txr->txr_lock); 3380 txevcnt->ev_count++; 3381 txmore = ixl_txeof(sc, txr, txlimit); 3382 mutex_exit(&txr->txr_lock); 3383 3384 mutex_enter(&rxr->rxr_lock); 3385 rxevcnt->ev_count++; 3386 rxmore = ixl_rxeof(sc, rxr, rxlimit); 3387 mutex_exit(&rxr->rxr_lock); 3388 3389 rv = txmore | (rxmore << 1); 3390 3391 return rv; 3392 } 3393 3394 static void 3395 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 3396 { 3397 3398 if (qp->qp_workqueue) 3399 workqueue_enqueue(sc->sc_workq_txrx, &qp->qp_work, NULL); 3400 else 3401 softint_schedule(qp->qp_si); 3402 } 3403 3404 static int 3405 ixl_intr(void *xsc) 3406 { 3407 struct ixl_softc *sc = xsc; 3408 struct ixl_tx_ring *txr; 3409 struct ixl_rx_ring *rxr; 3410 uint32_t icr, rxintr, txintr; 3411 int rv = 0; 3412 unsigned int i; 3413 3414 KASSERT(sc != NULL); 3415 3416 ixl_enable_other_intr(sc); 3417 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3418 3419 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3420 atomic_inc_64(&sc->sc_event_atq.ev_count); 3421 ixl_atq_done(sc); 3422 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3423 rv = 1; 3424 } 3425 3426 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3427 atomic_inc_64(&sc->sc_event_link.ev_count); 3428 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3429 rv = 1; 3430 } 3431 3432 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 3433 txintr = icr & I40E_INTR_NOTX_TX_MASK; 3434 3435 if (txintr || rxintr) { 3436 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 3437 txr = sc->sc_qps[i].qp_txr; 3438 rxr = sc->sc_qps[i].qp_rxr; 3439 3440 ixl_handle_queue_common(sc, &sc->sc_qps[i], 3441 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 3442 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 3443 } 3444 rv = 1; 3445 } 3446 3447 return rv; 3448 } 3449 3450 static int 3451 ixl_queue_intr(void *xqp) 3452 { 3453 struct ixl_queue_pair *qp = xqp; 3454 struct ixl_tx_ring *txr = qp->qp_txr; 3455 struct ixl_rx_ring *rxr = qp->qp_rxr; 3456 struct ixl_softc *sc = qp->qp_sc; 3457 u_int txlimit, rxlimit; 3458 int more; 3459 3460 txlimit = sc->sc_tx_intr_process_limit; 3461 rxlimit = sc->sc_rx_intr_process_limit; 3462 qp->qp_workqueue = sc->sc_txrx_workqueue; 3463 3464 more = ixl_handle_queue_common(sc, qp, 3465 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3466 3467 if (more != 0) { 3468 ixl_sched_handle_queue(sc, qp); 3469 } else { 3470 /* for ALTQ */ 3471 if (txr->txr_qid == 0) 3472 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3473 softint_schedule(txr->txr_si); 3474 3475 ixl_enable_queue_intr(sc, qp); 3476 } 3477 3478 return 1; 3479 } 3480 3481 static void 3482 ixl_handle_queue_wk(struct work *wk, void *xsc) 3483 { 3484 struct ixl_queue_pair *qp; 3485 3486 qp = container_of(wk, struct ixl_queue_pair, qp_work); 3487 ixl_handle_queue(qp); 3488 } 3489 3490 static void 3491 ixl_handle_queue(void *xqp) 3492 { 3493 struct ixl_queue_pair *qp = xqp; 3494 struct ixl_softc *sc = qp->qp_sc; 3495 struct ixl_tx_ring *txr = qp->qp_txr; 3496 struct ixl_rx_ring *rxr = qp->qp_rxr; 3497 u_int txlimit, rxlimit; 3498 int more; 3499 3500 txlimit = sc->sc_tx_process_limit; 3501 rxlimit = sc->sc_rx_process_limit; 3502 3503 more = ixl_handle_queue_common(sc, qp, 3504 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3505 3506 if (more != 0) 3507 ixl_sched_handle_queue(sc, qp); 3508 else 3509 ixl_enable_queue_intr(sc, qp); 3510 } 3511 3512 static inline void 3513 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3514 { 3515 uint32_t hmc_idx, hmc_isvf; 3516 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3517 3518 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3519 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3520 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3521 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3522 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3523 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3524 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3525 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3526 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3527 3528 device_printf(sc->sc_dev, 3529 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3530 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3531 } 3532 3533 static int 3534 ixl_other_intr(void *xsc) 3535 { 3536 struct ixl_softc *sc = xsc; 3537 uint32_t icr, mask, reg; 3538 int rv; 3539 3540 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3541 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3542 3543 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3544 atomic_inc_64(&sc->sc_event_atq.ev_count); 3545 ixl_atq_done(sc); 3546 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3547 rv = 1; 3548 } 3549 3550 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3551 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3552 device_printf(sc->sc_dev, "link stat changed\n"); 3553 3554 atomic_inc_64(&sc->sc_event_link.ev_count); 3555 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3556 rv = 1; 3557 } 3558 3559 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3560 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3561 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3562 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3563 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3564 3565 device_printf(sc->sc_dev, "GRST: %s\n", 3566 reg == I40E_RESET_CORER ? "CORER" : 3567 reg == I40E_RESET_GLOBR ? "GLOBR" : 3568 reg == I40E_RESET_EMPR ? "EMPR" : 3569 "POR"); 3570 } 3571 3572 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3573 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3574 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3575 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3576 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3577 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3578 3579 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3580 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3581 device_printf(sc->sc_dev, "critical error\n"); 3582 } 3583 3584 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3585 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3586 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3587 ixl_print_hmc_error(sc, reg); 3588 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3589 } 3590 3591 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3592 ixl_flush(sc); 3593 ixl_enable_other_intr(sc); 3594 return rv; 3595 } 3596 3597 static void 3598 ixl_get_link_status_done_work(void *xsc) 3599 { 3600 struct ixl_softc *sc = xsc; 3601 struct ixl_aq_desc *iaq, iaq_buf; 3602 3603 mutex_enter(&sc->sc_atq_lock); 3604 iaq = &sc->sc_link_state_atq.iatq_desc; 3605 iaq_buf = *iaq; 3606 mutex_exit(&sc->sc_atq_lock); 3607 3608 ixl_link_state_update(sc, &iaq_buf); 3609 3610 mutex_enter(&sc->sc_atq_lock); 3611 CLR(iaq->iaq_flags, htole16(IXL_AQ_DD)); 3612 ixl_wakeup(sc, iaq); 3613 mutex_exit(&sc->sc_atq_lock); 3614 } 3615 3616 static void 3617 ixl_get_link_status_done(struct ixl_softc *sc, 3618 const struct ixl_aq_desc *iaq) 3619 { 3620 3621 ixl_work_add(sc->sc_workq, &sc->sc_link_state_done_task); 3622 } 3623 3624 static int 3625 ixl_get_link_status(struct ixl_softc *sc, enum ixl_link_flags flags) 3626 { 3627 struct ixl_atq *iatq; 3628 struct ixl_aq_desc *iaq; 3629 struct ixl_aq_link_param *param; 3630 int error; 3631 3632 mutex_enter(&sc->sc_atq_lock); 3633 3634 iatq = &sc->sc_link_state_atq; 3635 iaq = &iatq->iatq_desc; 3636 3637 if (!sc->sc_link_state_atq.iatq_inuse && 3638 !ISSET(iaq->iaq_flags, htole16(IXL_AQ_DD))) { 3639 memset(iaq, 0, sizeof(*iaq)); 3640 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3641 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3642 param->notify = IXL_AQ_LINK_NOTIFY; 3643 3644 KASSERT(iatq->iatq_fn == ixl_get_link_status_done); 3645 error = ixl_atq_post_locked(sc, iatq); 3646 if (error != 0) 3647 goto out; 3648 } else { 3649 /* the previous command is not completed */ 3650 error = EBUSY; 3651 } 3652 3653 if (ISSET(flags, IXL_LINK_FLAG_WAITDONE)) { 3654 do { 3655 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3656 IXL_ATQ_EXEC_TIMEOUT); 3657 if (error == EWOULDBLOCK) 3658 break; 3659 } while (iatq->iatq_inuse || 3660 ISSET(iaq->iaq_flags, htole16(IXL_AQ_DD))); 3661 } 3662 3663 out: 3664 mutex_exit(&sc->sc_atq_lock); 3665 3666 return error; 3667 } 3668 3669 static void 3670 ixl_get_link_status_work(void *xsc) 3671 { 3672 struct ixl_softc *sc = xsc; 3673 3674 /* 3675 * IXL_LINK_FLAG_WAITDONE causes deadlock 3676 * because of doing ixl_gt_link_status_done_work() 3677 * in the same workqueue. 3678 */ 3679 (void)ixl_get_link_status(sc, IXL_LINK_NOFLAGS); 3680 } 3681 3682 static void 3683 ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3684 { 3685 struct ifnet *ifp = &sc->sc_ec.ec_if; 3686 int link_state; 3687 3688 mutex_enter(&sc->sc_cfg_lock); 3689 link_state = ixl_set_link_status_locked(sc, iaq); 3690 mutex_exit(&sc->sc_cfg_lock); 3691 3692 if (ifp->if_link_state != link_state) 3693 if_link_state_change(ifp, link_state); 3694 3695 if (link_state != LINK_STATE_DOWN) { 3696 kpreempt_disable(); 3697 if_schedule_deferred_start(ifp); 3698 kpreempt_enable(); 3699 } 3700 } 3701 3702 static void 3703 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3704 const char *msg) 3705 { 3706 char buf[512]; 3707 size_t len; 3708 3709 len = sizeof(buf); 3710 buf[--len] = '\0'; 3711 3712 device_printf(sc->sc_dev, "%s\n", msg); 3713 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3714 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3715 buf, le16toh(iaq->iaq_opcode)); 3716 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3717 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3718 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3719 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3720 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3721 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3722 } 3723 3724 static void 3725 ixl_arq(void *xsc) 3726 { 3727 struct ixl_softc *sc = xsc; 3728 struct ixl_aq_desc *arq, *iaq; 3729 struct ixl_aq_buf *aqb; 3730 unsigned int cons = sc->sc_arq_cons; 3731 unsigned int prod; 3732 int done = 0; 3733 3734 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3735 sc->sc_aq_regs->arq_head_mask; 3736 3737 if (cons == prod) 3738 goto done; 3739 3740 arq = IXL_DMA_KVA(&sc->sc_arq); 3741 3742 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3743 0, IXL_DMA_LEN(&sc->sc_arq), 3744 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3745 3746 do { 3747 iaq = &arq[cons]; 3748 aqb = sc->sc_arq_live[cons]; 3749 3750 KASSERT(aqb != NULL); 3751 3752 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3753 BUS_DMASYNC_POSTREAD); 3754 3755 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3756 ixl_aq_dump(sc, iaq, "arq event"); 3757 3758 switch (iaq->iaq_opcode) { 3759 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3760 ixl_link_state_update(sc, iaq); 3761 break; 3762 } 3763 3764 memset(iaq, 0, sizeof(*iaq)); 3765 sc->sc_arq_live[cons] = NULL; 3766 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3767 3768 cons++; 3769 cons &= IXL_AQ_MASK; 3770 3771 done = 1; 3772 } while (cons != prod); 3773 3774 if (done) { 3775 sc->sc_arq_cons = cons; 3776 ixl_arq_fill(sc); 3777 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3778 0, IXL_DMA_LEN(&sc->sc_arq), 3779 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3780 } 3781 3782 done: 3783 ixl_enable_other_intr(sc); 3784 } 3785 3786 static void 3787 ixl_atq_set(struct ixl_atq *iatq, 3788 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3789 { 3790 3791 iatq->iatq_fn = fn; 3792 } 3793 3794 static int 3795 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3796 { 3797 struct ixl_aq_desc *atq, *slot; 3798 unsigned int prod, cons, prod_next; 3799 3800 /* assert locked */ 3801 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3802 3803 atq = IXL_DMA_KVA(&sc->sc_atq); 3804 prod = sc->sc_atq_prod; 3805 cons = sc->sc_atq_cons; 3806 prod_next = (prod +1) & IXL_AQ_MASK; 3807 3808 if (cons == prod_next) 3809 return ENOMEM; 3810 3811 slot = &atq[prod]; 3812 3813 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3814 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3815 3816 KASSERT(iatq->iatq_fn != NULL); 3817 *slot = iatq->iatq_desc; 3818 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3819 3820 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3821 ixl_aq_dump(sc, slot, "atq command"); 3822 3823 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3824 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3825 3826 sc->sc_atq_prod = prod_next; 3827 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3828 iatq->iatq_inuse = true; 3829 3830 return 0; 3831 } 3832 3833 static void 3834 ixl_atq_done_locked(struct ixl_softc *sc) 3835 { 3836 struct ixl_aq_desc *atq, *slot; 3837 struct ixl_atq *iatq; 3838 unsigned int cons; 3839 unsigned int prod; 3840 3841 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3842 3843 prod = sc->sc_atq_prod; 3844 cons = sc->sc_atq_cons; 3845 3846 if (prod == cons) 3847 return; 3848 3849 atq = IXL_DMA_KVA(&sc->sc_atq); 3850 3851 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3852 0, IXL_DMA_LEN(&sc->sc_atq), 3853 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3854 3855 do { 3856 slot = &atq[cons]; 3857 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3858 break; 3859 3860 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3861 iatq->iatq_desc = *slot; 3862 iatq->iatq_inuse = false; 3863 3864 memset(slot, 0, sizeof(*slot)); 3865 3866 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3867 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3868 3869 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3870 3871 cons++; 3872 cons &= IXL_AQ_MASK; 3873 } while (cons != prod); 3874 3875 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3876 0, IXL_DMA_LEN(&sc->sc_atq), 3877 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3878 3879 sc->sc_atq_cons = cons; 3880 } 3881 3882 static void 3883 ixl_atq_done(struct ixl_softc *sc) 3884 { 3885 3886 mutex_enter(&sc->sc_atq_lock); 3887 ixl_atq_done_locked(sc); 3888 mutex_exit(&sc->sc_atq_lock); 3889 } 3890 3891 static void 3892 ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3893 { 3894 3895 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3896 3897 cv_broadcast(&sc->sc_atq_cv); 3898 } 3899 3900 static int 3901 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3902 { 3903 int error; 3904 3905 mutex_enter(&sc->sc_atq_lock); 3906 error = ixl_atq_exec_locked(sc, iatq); 3907 mutex_exit(&sc->sc_atq_lock); 3908 3909 return error; 3910 } 3911 3912 static int 3913 ixl_atq_exec_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3914 { 3915 int error; 3916 3917 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3918 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3919 3920 ixl_atq_set(iatq, ixl_wakeup); 3921 3922 error = ixl_atq_post_locked(sc, iatq); 3923 if (error) 3924 return error; 3925 3926 do { 3927 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3928 IXL_ATQ_EXEC_TIMEOUT); 3929 if (error == EWOULDBLOCK) 3930 break; 3931 } while (iatq->iatq_inuse); 3932 3933 return error; 3934 } 3935 3936 static int 3937 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3938 { 3939 struct ixl_aq_desc *atq, *slot; 3940 unsigned int prod; 3941 unsigned int t = 0; 3942 3943 mutex_enter(&sc->sc_atq_lock); 3944 3945 atq = IXL_DMA_KVA(&sc->sc_atq); 3946 prod = sc->sc_atq_prod; 3947 slot = atq + prod; 3948 3949 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3950 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3951 3952 *slot = *iaq; 3953 slot->iaq_flags |= htole16(IXL_AQ_SI); 3954 3955 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3956 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3957 3958 prod++; 3959 prod &= IXL_AQ_MASK; 3960 sc->sc_atq_prod = prod; 3961 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3962 3963 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3964 delaymsec(1); 3965 3966 if (t++ > tm) { 3967 mutex_exit(&sc->sc_atq_lock); 3968 return ETIMEDOUT; 3969 } 3970 } 3971 3972 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3973 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3974 *iaq = *slot; 3975 memset(slot, 0, sizeof(*slot)); 3976 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3977 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3978 3979 sc->sc_atq_cons = prod; 3980 3981 mutex_exit(&sc->sc_atq_lock); 3982 3983 return 0; 3984 } 3985 3986 static int 3987 ixl_get_version(struct ixl_softc *sc) 3988 { 3989 struct ixl_aq_desc iaq; 3990 uint32_t fwbuild, fwver, apiver; 3991 uint16_t api_maj_ver, api_min_ver; 3992 3993 memset(&iaq, 0, sizeof(iaq)); 3994 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3995 3996 iaq.iaq_retval = le16toh(23); 3997 3998 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3999 return ETIMEDOUT; 4000 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 4001 return EIO; 4002 4003 fwbuild = le32toh(iaq.iaq_param[1]); 4004 fwver = le32toh(iaq.iaq_param[2]); 4005 apiver = le32toh(iaq.iaq_param[3]); 4006 4007 api_maj_ver = (uint16_t)apiver; 4008 api_min_ver = (uint16_t)(apiver >> 16); 4009 4010 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 4011 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 4012 4013 if (sc->sc_mac_type == I40E_MAC_X722) { 4014 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK | 4015 IXL_SC_AQ_FLAG_NVMREAD); 4016 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 4017 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS); 4018 } 4019 4020 #define IXL_API_VER(maj, min) (((uint32_t)(maj) << 16) | (min)) 4021 if (IXL_API_VER(api_maj_ver, api_min_ver) >= IXL_API_VER(1, 5)) { 4022 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 4023 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK); 4024 } 4025 #undef IXL_API_VER 4026 4027 return 0; 4028 } 4029 4030 static int 4031 ixl_get_nvm_version(struct ixl_softc *sc) 4032 { 4033 uint16_t nvmver, cfg_ptr, eetrack_hi, eetrack_lo, oem_hi, oem_lo; 4034 uint32_t eetrack, oem; 4035 uint16_t nvm_maj_ver, nvm_min_ver, oem_build; 4036 uint8_t oem_ver, oem_patch; 4037 4038 nvmver = cfg_ptr = eetrack_hi = eetrack_lo = oem_hi = oem_lo = 0; 4039 ixl_rd16_nvm(sc, I40E_SR_NVM_DEV_STARTER_VERSION, &nvmver); 4040 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_HI, &eetrack_hi); 4041 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_LO, &eetrack_lo); 4042 ixl_rd16_nvm(sc, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr); 4043 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF, &oem_hi); 4044 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF + 1, &oem_lo); 4045 4046 nvm_maj_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_HI_MASK); 4047 nvm_min_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_LO_MASK); 4048 eetrack = ((uint32_t)eetrack_hi << 16) | eetrack_lo; 4049 oem = ((uint32_t)oem_hi << 16) | oem_lo; 4050 oem_ver = __SHIFTOUT(oem, IXL_NVM_OEMVERSION_MASK); 4051 oem_build = __SHIFTOUT(oem, IXL_NVM_OEMBUILD_MASK); 4052 oem_patch = __SHIFTOUT(oem, IXL_NVM_OEMPATCH_MASK); 4053 4054 aprint_normal(" nvm %x.%02x etid %08x oem %d.%d.%d", 4055 nvm_maj_ver, nvm_min_ver, eetrack, 4056 oem_ver, oem_build, oem_patch); 4057 4058 return 0; 4059 } 4060 4061 static int 4062 ixl_pxe_clear(struct ixl_softc *sc) 4063 { 4064 struct ixl_aq_desc iaq; 4065 int rv; 4066 4067 memset(&iaq, 0, sizeof(iaq)); 4068 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 4069 iaq.iaq_param[0] = htole32(0x2); 4070 4071 rv = ixl_atq_poll(sc, &iaq, 250); 4072 4073 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 4074 4075 if (rv != 0) 4076 return ETIMEDOUT; 4077 4078 switch (iaq.iaq_retval) { 4079 case htole16(IXL_AQ_RC_OK): 4080 case htole16(IXL_AQ_RC_EEXIST): 4081 break; 4082 default: 4083 return EIO; 4084 } 4085 4086 return 0; 4087 } 4088 4089 static int 4090 ixl_lldp_shut(struct ixl_softc *sc) 4091 { 4092 struct ixl_aq_desc iaq; 4093 4094 memset(&iaq, 0, sizeof(iaq)); 4095 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 4096 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 4097 4098 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4099 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 4100 return -1; 4101 } 4102 4103 switch (iaq.iaq_retval) { 4104 case htole16(IXL_AQ_RC_EMODE): 4105 case htole16(IXL_AQ_RC_EPERM): 4106 /* ignore silently */ 4107 default: 4108 break; 4109 } 4110 4111 return 0; 4112 } 4113 4114 static void 4115 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 4116 { 4117 uint16_t id; 4118 uint32_t number, logical_id; 4119 4120 id = le16toh(cap->cap_id); 4121 number = le32toh(cap->number); 4122 logical_id = le32toh(cap->logical_id); 4123 4124 switch (id) { 4125 case IXL_AQ_CAP_RSS: 4126 sc->sc_rss_table_size = number; 4127 sc->sc_rss_table_entry_width = logical_id; 4128 break; 4129 case IXL_AQ_CAP_RXQ: 4130 case IXL_AQ_CAP_TXQ: 4131 sc->sc_nqueue_pairs_device = MIN(number, 4132 sc->sc_nqueue_pairs_device); 4133 break; 4134 } 4135 } 4136 4137 static int 4138 ixl_get_hw_capabilities(struct ixl_softc *sc) 4139 { 4140 struct ixl_dmamem idm; 4141 struct ixl_aq_desc iaq; 4142 struct ixl_aq_capability *caps; 4143 size_t i, ncaps; 4144 bus_size_t caps_size; 4145 uint16_t status; 4146 int rv; 4147 4148 caps_size = sizeof(caps[0]) * 40; 4149 memset(&iaq, 0, sizeof(iaq)); 4150 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 4151 4152 do { 4153 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 4154 return -1; 4155 } 4156 4157 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4158 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4159 iaq.iaq_datalen = htole16(caps_size); 4160 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4161 4162 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4163 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 4164 4165 rv = ixl_atq_poll(sc, &iaq, 250); 4166 4167 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4168 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 4169 4170 if (rv != 0) { 4171 aprint_error(", HW capabilities timeout\n"); 4172 goto done; 4173 } 4174 4175 status = le16toh(iaq.iaq_retval); 4176 4177 if (status == IXL_AQ_RC_ENOMEM) { 4178 caps_size = le16toh(iaq.iaq_datalen); 4179 ixl_dmamem_free(sc, &idm); 4180 } 4181 } while (status == IXL_AQ_RC_ENOMEM); 4182 4183 if (status != IXL_AQ_RC_OK) { 4184 aprint_error(", HW capabilities error\n"); 4185 goto done; 4186 } 4187 4188 caps = IXL_DMA_KVA(&idm); 4189 ncaps = le16toh(iaq.iaq_param[1]); 4190 4191 for (i = 0; i < ncaps; i++) { 4192 ixl_parse_hw_capability(sc, &caps[i]); 4193 } 4194 4195 done: 4196 ixl_dmamem_free(sc, &idm); 4197 return rv; 4198 } 4199 4200 static int 4201 ixl_get_mac(struct ixl_softc *sc) 4202 { 4203 struct ixl_dmamem idm; 4204 struct ixl_aq_desc iaq; 4205 struct ixl_aq_mac_addresses *addrs; 4206 int rv; 4207 4208 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 4209 aprint_error(", unable to allocate mac addresses\n"); 4210 return -1; 4211 } 4212 4213 memset(&iaq, 0, sizeof(iaq)); 4214 iaq.iaq_flags = htole16(IXL_AQ_BUF); 4215 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 4216 iaq.iaq_datalen = htole16(sizeof(*addrs)); 4217 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4218 4219 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4220 BUS_DMASYNC_PREREAD); 4221 4222 rv = ixl_atq_poll(sc, &iaq, 250); 4223 4224 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4225 BUS_DMASYNC_POSTREAD); 4226 4227 if (rv != 0) { 4228 aprint_error(", MAC ADDRESS READ timeout\n"); 4229 rv = -1; 4230 goto done; 4231 } 4232 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4233 aprint_error(", MAC ADDRESS READ error\n"); 4234 rv = -1; 4235 goto done; 4236 } 4237 4238 addrs = IXL_DMA_KVA(&idm); 4239 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 4240 printf(", port address is not valid\n"); 4241 goto done; 4242 } 4243 4244 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 4245 rv = 0; 4246 4247 done: 4248 ixl_dmamem_free(sc, &idm); 4249 return rv; 4250 } 4251 4252 static int 4253 ixl_get_switch_config(struct ixl_softc *sc) 4254 { 4255 struct ixl_dmamem idm; 4256 struct ixl_aq_desc iaq; 4257 struct ixl_aq_switch_config *hdr; 4258 struct ixl_aq_switch_config_element *elms, *elm; 4259 unsigned int nelm, i; 4260 int rv; 4261 4262 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4263 aprint_error_dev(sc->sc_dev, 4264 "unable to allocate switch config buffer\n"); 4265 return -1; 4266 } 4267 4268 memset(&iaq, 0, sizeof(iaq)); 4269 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4270 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4271 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 4272 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 4273 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4274 4275 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4276 BUS_DMASYNC_PREREAD); 4277 4278 rv = ixl_atq_poll(sc, &iaq, 250); 4279 4280 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4281 BUS_DMASYNC_POSTREAD); 4282 4283 if (rv != 0) { 4284 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 4285 rv = -1; 4286 goto done; 4287 } 4288 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4289 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 4290 rv = -1; 4291 goto done; 4292 } 4293 4294 hdr = IXL_DMA_KVA(&idm); 4295 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 4296 4297 nelm = le16toh(hdr->num_reported); 4298 if (nelm < 1) { 4299 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 4300 rv = -1; 4301 goto done; 4302 } 4303 4304 for (i = 0; i < nelm; i++) { 4305 elm = &elms[i]; 4306 4307 aprint_debug_dev(sc->sc_dev, 4308 "type %x revision %u seid %04x\n", 4309 elm->type, elm->revision, le16toh(elm->seid)); 4310 aprint_debug_dev(sc->sc_dev, 4311 "uplink %04x downlink %04x\n", 4312 le16toh(elm->uplink_seid), 4313 le16toh(elm->downlink_seid)); 4314 aprint_debug_dev(sc->sc_dev, 4315 "conntype %x scheduler %04x extra %04x\n", 4316 elm->connection_type, 4317 le16toh(elm->scheduler_id), 4318 le16toh(elm->element_info)); 4319 } 4320 4321 elm = &elms[0]; 4322 4323 sc->sc_uplink_seid = elm->uplink_seid; 4324 sc->sc_downlink_seid = elm->downlink_seid; 4325 sc->sc_seid = elm->seid; 4326 4327 if ((sc->sc_uplink_seid == htole16(0)) != 4328 (sc->sc_downlink_seid == htole16(0))) { 4329 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 4330 rv = -1; 4331 goto done; 4332 } 4333 4334 done: 4335 ixl_dmamem_free(sc, &idm); 4336 return rv; 4337 } 4338 4339 static int 4340 ixl_phy_mask_ints(struct ixl_softc *sc) 4341 { 4342 struct ixl_aq_desc iaq; 4343 4344 memset(&iaq, 0, sizeof(iaq)); 4345 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 4346 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 4347 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 4348 IXL_AQ_PHY_EV_MEDIA_NA)); 4349 4350 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4351 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 4352 return -1; 4353 } 4354 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4355 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 4356 return -1; 4357 } 4358 4359 return 0; 4360 } 4361 4362 static int 4363 ixl_get_phy_abilities(struct ixl_softc *sc, struct ixl_dmamem *idm) 4364 { 4365 struct ixl_aq_desc iaq; 4366 int rv; 4367 4368 memset(&iaq, 0, sizeof(iaq)); 4369 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4370 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4371 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 4372 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 4373 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 4374 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 4375 4376 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4377 BUS_DMASYNC_PREREAD); 4378 4379 rv = ixl_atq_poll(sc, &iaq, 250); 4380 4381 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4382 BUS_DMASYNC_POSTREAD); 4383 4384 if (rv != 0) 4385 return -1; 4386 4387 return le16toh(iaq.iaq_retval); 4388 } 4389 4390 static int 4391 ixl_get_phy_info(struct ixl_softc *sc) 4392 { 4393 struct ixl_dmamem idm; 4394 struct ixl_aq_phy_abilities *phy; 4395 int rv; 4396 4397 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4398 aprint_error_dev(sc->sc_dev, 4399 "unable to allocate phy abilities buffer\n"); 4400 return -1; 4401 } 4402 4403 rv = ixl_get_phy_abilities(sc, &idm); 4404 switch (rv) { 4405 case -1: 4406 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 4407 goto done; 4408 case IXL_AQ_RC_OK: 4409 break; 4410 case IXL_AQ_RC_EIO: 4411 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 4412 goto done; 4413 default: 4414 aprint_error_dev(sc->sc_dev, 4415 "GET PHY ABILITIIES error %u\n", rv); 4416 goto done; 4417 } 4418 4419 phy = IXL_DMA_KVA(&idm); 4420 4421 sc->sc_phy_types = le32toh(phy->phy_type); 4422 sc->sc_phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 4423 4424 sc->sc_phy_abilities = phy->abilities; 4425 sc->sc_phy_linkspeed = phy->link_speed; 4426 sc->sc_phy_fec_cfg = phy->fec_cfg_curr_mod_ext_info & 4427 (IXL_AQ_ENABLE_FEC_KR | IXL_AQ_ENABLE_FEC_RS | 4428 IXL_AQ_REQUEST_FEC_KR | IXL_AQ_REQUEST_FEC_RS); 4429 sc->sc_eee_cap = phy->eee_capability; 4430 sc->sc_eeer_val = phy->eeer_val; 4431 sc->sc_d3_lpan = phy->d3_lpan; 4432 4433 rv = 0; 4434 4435 done: 4436 ixl_dmamem_free(sc, &idm); 4437 return rv; 4438 } 4439 4440 static int 4441 ixl_set_phy_config(struct ixl_softc *sc, 4442 uint8_t link_speed, uint8_t abilities, bool polling) 4443 { 4444 struct ixl_aq_phy_param *param; 4445 struct ixl_atq iatq; 4446 struct ixl_aq_desc *iaq; 4447 int error; 4448 4449 memset(&iatq, 0, sizeof(iatq)); 4450 4451 iaq = &iatq.iatq_desc; 4452 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_CONFIG); 4453 param = (struct ixl_aq_phy_param *)&iaq->iaq_param; 4454 param->phy_types = htole32((uint32_t)sc->sc_phy_types); 4455 param->phy_type_ext = (uint8_t)(sc->sc_phy_types >> 32); 4456 param->link_speed = link_speed; 4457 param->abilities = abilities | IXL_AQ_PHY_ABILITY_AUTO_LINK; 4458 param->fec_cfg = sc->sc_phy_fec_cfg; 4459 param->eee_capability = sc->sc_eee_cap; 4460 param->eeer_val = sc->sc_eeer_val; 4461 param->d3_lpan = sc->sc_d3_lpan; 4462 4463 if (polling) 4464 error = ixl_atq_poll(sc, iaq, 250); 4465 else 4466 error = ixl_atq_exec(sc, &iatq); 4467 4468 if (error != 0) 4469 return error; 4470 4471 switch (le16toh(iaq->iaq_retval)) { 4472 case IXL_AQ_RC_OK: 4473 break; 4474 case IXL_AQ_RC_EPERM: 4475 return EPERM; 4476 default: 4477 return EIO; 4478 } 4479 4480 return 0; 4481 } 4482 4483 static int 4484 ixl_set_phy_autoselect(struct ixl_softc *sc) 4485 { 4486 uint8_t link_speed, abilities; 4487 4488 link_speed = sc->sc_phy_linkspeed; 4489 abilities = IXL_PHY_ABILITY_LINKUP | IXL_PHY_ABILITY_AUTONEGO; 4490 4491 return ixl_set_phy_config(sc, link_speed, abilities, true); 4492 } 4493 4494 static int 4495 ixl_get_link_status_poll(struct ixl_softc *sc, int *l) 4496 { 4497 struct ixl_aq_desc iaq; 4498 struct ixl_aq_link_param *param; 4499 int link; 4500 4501 memset(&iaq, 0, sizeof(iaq)); 4502 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 4503 param = (struct ixl_aq_link_param *)iaq.iaq_param; 4504 param->notify = IXL_AQ_LINK_NOTIFY; 4505 4506 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4507 return ETIMEDOUT; 4508 } 4509 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4510 return EIO; 4511 } 4512 4513 /* It is unneccessary to hold lock */ 4514 link = ixl_set_link_status_locked(sc, &iaq); 4515 4516 if (l != NULL) 4517 *l = link; 4518 4519 return 0; 4520 } 4521 4522 static int 4523 ixl_get_vsi(struct ixl_softc *sc) 4524 { 4525 struct ixl_dmamem *vsi = &sc->sc_scratch; 4526 struct ixl_aq_desc iaq; 4527 struct ixl_aq_vsi_param *param; 4528 struct ixl_aq_vsi_reply *reply; 4529 struct ixl_aq_vsi_data *data; 4530 int rv; 4531 4532 /* grumble, vsi info isn't "known" at compile time */ 4533 4534 memset(&iaq, 0, sizeof(iaq)); 4535 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4536 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4537 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 4538 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4539 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4540 4541 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4542 param->uplink_seid = sc->sc_seid; 4543 4544 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4545 BUS_DMASYNC_PREREAD); 4546 4547 rv = ixl_atq_poll(sc, &iaq, 250); 4548 4549 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4550 BUS_DMASYNC_POSTREAD); 4551 4552 if (rv != 0) { 4553 return ETIMEDOUT; 4554 } 4555 4556 switch (le16toh(iaq.iaq_retval)) { 4557 case IXL_AQ_RC_OK: 4558 break; 4559 case IXL_AQ_RC_ENOENT: 4560 return ENOENT; 4561 case IXL_AQ_RC_EACCES: 4562 return EACCES; 4563 default: 4564 return EIO; 4565 } 4566 4567 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 4568 sc->sc_vsi_number = le16toh(reply->vsi_number); 4569 data = IXL_DMA_KVA(vsi); 4570 sc->sc_vsi_stat_counter_idx = le16toh(data->stat_counter_idx); 4571 4572 return 0; 4573 } 4574 4575 static int 4576 ixl_set_vsi(struct ixl_softc *sc) 4577 { 4578 struct ixl_dmamem *vsi = &sc->sc_scratch; 4579 struct ixl_aq_desc iaq; 4580 struct ixl_aq_vsi_param *param; 4581 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 4582 unsigned int qnum; 4583 uint16_t val; 4584 int rv; 4585 4586 qnum = sc->sc_nqueue_pairs - 1; 4587 4588 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 4589 IXL_AQ_VSI_VALID_VLAN); 4590 4591 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 4592 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 4593 data->queue_mapping[0] = htole16(0); 4594 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 4595 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 4596 4597 val = le16toh(data->port_vlan_flags); 4598 CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK); 4599 SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL); 4600 4601 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { 4602 SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH); 4603 } else { 4604 SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING); 4605 } 4606 4607 data->port_vlan_flags = htole16(val); 4608 4609 /* grumble, vsi info isn't "known" at compile time */ 4610 4611 memset(&iaq, 0, sizeof(iaq)); 4612 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4613 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4614 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 4615 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4616 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4617 4618 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4619 param->uplink_seid = sc->sc_seid; 4620 4621 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4622 BUS_DMASYNC_PREWRITE); 4623 4624 rv = ixl_atq_poll(sc, &iaq, 250); 4625 4626 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4627 BUS_DMASYNC_POSTWRITE); 4628 4629 if (rv != 0) { 4630 return ETIMEDOUT; 4631 } 4632 4633 switch (le16toh(iaq.iaq_retval)) { 4634 case IXL_AQ_RC_OK: 4635 break; 4636 case IXL_AQ_RC_ENOENT: 4637 return ENOENT; 4638 case IXL_AQ_RC_EACCES: 4639 return EACCES; 4640 default: 4641 return EIO; 4642 } 4643 4644 return 0; 4645 } 4646 4647 static void 4648 ixl_set_filter_control(struct ixl_softc *sc) 4649 { 4650 uint32_t reg; 4651 4652 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4653 4654 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4655 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4656 4657 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4658 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4659 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4660 4661 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4662 } 4663 4664 static inline void 4665 ixl_get_default_rss_key(uint32_t *buf, size_t len) 4666 { 4667 size_t cplen; 4668 uint8_t rss_seed[RSS_KEYSIZE]; 4669 4670 rss_getkey(rss_seed); 4671 memset(buf, 0, len); 4672 4673 cplen = MIN(len, sizeof(rss_seed)); 4674 memcpy(buf, rss_seed, cplen); 4675 } 4676 4677 static int 4678 ixl_set_rss_key(struct ixl_softc *sc, uint8_t *key, size_t keylen) 4679 { 4680 struct ixl_dmamem *idm; 4681 struct ixl_atq iatq; 4682 struct ixl_aq_desc *iaq; 4683 struct ixl_aq_rss_key_param *param; 4684 struct ixl_aq_rss_key_data *data; 4685 size_t len, datalen, stdlen, extlen; 4686 uint16_t vsi_id; 4687 int rv; 4688 4689 memset(&iatq, 0, sizeof(iatq)); 4690 iaq = &iatq.iatq_desc; 4691 idm = &sc->sc_aqbuf; 4692 4693 datalen = sizeof(*data); 4694 4695 /*XXX The buf size has to be less than the size of the register */ 4696 datalen = MIN(IXL_RSS_KEY_SIZE_REG * sizeof(uint32_t), datalen); 4697 4698 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4699 (datalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4700 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_KEY); 4701 iaq->iaq_datalen = htole16(datalen); 4702 4703 param = (struct ixl_aq_rss_key_param *)iaq->iaq_param; 4704 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSKEY_VSI_ID_SHIFT) | 4705 IXL_AQ_RSSKEY_VSI_VALID; 4706 param->vsi_id = htole16(vsi_id); 4707 4708 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4709 data = IXL_DMA_KVA(idm); 4710 4711 len = MIN(keylen, datalen); 4712 stdlen = MIN(sizeof(data->standard_rss_key), len); 4713 memcpy(data->standard_rss_key, key, stdlen); 4714 len = (len > stdlen) ? (len - stdlen) : 0; 4715 4716 extlen = MIN(sizeof(data->extended_hash_key), len); 4717 extlen = (stdlen < keylen) ? 0 : keylen - stdlen; 4718 memcpy(data->extended_hash_key, key + stdlen, extlen); 4719 4720 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4721 4722 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4723 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4724 4725 rv = ixl_atq_exec(sc, &iatq); 4726 4727 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4728 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4729 4730 if (rv != 0) { 4731 return ETIMEDOUT; 4732 } 4733 4734 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4735 return EIO; 4736 } 4737 4738 return 0; 4739 } 4740 4741 static int 4742 ixl_set_rss_lut(struct ixl_softc *sc, uint8_t *lut, size_t lutlen) 4743 { 4744 struct ixl_dmamem *idm; 4745 struct ixl_atq iatq; 4746 struct ixl_aq_desc *iaq; 4747 struct ixl_aq_rss_lut_param *param; 4748 uint16_t vsi_id; 4749 uint8_t *data; 4750 size_t dmalen; 4751 int rv; 4752 4753 memset(&iatq, 0, sizeof(iatq)); 4754 iaq = &iatq.iatq_desc; 4755 idm = &sc->sc_aqbuf; 4756 4757 dmalen = MIN(lutlen, IXL_DMA_LEN(idm)); 4758 4759 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4760 (dmalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4761 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_LUT); 4762 iaq->iaq_datalen = htole16(dmalen); 4763 4764 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4765 data = IXL_DMA_KVA(idm); 4766 memcpy(data, lut, dmalen); 4767 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4768 4769 param = (struct ixl_aq_rss_lut_param *)iaq->iaq_param; 4770 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSLUT_VSI_ID_SHIFT) | 4771 IXL_AQ_RSSLUT_VSI_VALID; 4772 param->vsi_id = htole16(vsi_id); 4773 param->flags = htole16(IXL_AQ_RSSLUT_TABLE_TYPE_PF << 4774 IXL_AQ_RSSLUT_TABLE_TYPE_SHIFT); 4775 4776 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4777 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4778 4779 rv = ixl_atq_exec(sc, &iatq); 4780 4781 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4782 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4783 4784 if (rv != 0) { 4785 return ETIMEDOUT; 4786 } 4787 4788 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4789 return EIO; 4790 } 4791 4792 return 0; 4793 } 4794 4795 static int 4796 ixl_register_rss_key(struct ixl_softc *sc) 4797 { 4798 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4799 int rv; 4800 size_t i; 4801 4802 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4803 4804 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4805 rv = ixl_set_rss_key(sc, (uint8_t*)rss_seed, 4806 sizeof(rss_seed)); 4807 } else { 4808 rv = 0; 4809 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4810 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4811 } 4812 } 4813 4814 return rv; 4815 } 4816 4817 static void 4818 ixl_register_rss_pctype(struct ixl_softc *sc) 4819 { 4820 uint64_t set_hena = 0; 4821 uint32_t hena0, hena1; 4822 4823 /* 4824 * We use TCP/UDP with IPv4/IPv6 by default. 4825 * Note: the device can not use just IP header in each 4826 * TCP/UDP packets for the RSS hash calculation. 4827 */ 4828 if (sc->sc_mac_type == I40E_MAC_X722) 4829 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4830 else 4831 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4832 4833 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4834 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4835 4836 SET(hena0, set_hena); 4837 SET(hena1, set_hena >> 32); 4838 4839 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4840 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4841 } 4842 4843 static int 4844 ixl_register_rss_hlut(struct ixl_softc *sc) 4845 { 4846 unsigned int qid; 4847 uint8_t hlut_buf[512], lut_mask; 4848 uint32_t *hluts; 4849 size_t i, hluts_num; 4850 int rv; 4851 4852 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4853 4854 for (i = 0; i < sc->sc_rss_table_size; i++) { 4855 qid = i % sc->sc_nqueue_pairs; 4856 hlut_buf[i] = qid & lut_mask; 4857 } 4858 4859 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4860 rv = ixl_set_rss_lut(sc, hlut_buf, sizeof(hlut_buf)); 4861 } else { 4862 rv = 0; 4863 hluts = (uint32_t *)hlut_buf; 4864 hluts_num = sc->sc_rss_table_size >> 2; 4865 for (i = 0; i < hluts_num; i++) { 4866 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4867 } 4868 ixl_flush(sc); 4869 } 4870 4871 return rv; 4872 } 4873 4874 static void 4875 ixl_config_rss(struct ixl_softc *sc) 4876 { 4877 4878 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4879 4880 ixl_register_rss_key(sc); 4881 ixl_register_rss_pctype(sc); 4882 ixl_register_rss_hlut(sc); 4883 } 4884 4885 static const struct ixl_phy_type * 4886 ixl_search_phy_type(uint8_t phy_type) 4887 { 4888 const struct ixl_phy_type *itype; 4889 uint64_t mask; 4890 unsigned int i; 4891 4892 if (phy_type >= 64) 4893 return NULL; 4894 4895 mask = 1ULL << phy_type; 4896 4897 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4898 itype = &ixl_phy_type_map[i]; 4899 4900 if (ISSET(itype->phy_type, mask)) 4901 return itype; 4902 } 4903 4904 return NULL; 4905 } 4906 4907 static uint64_t 4908 ixl_search_link_speed(uint8_t link_speed) 4909 { 4910 const struct ixl_speed_type *type; 4911 unsigned int i; 4912 4913 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4914 type = &ixl_speed_type_map[i]; 4915 4916 if (ISSET(type->dev_speed, link_speed)) 4917 return type->net_speed; 4918 } 4919 4920 return 0; 4921 } 4922 4923 static uint8_t 4924 ixl_search_baudrate(uint64_t baudrate) 4925 { 4926 const struct ixl_speed_type *type; 4927 unsigned int i; 4928 4929 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4930 type = &ixl_speed_type_map[i]; 4931 4932 if (type->net_speed == baudrate) { 4933 return type->dev_speed; 4934 } 4935 } 4936 4937 return 0; 4938 } 4939 4940 static int 4941 ixl_restart_an(struct ixl_softc *sc) 4942 { 4943 struct ixl_aq_desc iaq; 4944 4945 memset(&iaq, 0, sizeof(iaq)); 4946 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4947 iaq.iaq_param[0] = 4948 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4949 4950 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4951 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4952 return -1; 4953 } 4954 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4955 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4956 return -1; 4957 } 4958 4959 return 0; 4960 } 4961 4962 static int 4963 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4964 uint16_t vlan, uint16_t flags) 4965 { 4966 struct ixl_aq_desc iaq; 4967 struct ixl_aq_add_macvlan *param; 4968 struct ixl_aq_add_macvlan_elem *elem; 4969 4970 memset(&iaq, 0, sizeof(iaq)); 4971 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4972 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4973 iaq.iaq_datalen = htole16(sizeof(*elem)); 4974 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4975 4976 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4977 param->num_addrs = htole16(1); 4978 param->seid0 = htole16(0x8000) | sc->sc_seid; 4979 param->seid1 = 0; 4980 param->seid2 = 0; 4981 4982 elem = IXL_DMA_KVA(&sc->sc_scratch); 4983 memset(elem, 0, sizeof(*elem)); 4984 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4985 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4986 elem->vlan = htole16(vlan); 4987 4988 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4989 return IXL_AQ_RC_EINVAL; 4990 } 4991 4992 switch (le16toh(iaq.iaq_retval)) { 4993 case IXL_AQ_RC_OK: 4994 break; 4995 case IXL_AQ_RC_ENOSPC: 4996 return ENOSPC; 4997 case IXL_AQ_RC_ENOENT: 4998 return ENOENT; 4999 case IXL_AQ_RC_EACCES: 5000 return EACCES; 5001 case IXL_AQ_RC_EEXIST: 5002 return EEXIST; 5003 case IXL_AQ_RC_EINVAL: 5004 return EINVAL; 5005 default: 5006 return EIO; 5007 } 5008 5009 return 0; 5010 } 5011 5012 static int 5013 ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 5014 uint16_t vlan, uint16_t flags) 5015 { 5016 struct ixl_aq_desc iaq; 5017 struct ixl_aq_remove_macvlan *param; 5018 struct ixl_aq_remove_macvlan_elem *elem; 5019 5020 memset(&iaq, 0, sizeof(iaq)); 5021 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 5022 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 5023 iaq.iaq_datalen = htole16(sizeof(*elem)); 5024 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 5025 5026 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 5027 param->num_addrs = htole16(1); 5028 param->seid0 = htole16(0x8000) | sc->sc_seid; 5029 param->seid1 = 0; 5030 param->seid2 = 0; 5031 5032 elem = IXL_DMA_KVA(&sc->sc_scratch); 5033 memset(elem, 0, sizeof(*elem)); 5034 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 5035 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 5036 elem->vlan = htole16(vlan); 5037 5038 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 5039 return EINVAL; 5040 } 5041 5042 switch (le16toh(iaq.iaq_retval)) { 5043 case IXL_AQ_RC_OK: 5044 break; 5045 case IXL_AQ_RC_ENOENT: 5046 return ENOENT; 5047 case IXL_AQ_RC_EACCES: 5048 return EACCES; 5049 case IXL_AQ_RC_EINVAL: 5050 return EINVAL; 5051 default: 5052 return EIO; 5053 } 5054 5055 return 0; 5056 } 5057 5058 static int 5059 ixl_hmc(struct ixl_softc *sc) 5060 { 5061 struct { 5062 uint32_t count; 5063 uint32_t minsize; 5064 bus_size_t objsiz; 5065 bus_size_t setoff; 5066 bus_size_t setcnt; 5067 } regs[] = { 5068 { 5069 0, 5070 IXL_HMC_TXQ_MINSIZE, 5071 I40E_GLHMC_LANTXOBJSZ, 5072 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 5073 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 5074 }, 5075 { 5076 0, 5077 IXL_HMC_RXQ_MINSIZE, 5078 I40E_GLHMC_LANRXOBJSZ, 5079 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 5080 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 5081 }, 5082 { 5083 0, 5084 0, 5085 I40E_GLHMC_FCOEDDPOBJSZ, 5086 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 5087 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 5088 }, 5089 { 5090 0, 5091 0, 5092 I40E_GLHMC_FCOEFOBJSZ, 5093 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 5094 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 5095 }, 5096 }; 5097 struct ixl_hmc_entry *e; 5098 uint64_t size, dva; 5099 uint8_t *kva; 5100 uint64_t *sdpage; 5101 unsigned int i; 5102 int npages, tables; 5103 uint32_t reg; 5104 5105 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 5106 5107 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 5108 ixl_rd(sc, I40E_GLHMC_LANQMAX); 5109 5110 size = 0; 5111 for (i = 0; i < __arraycount(regs); i++) { 5112 e = &sc->sc_hmc_entries[i]; 5113 5114 e->hmc_count = regs[i].count; 5115 reg = ixl_rd(sc, regs[i].objsiz); 5116 e->hmc_size = IXL_BIT_ULL(0x3F & reg); 5117 e->hmc_base = size; 5118 5119 if ((e->hmc_size * 8) < regs[i].minsize) { 5120 aprint_error_dev(sc->sc_dev, 5121 "kernel hmc entry is too big\n"); 5122 return -1; 5123 } 5124 5125 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 5126 } 5127 size = roundup(size, IXL_HMC_PGSIZE); 5128 npages = size / IXL_HMC_PGSIZE; 5129 5130 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 5131 5132 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 5133 aprint_error_dev(sc->sc_dev, 5134 "unable to allocate hmc pd memory\n"); 5135 return -1; 5136 } 5137 5138 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 5139 IXL_HMC_PGSIZE) != 0) { 5140 aprint_error_dev(sc->sc_dev, 5141 "unable to allocate hmc sd memory\n"); 5142 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5143 return -1; 5144 } 5145 5146 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 5147 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 5148 5149 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 5150 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 5151 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5152 5153 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 5154 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 5155 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 5156 5157 for (i = 0; (int)i < npages; i++) { 5158 *sdpage = htole64(dva | IXL_HMC_PDVALID); 5159 sdpage++; 5160 5161 dva += IXL_HMC_PGSIZE; 5162 } 5163 5164 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 5165 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 5166 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5167 5168 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 5169 for (i = 0; (int)i < tables; i++) { 5170 uint32_t count; 5171 5172 KASSERT(npages >= 0); 5173 5174 count = ((unsigned int)npages > IXL_HMC_PGS) ? 5175 IXL_HMC_PGS : (unsigned int)npages; 5176 5177 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 5178 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 5179 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 5180 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 5181 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 5182 ixl_wr(sc, I40E_PFHMC_SDCMD, 5183 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 5184 5185 npages -= IXL_HMC_PGS; 5186 dva += IXL_HMC_PGSIZE; 5187 } 5188 5189 for (i = 0; i < __arraycount(regs); i++) { 5190 e = &sc->sc_hmc_entries[i]; 5191 5192 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 5193 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 5194 } 5195 5196 return 0; 5197 } 5198 5199 static void 5200 ixl_hmc_free(struct ixl_softc *sc) 5201 { 5202 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 5203 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5204 } 5205 5206 static void 5207 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 5208 unsigned int npacking) 5209 { 5210 uint8_t *dst = d; 5211 const uint8_t *src = s; 5212 unsigned int i; 5213 5214 for (i = 0; i < npacking; i++) { 5215 const struct ixl_hmc_pack *pack = &packing[i]; 5216 unsigned int offset = pack->lsb / 8; 5217 unsigned int align = pack->lsb % 8; 5218 const uint8_t *in = src + pack->offset; 5219 uint8_t *out = dst + offset; 5220 int width = pack->width; 5221 unsigned int inbits = 0; 5222 5223 if (align) { 5224 inbits = (*in++) << align; 5225 *out++ |= (inbits & 0xff); 5226 inbits >>= 8; 5227 5228 width -= 8 - align; 5229 } 5230 5231 while (width >= 8) { 5232 inbits |= (*in++) << align; 5233 *out++ = (inbits & 0xff); 5234 inbits >>= 8; 5235 5236 width -= 8; 5237 } 5238 5239 if (width > 0) { 5240 inbits |= (*in) << align; 5241 *out |= (inbits & ((1 << width) - 1)); 5242 } 5243 } 5244 } 5245 5246 static struct ixl_aq_buf * 5247 ixl_aqb_alloc(struct ixl_softc *sc) 5248 { 5249 struct ixl_aq_buf *aqb; 5250 5251 aqb = kmem_alloc(sizeof(*aqb), KM_SLEEP); 5252 5253 aqb->aqb_size = IXL_AQ_BUFLEN; 5254 5255 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 5256 aqb->aqb_size, 0, 5257 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 5258 goto free; 5259 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 5260 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 5261 BUS_DMA_WAITOK) != 0) 5262 goto destroy; 5263 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 5264 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 5265 goto dma_free; 5266 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 5267 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 5268 goto unmap; 5269 5270 return aqb; 5271 unmap: 5272 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5273 dma_free: 5274 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5275 destroy: 5276 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5277 free: 5278 kmem_free(aqb, sizeof(*aqb)); 5279 5280 return NULL; 5281 } 5282 5283 static void 5284 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 5285 { 5286 5287 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 5288 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5289 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5290 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5291 kmem_free(aqb, sizeof(*aqb)); 5292 } 5293 5294 static int 5295 ixl_arq_fill(struct ixl_softc *sc) 5296 { 5297 struct ixl_aq_buf *aqb; 5298 struct ixl_aq_desc *arq, *iaq; 5299 unsigned int prod = sc->sc_arq_prod; 5300 unsigned int n; 5301 int post = 0; 5302 5303 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 5304 IXL_AQ_NUM); 5305 arq = IXL_DMA_KVA(&sc->sc_arq); 5306 5307 if (__predict_false(n <= 0)) 5308 return 0; 5309 5310 do { 5311 aqb = sc->sc_arq_live[prod]; 5312 iaq = &arq[prod]; 5313 5314 if (aqb == NULL) { 5315 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 5316 if (aqb != NULL) { 5317 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5318 ixl_aq_buf, aqb_entry); 5319 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 5320 break; 5321 } 5322 5323 sc->sc_arq_live[prod] = aqb; 5324 memset(aqb->aqb_data, 0, aqb->aqb_size); 5325 5326 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 5327 aqb->aqb_size, BUS_DMASYNC_PREREAD); 5328 5329 iaq->iaq_flags = htole16(IXL_AQ_BUF | 5330 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 5331 IXL_AQ_LB : 0)); 5332 iaq->iaq_opcode = 0; 5333 iaq->iaq_datalen = htole16(aqb->aqb_size); 5334 iaq->iaq_retval = 0; 5335 iaq->iaq_cookie = 0; 5336 iaq->iaq_param[0] = 0; 5337 iaq->iaq_param[1] = 0; 5338 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 5339 } 5340 5341 prod++; 5342 prod &= IXL_AQ_MASK; 5343 5344 post = 1; 5345 5346 } while (--n); 5347 5348 if (post) { 5349 sc->sc_arq_prod = prod; 5350 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 5351 } 5352 5353 return post; 5354 } 5355 5356 static void 5357 ixl_arq_unfill(struct ixl_softc *sc) 5358 { 5359 struct ixl_aq_buf *aqb; 5360 unsigned int i; 5361 5362 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 5363 aqb = sc->sc_arq_live[i]; 5364 if (aqb == NULL) 5365 continue; 5366 5367 sc->sc_arq_live[i] = NULL; 5368 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 5369 BUS_DMASYNC_POSTREAD); 5370 ixl_aqb_free(sc, aqb); 5371 } 5372 5373 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 5374 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5375 ixl_aq_buf, aqb_entry); 5376 ixl_aqb_free(sc, aqb); 5377 } 5378 } 5379 5380 static void 5381 ixl_clear_hw(struct ixl_softc *sc) 5382 { 5383 uint32_t num_queues, base_queue; 5384 uint32_t num_pf_int; 5385 uint32_t num_vf_int; 5386 uint32_t num_vfs; 5387 uint32_t i, j; 5388 uint32_t val; 5389 uint32_t eol = 0x7ff; 5390 5391 /* get number of interrupts, queues, and vfs */ 5392 val = ixl_rd(sc, I40E_GLPCI_CNF2); 5393 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 5394 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 5395 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 5396 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 5397 5398 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 5399 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 5400 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 5401 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 5402 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 5403 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 5404 num_queues = (j - base_queue) + 1; 5405 else 5406 num_queues = 0; 5407 5408 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 5409 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 5410 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 5411 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 5412 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 5413 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 5414 num_vfs = (j - i) + 1; 5415 else 5416 num_vfs = 0; 5417 5418 /* stop all the interrupts */ 5419 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5420 ixl_flush(sc); 5421 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 5422 for (i = 0; i < num_pf_int - 2; i++) 5423 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 5424 ixl_flush(sc); 5425 5426 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 5427 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5428 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 5429 for (i = 0; i < num_pf_int - 2; i++) 5430 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 5431 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5432 for (i = 0; i < num_vfs; i++) 5433 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 5434 for (i = 0; i < num_vf_int - 2; i++) 5435 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 5436 5437 /* warn the HW of the coming Tx disables */ 5438 for (i = 0; i < num_queues; i++) { 5439 uint32_t abs_queue_idx = base_queue + i; 5440 uint32_t reg_block = 0; 5441 5442 if (abs_queue_idx >= 128) { 5443 reg_block = abs_queue_idx / 128; 5444 abs_queue_idx %= 128; 5445 } 5446 5447 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 5448 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 5449 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 5450 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 5451 5452 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 5453 } 5454 delaymsec(400); 5455 5456 /* stop all the queues */ 5457 for (i = 0; i < num_queues; i++) { 5458 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 5459 ixl_wr(sc, I40E_QTX_ENA(i), 0); 5460 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 5461 ixl_wr(sc, I40E_QRX_ENA(i), 0); 5462 } 5463 5464 /* short wait for all queue disables to settle */ 5465 delaymsec(50); 5466 } 5467 5468 static int 5469 ixl_pf_reset(struct ixl_softc *sc) 5470 { 5471 uint32_t cnt = 0; 5472 uint32_t cnt1 = 0; 5473 uint32_t reg = 0, reg0 = 0; 5474 uint32_t grst_del; 5475 5476 /* 5477 * Poll for Global Reset steady state in case of recent GRST. 5478 * The grst delay value is in 100ms units, and we'll wait a 5479 * couple counts longer to be sure we don't just miss the end. 5480 */ 5481 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 5482 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 5483 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 5484 5485 grst_del = grst_del * 20; 5486 5487 for (cnt = 0; cnt < grst_del; cnt++) { 5488 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 5489 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 5490 break; 5491 delaymsec(100); 5492 } 5493 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5494 aprint_error(", Global reset polling failed to complete\n"); 5495 return -1; 5496 } 5497 5498 /* Now Wait for the FW to be ready */ 5499 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 5500 reg = ixl_rd(sc, I40E_GLNVM_ULD); 5501 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5502 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 5503 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5504 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 5505 break; 5506 5507 delaymsec(10); 5508 } 5509 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5510 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 5511 aprint_error(", wait for FW Reset complete timed out " 5512 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 5513 return -1; 5514 } 5515 5516 /* 5517 * If there was a Global Reset in progress when we got here, 5518 * we don't need to do the PF Reset 5519 */ 5520 if (cnt == 0) { 5521 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5522 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 5523 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 5524 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5525 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 5526 break; 5527 delaymsec(1); 5528 5529 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 5530 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5531 aprint_error(", Core reset upcoming." 5532 " Skipping PF reset reset request\n"); 5533 return -1; 5534 } 5535 } 5536 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 5537 aprint_error(", PF reset polling failed to complete" 5538 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 5539 return -1; 5540 } 5541 } 5542 5543 return 0; 5544 } 5545 5546 static int 5547 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 5548 bus_size_t size, bus_size_t align) 5549 { 5550 ixm->ixm_size = size; 5551 5552 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 5553 ixm->ixm_size, 0, 5554 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 5555 &ixm->ixm_map) != 0) 5556 return 1; 5557 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 5558 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 5559 BUS_DMA_WAITOK) != 0) 5560 goto destroy; 5561 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 5562 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 5563 goto free; 5564 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 5565 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 5566 goto unmap; 5567 5568 memset(ixm->ixm_kva, 0, ixm->ixm_size); 5569 5570 return 0; 5571 unmap: 5572 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5573 free: 5574 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5575 destroy: 5576 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5577 return 1; 5578 } 5579 5580 static void 5581 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 5582 { 5583 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 5584 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5585 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5586 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5587 } 5588 5589 static int 5590 ixl_setup_vlan_hwfilter(struct ixl_softc *sc) 5591 { 5592 struct ethercom *ec = &sc->sc_ec; 5593 struct vlanid_list *vlanidp; 5594 int rv; 5595 5596 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5597 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5598 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5599 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5600 5601 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5602 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5603 if (rv != 0) 5604 return rv; 5605 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5606 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5607 if (rv != 0) 5608 return rv; 5609 5610 ETHER_LOCK(ec); 5611 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5612 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 5613 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5614 if (rv != 0) 5615 break; 5616 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 5617 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5618 if (rv != 0) 5619 break; 5620 } 5621 ETHER_UNLOCK(ec); 5622 5623 return rv; 5624 } 5625 5626 static void 5627 ixl_teardown_vlan_hwfilter(struct ixl_softc *sc) 5628 { 5629 struct vlanid_list *vlanidp; 5630 struct ethercom *ec = &sc->sc_ec; 5631 5632 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5633 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5634 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5635 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5636 5637 ETHER_LOCK(ec); 5638 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5639 ixl_remove_macvlan(sc, sc->sc_enaddr, 5640 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5641 ixl_remove_macvlan(sc, etherbroadcastaddr, 5642 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5643 } 5644 ETHER_UNLOCK(ec); 5645 5646 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5647 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5648 ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5649 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5650 } 5651 5652 static int 5653 ixl_update_macvlan(struct ixl_softc *sc) 5654 { 5655 int rv = 0; 5656 int next_ec_capenable = sc->sc_ec.ec_capenable; 5657 5658 if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 5659 rv = ixl_setup_vlan_hwfilter(sc); 5660 if (rv != 0) 5661 ixl_teardown_vlan_hwfilter(sc); 5662 } else { 5663 ixl_teardown_vlan_hwfilter(sc); 5664 } 5665 5666 return rv; 5667 } 5668 5669 static int 5670 ixl_ifflags_cb(struct ethercom *ec) 5671 { 5672 struct ifnet *ifp = &ec->ec_if; 5673 struct ixl_softc *sc = ifp->if_softc; 5674 int rv, change; 5675 5676 mutex_enter(&sc->sc_cfg_lock); 5677 5678 change = ec->ec_capenable ^ sc->sc_cur_ec_capenable; 5679 5680 if (ISSET(change, ETHERCAP_VLAN_HWTAGGING)) { 5681 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 5682 rv = ENETRESET; 5683 goto out; 5684 } 5685 5686 if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) { 5687 rv = ixl_update_macvlan(sc); 5688 if (rv == 0) { 5689 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 5690 } else { 5691 CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER); 5692 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 5693 } 5694 } 5695 5696 rv = ixl_iff(sc); 5697 out: 5698 mutex_exit(&sc->sc_cfg_lock); 5699 5700 return rv; 5701 } 5702 5703 static int 5704 ixl_set_link_status_locked(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 5705 { 5706 const struct ixl_aq_link_status *status; 5707 const struct ixl_phy_type *itype; 5708 5709 uint64_t ifm_active = IFM_ETHER; 5710 uint64_t ifm_status = IFM_AVALID; 5711 int link_state = LINK_STATE_DOWN; 5712 uint64_t baudrate = 0; 5713 5714 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 5715 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) { 5716 ifm_active |= IFM_NONE; 5717 goto done; 5718 } 5719 5720 ifm_active |= IFM_FDX; 5721 ifm_status |= IFM_ACTIVE; 5722 link_state = LINK_STATE_UP; 5723 5724 itype = ixl_search_phy_type(status->phy_type); 5725 if (itype != NULL) 5726 ifm_active |= itype->ifm_type; 5727 5728 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 5729 ifm_active |= IFM_ETH_TXPAUSE; 5730 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 5731 ifm_active |= IFM_ETH_RXPAUSE; 5732 5733 baudrate = ixl_search_link_speed(status->link_speed); 5734 5735 done: 5736 /* sc->sc_cfg_lock held expect during attach */ 5737 sc->sc_media_active = ifm_active; 5738 sc->sc_media_status = ifm_status; 5739 5740 sc->sc_ec.ec_if.if_baudrate = baudrate; 5741 5742 return link_state; 5743 } 5744 5745 static int 5746 ixl_establish_intx(struct ixl_softc *sc) 5747 { 5748 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5749 pci_intr_handle_t *intr; 5750 char xnamebuf[32]; 5751 char intrbuf[PCI_INTRSTR_LEN]; 5752 char const *intrstr; 5753 5754 KASSERT(sc->sc_nintrs == 1); 5755 5756 intr = &sc->sc_ihp[0]; 5757 5758 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 5759 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 5760 device_xname(sc->sc_dev)); 5761 5762 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 5763 sc, xnamebuf); 5764 5765 if (sc->sc_ihs[0] == NULL) { 5766 aprint_error_dev(sc->sc_dev, 5767 "unable to establish interrupt at %s\n", intrstr); 5768 return -1; 5769 } 5770 5771 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 5772 return 0; 5773 } 5774 5775 static int 5776 ixl_establish_msix(struct ixl_softc *sc) 5777 { 5778 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5779 kcpuset_t *affinity; 5780 unsigned int vector = 0; 5781 unsigned int i; 5782 int affinity_to, r; 5783 char xnamebuf[32]; 5784 char intrbuf[PCI_INTRSTR_LEN]; 5785 char const *intrstr; 5786 5787 kcpuset_create(&affinity, false); 5788 5789 /* the "other" intr is mapped to vector 0 */ 5790 vector = 0; 5791 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5792 intrbuf, sizeof(intrbuf)); 5793 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 5794 device_xname(sc->sc_dev)); 5795 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5796 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 5797 sc, xnamebuf); 5798 if (sc->sc_ihs[vector] == NULL) { 5799 aprint_error_dev(sc->sc_dev, 5800 "unable to establish interrupt at %s\n", intrstr); 5801 goto fail; 5802 } 5803 5804 aprint_normal_dev(sc->sc_dev, "other interrupt at %s", intrstr); 5805 5806 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5807 affinity_to = (affinity_to + sc->sc_nqueue_pairs_max) % ncpu; 5808 5809 kcpuset_zero(affinity); 5810 kcpuset_set(affinity, affinity_to); 5811 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5812 if (r == 0) { 5813 aprint_normal(", affinity to %u", affinity_to); 5814 } 5815 aprint_normal("\n"); 5816 vector++; 5817 5818 sc->sc_msix_vector_queue = vector; 5819 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5820 5821 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5822 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5823 intrbuf, sizeof(intrbuf)); 5824 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 5825 device_xname(sc->sc_dev), i); 5826 5827 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5828 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 5829 (void *)&sc->sc_qps[i], xnamebuf); 5830 5831 if (sc->sc_ihs[vector] == NULL) { 5832 aprint_error_dev(sc->sc_dev, 5833 "unable to establish interrupt at %s\n", intrstr); 5834 goto fail; 5835 } 5836 5837 aprint_normal_dev(sc->sc_dev, 5838 "for TXRX%d interrupt at %s", i, intrstr); 5839 5840 kcpuset_zero(affinity); 5841 kcpuset_set(affinity, affinity_to); 5842 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5843 if (r == 0) { 5844 aprint_normal(", affinity to %u", affinity_to); 5845 affinity_to = (affinity_to + 1) % ncpu; 5846 } 5847 aprint_normal("\n"); 5848 vector++; 5849 } 5850 5851 kcpuset_destroy(affinity); 5852 5853 return 0; 5854 fail: 5855 for (i = 0; i < vector; i++) { 5856 pci_intr_disestablish(pc, sc->sc_ihs[i]); 5857 } 5858 5859 sc->sc_msix_vector_queue = 0; 5860 sc->sc_msix_vector_queue = 0; 5861 kcpuset_destroy(affinity); 5862 5863 return -1; 5864 } 5865 5866 static void 5867 ixl_config_queue_intr(struct ixl_softc *sc) 5868 { 5869 unsigned int i, vector; 5870 5871 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5872 vector = sc->sc_msix_vector_queue; 5873 } else { 5874 vector = I40E_INTR_NOTX_INTR; 5875 5876 ixl_wr(sc, I40E_PFINT_LNKLST0, 5877 (I40E_INTR_NOTX_QUEUE << 5878 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5879 (I40E_QUEUE_TYPE_RX << 5880 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5881 } 5882 5883 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5884 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5885 ixl_flush(sc); 5886 5887 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5888 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5889 (I40E_QUEUE_TYPE_RX << 5890 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5891 5892 ixl_wr(sc, I40E_QINT_RQCTL(i), 5893 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5894 (I40E_ITR_INDEX_RX << 5895 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5896 (I40E_INTR_NOTX_RX_QUEUE << 5897 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5898 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5899 (I40E_QUEUE_TYPE_TX << 5900 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5901 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5902 5903 ixl_wr(sc, I40E_QINT_TQCTL(i), 5904 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5905 (I40E_ITR_INDEX_TX << 5906 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5907 (I40E_INTR_NOTX_TX_QUEUE << 5908 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5909 (I40E_QUEUE_TYPE_EOL << 5910 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5911 (I40E_QUEUE_TYPE_RX << 5912 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5913 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5914 5915 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5916 ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_RX, i), 5917 sc->sc_itr_rx); 5918 ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_TX, i), 5919 sc->sc_itr_tx); 5920 vector++; 5921 } 5922 } 5923 ixl_flush(sc); 5924 5925 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), sc->sc_itr_rx); 5926 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), sc->sc_itr_tx); 5927 ixl_flush(sc); 5928 } 5929 5930 static void 5931 ixl_config_other_intr(struct ixl_softc *sc) 5932 { 5933 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5934 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5935 5936 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5937 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5938 I40E_PFINT_ICR0_ENA_GRST_MASK | 5939 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5940 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5941 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5942 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5943 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5944 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5945 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5946 5947 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5948 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5949 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5950 (I40E_ITR_INDEX_OTHER << 5951 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5952 ixl_flush(sc); 5953 } 5954 5955 static int 5956 ixl_setup_interrupts(struct ixl_softc *sc) 5957 { 5958 struct pci_attach_args *pa = &sc->sc_pa; 5959 pci_intr_type_t max_type, intr_type; 5960 int counts[PCI_INTR_TYPE_SIZE]; 5961 int error; 5962 unsigned int i; 5963 bool retry; 5964 5965 memset(counts, 0, sizeof(counts)); 5966 max_type = PCI_INTR_TYPE_MSIX; 5967 /* QPs + other interrupt */ 5968 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5969 counts[PCI_INTR_TYPE_INTX] = 1; 5970 5971 if (ixl_param_nomsix) 5972 counts[PCI_INTR_TYPE_MSIX] = 0; 5973 5974 do { 5975 retry = false; 5976 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5977 if (error != 0) { 5978 aprint_error_dev(sc->sc_dev, 5979 "couldn't map interrupt\n"); 5980 break; 5981 } 5982 5983 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5984 sc->sc_nintrs = counts[intr_type]; 5985 KASSERT(sc->sc_nintrs > 0); 5986 5987 for (i = 0; i < sc->sc_nintrs; i++) { 5988 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5989 PCI_INTR_MPSAFE, true); 5990 } 5991 5992 sc->sc_ihs = kmem_zalloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5993 KM_SLEEP); 5994 5995 if (intr_type == PCI_INTR_TYPE_MSIX) { 5996 error = ixl_establish_msix(sc); 5997 if (error) { 5998 counts[PCI_INTR_TYPE_MSIX] = 0; 5999 retry = true; 6000 } 6001 } else if (intr_type == PCI_INTR_TYPE_INTX) { 6002 error = ixl_establish_intx(sc); 6003 } else { 6004 error = -1; 6005 } 6006 6007 if (error) { 6008 kmem_free(sc->sc_ihs, 6009 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 6010 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 6011 } else { 6012 sc->sc_intrtype = intr_type; 6013 } 6014 } while (retry); 6015 6016 return error; 6017 } 6018 6019 static void 6020 ixl_teardown_interrupts(struct ixl_softc *sc) 6021 { 6022 struct pci_attach_args *pa = &sc->sc_pa; 6023 unsigned int i; 6024 6025 for (i = 0; i < sc->sc_nintrs; i++) { 6026 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 6027 } 6028 6029 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 6030 6031 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 6032 sc->sc_ihs = NULL; 6033 sc->sc_nintrs = 0; 6034 } 6035 6036 static int 6037 ixl_setup_stats(struct ixl_softc *sc) 6038 { 6039 struct ixl_queue_pair *qp; 6040 struct ixl_tx_ring *txr; 6041 struct ixl_rx_ring *rxr; 6042 struct ixl_stats_counters *isc; 6043 unsigned int i; 6044 6045 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6046 qp = &sc->sc_qps[i]; 6047 txr = qp->qp_txr; 6048 rxr = qp->qp_rxr; 6049 6050 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 6051 NULL, qp->qp_name, "m_defrag successed"); 6052 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 6053 NULL, qp->qp_name, "m_defrag_failed"); 6054 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 6055 NULL, qp->qp_name, "Dropped in pcq"); 6056 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 6057 NULL, qp->qp_name, "Deferred transmit"); 6058 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 6059 NULL, qp->qp_name, "Interrupt on queue"); 6060 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 6061 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6062 6063 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 6064 NULL, qp->qp_name, "MGETHDR failed"); 6065 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 6066 NULL, qp->qp_name, "MCLGET failed"); 6067 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 6068 EVCNT_TYPE_MISC, NULL, qp->qp_name, 6069 "bus_dmamap_load_mbuf failed"); 6070 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 6071 NULL, qp->qp_name, "Interrupt on queue"); 6072 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 6073 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6074 } 6075 6076 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 6077 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 6078 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 6079 NULL, device_xname(sc->sc_dev), "Link status event"); 6080 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 6081 NULL, device_xname(sc->sc_dev), "ECC error"); 6082 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 6083 NULL, device_xname(sc->sc_dev), "PCI exception"); 6084 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 6085 NULL, device_xname(sc->sc_dev), "Critical error"); 6086 6087 isc = &sc->sc_stats_counters; 6088 evcnt_attach_dynamic(&isc->isc_crc_errors, EVCNT_TYPE_MISC, 6089 NULL, device_xname(sc->sc_dev), "CRC errors"); 6090 evcnt_attach_dynamic(&isc->isc_illegal_bytes, EVCNT_TYPE_MISC, 6091 NULL, device_xname(sc->sc_dev), "Illegal bytes"); 6092 evcnt_attach_dynamic(&isc->isc_mac_local_faults, EVCNT_TYPE_MISC, 6093 NULL, device_xname(sc->sc_dev), "Mac local faults"); 6094 evcnt_attach_dynamic(&isc->isc_mac_remote_faults, EVCNT_TYPE_MISC, 6095 NULL, device_xname(sc->sc_dev), "Mac remote faults"); 6096 evcnt_attach_dynamic(&isc->isc_link_xon_rx, EVCNT_TYPE_MISC, 6097 NULL, device_xname(sc->sc_dev), "Rx xon"); 6098 evcnt_attach_dynamic(&isc->isc_link_xon_tx, EVCNT_TYPE_MISC, 6099 NULL, device_xname(sc->sc_dev), "Tx xon"); 6100 evcnt_attach_dynamic(&isc->isc_link_xoff_rx, EVCNT_TYPE_MISC, 6101 NULL, device_xname(sc->sc_dev), "Rx xoff"); 6102 evcnt_attach_dynamic(&isc->isc_link_xoff_tx, EVCNT_TYPE_MISC, 6103 NULL, device_xname(sc->sc_dev), "Tx xoff"); 6104 evcnt_attach_dynamic(&isc->isc_rx_fragments, EVCNT_TYPE_MISC, 6105 NULL, device_xname(sc->sc_dev), "Rx fragments"); 6106 evcnt_attach_dynamic(&isc->isc_rx_jabber, EVCNT_TYPE_MISC, 6107 NULL, device_xname(sc->sc_dev), "Rx jabber"); 6108 6109 evcnt_attach_dynamic(&isc->isc_rx_size_64, EVCNT_TYPE_MISC, 6110 NULL, device_xname(sc->sc_dev), "Rx size 64"); 6111 evcnt_attach_dynamic(&isc->isc_rx_size_127, EVCNT_TYPE_MISC, 6112 NULL, device_xname(sc->sc_dev), "Rx size 127"); 6113 evcnt_attach_dynamic(&isc->isc_rx_size_255, EVCNT_TYPE_MISC, 6114 NULL, device_xname(sc->sc_dev), "Rx size 255"); 6115 evcnt_attach_dynamic(&isc->isc_rx_size_511, EVCNT_TYPE_MISC, 6116 NULL, device_xname(sc->sc_dev), "Rx size 511"); 6117 evcnt_attach_dynamic(&isc->isc_rx_size_1023, EVCNT_TYPE_MISC, 6118 NULL, device_xname(sc->sc_dev), "Rx size 1023"); 6119 evcnt_attach_dynamic(&isc->isc_rx_size_1522, EVCNT_TYPE_MISC, 6120 NULL, device_xname(sc->sc_dev), "Rx size 1522"); 6121 evcnt_attach_dynamic(&isc->isc_rx_size_big, EVCNT_TYPE_MISC, 6122 NULL, device_xname(sc->sc_dev), "Rx jumbo packets"); 6123 evcnt_attach_dynamic(&isc->isc_rx_undersize, EVCNT_TYPE_MISC, 6124 NULL, device_xname(sc->sc_dev), "Rx under size"); 6125 evcnt_attach_dynamic(&isc->isc_rx_oversize, EVCNT_TYPE_MISC, 6126 NULL, device_xname(sc->sc_dev), "Rx over size"); 6127 6128 evcnt_attach_dynamic(&isc->isc_rx_bytes, EVCNT_TYPE_MISC, 6129 NULL, device_xname(sc->sc_dev), "Rx bytes / port"); 6130 evcnt_attach_dynamic(&isc->isc_rx_discards, EVCNT_TYPE_MISC, 6131 NULL, device_xname(sc->sc_dev), "Rx discards / port"); 6132 evcnt_attach_dynamic(&isc->isc_rx_unicast, EVCNT_TYPE_MISC, 6133 NULL, device_xname(sc->sc_dev), "Rx unicast / port"); 6134 evcnt_attach_dynamic(&isc->isc_rx_multicast, EVCNT_TYPE_MISC, 6135 NULL, device_xname(sc->sc_dev), "Rx multicast / port"); 6136 evcnt_attach_dynamic(&isc->isc_rx_broadcast, EVCNT_TYPE_MISC, 6137 NULL, device_xname(sc->sc_dev), "Rx broadcast / port"); 6138 6139 evcnt_attach_dynamic(&isc->isc_vsi_rx_bytes, EVCNT_TYPE_MISC, 6140 NULL, device_xname(sc->sc_dev), "Rx bytes / vsi"); 6141 evcnt_attach_dynamic(&isc->isc_vsi_rx_discards, EVCNT_TYPE_MISC, 6142 NULL, device_xname(sc->sc_dev), "Rx discards / vsi"); 6143 evcnt_attach_dynamic(&isc->isc_vsi_rx_unicast, EVCNT_TYPE_MISC, 6144 NULL, device_xname(sc->sc_dev), "Rx unicast / vsi"); 6145 evcnt_attach_dynamic(&isc->isc_vsi_rx_multicast, EVCNT_TYPE_MISC, 6146 NULL, device_xname(sc->sc_dev), "Rx multicast / vsi"); 6147 evcnt_attach_dynamic(&isc->isc_vsi_rx_broadcast, EVCNT_TYPE_MISC, 6148 NULL, device_xname(sc->sc_dev), "Rx broadcast / vsi"); 6149 6150 evcnt_attach_dynamic(&isc->isc_tx_size_64, EVCNT_TYPE_MISC, 6151 NULL, device_xname(sc->sc_dev), "Tx size 64"); 6152 evcnt_attach_dynamic(&isc->isc_tx_size_127, EVCNT_TYPE_MISC, 6153 NULL, device_xname(sc->sc_dev), "Tx size 127"); 6154 evcnt_attach_dynamic(&isc->isc_tx_size_255, EVCNT_TYPE_MISC, 6155 NULL, device_xname(sc->sc_dev), "Tx size 255"); 6156 evcnt_attach_dynamic(&isc->isc_tx_size_511, EVCNT_TYPE_MISC, 6157 NULL, device_xname(sc->sc_dev), "Tx size 511"); 6158 evcnt_attach_dynamic(&isc->isc_tx_size_1023, EVCNT_TYPE_MISC, 6159 NULL, device_xname(sc->sc_dev), "Tx size 1023"); 6160 evcnt_attach_dynamic(&isc->isc_tx_size_1522, EVCNT_TYPE_MISC, 6161 NULL, device_xname(sc->sc_dev), "Tx size 1522"); 6162 evcnt_attach_dynamic(&isc->isc_tx_size_big, EVCNT_TYPE_MISC, 6163 NULL, device_xname(sc->sc_dev), "Tx jumbo packets"); 6164 6165 evcnt_attach_dynamic(&isc->isc_tx_bytes, EVCNT_TYPE_MISC, 6166 NULL, device_xname(sc->sc_dev), "Tx bytes / port"); 6167 evcnt_attach_dynamic(&isc->isc_tx_dropped_link_down, EVCNT_TYPE_MISC, 6168 NULL, device_xname(sc->sc_dev), 6169 "Tx dropped due to link down / port"); 6170 evcnt_attach_dynamic(&isc->isc_tx_unicast, EVCNT_TYPE_MISC, 6171 NULL, device_xname(sc->sc_dev), "Tx unicast / port"); 6172 evcnt_attach_dynamic(&isc->isc_tx_multicast, EVCNT_TYPE_MISC, 6173 NULL, device_xname(sc->sc_dev), "Tx multicast / port"); 6174 evcnt_attach_dynamic(&isc->isc_tx_broadcast, EVCNT_TYPE_MISC, 6175 NULL, device_xname(sc->sc_dev), "Tx broadcast / port"); 6176 6177 evcnt_attach_dynamic(&isc->isc_vsi_tx_bytes, EVCNT_TYPE_MISC, 6178 NULL, device_xname(sc->sc_dev), "Tx bytes / vsi"); 6179 evcnt_attach_dynamic(&isc->isc_vsi_tx_errors, EVCNT_TYPE_MISC, 6180 NULL, device_xname(sc->sc_dev), "Tx errors / vsi"); 6181 evcnt_attach_dynamic(&isc->isc_vsi_tx_unicast, EVCNT_TYPE_MISC, 6182 NULL, device_xname(sc->sc_dev), "Tx unicast / vsi"); 6183 evcnt_attach_dynamic(&isc->isc_vsi_tx_multicast, EVCNT_TYPE_MISC, 6184 NULL, device_xname(sc->sc_dev), "Tx multicast / vsi"); 6185 evcnt_attach_dynamic(&isc->isc_vsi_tx_broadcast, EVCNT_TYPE_MISC, 6186 NULL, device_xname(sc->sc_dev), "Tx broadcast / vsi"); 6187 6188 sc->sc_stats_intval = ixl_param_stats_interval; 6189 callout_init(&sc->sc_stats_callout, CALLOUT_MPSAFE); 6190 callout_setfunc(&sc->sc_stats_callout, ixl_stats_callout, sc); 6191 ixl_work_set(&sc->sc_stats_task, ixl_stats_update, sc); 6192 6193 return 0; 6194 } 6195 6196 static void 6197 ixl_teardown_stats(struct ixl_softc *sc) 6198 { 6199 struct ixl_tx_ring *txr; 6200 struct ixl_rx_ring *rxr; 6201 struct ixl_stats_counters *isc; 6202 unsigned int i; 6203 6204 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6205 txr = sc->sc_qps[i].qp_txr; 6206 rxr = sc->sc_qps[i].qp_rxr; 6207 6208 evcnt_detach(&txr->txr_defragged); 6209 evcnt_detach(&txr->txr_defrag_failed); 6210 evcnt_detach(&txr->txr_pcqdrop); 6211 evcnt_detach(&txr->txr_transmitdef); 6212 evcnt_detach(&txr->txr_intr); 6213 evcnt_detach(&txr->txr_defer); 6214 6215 evcnt_detach(&rxr->rxr_mgethdr_failed); 6216 evcnt_detach(&rxr->rxr_mgetcl_failed); 6217 evcnt_detach(&rxr->rxr_mbuf_load_failed); 6218 evcnt_detach(&rxr->rxr_intr); 6219 evcnt_detach(&rxr->rxr_defer); 6220 } 6221 6222 isc = &sc->sc_stats_counters; 6223 evcnt_detach(&isc->isc_crc_errors); 6224 evcnt_detach(&isc->isc_illegal_bytes); 6225 evcnt_detach(&isc->isc_mac_local_faults); 6226 evcnt_detach(&isc->isc_mac_remote_faults); 6227 evcnt_detach(&isc->isc_link_xon_rx); 6228 evcnt_detach(&isc->isc_link_xon_tx); 6229 evcnt_detach(&isc->isc_link_xoff_rx); 6230 evcnt_detach(&isc->isc_link_xoff_tx); 6231 evcnt_detach(&isc->isc_rx_fragments); 6232 evcnt_detach(&isc->isc_rx_jabber); 6233 evcnt_detach(&isc->isc_rx_bytes); 6234 evcnt_detach(&isc->isc_rx_discards); 6235 evcnt_detach(&isc->isc_rx_unicast); 6236 evcnt_detach(&isc->isc_rx_multicast); 6237 evcnt_detach(&isc->isc_rx_broadcast); 6238 evcnt_detach(&isc->isc_rx_size_64); 6239 evcnt_detach(&isc->isc_rx_size_127); 6240 evcnt_detach(&isc->isc_rx_size_255); 6241 evcnt_detach(&isc->isc_rx_size_511); 6242 evcnt_detach(&isc->isc_rx_size_1023); 6243 evcnt_detach(&isc->isc_rx_size_1522); 6244 evcnt_detach(&isc->isc_rx_size_big); 6245 evcnt_detach(&isc->isc_rx_undersize); 6246 evcnt_detach(&isc->isc_rx_oversize); 6247 evcnt_detach(&isc->isc_tx_bytes); 6248 evcnt_detach(&isc->isc_tx_dropped_link_down); 6249 evcnt_detach(&isc->isc_tx_unicast); 6250 evcnt_detach(&isc->isc_tx_multicast); 6251 evcnt_detach(&isc->isc_tx_broadcast); 6252 evcnt_detach(&isc->isc_tx_size_64); 6253 evcnt_detach(&isc->isc_tx_size_127); 6254 evcnt_detach(&isc->isc_tx_size_255); 6255 evcnt_detach(&isc->isc_tx_size_511); 6256 evcnt_detach(&isc->isc_tx_size_1023); 6257 evcnt_detach(&isc->isc_tx_size_1522); 6258 evcnt_detach(&isc->isc_tx_size_big); 6259 evcnt_detach(&isc->isc_vsi_rx_discards); 6260 evcnt_detach(&isc->isc_vsi_rx_bytes); 6261 evcnt_detach(&isc->isc_vsi_rx_unicast); 6262 evcnt_detach(&isc->isc_vsi_rx_multicast); 6263 evcnt_detach(&isc->isc_vsi_rx_broadcast); 6264 evcnt_detach(&isc->isc_vsi_tx_errors); 6265 evcnt_detach(&isc->isc_vsi_tx_bytes); 6266 evcnt_detach(&isc->isc_vsi_tx_unicast); 6267 evcnt_detach(&isc->isc_vsi_tx_multicast); 6268 evcnt_detach(&isc->isc_vsi_tx_broadcast); 6269 6270 evcnt_detach(&sc->sc_event_atq); 6271 evcnt_detach(&sc->sc_event_link); 6272 evcnt_detach(&sc->sc_event_ecc_err); 6273 evcnt_detach(&sc->sc_event_pci_exception); 6274 evcnt_detach(&sc->sc_event_crit_err); 6275 6276 callout_destroy(&sc->sc_stats_callout); 6277 } 6278 6279 static void 6280 ixl_stats_callout(void *xsc) 6281 { 6282 struct ixl_softc *sc = xsc; 6283 6284 ixl_work_add(sc->sc_workq, &sc->sc_stats_task); 6285 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 6286 } 6287 6288 static uint64_t 6289 ixl_stat_delta(struct ixl_softc *sc, uint32_t reg_hi, uint32_t reg_lo, 6290 uint64_t *offset, bool has_offset) 6291 { 6292 uint64_t value, delta; 6293 int bitwidth; 6294 6295 bitwidth = reg_hi == 0 ? 32 : 48; 6296 6297 value = ixl_rd(sc, reg_lo); 6298 6299 if (bitwidth > 32) { 6300 value |= ((uint64_t)ixl_rd(sc, reg_hi) << 32); 6301 } 6302 6303 if (__predict_true(has_offset)) { 6304 delta = value; 6305 if (value < *offset) 6306 delta += ((uint64_t)1 << bitwidth); 6307 delta -= *offset; 6308 } else { 6309 delta = 0; 6310 } 6311 atomic_swap_64(offset, value); 6312 6313 return delta; 6314 } 6315 6316 static void 6317 ixl_stats_update(void *xsc) 6318 { 6319 struct ixl_softc *sc = xsc; 6320 struct ixl_stats_counters *isc; 6321 uint64_t delta; 6322 6323 isc = &sc->sc_stats_counters; 6324 6325 /* errors */ 6326 delta = ixl_stat_delta(sc, 6327 0, I40E_GLPRT_CRCERRS(sc->sc_port), 6328 &isc->isc_crc_errors_offset, isc->isc_has_offset); 6329 atomic_add_64(&isc->isc_crc_errors.ev_count, delta); 6330 6331 delta = ixl_stat_delta(sc, 6332 0, I40E_GLPRT_ILLERRC(sc->sc_port), 6333 &isc->isc_illegal_bytes_offset, isc->isc_has_offset); 6334 atomic_add_64(&isc->isc_illegal_bytes.ev_count, delta); 6335 6336 /* rx */ 6337 delta = ixl_stat_delta(sc, 6338 I40E_GLPRT_GORCH(sc->sc_port), I40E_GLPRT_GORCL(sc->sc_port), 6339 &isc->isc_rx_bytes_offset, isc->isc_has_offset); 6340 atomic_add_64(&isc->isc_rx_bytes.ev_count, delta); 6341 6342 delta = ixl_stat_delta(sc, 6343 0, I40E_GLPRT_RDPC(sc->sc_port), 6344 &isc->isc_rx_discards_offset, isc->isc_has_offset); 6345 atomic_add_64(&isc->isc_rx_discards.ev_count, delta); 6346 6347 delta = ixl_stat_delta(sc, 6348 I40E_GLPRT_UPRCH(sc->sc_port), I40E_GLPRT_UPRCL(sc->sc_port), 6349 &isc->isc_rx_unicast_offset, isc->isc_has_offset); 6350 atomic_add_64(&isc->isc_rx_unicast.ev_count, delta); 6351 6352 delta = ixl_stat_delta(sc, 6353 I40E_GLPRT_MPRCH(sc->sc_port), I40E_GLPRT_MPRCL(sc->sc_port), 6354 &isc->isc_rx_multicast_offset, isc->isc_has_offset); 6355 atomic_add_64(&isc->isc_rx_multicast.ev_count, delta); 6356 6357 delta = ixl_stat_delta(sc, 6358 I40E_GLPRT_BPRCH(sc->sc_port), I40E_GLPRT_BPRCL(sc->sc_port), 6359 &isc->isc_rx_broadcast_offset, isc->isc_has_offset); 6360 atomic_add_64(&isc->isc_rx_broadcast.ev_count, delta); 6361 6362 /* Packet size stats rx */ 6363 delta = ixl_stat_delta(sc, 6364 I40E_GLPRT_PRC64H(sc->sc_port), I40E_GLPRT_PRC64L(sc->sc_port), 6365 &isc->isc_rx_size_64_offset, isc->isc_has_offset); 6366 atomic_add_64(&isc->isc_rx_size_64.ev_count, delta); 6367 6368 delta = ixl_stat_delta(sc, 6369 I40E_GLPRT_PRC127H(sc->sc_port), I40E_GLPRT_PRC127L(sc->sc_port), 6370 &isc->isc_rx_size_127_offset, isc->isc_has_offset); 6371 atomic_add_64(&isc->isc_rx_size_127.ev_count, delta); 6372 6373 delta = ixl_stat_delta(sc, 6374 I40E_GLPRT_PRC255H(sc->sc_port), I40E_GLPRT_PRC255L(sc->sc_port), 6375 &isc->isc_rx_size_255_offset, isc->isc_has_offset); 6376 atomic_add_64(&isc->isc_rx_size_255.ev_count, delta); 6377 6378 delta = ixl_stat_delta(sc, 6379 I40E_GLPRT_PRC511H(sc->sc_port), I40E_GLPRT_PRC511L(sc->sc_port), 6380 &isc->isc_rx_size_511_offset, isc->isc_has_offset); 6381 atomic_add_64(&isc->isc_rx_size_511.ev_count, delta); 6382 6383 delta = ixl_stat_delta(sc, 6384 I40E_GLPRT_PRC1023H(sc->sc_port), I40E_GLPRT_PRC1023L(sc->sc_port), 6385 &isc->isc_rx_size_1023_offset, isc->isc_has_offset); 6386 atomic_add_64(&isc->isc_rx_size_1023.ev_count, delta); 6387 6388 delta = ixl_stat_delta(sc, 6389 I40E_GLPRT_PRC1522H(sc->sc_port), I40E_GLPRT_PRC1522L(sc->sc_port), 6390 &isc->isc_rx_size_1522_offset, isc->isc_has_offset); 6391 atomic_add_64(&isc->isc_rx_size_1522.ev_count, delta); 6392 6393 delta = ixl_stat_delta(sc, 6394 I40E_GLPRT_PRC9522H(sc->sc_port), I40E_GLPRT_PRC9522L(sc->sc_port), 6395 &isc->isc_rx_size_big_offset, isc->isc_has_offset); 6396 atomic_add_64(&isc->isc_rx_size_big.ev_count, delta); 6397 6398 delta = ixl_stat_delta(sc, 6399 0, I40E_GLPRT_RUC(sc->sc_port), 6400 &isc->isc_rx_undersize_offset, isc->isc_has_offset); 6401 atomic_add_64(&isc->isc_rx_undersize.ev_count, delta); 6402 6403 delta = ixl_stat_delta(sc, 6404 0, I40E_GLPRT_ROC(sc->sc_port), 6405 &isc->isc_rx_oversize_offset, isc->isc_has_offset); 6406 atomic_add_64(&isc->isc_rx_oversize.ev_count, delta); 6407 6408 /* tx */ 6409 delta = ixl_stat_delta(sc, 6410 I40E_GLPRT_GOTCH(sc->sc_port), I40E_GLPRT_GOTCL(sc->sc_port), 6411 &isc->isc_tx_bytes_offset, isc->isc_has_offset); 6412 atomic_add_64(&isc->isc_tx_bytes.ev_count, delta); 6413 6414 delta = ixl_stat_delta(sc, 6415 0, I40E_GLPRT_TDOLD(sc->sc_port), 6416 &isc->isc_tx_dropped_link_down_offset, isc->isc_has_offset); 6417 atomic_add_64(&isc->isc_tx_dropped_link_down.ev_count, delta); 6418 6419 delta = ixl_stat_delta(sc, 6420 I40E_GLPRT_UPTCH(sc->sc_port), I40E_GLPRT_UPTCL(sc->sc_port), 6421 &isc->isc_tx_unicast_offset, isc->isc_has_offset); 6422 atomic_add_64(&isc->isc_tx_unicast.ev_count, delta); 6423 6424 delta = ixl_stat_delta(sc, 6425 I40E_GLPRT_MPTCH(sc->sc_port), I40E_GLPRT_MPTCL(sc->sc_port), 6426 &isc->isc_tx_multicast_offset, isc->isc_has_offset); 6427 atomic_add_64(&isc->isc_tx_multicast.ev_count, delta); 6428 6429 delta = ixl_stat_delta(sc, 6430 I40E_GLPRT_BPTCH(sc->sc_port), I40E_GLPRT_BPTCL(sc->sc_port), 6431 &isc->isc_tx_broadcast_offset, isc->isc_has_offset); 6432 atomic_add_64(&isc->isc_tx_broadcast.ev_count, delta); 6433 6434 /* Packet size stats tx */ 6435 delta = ixl_stat_delta(sc, 6436 I40E_GLPRT_PTC64L(sc->sc_port), I40E_GLPRT_PTC64L(sc->sc_port), 6437 &isc->isc_tx_size_64_offset, isc->isc_has_offset); 6438 atomic_add_64(&isc->isc_tx_size_64.ev_count, delta); 6439 6440 delta = ixl_stat_delta(sc, 6441 I40E_GLPRT_PTC127H(sc->sc_port), I40E_GLPRT_PTC127L(sc->sc_port), 6442 &isc->isc_tx_size_127_offset, isc->isc_has_offset); 6443 atomic_add_64(&isc->isc_tx_size_127.ev_count, delta); 6444 6445 delta = ixl_stat_delta(sc, 6446 I40E_GLPRT_PTC255H(sc->sc_port), I40E_GLPRT_PTC255L(sc->sc_port), 6447 &isc->isc_tx_size_255_offset, isc->isc_has_offset); 6448 atomic_add_64(&isc->isc_tx_size_255.ev_count, delta); 6449 6450 delta = ixl_stat_delta(sc, 6451 I40E_GLPRT_PTC511H(sc->sc_port), I40E_GLPRT_PTC511L(sc->sc_port), 6452 &isc->isc_tx_size_511_offset, isc->isc_has_offset); 6453 atomic_add_64(&isc->isc_tx_size_511.ev_count, delta); 6454 6455 delta = ixl_stat_delta(sc, 6456 I40E_GLPRT_PTC1023H(sc->sc_port), I40E_GLPRT_PTC1023L(sc->sc_port), 6457 &isc->isc_tx_size_1023_offset, isc->isc_has_offset); 6458 atomic_add_64(&isc->isc_tx_size_1023.ev_count, delta); 6459 6460 delta = ixl_stat_delta(sc, 6461 I40E_GLPRT_PTC1522H(sc->sc_port), I40E_GLPRT_PTC1522L(sc->sc_port), 6462 &isc->isc_tx_size_1522_offset, isc->isc_has_offset); 6463 atomic_add_64(&isc->isc_tx_size_1522.ev_count, delta); 6464 6465 delta = ixl_stat_delta(sc, 6466 I40E_GLPRT_PTC9522H(sc->sc_port), I40E_GLPRT_PTC9522L(sc->sc_port), 6467 &isc->isc_tx_size_big_offset, isc->isc_has_offset); 6468 atomic_add_64(&isc->isc_tx_size_big.ev_count, delta); 6469 6470 /* mac faults */ 6471 delta = ixl_stat_delta(sc, 6472 0, I40E_GLPRT_MLFC(sc->sc_port), 6473 &isc->isc_mac_local_faults_offset, isc->isc_has_offset); 6474 atomic_add_64(&isc->isc_mac_local_faults.ev_count, delta); 6475 6476 delta = ixl_stat_delta(sc, 6477 0, I40E_GLPRT_MRFC(sc->sc_port), 6478 &isc->isc_mac_remote_faults_offset, isc->isc_has_offset); 6479 atomic_add_64(&isc->isc_mac_remote_faults.ev_count, delta); 6480 6481 /* Flow control (LFC) stats */ 6482 delta = ixl_stat_delta(sc, 6483 0, I40E_GLPRT_LXONRXC(sc->sc_port), 6484 &isc->isc_link_xon_rx_offset, isc->isc_has_offset); 6485 atomic_add_64(&isc->isc_link_xon_rx.ev_count, delta); 6486 6487 delta = ixl_stat_delta(sc, 6488 0, I40E_GLPRT_LXONTXC(sc->sc_port), 6489 &isc->isc_link_xon_tx_offset, isc->isc_has_offset); 6490 atomic_add_64(&isc->isc_link_xon_tx.ev_count, delta); 6491 6492 delta = ixl_stat_delta(sc, 6493 0, I40E_GLPRT_LXOFFRXC(sc->sc_port), 6494 &isc->isc_link_xoff_rx_offset, isc->isc_has_offset); 6495 atomic_add_64(&isc->isc_link_xoff_rx.ev_count, delta); 6496 6497 delta = ixl_stat_delta(sc, 6498 0, I40E_GLPRT_LXOFFTXC(sc->sc_port), 6499 &isc->isc_link_xoff_tx_offset, isc->isc_has_offset); 6500 atomic_add_64(&isc->isc_link_xoff_tx.ev_count, delta); 6501 6502 /* fragments */ 6503 delta = ixl_stat_delta(sc, 6504 0, I40E_GLPRT_RFC(sc->sc_port), 6505 &isc->isc_rx_fragments_offset, isc->isc_has_offset); 6506 atomic_add_64(&isc->isc_rx_fragments.ev_count, delta); 6507 6508 delta = ixl_stat_delta(sc, 6509 0, I40E_GLPRT_RJC(sc->sc_port), 6510 &isc->isc_rx_jabber_offset, isc->isc_has_offset); 6511 atomic_add_64(&isc->isc_rx_jabber.ev_count, delta); 6512 6513 /* VSI rx counters */ 6514 delta = ixl_stat_delta(sc, 6515 0, I40E_GLV_RDPC(sc->sc_vsi_stat_counter_idx), 6516 &isc->isc_vsi_rx_discards_offset, isc->isc_has_offset); 6517 atomic_add_64(&isc->isc_vsi_rx_discards.ev_count, delta); 6518 6519 delta = ixl_stat_delta(sc, 6520 I40E_GLV_GORCH(sc->sc_vsi_stat_counter_idx), 6521 I40E_GLV_GORCL(sc->sc_vsi_stat_counter_idx), 6522 &isc->isc_vsi_rx_bytes_offset, isc->isc_has_offset); 6523 atomic_add_64(&isc->isc_vsi_rx_bytes.ev_count, delta); 6524 6525 delta = ixl_stat_delta(sc, 6526 I40E_GLV_UPRCH(sc->sc_vsi_stat_counter_idx), 6527 I40E_GLV_UPRCL(sc->sc_vsi_stat_counter_idx), 6528 &isc->isc_vsi_rx_unicast_offset, isc->isc_has_offset); 6529 atomic_add_64(&isc->isc_vsi_rx_unicast.ev_count, delta); 6530 6531 delta = ixl_stat_delta(sc, 6532 I40E_GLV_MPRCH(sc->sc_vsi_stat_counter_idx), 6533 I40E_GLV_MPRCL(sc->sc_vsi_stat_counter_idx), 6534 &isc->isc_vsi_rx_multicast_offset, isc->isc_has_offset); 6535 atomic_add_64(&isc->isc_vsi_rx_multicast.ev_count, delta); 6536 6537 delta = ixl_stat_delta(sc, 6538 I40E_GLV_BPRCH(sc->sc_vsi_stat_counter_idx), 6539 I40E_GLV_BPRCL(sc->sc_vsi_stat_counter_idx), 6540 &isc->isc_vsi_rx_broadcast_offset, isc->isc_has_offset); 6541 atomic_add_64(&isc->isc_vsi_rx_broadcast.ev_count, delta); 6542 6543 /* VSI tx counters */ 6544 delta = ixl_stat_delta(sc, 6545 0, I40E_GLV_TEPC(sc->sc_vsi_stat_counter_idx), 6546 &isc->isc_vsi_tx_errors_offset, isc->isc_has_offset); 6547 atomic_add_64(&isc->isc_vsi_tx_errors.ev_count, delta); 6548 6549 delta = ixl_stat_delta(sc, 6550 I40E_GLV_GOTCH(sc->sc_vsi_stat_counter_idx), 6551 I40E_GLV_GOTCL(sc->sc_vsi_stat_counter_idx), 6552 &isc->isc_vsi_tx_bytes_offset, isc->isc_has_offset); 6553 atomic_add_64(&isc->isc_vsi_tx_bytes.ev_count, delta); 6554 6555 delta = ixl_stat_delta(sc, 6556 I40E_GLV_UPTCH(sc->sc_vsi_stat_counter_idx), 6557 I40E_GLV_UPTCL(sc->sc_vsi_stat_counter_idx), 6558 &isc->isc_vsi_tx_unicast_offset, isc->isc_has_offset); 6559 atomic_add_64(&isc->isc_vsi_tx_unicast.ev_count, delta); 6560 6561 delta = ixl_stat_delta(sc, 6562 I40E_GLV_MPTCH(sc->sc_vsi_stat_counter_idx), 6563 I40E_GLV_MPTCL(sc->sc_vsi_stat_counter_idx), 6564 &isc->isc_vsi_tx_multicast_offset, isc->isc_has_offset); 6565 atomic_add_64(&isc->isc_vsi_tx_multicast.ev_count, delta); 6566 6567 delta = ixl_stat_delta(sc, 6568 I40E_GLV_BPTCH(sc->sc_vsi_stat_counter_idx), 6569 I40E_GLV_BPTCL(sc->sc_vsi_stat_counter_idx), 6570 &isc->isc_vsi_tx_broadcast_offset, isc->isc_has_offset); 6571 atomic_add_64(&isc->isc_vsi_tx_broadcast.ev_count, delta); 6572 } 6573 6574 static int 6575 ixl_setup_sysctls(struct ixl_softc *sc) 6576 { 6577 const char *devname; 6578 struct sysctllog **log; 6579 const struct sysctlnode *rnode, *rxnode, *txnode; 6580 int error; 6581 6582 log = &sc->sc_sysctllog; 6583 devname = device_xname(sc->sc_dev); 6584 6585 error = sysctl_createv(log, 0, NULL, &rnode, 6586 0, CTLTYPE_NODE, devname, 6587 SYSCTL_DESCR("ixl information and settings"), 6588 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 6589 if (error) 6590 goto out; 6591 6592 error = sysctl_createv(log, 0, &rnode, NULL, 6593 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 6594 SYSCTL_DESCR("Use workqueue for packet processing"), 6595 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 6596 if (error) 6597 goto out; 6598 6599 error = sysctl_createv(log, 0, &rnode, NULL, 6600 CTLFLAG_READONLY, CTLTYPE_INT, "stats_interval", 6601 SYSCTL_DESCR("Statistics collection interval in milliseconds"), 6602 NULL, 0, &sc->sc_stats_intval, 0, CTL_CREATE, CTL_EOL); 6603 6604 error = sysctl_createv(log, 0, &rnode, &rxnode, 6605 0, CTLTYPE_NODE, "rx", 6606 SYSCTL_DESCR("ixl information and settings for Rx"), 6607 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6608 if (error) 6609 goto out; 6610 6611 error = sysctl_createv(log, 0, &rxnode, NULL, 6612 CTLFLAG_READWRITE, CTLTYPE_INT, "itr", 6613 SYSCTL_DESCR("Interrupt Throttling"), 6614 ixl_sysctl_itr_handler, 0, 6615 (void *)sc, 0, CTL_CREATE, CTL_EOL); 6616 if (error) 6617 goto out; 6618 6619 error = sysctl_createv(log, 0, &rxnode, NULL, 6620 CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num", 6621 SYSCTL_DESCR("the number of rx descriptors"), 6622 NULL, 0, &sc->sc_rx_ring_ndescs, 0, CTL_CREATE, CTL_EOL); 6623 if (error) 6624 goto out; 6625 6626 error = sysctl_createv(log, 0, &rxnode, NULL, 6627 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6628 SYSCTL_DESCR("max number of Rx packets" 6629 " to process for interrupt processing"), 6630 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6631 if (error) 6632 goto out; 6633 6634 error = sysctl_createv(log, 0, &rxnode, NULL, 6635 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6636 SYSCTL_DESCR("max number of Rx packets" 6637 " to process for deferred processing"), 6638 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 6639 if (error) 6640 goto out; 6641 6642 error = sysctl_createv(log, 0, &rnode, &txnode, 6643 0, CTLTYPE_NODE, "tx", 6644 SYSCTL_DESCR("ixl information and settings for Tx"), 6645 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6646 if (error) 6647 goto out; 6648 6649 error = sysctl_createv(log, 0, &txnode, NULL, 6650 CTLFLAG_READWRITE, CTLTYPE_INT, "itr", 6651 SYSCTL_DESCR("Interrupt Throttling"), 6652 ixl_sysctl_itr_handler, 0, 6653 (void *)sc, 0, CTL_CREATE, CTL_EOL); 6654 if (error) 6655 goto out; 6656 6657 error = sysctl_createv(log, 0, &txnode, NULL, 6658 CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num", 6659 SYSCTL_DESCR("the number of tx descriptors"), 6660 NULL, 0, &sc->sc_tx_ring_ndescs, 0, CTL_CREATE, CTL_EOL); 6661 if (error) 6662 goto out; 6663 6664 error = sysctl_createv(log, 0, &txnode, NULL, 6665 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6666 SYSCTL_DESCR("max number of Tx packets" 6667 " to process for interrupt processing"), 6668 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6669 if (error) 6670 goto out; 6671 6672 error = sysctl_createv(log, 0, &txnode, NULL, 6673 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6674 SYSCTL_DESCR("max number of Tx packets" 6675 " to process for deferred processing"), 6676 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 6677 if (error) 6678 goto out; 6679 6680 out: 6681 if (error) { 6682 aprint_error_dev(sc->sc_dev, 6683 "unable to create sysctl node\n"); 6684 sysctl_teardown(log); 6685 } 6686 6687 return error; 6688 } 6689 6690 static void 6691 ixl_teardown_sysctls(struct ixl_softc *sc) 6692 { 6693 6694 sysctl_teardown(&sc->sc_sysctllog); 6695 } 6696 6697 static bool 6698 ixl_sysctlnode_is_rx(struct sysctlnode *node) 6699 { 6700 6701 if (strstr(node->sysctl_parent->sysctl_name, "rx") != NULL) 6702 return true; 6703 6704 return false; 6705 } 6706 6707 static int 6708 ixl_sysctl_itr_handler(SYSCTLFN_ARGS) 6709 { 6710 struct sysctlnode node = *rnode; 6711 struct ixl_softc *sc = (struct ixl_softc *)node.sysctl_data; 6712 struct ifnet *ifp = &sc->sc_ec.ec_if; 6713 uint32_t newitr, *itrptr; 6714 int error; 6715 6716 if (ixl_sysctlnode_is_rx(&node)) { 6717 itrptr = &sc->sc_itr_rx; 6718 } else { 6719 itrptr = &sc->sc_itr_tx; 6720 } 6721 6722 newitr = *itrptr; 6723 node.sysctl_data = &newitr; 6724 node.sysctl_size = sizeof(newitr); 6725 6726 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6727 6728 if (error || newp == NULL) 6729 return error; 6730 6731 /* ITRs are applied in ixl_init() for simple implementation */ 6732 if (ISSET(ifp->if_flags, IFF_RUNNING)) 6733 return EBUSY; 6734 6735 if (newitr > 0x07ff) 6736 return EINVAL; 6737 6738 *itrptr = newitr; 6739 6740 return 0; 6741 } 6742 6743 static struct workqueue * 6744 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 6745 { 6746 struct workqueue *wq; 6747 int error; 6748 6749 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 6750 prio, ipl, flags); 6751 6752 if (error) 6753 return NULL; 6754 6755 return wq; 6756 } 6757 6758 static void 6759 ixl_workq_destroy(struct workqueue *wq) 6760 { 6761 6762 workqueue_destroy(wq); 6763 } 6764 6765 static void 6766 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 6767 { 6768 6769 memset(work, 0, sizeof(*work)); 6770 work->ixw_func = func; 6771 work->ixw_arg = arg; 6772 } 6773 6774 static void 6775 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 6776 { 6777 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 6778 return; 6779 6780 kpreempt_disable(); 6781 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 6782 kpreempt_enable(); 6783 } 6784 6785 static void 6786 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 6787 { 6788 6789 workqueue_wait(wq, &work->ixw_cookie); 6790 } 6791 6792 static void 6793 ixl_workq_work(struct work *wk, void *context) 6794 { 6795 struct ixl_work *work; 6796 6797 work = container_of(wk, struct ixl_work, ixw_cookie); 6798 6799 atomic_swap_uint(&work->ixw_added, 0); 6800 work->ixw_func(work->ixw_arg); 6801 } 6802 6803 static int 6804 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 6805 { 6806 struct ixl_aq_desc iaq; 6807 6808 memset(&iaq, 0, sizeof(iaq)); 6809 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 6810 iaq.iaq_param[1] = htole32(reg); 6811 6812 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6813 return ETIMEDOUT; 6814 6815 switch (htole16(iaq.iaq_retval)) { 6816 case IXL_AQ_RC_OK: 6817 /* success */ 6818 break; 6819 case IXL_AQ_RC_EACCES: 6820 return EPERM; 6821 case IXL_AQ_RC_EAGAIN: 6822 return EAGAIN; 6823 default: 6824 return EIO; 6825 } 6826 6827 *rv = htole32(iaq.iaq_param[3]); 6828 return 0; 6829 } 6830 6831 static uint32_t 6832 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 6833 { 6834 uint32_t val; 6835 int rv, retry, retry_limit; 6836 6837 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6838 retry_limit = 5; 6839 } else { 6840 retry_limit = 0; 6841 } 6842 6843 for (retry = 0; retry < retry_limit; retry++) { 6844 rv = ixl_rx_ctl_read(sc, reg, &val); 6845 if (rv == 0) 6846 return val; 6847 else if (rv == EAGAIN) 6848 delaymsec(1); 6849 else 6850 break; 6851 } 6852 6853 val = ixl_rd(sc, reg); 6854 6855 return val; 6856 } 6857 6858 static int 6859 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6860 { 6861 struct ixl_aq_desc iaq; 6862 6863 memset(&iaq, 0, sizeof(iaq)); 6864 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 6865 iaq.iaq_param[1] = htole32(reg); 6866 iaq.iaq_param[3] = htole32(value); 6867 6868 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6869 return ETIMEDOUT; 6870 6871 switch (htole16(iaq.iaq_retval)) { 6872 case IXL_AQ_RC_OK: 6873 /* success */ 6874 break; 6875 case IXL_AQ_RC_EACCES: 6876 return EPERM; 6877 case IXL_AQ_RC_EAGAIN: 6878 return EAGAIN; 6879 default: 6880 return EIO; 6881 } 6882 6883 return 0; 6884 } 6885 6886 static void 6887 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6888 { 6889 int rv, retry, retry_limit; 6890 6891 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6892 retry_limit = 5; 6893 } else { 6894 retry_limit = 0; 6895 } 6896 6897 for (retry = 0; retry < retry_limit; retry++) { 6898 rv = ixl_rx_ctl_write(sc, reg, value); 6899 if (rv == 0) 6900 return; 6901 else if (rv == EAGAIN) 6902 delaymsec(1); 6903 else 6904 break; 6905 } 6906 6907 ixl_wr(sc, reg, value); 6908 } 6909 6910 static int 6911 ixl_nvm_lock(struct ixl_softc *sc, char rw) 6912 { 6913 struct ixl_aq_desc iaq; 6914 struct ixl_aq_req_resource_param *param; 6915 int rv; 6916 6917 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6918 return 0; 6919 6920 memset(&iaq, 0, sizeof(iaq)); 6921 iaq.iaq_opcode = htole16(IXL_AQ_OP_REQUEST_RESOURCE); 6922 6923 param = (struct ixl_aq_req_resource_param *)&iaq.iaq_param; 6924 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6925 if (rw == 'R') { 6926 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_READ); 6927 } else { 6928 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_WRITE); 6929 } 6930 6931 rv = ixl_atq_poll(sc, &iaq, 250); 6932 6933 if (rv != 0) 6934 return ETIMEDOUT; 6935 6936 switch (le16toh(iaq.iaq_retval)) { 6937 case IXL_AQ_RC_OK: 6938 break; 6939 case IXL_AQ_RC_EACCES: 6940 return EACCES; 6941 case IXL_AQ_RC_EBUSY: 6942 return EBUSY; 6943 case IXL_AQ_RC_EPERM: 6944 return EPERM; 6945 } 6946 6947 return 0; 6948 } 6949 6950 static int 6951 ixl_nvm_unlock(struct ixl_softc *sc) 6952 { 6953 struct ixl_aq_desc iaq; 6954 struct ixl_aq_rel_resource_param *param; 6955 int rv; 6956 6957 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6958 return 0; 6959 6960 memset(&iaq, 0, sizeof(iaq)); 6961 iaq.iaq_opcode = htole16(IXL_AQ_OP_RELEASE_RESOURCE); 6962 6963 param = (struct ixl_aq_rel_resource_param *)&iaq.iaq_param; 6964 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6965 6966 rv = ixl_atq_poll(sc, &iaq, 250); 6967 6968 if (rv != 0) 6969 return ETIMEDOUT; 6970 6971 switch (le16toh(iaq.iaq_retval)) { 6972 case IXL_AQ_RC_OK: 6973 break; 6974 default: 6975 return EIO; 6976 } 6977 return 0; 6978 } 6979 6980 static int 6981 ixl_srdone_poll(struct ixl_softc *sc) 6982 { 6983 int wait_count; 6984 uint32_t reg; 6985 6986 for (wait_count = 0; wait_count < IXL_SRRD_SRCTL_ATTEMPTS; 6987 wait_count++) { 6988 reg = ixl_rd(sc, I40E_GLNVM_SRCTL); 6989 if (ISSET(reg, I40E_GLNVM_SRCTL_DONE_MASK)) 6990 break; 6991 6992 delaymsec(5); 6993 } 6994 6995 if (wait_count == IXL_SRRD_SRCTL_ATTEMPTS) 6996 return -1; 6997 6998 return 0; 6999 } 7000 7001 static int 7002 ixl_nvm_read_srctl(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 7003 { 7004 uint32_t reg; 7005 7006 if (ixl_srdone_poll(sc) != 0) 7007 return ETIMEDOUT; 7008 7009 reg = ((uint32_t)offset << I40E_GLNVM_SRCTL_ADDR_SHIFT) | 7010 __BIT(I40E_GLNVM_SRCTL_START_SHIFT); 7011 ixl_wr(sc, I40E_GLNVM_SRCTL, reg); 7012 7013 if (ixl_srdone_poll(sc) != 0) { 7014 aprint_debug("NVM read error: couldn't access " 7015 "Shadow RAM address: 0x%x\n", offset); 7016 return ETIMEDOUT; 7017 } 7018 7019 reg = ixl_rd(sc, I40E_GLNVM_SRDATA); 7020 *data = (uint16_t)__SHIFTOUT(reg, I40E_GLNVM_SRDATA_RDDATA_MASK); 7021 7022 return 0; 7023 } 7024 7025 static int 7026 ixl_nvm_read_aq(struct ixl_softc *sc, uint16_t offset_word, 7027 void *data, size_t len) 7028 { 7029 struct ixl_dmamem *idm; 7030 struct ixl_aq_desc iaq; 7031 struct ixl_aq_nvm_param *param; 7032 uint32_t offset_bytes; 7033 int rv; 7034 7035 idm = &sc->sc_aqbuf; 7036 if (len > IXL_DMA_LEN(idm)) 7037 return ENOMEM; 7038 7039 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 7040 memset(&iaq, 0, sizeof(iaq)); 7041 iaq.iaq_opcode = htole16(IXL_AQ_OP_NVM_READ); 7042 iaq.iaq_flags = htole16(IXL_AQ_BUF | 7043 ((len > I40E_AQ_LARGE_BUF) ? IXL_AQ_LB : 0)); 7044 iaq.iaq_datalen = htole16(len); 7045 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 7046 7047 param = (struct ixl_aq_nvm_param *)iaq.iaq_param; 7048 param->command_flags = IXL_AQ_NVM_LAST_CMD; 7049 param->module_pointer = 0; 7050 param->length = htole16(len); 7051 offset_bytes = (uint32_t)offset_word * 2; 7052 offset_bytes &= 0x00FFFFFF; 7053 param->offset = htole32(offset_bytes); 7054 7055 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 7056 BUS_DMASYNC_PREREAD); 7057 7058 rv = ixl_atq_poll(sc, &iaq, 250); 7059 7060 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 7061 BUS_DMASYNC_POSTREAD); 7062 7063 if (rv != 0) { 7064 return ETIMEDOUT; 7065 } 7066 7067 switch (le16toh(iaq.iaq_retval)) { 7068 case IXL_AQ_RC_OK: 7069 break; 7070 case IXL_AQ_RC_EPERM: 7071 return EPERM; 7072 case IXL_AQ_RC_EINVAL: 7073 return EINVAL; 7074 case IXL_AQ_RC_EBUSY: 7075 return EBUSY; 7076 case IXL_AQ_RC_EIO: 7077 default: 7078 return EIO; 7079 } 7080 7081 memcpy(data, IXL_DMA_KVA(idm), len); 7082 7083 return 0; 7084 } 7085 7086 static int 7087 ixl_rd16_nvm(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 7088 { 7089 int error; 7090 uint16_t buf; 7091 7092 error = ixl_nvm_lock(sc, 'R'); 7093 if (error) 7094 return error; 7095 7096 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMREAD)) { 7097 error = ixl_nvm_read_aq(sc, offset, 7098 &buf, sizeof(buf)); 7099 if (error == 0) 7100 *data = le16toh(buf); 7101 } else { 7102 error = ixl_nvm_read_srctl(sc, offset, &buf); 7103 if (error == 0) 7104 *data = buf; 7105 } 7106 7107 ixl_nvm_unlock(sc); 7108 7109 return error; 7110 } 7111 7112 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 7113 7114 #ifdef _MODULE 7115 #include "ioconf.c" 7116 #endif 7117 7118 #ifdef _MODULE 7119 static void 7120 ixl_parse_modprop(prop_dictionary_t dict) 7121 { 7122 prop_object_t obj; 7123 int64_t val; 7124 uint64_t uval; 7125 7126 if (dict == NULL) 7127 return; 7128 7129 obj = prop_dictionary_get(dict, "nomsix"); 7130 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_BOOL) { 7131 ixl_param_nomsix = prop_bool_true((prop_bool_t)obj); 7132 } 7133 7134 obj = prop_dictionary_get(dict, "stats_interval"); 7135 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7136 val = prop_number_signed_value((prop_number_t)obj); 7137 7138 /* the range has no reason */ 7139 if (100 < val && val < 180000) { 7140 ixl_param_stats_interval = val; 7141 } 7142 } 7143 7144 obj = prop_dictionary_get(dict, "nqps_limit"); 7145 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7146 val = prop_number_signed_value((prop_number_t)obj); 7147 7148 if (val <= INT32_MAX) 7149 ixl_param_nqps_limit = val; 7150 } 7151 7152 obj = prop_dictionary_get(dict, "rx_ndescs"); 7153 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7154 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7155 7156 if (uval > 8) 7157 ixl_param_rx_ndescs = uval; 7158 } 7159 7160 obj = prop_dictionary_get(dict, "tx_ndescs"); 7161 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7162 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7163 7164 if (uval > IXL_TX_PKT_DESCS) 7165 ixl_param_tx_ndescs = uval; 7166 } 7167 7168 } 7169 #endif 7170 7171 static int 7172 if_ixl_modcmd(modcmd_t cmd, void *opaque) 7173 { 7174 int error = 0; 7175 7176 #ifdef _MODULE 7177 switch (cmd) { 7178 case MODULE_CMD_INIT: 7179 ixl_parse_modprop((prop_dictionary_t)opaque); 7180 error = config_init_component(cfdriver_ioconf_if_ixl, 7181 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7182 break; 7183 case MODULE_CMD_FINI: 7184 error = config_fini_component(cfdriver_ioconf_if_ixl, 7185 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7186 break; 7187 default: 7188 error = ENOTTY; 7189 break; 7190 } 7191 #endif 7192 7193 return error; 7194 } 7195
Indexes created Tue Oct 14 01:09:49 GMT 2025