if_ixl.c revision 1.35
1 /* $NetBSD: if_ixl.c,v 1.35 2020/02/01 12:45:05 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 78 #ifdef _KERNEL_OPT 79 #include "opt_net_mpsafe.h" 80 #include "opt_if_ixl.h" 81 #endif 82 83 #include <sys/param.h> 84 #include <sys/types.h> 85 86 #include <sys/cpu.h> 87 #include <sys/device.h> 88 #include <sys/evcnt.h> 89 #include <sys/interrupt.h> 90 #include <sys/kmem.h> 91 #include <sys/malloc.h> 92 #include <sys/module.h> 93 #include <sys/mutex.h> 94 #include <sys/pcq.h> 95 #include <sys/syslog.h> 96 #include <sys/workqueue.h> 97 98 #include <sys/bus.h> 99 100 #include <net/bpf.h> 101 #include <net/if.h> 102 #include <net/if_dl.h> 103 #include <net/if_media.h> 104 #include <net/if_ether.h> 105 #include <net/rss_config.h> 106 107 #include <netinet/tcp.h> /* for struct tcphdr */ 108 #include <netinet/udp.h> /* for struct udphdr */ 109 110 #include <dev/pci/pcivar.h> 111 #include <dev/pci/pcidevs.h> 112 113 #include <dev/pci/if_ixlreg.h> 114 #include <dev/pci/if_ixlvar.h> 115 116 #include <prop/proplib.h> 117 118 struct ixl_softc; /* defined */ 119 120 #define I40E_PF_RESET_WAIT_COUNT 200 121 #define I40E_AQ_LARGE_BUF 512 122 123 /* bitfields for Tx queue mapping in QTX_CTL */ 124 #define I40E_QTX_CTL_VF_QUEUE 0x0 125 #define I40E_QTX_CTL_VM_QUEUE 0x1 126 #define I40E_QTX_CTL_PF_QUEUE 0x2 127 128 #define I40E_QUEUE_TYPE_EOL 0x7ff 129 #define I40E_INTR_NOTX_QUEUE 0 130 131 #define I40E_QUEUE_TYPE_RX 0x0 132 #define I40E_QUEUE_TYPE_TX 0x1 133 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 134 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 135 136 #define I40E_ITR_INDEX_RX 0x0 137 #define I40E_ITR_INDEX_TX 0x1 138 #define I40E_ITR_INDEX_OTHER 0x2 139 #define I40E_ITR_INDEX_NONE 0x3 140 141 #define I40E_INTR_NOTX_QUEUE 0 142 #define I40E_INTR_NOTX_INTR 0 143 #define I40E_INTR_NOTX_RX_QUEUE 0 144 #define I40E_INTR_NOTX_TX_QUEUE 1 145 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 146 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 147 148 #define BIT_ULL(a) (1ULL << (a)) 149 #define IXL_RSS_HENA_DEFAULT_BASE \ 150 (BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 151 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 152 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 153 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 154 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 155 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 156 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 157 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 158 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 159 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 160 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 161 #define IXL_RSS_HENA_DEFAULT_XL710 IXL_RSS_HENA_DEFAULT_BASE 162 #define IXL_RSS_HENA_DEFAULT_X722 (IXL_RSS_HENA_DEFAULT_XL710 | \ 163 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 164 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 165 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 166 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 167 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 168 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 169 #define I40E_HASH_LUT_SIZE_128 0 170 #define IXL_RSS_KEY_SIZE_REG 13 171 172 #define IXL_ICR0_CRIT_ERR_MASK \ 173 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 174 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 175 I40E_PFINT_ICR0_PE_CRITERR_MASK) 176 177 #define IXL_TX_PKT_DESCS 8 178 #define IXL_TX_PKT_MAXSIZE (MCLBYTES * IXL_TX_PKT_DESCS) 179 #define IXL_TX_QUEUE_ALIGN 128 180 #define IXL_RX_QUEUE_ALIGN 128 181 182 #define IXL_MCLBYTES (MCLBYTES - ETHER_ALIGN) 183 #define IXL_MTU_ETHERLEN ETHER_HDR_LEN \ 184 + ETHER_CRC_LEN 185 #if 0 186 #define IXL_MAX_MTU (9728 - IXL_MTU_ETHERLEN) 187 #else 188 /* (dbuff * 5) - ETHER_HDR_LEN - ETHER_CRC_LEN */ 189 #define IXL_MAX_MTU (9600 - IXL_MTU_ETHERLEN) 190 #endif 191 #define IXL_MIN_MTU (ETHER_MIN_LEN - ETHER_CRC_LEN) 192 193 #define IXL_PCIREG PCI_MAPREG_START 194 195 #define IXL_ITR0 0x0 196 #define IXL_ITR1 0x1 197 #define IXL_ITR2 0x2 198 #define IXL_NOITR 0x3 199 200 #define IXL_AQ_NUM 256 201 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 202 #define IXL_AQ_ALIGN 64 /* lol */ 203 #define IXL_AQ_BUFLEN 4096 204 205 #define IXL_HMC_ROUNDUP 512 206 #define IXL_HMC_PGSIZE 4096 207 #define IXL_HMC_DVASZ sizeof(uint64_t) 208 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 209 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 210 #define IXL_HMC_PDVALID 1ULL 211 212 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 213 214 #define IXL_SRRD_SRCTL_ATTEMPTS 100000 215 216 struct ixl_aq_regs { 217 bus_size_t atq_tail; 218 bus_size_t atq_head; 219 bus_size_t atq_len; 220 bus_size_t atq_bal; 221 bus_size_t atq_bah; 222 223 bus_size_t arq_tail; 224 bus_size_t arq_head; 225 bus_size_t arq_len; 226 bus_size_t arq_bal; 227 bus_size_t arq_bah; 228 229 uint32_t atq_len_enable; 230 uint32_t atq_tail_mask; 231 uint32_t atq_head_mask; 232 233 uint32_t arq_len_enable; 234 uint32_t arq_tail_mask; 235 uint32_t arq_head_mask; 236 }; 237 238 struct ixl_phy_type { 239 uint64_t phy_type; 240 uint64_t ifm_type; 241 }; 242 243 struct ixl_speed_type { 244 uint8_t dev_speed; 245 uint64_t net_speed; 246 }; 247 248 struct ixl_aq_buf { 249 SIMPLEQ_ENTRY(ixl_aq_buf) 250 aqb_entry; 251 void *aqb_data; 252 bus_dmamap_t aqb_map; 253 bus_dma_segment_t aqb_seg; 254 size_t aqb_size; 255 int aqb_nsegs; 256 }; 257 SIMPLEQ_HEAD(ixl_aq_bufs, ixl_aq_buf); 258 259 struct ixl_dmamem { 260 bus_dmamap_t ixm_map; 261 bus_dma_segment_t ixm_seg; 262 int ixm_nsegs; 263 size_t ixm_size; 264 void *ixm_kva; 265 }; 266 267 #define IXL_DMA_MAP(_ixm) ((_ixm)->ixm_map) 268 #define IXL_DMA_DVA(_ixm) ((_ixm)->ixm_map->dm_segs[0].ds_addr) 269 #define IXL_DMA_KVA(_ixm) ((void *)(_ixm)->ixm_kva) 270 #define IXL_DMA_LEN(_ixm) ((_ixm)->ixm_size) 271 272 struct ixl_hmc_entry { 273 uint64_t hmc_base; 274 uint32_t hmc_count; 275 uint64_t hmc_size; 276 }; 277 278 enum ixl_hmc_types { 279 IXL_HMC_LAN_TX = 0, 280 IXL_HMC_LAN_RX, 281 IXL_HMC_FCOE_CTX, 282 IXL_HMC_FCOE_FILTER, 283 IXL_HMC_COUNT 284 }; 285 286 struct ixl_hmc_pack { 287 uint16_t offset; 288 uint16_t width; 289 uint16_t lsb; 290 }; 291 292 /* 293 * these hmc objects have weird sizes and alignments, so these are abstract 294 * representations of them that are nice for c to populate. 295 * 296 * the packing code relies on little-endian values being stored in the fields, 297 * no high bits in the fields being set, and the fields must be packed in the 298 * same order as they are in the ctx structure. 299 */ 300 301 struct ixl_hmc_rxq { 302 uint16_t head; 303 uint8_t cpuid; 304 uint64_t base; 305 #define IXL_HMC_RXQ_BASE_UNIT 128 306 uint16_t qlen; 307 uint16_t dbuff; 308 #define IXL_HMC_RXQ_DBUFF_UNIT 128 309 uint8_t hbuff; 310 #define IXL_HMC_RXQ_HBUFF_UNIT 64 311 uint8_t dtype; 312 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 313 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 314 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 315 uint8_t dsize; 316 #define IXL_HMC_RXQ_DSIZE_16 0 317 #define IXL_HMC_RXQ_DSIZE_32 1 318 uint8_t crcstrip; 319 uint8_t fc_ena; 320 uint8_t l2sel; 321 uint8_t hsplit_0; 322 uint8_t hsplit_1; 323 uint8_t showiv; 324 uint16_t rxmax; 325 uint8_t tphrdesc_ena; 326 uint8_t tphwdesc_ena; 327 uint8_t tphdata_ena; 328 uint8_t tphhead_ena; 329 uint8_t lrxqthresh; 330 uint8_t prefena; 331 }; 332 333 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 334 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 335 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 336 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 337 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 338 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 339 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 340 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 341 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 342 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 343 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 344 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 345 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 346 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 347 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 348 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 349 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 350 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 351 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 352 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 353 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 354 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 355 }; 356 357 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 358 359 struct ixl_hmc_txq { 360 uint16_t head; 361 uint8_t new_context; 362 uint64_t base; 363 #define IXL_HMC_TXQ_BASE_UNIT 128 364 uint8_t fc_ena; 365 uint8_t timesync_ena; 366 uint8_t fd_ena; 367 uint8_t alt_vlan_ena; 368 uint16_t thead_wb; 369 uint8_t cpuid; 370 uint8_t head_wb_ena; 371 #define IXL_HMC_TXQ_DESC_WB 0 372 #define IXL_HMC_TXQ_HEAD_WB 1 373 uint16_t qlen; 374 uint8_t tphrdesc_ena; 375 uint8_t tphrpacket_ena; 376 uint8_t tphwdesc_ena; 377 uint64_t head_wb_addr; 378 uint32_t crc; 379 uint16_t rdylist; 380 uint8_t rdylist_act; 381 }; 382 383 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 384 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 385 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 386 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 387 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 388 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 389 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 390 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 391 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 392 /* line 1 */ 393 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 394 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 395 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 396 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 397 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 398 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 399 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 400 /* line 7 */ 401 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 402 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 403 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 404 }; 405 406 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 407 408 struct ixl_work { 409 struct work ixw_cookie; 410 void (*ixw_func)(void *); 411 void *ixw_arg; 412 unsigned int ixw_added; 413 }; 414 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 415 416 struct ixl_tx_map { 417 struct mbuf *txm_m; 418 bus_dmamap_t txm_map; 419 unsigned int txm_eop; 420 }; 421 422 struct ixl_tx_ring { 423 kmutex_t txr_lock; 424 struct ixl_softc *txr_sc; 425 426 unsigned int txr_prod; 427 unsigned int txr_cons; 428 429 struct ixl_tx_map *txr_maps; 430 struct ixl_dmamem txr_mem; 431 432 bus_size_t txr_tail; 433 unsigned int txr_qid; 434 pcq_t *txr_intrq; 435 void *txr_si; 436 437 struct evcnt txr_defragged; 438 struct evcnt txr_defrag_failed; 439 struct evcnt txr_pcqdrop; 440 struct evcnt txr_transmitdef; 441 struct evcnt txr_intr; 442 struct evcnt txr_defer; 443 }; 444 445 struct ixl_rx_map { 446 struct mbuf *rxm_m; 447 bus_dmamap_t rxm_map; 448 }; 449 450 struct ixl_rx_ring { 451 kmutex_t rxr_lock; 452 453 unsigned int rxr_prod; 454 unsigned int rxr_cons; 455 456 struct ixl_rx_map *rxr_maps; 457 struct ixl_dmamem rxr_mem; 458 459 struct mbuf *rxr_m_head; 460 struct mbuf **rxr_m_tail; 461 462 bus_size_t rxr_tail; 463 unsigned int rxr_qid; 464 465 struct evcnt rxr_mgethdr_failed; 466 struct evcnt rxr_mgetcl_failed; 467 struct evcnt rxr_mbuf_load_failed; 468 struct evcnt rxr_intr; 469 struct evcnt rxr_defer; 470 }; 471 472 struct ixl_queue_pair { 473 struct ixl_softc *qp_sc; 474 struct ixl_tx_ring *qp_txr; 475 struct ixl_rx_ring *qp_rxr; 476 477 char qp_name[16]; 478 479 void *qp_si; 480 struct ixl_work qp_task; 481 bool qp_workqueue; 482 }; 483 484 struct ixl_atq { 485 struct ixl_aq_desc iatq_desc; 486 void (*iatq_fn)(struct ixl_softc *, 487 const struct ixl_aq_desc *); 488 }; 489 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 490 491 struct ixl_product { 492 unsigned int vendor_id; 493 unsigned int product_id; 494 }; 495 496 struct ixl_stats_counters { 497 bool isc_has_offset; 498 struct evcnt isc_crc_errors; 499 uint64_t isc_crc_errors_offset; 500 struct evcnt isc_illegal_bytes; 501 uint64_t isc_illegal_bytes_offset; 502 struct evcnt isc_rx_bytes; 503 uint64_t isc_rx_bytes_offset; 504 struct evcnt isc_rx_discards; 505 uint64_t isc_rx_discards_offset; 506 struct evcnt isc_rx_unicast; 507 uint64_t isc_rx_unicast_offset; 508 struct evcnt isc_rx_multicast; 509 uint64_t isc_rx_multicast_offset; 510 struct evcnt isc_rx_broadcast; 511 uint64_t isc_rx_broadcast_offset; 512 struct evcnt isc_rx_size_64; 513 uint64_t isc_rx_size_64_offset; 514 struct evcnt isc_rx_size_127; 515 uint64_t isc_rx_size_127_offset; 516 struct evcnt isc_rx_size_255; 517 uint64_t isc_rx_size_255_offset; 518 struct evcnt isc_rx_size_511; 519 uint64_t isc_rx_size_511_offset; 520 struct evcnt isc_rx_size_1023; 521 uint64_t isc_rx_size_1023_offset; 522 struct evcnt isc_rx_size_1522; 523 uint64_t isc_rx_size_1522_offset; 524 struct evcnt isc_rx_size_big; 525 uint64_t isc_rx_size_big_offset; 526 struct evcnt isc_rx_undersize; 527 uint64_t isc_rx_undersize_offset; 528 struct evcnt isc_rx_oversize; 529 uint64_t isc_rx_oversize_offset; 530 struct evcnt isc_rx_fragments; 531 uint64_t isc_rx_fragments_offset; 532 struct evcnt isc_rx_jabber; 533 uint64_t isc_rx_jabber_offset; 534 struct evcnt isc_tx_bytes; 535 uint64_t isc_tx_bytes_offset; 536 struct evcnt isc_tx_dropped_link_down; 537 uint64_t isc_tx_dropped_link_down_offset; 538 struct evcnt isc_tx_unicast; 539 uint64_t isc_tx_unicast_offset; 540 struct evcnt isc_tx_multicast; 541 uint64_t isc_tx_multicast_offset; 542 struct evcnt isc_tx_broadcast; 543 uint64_t isc_tx_broadcast_offset; 544 struct evcnt isc_tx_size_64; 545 uint64_t isc_tx_size_64_offset; 546 struct evcnt isc_tx_size_127; 547 uint64_t isc_tx_size_127_offset; 548 struct evcnt isc_tx_size_255; 549 uint64_t isc_tx_size_255_offset; 550 struct evcnt isc_tx_size_511; 551 uint64_t isc_tx_size_511_offset; 552 struct evcnt isc_tx_size_1023; 553 uint64_t isc_tx_size_1023_offset; 554 struct evcnt isc_tx_size_1522; 555 uint64_t isc_tx_size_1522_offset; 556 struct evcnt isc_tx_size_big; 557 uint64_t isc_tx_size_big_offset; 558 struct evcnt isc_mac_local_faults; 559 uint64_t isc_mac_local_faults_offset; 560 struct evcnt isc_mac_remote_faults; 561 uint64_t isc_mac_remote_faults_offset; 562 struct evcnt isc_link_xon_rx; 563 uint64_t isc_link_xon_rx_offset; 564 struct evcnt isc_link_xon_tx; 565 uint64_t isc_link_xon_tx_offset; 566 struct evcnt isc_link_xoff_rx; 567 uint64_t isc_link_xoff_rx_offset; 568 struct evcnt isc_link_xoff_tx; 569 uint64_t isc_link_xoff_tx_offset; 570 struct evcnt isc_vsi_rx_discards; 571 uint64_t isc_vsi_rx_discards_offset; 572 struct evcnt isc_vsi_rx_bytes; 573 uint64_t isc_vsi_rx_bytes_offset; 574 struct evcnt isc_vsi_rx_unicast; 575 uint64_t isc_vsi_rx_unicast_offset; 576 struct evcnt isc_vsi_rx_multicast; 577 uint64_t isc_vsi_rx_multicast_offset; 578 struct evcnt isc_vsi_rx_broadcast; 579 uint64_t isc_vsi_rx_broadcast_offset; 580 struct evcnt isc_vsi_tx_errors; 581 uint64_t isc_vsi_tx_errors_offset; 582 struct evcnt isc_vsi_tx_bytes; 583 uint64_t isc_vsi_tx_bytes_offset; 584 struct evcnt isc_vsi_tx_unicast; 585 uint64_t isc_vsi_tx_unicast_offset; 586 struct evcnt isc_vsi_tx_multicast; 587 uint64_t isc_vsi_tx_multicast_offset; 588 struct evcnt isc_vsi_tx_broadcast; 589 uint64_t isc_vsi_tx_broadcast_offset; 590 }; 591 592 /* 593 * Locking notes: 594 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 595 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 596 * - more than one lock of them cannot be held at once. 597 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 598 * (a spin mutex). 599 * - the lock cannot held with txr_lock or rxr_lock. 600 * + a field named sc_arq_* is not protected by any lock. 601 * - operations for sc_arq_* is done in one context related to 602 * sc_arq_task. 603 * + other fields in ixl_softc is protected by sc_cfg_lock 604 * (an adaptive mutex) 605 * - It must be held before another lock is held, and It can be 606 * released after the other lock is released. 607 * */ 608 609 struct ixl_softc { 610 device_t sc_dev; 611 struct ethercom sc_ec; 612 bool sc_attached; 613 bool sc_dead; 614 uint32_t sc_port; 615 struct sysctllog *sc_sysctllog; 616 struct workqueue *sc_workq; 617 struct workqueue *sc_workq_txrx; 618 int sc_stats_intval; 619 callout_t sc_stats_callout; 620 struct ixl_work sc_stats_task; 621 struct ixl_stats_counters 622 sc_stats_counters; 623 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 624 struct ifmedia sc_media; 625 uint64_t sc_media_status; 626 uint64_t sc_media_active; 627 uint64_t sc_phy_types; 628 uint8_t sc_phy_abilities; 629 uint8_t sc_phy_linkspeed; 630 uint8_t sc_phy_fec_cfg; 631 uint16_t sc_eee_cap; 632 uint32_t sc_eeer_val; 633 uint8_t sc_d3_lpan; 634 kmutex_t sc_cfg_lock; 635 enum i40e_mac_type sc_mac_type; 636 uint32_t sc_rss_table_size; 637 uint32_t sc_rss_table_entry_width; 638 bool sc_txrx_workqueue; 639 u_int sc_tx_process_limit; 640 u_int sc_rx_process_limit; 641 u_int sc_tx_intr_process_limit; 642 u_int sc_rx_intr_process_limit; 643 644 int sc_cur_ec_capenable; 645 646 struct pci_attach_args sc_pa; 647 pci_intr_handle_t *sc_ihp; 648 void **sc_ihs; 649 unsigned int sc_nintrs; 650 651 bus_dma_tag_t sc_dmat; 652 bus_space_tag_t sc_memt; 653 bus_space_handle_t sc_memh; 654 bus_size_t sc_mems; 655 656 uint8_t sc_pf_id; 657 uint16_t sc_uplink_seid; /* le */ 658 uint16_t sc_downlink_seid; /* le */ 659 uint16_t sc_vsi_number; /* le */ 660 uint16_t sc_vsi_stat_counter_idx; 661 uint16_t sc_seid; 662 unsigned int sc_base_queue; 663 664 pci_intr_type_t sc_intrtype; 665 unsigned int sc_msix_vector_queue; 666 667 struct ixl_dmamem sc_scratch; 668 struct ixl_dmamem sc_aqbuf; 669 670 const struct ixl_aq_regs * 671 sc_aq_regs; 672 uint32_t sc_aq_flags; 673 #define IXL_SC_AQ_FLAG_RXCTL __BIT(0) 674 #define IXL_SC_AQ_FLAG_NVMLOCK __BIT(1) 675 #define IXL_SC_AQ_FLAG_NVMREAD __BIT(2) 676 677 kmutex_t sc_atq_lock; 678 kcondvar_t sc_atq_cv; 679 struct ixl_dmamem sc_atq; 680 unsigned int sc_atq_prod; 681 unsigned int sc_atq_cons; 682 683 struct ixl_dmamem sc_arq; 684 struct ixl_work sc_arq_task; 685 struct ixl_aq_bufs sc_arq_idle; 686 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 687 unsigned int sc_arq_prod; 688 unsigned int sc_arq_cons; 689 690 struct ixl_work sc_link_state_task; 691 struct ixl_atq sc_link_state_atq; 692 693 struct ixl_dmamem sc_hmc_sd; 694 struct ixl_dmamem sc_hmc_pd; 695 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 696 697 unsigned int sc_tx_ring_ndescs; 698 unsigned int sc_rx_ring_ndescs; 699 unsigned int sc_nqueue_pairs; 700 unsigned int sc_nqueue_pairs_max; 701 unsigned int sc_nqueue_pairs_device; 702 struct ixl_queue_pair *sc_qps; 703 704 struct evcnt sc_event_atq; 705 struct evcnt sc_event_link; 706 struct evcnt sc_event_ecc_err; 707 struct evcnt sc_event_pci_exception; 708 struct evcnt sc_event_crit_err; 709 }; 710 711 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 712 #define IXL_TX_PROCESS_LIMIT 256 713 #define IXL_RX_PROCESS_LIMIT 256 714 #define IXL_TX_INTR_PROCESS_LIMIT 256 715 #define IXL_RX_INTR_PROCESS_LIMIT 0U 716 717 #define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx | \ 718 IFCAP_CSUM_TCPv4_Rx | \ 719 IFCAP_CSUM_UDPv4_Rx | \ 720 IFCAP_CSUM_TCPv6_Rx | \ 721 IFCAP_CSUM_UDPv6_Rx) 722 #define IXL_IFCAP_TXCSUM (IFCAP_CSUM_IPv4_Tx | \ 723 IFCAP_CSUM_TCPv4_Tx | \ 724 IFCAP_CSUM_UDPv4_Tx | \ 725 IFCAP_CSUM_TCPv6_Tx | \ 726 IFCAP_CSUM_UDPv6_Tx) 727 #define IXL_CSUM_ALL_OFFLOAD (M_CSUM_IPv4 | \ 728 M_CSUM_TCPv4 | M_CSUM_TCPv6 | \ 729 M_CSUM_UDPv4 | M_CSUM_UDPv6) 730 731 #define delaymsec(_x) DELAY(1000 * (_x)) 732 #ifdef IXL_DEBUG 733 #define DDPRINTF(sc, fmt, args...) \ 734 do { \ 735 if ((sc) != NULL) { \ 736 device_printf( \ 737 ((struct ixl_softc *)(sc))->sc_dev, \ 738 ""); \ 739 } \ 740 printf("%s:\t" fmt, __func__, ##args); \ 741 } while (0) 742 #else 743 #define DDPRINTF(sc, fmt, args...) __nothing 744 #endif 745 #ifndef IXL_STATS_INTERVAL_MSEC 746 #define IXL_STATS_INTERVAL_MSEC 10000 747 #endif 748 #ifndef IXL_QUEUE_NUM 749 #define IXL_QUEUE_NUM 0 750 #endif 751 752 static bool ixl_param_nomsix = false; 753 static int ixl_param_stats_interval = IXL_STATS_INTERVAL_MSEC; 754 static int ixl_param_nqps_limit = IXL_QUEUE_NUM; 755 static unsigned int ixl_param_tx_ndescs = 1024; 756 static unsigned int ixl_param_rx_ndescs = 1024; 757 758 static enum i40e_mac_type 759 ixl_mactype(pci_product_id_t); 760 static void ixl_clear_hw(struct ixl_softc *); 761 static int ixl_pf_reset(struct ixl_softc *); 762 763 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 764 bus_size_t, bus_size_t); 765 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 766 767 static int ixl_arq_fill(struct ixl_softc *); 768 static void ixl_arq_unfill(struct ixl_softc *); 769 770 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 771 unsigned int); 772 static void ixl_atq_set(struct ixl_atq *, 773 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 774 static int ixl_atq_post(struct ixl_softc *, struct ixl_atq *); 775 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 776 static void ixl_atq_done(struct ixl_softc *); 777 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 778 static int ixl_get_version(struct ixl_softc *); 779 static int ixl_get_nvm_version(struct ixl_softc *); 780 static int ixl_get_hw_capabilities(struct ixl_softc *); 781 static int ixl_pxe_clear(struct ixl_softc *); 782 static int ixl_lldp_shut(struct ixl_softc *); 783 static int ixl_get_mac(struct ixl_softc *); 784 static int ixl_get_switch_config(struct ixl_softc *); 785 static int ixl_phy_mask_ints(struct ixl_softc *); 786 static int ixl_get_phy_info(struct ixl_softc *); 787 static int ixl_set_phy_config(struct ixl_softc *, uint8_t, uint8_t, bool); 788 static int ixl_set_phy_autoselect(struct ixl_softc *); 789 static int ixl_restart_an(struct ixl_softc *); 790 static int ixl_hmc(struct ixl_softc *); 791 static void ixl_hmc_free(struct ixl_softc *); 792 static int ixl_get_vsi(struct ixl_softc *); 793 static int ixl_set_vsi(struct ixl_softc *); 794 static void ixl_set_filter_control(struct ixl_softc *); 795 static void ixl_get_link_status(void *); 796 static int ixl_get_link_status_poll(struct ixl_softc *); 797 static int ixl_set_link_status(struct ixl_softc *, 798 const struct ixl_aq_desc *); 799 static uint64_t ixl_search_link_speed(uint8_t); 800 static uint8_t ixl_search_baudrate(uint64_t); 801 static void ixl_config_rss(struct ixl_softc *); 802 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 803 uint16_t, uint16_t); 804 static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *, 805 uint16_t, uint16_t); 806 static void ixl_arq(void *); 807 static void ixl_hmc_pack(void *, const void *, 808 const struct ixl_hmc_pack *, unsigned int); 809 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 810 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 811 static int ixl_rd16_nvm(struct ixl_softc *, uint16_t, uint16_t *); 812 813 static int ixl_match(device_t, cfdata_t, void *); 814 static void ixl_attach(device_t, device_t, void *); 815 static int ixl_detach(device_t, int); 816 817 static void ixl_media_add(struct ixl_softc *); 818 static int ixl_media_change(struct ifnet *); 819 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 820 static void ixl_watchdog(struct ifnet *); 821 static int ixl_ioctl(struct ifnet *, u_long, void *); 822 static void ixl_start(struct ifnet *); 823 static int ixl_transmit(struct ifnet *, struct mbuf *); 824 static void ixl_deferred_transmit(void *); 825 static int ixl_intr(void *); 826 static int ixl_queue_intr(void *); 827 static int ixl_other_intr(void *); 828 static void ixl_handle_queue(void *); 829 static void ixl_sched_handle_queue(struct ixl_softc *, 830 struct ixl_queue_pair *); 831 static int ixl_init(struct ifnet *); 832 static int ixl_init_locked(struct ixl_softc *); 833 static void ixl_stop(struct ifnet *, int); 834 static void ixl_stop_locked(struct ixl_softc *); 835 static int ixl_iff(struct ixl_softc *); 836 static int ixl_ifflags_cb(struct ethercom *); 837 static int ixl_setup_interrupts(struct ixl_softc *); 838 static int ixl_establish_intx(struct ixl_softc *); 839 static int ixl_establish_msix(struct ixl_softc *); 840 static void ixl_enable_queue_intr(struct ixl_softc *, 841 struct ixl_queue_pair *); 842 static void ixl_disable_queue_intr(struct ixl_softc *, 843 struct ixl_queue_pair *); 844 static void ixl_enable_other_intr(struct ixl_softc *); 845 static void ixl_disable_other_intr(struct ixl_softc *); 846 static void ixl_config_queue_intr(struct ixl_softc *); 847 static void ixl_config_other_intr(struct ixl_softc *); 848 849 static struct ixl_tx_ring * 850 ixl_txr_alloc(struct ixl_softc *, unsigned int); 851 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 852 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 853 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 854 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 855 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 856 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 857 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 858 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 859 860 static struct ixl_rx_ring * 861 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 862 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 863 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 864 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 865 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 866 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 867 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 868 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 869 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 870 871 static struct workqueue * 872 ixl_workq_create(const char *, pri_t, int, int); 873 static void ixl_workq_destroy(struct workqueue *); 874 static int ixl_workqs_teardown(device_t); 875 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 876 static void ixl_work_add(struct workqueue *, struct ixl_work *); 877 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 878 static void ixl_workq_work(struct work *, void *); 879 static const struct ixl_product * 880 ixl_lookup(const struct pci_attach_args *pa); 881 static void ixl_link_state_update(struct ixl_softc *, 882 const struct ixl_aq_desc *); 883 static int ixl_vlan_cb(struct ethercom *, uint16_t, bool); 884 static int ixl_setup_vlan_hwfilter(struct ixl_softc *); 885 static void ixl_teardown_vlan_hwfilter(struct ixl_softc *); 886 static int ixl_update_macvlan(struct ixl_softc *); 887 static int ixl_setup_interrupts(struct ixl_softc *);; 888 static void ixl_teardown_interrupts(struct ixl_softc *); 889 static int ixl_setup_stats(struct ixl_softc *); 890 static void ixl_teardown_stats(struct ixl_softc *); 891 static void ixl_stats_callout(void *); 892 static void ixl_stats_update(void *); 893 static int ixl_setup_sysctls(struct ixl_softc *); 894 static void ixl_teardown_sysctls(struct ixl_softc *); 895 static int ixl_queue_pairs_alloc(struct ixl_softc *); 896 static void ixl_queue_pairs_free(struct ixl_softc *); 897 898 static const struct ixl_phy_type ixl_phy_type_map[] = { 899 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 900 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 901 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 902 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 903 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 904 { 1ULL << IXL_PHY_TYPE_XAUI | 905 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 906 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 907 { 1ULL << IXL_PHY_TYPE_XLAUI | 908 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 909 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 910 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 911 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 912 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 913 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 914 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 915 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 916 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 917 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 918 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 919 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 920 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 921 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 922 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 923 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 924 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 925 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 926 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 927 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 928 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 929 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 930 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 931 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 932 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_CR }, 933 }; 934 935 static const struct ixl_speed_type ixl_speed_type_map[] = { 936 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 937 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 938 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 939 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 940 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 941 }; 942 943 static const struct ixl_aq_regs ixl_pf_aq_regs = { 944 .atq_tail = I40E_PF_ATQT, 945 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 946 .atq_head = I40E_PF_ATQH, 947 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 948 .atq_len = I40E_PF_ATQLEN, 949 .atq_bal = I40E_PF_ATQBAL, 950 .atq_bah = I40E_PF_ATQBAH, 951 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 952 953 .arq_tail = I40E_PF_ARQT, 954 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 955 .arq_head = I40E_PF_ARQH, 956 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 957 .arq_len = I40E_PF_ARQLEN, 958 .arq_bal = I40E_PF_ARQBAL, 959 .arq_bah = I40E_PF_ARQBAH, 960 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 961 }; 962 963 #define ixl_rd(_s, _r) \ 964 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 965 #define ixl_wr(_s, _r, _v) \ 966 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 967 #define ixl_barrier(_s, _r, _l, _o) \ 968 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 969 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 970 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 971 972 static inline uint32_t 973 ixl_dmamem_hi(struct ixl_dmamem *ixm) 974 { 975 uint32_t retval; 976 uint64_t val; 977 978 if (sizeof(IXL_DMA_DVA(ixm)) > 4) { 979 val = (intptr_t)IXL_DMA_DVA(ixm); 980 retval = (uint32_t)(val >> 32); 981 } else { 982 retval = 0; 983 } 984 985 return retval; 986 } 987 988 static inline uint32_t 989 ixl_dmamem_lo(struct ixl_dmamem *ixm) 990 { 991 992 return (uint32_t)IXL_DMA_DVA(ixm); 993 } 994 995 static inline void 996 ixl_aq_dva(struct ixl_aq_desc *iaq, bus_addr_t addr) 997 { 998 uint64_t val; 999 1000 if (sizeof(addr) > 4) { 1001 val = (intptr_t)addr; 1002 iaq->iaq_param[2] = htole32(val >> 32); 1003 } else { 1004 iaq->iaq_param[2] = htole32(0); 1005 } 1006 1007 iaq->iaq_param[3] = htole32(addr); 1008 } 1009 1010 static inline unsigned int 1011 ixl_rxr_unrefreshed(unsigned int prod, unsigned int cons, unsigned int ndescs) 1012 { 1013 unsigned int num; 1014 1015 if (prod < cons) 1016 num = cons - prod; 1017 else 1018 num = (ndescs - prod) + cons; 1019 1020 if (__predict_true(num > 0)) { 1021 /* device cannot receive packets if all descripter is filled */ 1022 num -= 1; 1023 } 1024 1025 return num; 1026 } 1027 1028 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 1029 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 1030 DVF_DETACH_SHUTDOWN); 1031 1032 static const struct ixl_product ixl_products[] = { 1033 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 1034 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 1035 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 1036 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 1037 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 1038 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 1039 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T }, 1040 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 1041 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 1042 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 1043 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 1044 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 1045 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 1046 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 1047 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 1048 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 1049 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 1050 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 1051 /* required last entry */ 1052 {0, 0} 1053 }; 1054 1055 static const struct ixl_product * 1056 ixl_lookup(const struct pci_attach_args *pa) 1057 { 1058 const struct ixl_product *ixlp; 1059 1060 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 1061 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 1062 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 1063 return ixlp; 1064 } 1065 1066 return NULL; 1067 } 1068 1069 static int 1070 ixl_match(device_t parent, cfdata_t match, void *aux) 1071 { 1072 const struct pci_attach_args *pa = aux; 1073 1074 return (ixl_lookup(pa) != NULL) ? 1 : 0; 1075 } 1076 1077 static void 1078 ixl_attach(device_t parent, device_t self, void *aux) 1079 { 1080 struct ixl_softc *sc; 1081 struct pci_attach_args *pa = aux; 1082 struct ifnet *ifp; 1083 pcireg_t memtype; 1084 uint32_t firstq, port, ari, func; 1085 char xnamebuf[32]; 1086 int tries, rv; 1087 1088 sc = device_private(self); 1089 sc->sc_dev = self; 1090 ifp = &sc->sc_ec.ec_if; 1091 1092 sc->sc_pa = *pa; 1093 sc->sc_dmat = (pci_dma64_available(pa)) ? 1094 pa->pa_dmat64 : pa->pa_dmat; 1095 sc->sc_aq_regs = &ixl_pf_aq_regs; 1096 1097 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(pa->pa_id)); 1098 1099 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 1100 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 1101 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 1102 aprint_error(": unable to map registers\n"); 1103 return; 1104 } 1105 1106 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 1107 1108 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 1109 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 1110 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 1111 sc->sc_base_queue = firstq; 1112 1113 ixl_clear_hw(sc); 1114 if (ixl_pf_reset(sc) == -1) { 1115 /* error printed by ixl pf_reset */ 1116 goto unmap; 1117 } 1118 1119 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 1120 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 1121 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 1122 sc->sc_port = port; 1123 aprint_normal(": port %u", sc->sc_port); 1124 1125 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 1126 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 1127 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 1128 1129 func = ixl_rd(sc, I40E_PF_FUNC_RID); 1130 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 1131 1132 /* initialise the adminq */ 1133 1134 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 1135 1136 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 1137 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1138 aprint_error("\n" "%s: unable to allocate atq\n", 1139 device_xname(self)); 1140 goto unmap; 1141 } 1142 1143 SIMPLEQ_INIT(&sc->sc_arq_idle); 1144 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 1145 sc->sc_arq_cons = 0; 1146 sc->sc_arq_prod = 0; 1147 1148 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 1149 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1150 aprint_error("\n" "%s: unable to allocate arq\n", 1151 device_xname(self)); 1152 goto free_atq; 1153 } 1154 1155 if (!ixl_arq_fill(sc)) { 1156 aprint_error("\n" "%s: unable to fill arq descriptors\n", 1157 device_xname(self)); 1158 goto free_arq; 1159 } 1160 1161 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1162 0, IXL_DMA_LEN(&sc->sc_atq), 1163 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1164 1165 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1166 0, IXL_DMA_LEN(&sc->sc_arq), 1167 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1168 1169 for (tries = 0; tries < 10; tries++) { 1170 sc->sc_atq_cons = 0; 1171 sc->sc_atq_prod = 0; 1172 1173 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1174 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1175 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1176 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1177 1178 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1179 1180 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1181 ixl_dmamem_lo(&sc->sc_atq)); 1182 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1183 ixl_dmamem_hi(&sc->sc_atq)); 1184 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1185 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1186 1187 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1188 ixl_dmamem_lo(&sc->sc_arq)); 1189 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1190 ixl_dmamem_hi(&sc->sc_arq)); 1191 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1192 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1193 1194 rv = ixl_get_version(sc); 1195 if (rv == 0) 1196 break; 1197 if (rv != ETIMEDOUT) { 1198 aprint_error(", unable to get firmware version\n"); 1199 goto shutdown; 1200 } 1201 1202 delaymsec(100); 1203 } 1204 1205 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1206 1207 if (ixl_dmamem_alloc(sc, &sc->sc_aqbuf, IXL_AQ_BUFLEN, 0) != 0) { 1208 aprint_error_dev(self, ", unable to allocate nvm buffer\n"); 1209 goto shutdown; 1210 } 1211 1212 ixl_get_nvm_version(sc); 1213 1214 if (sc->sc_mac_type == I40E_MAC_X722) 1215 sc->sc_nqueue_pairs_device = 128; 1216 else 1217 sc->sc_nqueue_pairs_device = 64; 1218 1219 rv = ixl_get_hw_capabilities(sc); 1220 if (rv != 0) { 1221 aprint_error(", GET HW CAPABILITIES %s\n", 1222 rv == ETIMEDOUT ? "timeout" : "error"); 1223 goto free_aqbuf; 1224 } 1225 1226 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1227 if (ixl_param_nqps_limit > 0) { 1228 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_max, 1229 ixl_param_nqps_limit); 1230 } 1231 1232 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1233 sc->sc_tx_ring_ndescs = ixl_param_tx_ndescs; 1234 sc->sc_rx_ring_ndescs = ixl_param_rx_ndescs; 1235 1236 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1237 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1238 1239 if (ixl_get_mac(sc) != 0) { 1240 /* error printed by ixl_get_mac */ 1241 goto free_aqbuf; 1242 } 1243 1244 aprint_normal("\n"); 1245 aprint_naive("\n"); 1246 1247 aprint_normal_dev(self, "Ethernet address %s\n", 1248 ether_sprintf(sc->sc_enaddr)); 1249 1250 rv = ixl_pxe_clear(sc); 1251 if (rv != 0) { 1252 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1253 rv == ETIMEDOUT ? "timeout" : "error"); 1254 } 1255 1256 ixl_set_filter_control(sc); 1257 1258 if (ixl_hmc(sc) != 0) { 1259 /* error printed by ixl_hmc */ 1260 goto free_aqbuf; 1261 } 1262 1263 if (ixl_lldp_shut(sc) != 0) { 1264 /* error printed by ixl_lldp_shut */ 1265 goto free_hmc; 1266 } 1267 1268 if (ixl_phy_mask_ints(sc) != 0) { 1269 /* error printed by ixl_phy_mask_ints */ 1270 goto free_hmc; 1271 } 1272 1273 if (ixl_restart_an(sc) != 0) { 1274 /* error printed by ixl_restart_an */ 1275 goto free_hmc; 1276 } 1277 1278 if (ixl_get_switch_config(sc) != 0) { 1279 /* error printed by ixl_get_switch_config */ 1280 goto free_hmc; 1281 } 1282 1283 if (ixl_get_phy_info(sc) != 0) { 1284 /* error printed by ixl_get_phy_info */ 1285 goto free_hmc; 1286 } 1287 1288 rv = ixl_get_link_status_poll(sc); 1289 if (rv != 0) { 1290 aprint_error_dev(self, "GET LINK STATUS %s\n", 1291 rv == ETIMEDOUT ? "timeout" : "error"); 1292 goto free_hmc; 1293 } 1294 1295 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1296 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1297 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1298 goto free_hmc; 1299 } 1300 1301 rv = ixl_get_vsi(sc); 1302 if (rv != 0) { 1303 aprint_error_dev(self, "GET VSI %s %d\n", 1304 rv == ETIMEDOUT ? "timeout" : "error", rv); 1305 goto free_scratch; 1306 } 1307 1308 rv = ixl_set_vsi(sc); 1309 if (rv != 0) { 1310 aprint_error_dev(self, "UPDATE VSI error %s %d\n", 1311 rv == ETIMEDOUT ? "timeout" : "error", rv); 1312 goto free_scratch; 1313 } 1314 1315 if (ixl_queue_pairs_alloc(sc) != 0) { 1316 /* error printed by ixl_queue_pairs_alloc */ 1317 goto free_scratch; 1318 } 1319 1320 if (ixl_setup_interrupts(sc) != 0) { 1321 /* error printed by ixl_setup_interrupts */ 1322 goto free_queue_pairs; 1323 } 1324 1325 if (ixl_setup_stats(sc) != 0) { 1326 aprint_error_dev(self, "failed to setup event counters\n"); 1327 goto teardown_intrs; 1328 } 1329 1330 if (ixl_setup_sysctls(sc) != 0) { 1331 /* error printed by ixl_setup_sysctls */ 1332 goto teardown_stats; 1333 } 1334 1335 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1336 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1337 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1338 if (sc->sc_workq == NULL) 1339 goto teardown_sysctls; 1340 1341 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1342 sc->sc_workq_txrx = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1343 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1344 if (sc->sc_workq_txrx == NULL) 1345 goto teardown_wqs; 1346 1347 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1348 cv_init(&sc->sc_atq_cv, xnamebuf); 1349 1350 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1351 1352 ifp->if_softc = sc; 1353 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1354 ifp->if_extflags = IFEF_MPSAFE; 1355 ifp->if_ioctl = ixl_ioctl; 1356 ifp->if_start = ixl_start; 1357 ifp->if_transmit = ixl_transmit; 1358 ifp->if_watchdog = ixl_watchdog; 1359 ifp->if_init = ixl_init; 1360 ifp->if_stop = ixl_stop; 1361 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1362 IFQ_SET_READY(&ifp->if_snd); 1363 ifp->if_capabilities |= IXL_IFCAP_RXCSUM; 1364 ifp->if_capabilities |= IXL_IFCAP_TXCSUM; 1365 #if 0 1366 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1367 #endif 1368 ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb); 1369 sc->sc_ec.ec_capabilities |= ETHERCAP_JUMBO_MTU; 1370 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1371 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1372 1373 sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities; 1374 /* Disable VLAN_HWFILTER by default */ 1375 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1376 1377 sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable; 1378 1379 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1380 ifmedia_init(&sc->sc_media, IFM_IMASK, ixl_media_change, 1381 ixl_media_status); 1382 1383 ixl_media_add(sc); 1384 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1385 if (ISSET(sc->sc_phy_abilities, 1386 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1387 ifmedia_add(&sc->sc_media, 1388 IFM_ETHER | IFM_AUTO | IFM_FLOW, 0, NULL); 1389 } 1390 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_NONE, 0, NULL); 1391 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1392 1393 if_attach(ifp); 1394 if_deferred_start_init(ifp, NULL); 1395 ether_ifattach(ifp, sc->sc_enaddr); 1396 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1397 1398 (void)ixl_get_link_status_poll(sc); 1399 ixl_work_set(&sc->sc_link_state_task, ixl_get_link_status, sc); 1400 1401 ixl_config_other_intr(sc); 1402 ixl_enable_other_intr(sc); 1403 1404 ixl_set_phy_autoselect(sc); 1405 1406 /* remove default mac filter and replace it so we can see vlans */ 1407 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 1408 if (rv != ENOENT) { 1409 aprint_debug_dev(self, 1410 "unable to remove macvlan %u\n", rv); 1411 } 1412 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1413 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1414 if (rv != ENOENT) { 1415 aprint_debug_dev(self, 1416 "unable to remove macvlan, ignore vlan %u\n", rv); 1417 } 1418 1419 if (ixl_update_macvlan(sc) != 0) { 1420 aprint_debug_dev(self, 1421 "couldn't enable vlan hardware filter\n"); 1422 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1423 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1424 } 1425 1426 sc->sc_txrx_workqueue = true; 1427 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1428 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1429 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1430 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1431 1432 ixl_stats_update(sc); 1433 sc->sc_stats_counters.isc_has_offset = true; 1434 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 1435 1436 if (pmf_device_register(self, NULL, NULL) != true) 1437 aprint_debug_dev(self, "couldn't establish power handler\n"); 1438 sc->sc_attached = true; 1439 return; 1440 1441 teardown_wqs: 1442 config_finalize_register(self, ixl_workqs_teardown); 1443 teardown_sysctls: 1444 ixl_teardown_sysctls(sc); 1445 teardown_stats: 1446 ixl_teardown_stats(sc); 1447 teardown_intrs: 1448 ixl_teardown_interrupts(sc); 1449 free_queue_pairs: 1450 ixl_queue_pairs_free(sc); 1451 free_scratch: 1452 ixl_dmamem_free(sc, &sc->sc_scratch); 1453 free_hmc: 1454 ixl_hmc_free(sc); 1455 free_aqbuf: 1456 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1457 shutdown: 1458 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1459 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1460 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1461 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1462 1463 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1464 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1465 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1466 1467 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1468 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1469 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1470 1471 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1472 0, IXL_DMA_LEN(&sc->sc_arq), 1473 BUS_DMASYNC_POSTREAD); 1474 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1475 0, IXL_DMA_LEN(&sc->sc_atq), 1476 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1477 1478 ixl_arq_unfill(sc); 1479 free_arq: 1480 ixl_dmamem_free(sc, &sc->sc_arq); 1481 free_atq: 1482 ixl_dmamem_free(sc, &sc->sc_atq); 1483 unmap: 1484 mutex_destroy(&sc->sc_atq_lock); 1485 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1486 mutex_destroy(&sc->sc_cfg_lock); 1487 sc->sc_mems = 0; 1488 1489 sc->sc_attached = false; 1490 } 1491 1492 static int 1493 ixl_detach(device_t self, int flags) 1494 { 1495 struct ixl_softc *sc = device_private(self); 1496 struct ifnet *ifp = &sc->sc_ec.ec_if; 1497 1498 if (!sc->sc_attached) 1499 return 0; 1500 1501 ixl_stop(ifp, 1); 1502 1503 ixl_disable_other_intr(sc); 1504 1505 callout_stop(&sc->sc_stats_callout); 1506 ixl_work_wait(sc->sc_workq, &sc->sc_stats_task); 1507 1508 /* wait for ATQ handler */ 1509 mutex_enter(&sc->sc_atq_lock); 1510 mutex_exit(&sc->sc_atq_lock); 1511 1512 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1513 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1514 1515 if (sc->sc_workq != NULL) { 1516 ixl_workq_destroy(sc->sc_workq); 1517 sc->sc_workq = NULL; 1518 } 1519 1520 if (sc->sc_workq_txrx != NULL) { 1521 ixl_workq_destroy(sc->sc_workq_txrx); 1522 sc->sc_workq_txrx = NULL; 1523 } 1524 1525 ifmedia_delete_instance(&sc->sc_media, IFM_INST_ANY); 1526 ether_ifdetach(ifp); 1527 if_detach(ifp); 1528 1529 ixl_teardown_interrupts(sc); 1530 ixl_teardown_stats(sc); 1531 ixl_teardown_sysctls(sc); 1532 1533 ixl_queue_pairs_free(sc); 1534 1535 ixl_dmamem_free(sc, &sc->sc_scratch); 1536 ixl_hmc_free(sc); 1537 1538 /* shutdown */ 1539 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1540 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1541 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1542 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1543 1544 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1545 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1546 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1547 1548 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1549 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1550 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1551 1552 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1553 0, IXL_DMA_LEN(&sc->sc_arq), 1554 BUS_DMASYNC_POSTREAD); 1555 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1556 0, IXL_DMA_LEN(&sc->sc_atq), 1557 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1558 1559 ixl_arq_unfill(sc); 1560 1561 ixl_dmamem_free(sc, &sc->sc_arq); 1562 ixl_dmamem_free(sc, &sc->sc_atq); 1563 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1564 1565 cv_destroy(&sc->sc_atq_cv); 1566 mutex_destroy(&sc->sc_atq_lock); 1567 1568 if (sc->sc_mems != 0) { 1569 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1570 sc->sc_mems = 0; 1571 } 1572 1573 mutex_destroy(&sc->sc_cfg_lock); 1574 1575 return 0; 1576 } 1577 1578 static int 1579 ixl_workqs_teardown(device_t self) 1580 { 1581 struct ixl_softc *sc = device_private(self); 1582 1583 if (sc->sc_workq != NULL) { 1584 ixl_workq_destroy(sc->sc_workq); 1585 sc->sc_workq = NULL; 1586 } 1587 1588 if (sc->sc_workq_txrx != NULL) { 1589 ixl_workq_destroy(sc->sc_workq_txrx); 1590 sc->sc_workq_txrx = NULL; 1591 } 1592 1593 return 0; 1594 } 1595 1596 static int 1597 ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 1598 { 1599 struct ifnet *ifp = &ec->ec_if; 1600 struct ixl_softc *sc = ifp->if_softc; 1601 int rv; 1602 1603 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1604 return 0; 1605 } 1606 1607 if (set) { 1608 rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid, 1609 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1610 if (rv == 0) { 1611 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 1612 vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1613 } 1614 } else { 1615 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid, 1616 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1617 (void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid, 1618 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1619 } 1620 1621 return rv; 1622 } 1623 1624 static void 1625 ixl_media_add(struct ixl_softc *sc) 1626 { 1627 struct ifmedia *ifm = &sc->sc_media; 1628 const struct ixl_phy_type *itype; 1629 unsigned int i; 1630 bool flow; 1631 1632 if (ISSET(sc->sc_phy_abilities, 1633 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1634 flow = true; 1635 } else { 1636 flow = false; 1637 } 1638 1639 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1640 itype = &ixl_phy_type_map[i]; 1641 1642 if (ISSET(sc->sc_phy_types, itype->phy_type)) { 1643 ifmedia_add(ifm, 1644 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1645 1646 if (flow) { 1647 ifmedia_add(ifm, 1648 IFM_ETHER | IFM_FDX | IFM_FLOW | 1649 itype->ifm_type, 0, NULL); 1650 } 1651 1652 if (itype->ifm_type != IFM_100_TX) 1653 continue; 1654 1655 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1656 0, NULL); 1657 if (flow) { 1658 ifmedia_add(ifm, 1659 IFM_ETHER | IFM_FLOW | itype->ifm_type, 1660 0, NULL); 1661 } 1662 } 1663 } 1664 } 1665 1666 static void 1667 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1668 { 1669 struct ixl_softc *sc = ifp->if_softc; 1670 1671 ifmr->ifm_status = sc->sc_media_status; 1672 ifmr->ifm_active = sc->sc_media_active; 1673 1674 mutex_enter(&sc->sc_cfg_lock); 1675 if (ifp->if_link_state == LINK_STATE_UP) 1676 SET(ifmr->ifm_status, IFM_ACTIVE); 1677 mutex_exit(&sc->sc_cfg_lock); 1678 } 1679 1680 static int 1681 ixl_media_change(struct ifnet *ifp) 1682 { 1683 struct ixl_softc *sc = ifp->if_softc; 1684 struct ifmedia *ifm = &sc->sc_media; 1685 uint64_t ifm_active = sc->sc_media_active; 1686 uint8_t link_speed, abilities; 1687 1688 switch (IFM_SUBTYPE(ifm_active)) { 1689 case IFM_1000_SGMII: 1690 case IFM_1000_KX: 1691 case IFM_10G_KX4: 1692 case IFM_10G_KR: 1693 case IFM_40G_KR4: 1694 case IFM_20G_KR2: 1695 case IFM_25G_KR: 1696 /* backplanes */ 1697 return EINVAL; 1698 } 1699 1700 abilities = IXL_PHY_ABILITY_AUTONEGO | IXL_PHY_ABILITY_LINKUP; 1701 1702 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1703 case IFM_AUTO: 1704 link_speed = sc->sc_phy_linkspeed; 1705 break; 1706 case IFM_NONE: 1707 link_speed = 0; 1708 CLR(abilities, IXL_PHY_ABILITY_LINKUP); 1709 break; 1710 default: 1711 link_speed = ixl_search_baudrate( 1712 ifmedia_baudrate(ifm->ifm_media)); 1713 } 1714 1715 if (ISSET(abilities, IXL_PHY_ABILITY_LINKUP)) { 1716 if (ISSET(link_speed, sc->sc_phy_linkspeed) == 0) 1717 return EINVAL; 1718 } 1719 1720 if (ifm->ifm_media & IFM_FLOW) { 1721 abilities |= sc->sc_phy_abilities & 1722 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX); 1723 } 1724 1725 return ixl_set_phy_config(sc, link_speed, abilities, false); 1726 } 1727 1728 static void 1729 ixl_watchdog(struct ifnet *ifp) 1730 { 1731 1732 } 1733 1734 static void 1735 ixl_del_all_multiaddr(struct ixl_softc *sc) 1736 { 1737 struct ethercom *ec = &sc->sc_ec; 1738 struct ether_multi *enm; 1739 struct ether_multistep step; 1740 1741 ETHER_LOCK(ec); 1742 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1743 ETHER_NEXT_MULTI(step, enm)) { 1744 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1745 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1746 } 1747 ETHER_UNLOCK(ec); 1748 } 1749 1750 static int 1751 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1752 { 1753 struct ifnet *ifp = &sc->sc_ec.ec_if; 1754 int rv; 1755 1756 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1757 return 0; 1758 1759 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1760 ixl_del_all_multiaddr(sc); 1761 SET(ifp->if_flags, IFF_ALLMULTI); 1762 return ENETRESET; 1763 } 1764 1765 /* multicast address can not use VLAN HWFILTER */ 1766 rv = ixl_add_macvlan(sc, addrlo, 0, 1767 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1768 1769 if (rv == ENOSPC) { 1770 ixl_del_all_multiaddr(sc); 1771 SET(ifp->if_flags, IFF_ALLMULTI); 1772 return ENETRESET; 1773 } 1774 1775 return rv; 1776 } 1777 1778 static int 1779 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1780 { 1781 struct ifnet *ifp = &sc->sc_ec.ec_if; 1782 struct ethercom *ec = &sc->sc_ec; 1783 struct ether_multi *enm, *enm_last; 1784 struct ether_multistep step; 1785 int error, rv = 0; 1786 1787 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1788 ixl_remove_macvlan(sc, addrlo, 0, 1789 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1790 return 0; 1791 } 1792 1793 ETHER_LOCK(ec); 1794 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1795 ETHER_NEXT_MULTI(step, enm)) { 1796 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1797 ETHER_ADDR_LEN) != 0) { 1798 goto out; 1799 } 1800 } 1801 1802 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1803 ETHER_NEXT_MULTI(step, enm)) { 1804 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1805 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1806 if (error != 0) 1807 break; 1808 } 1809 1810 if (enm != NULL) { 1811 enm_last = enm; 1812 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1813 ETHER_NEXT_MULTI(step, enm)) { 1814 if (enm == enm_last) 1815 break; 1816 1817 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1818 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1819 } 1820 } else { 1821 CLR(ifp->if_flags, IFF_ALLMULTI); 1822 rv = ENETRESET; 1823 } 1824 1825 out: 1826 ETHER_UNLOCK(ec); 1827 return rv; 1828 } 1829 1830 static int 1831 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1832 { 1833 struct ifreq *ifr = (struct ifreq *)data; 1834 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1835 const struct sockaddr *sa; 1836 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1837 int s, error = 0; 1838 unsigned int nmtu; 1839 1840 switch (cmd) { 1841 case SIOCSIFMTU: 1842 nmtu = ifr->ifr_mtu; 1843 1844 if (nmtu < IXL_MIN_MTU || nmtu > IXL_MAX_MTU) { 1845 error = EINVAL; 1846 break; 1847 } 1848 if (ifp->if_mtu != nmtu) { 1849 s = splnet(); 1850 error = ether_ioctl(ifp, cmd, data); 1851 splx(s); 1852 if (error == ENETRESET) 1853 error = ixl_init(ifp); 1854 } 1855 break; 1856 case SIOCADDMULTI: 1857 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1858 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1859 error = ether_multiaddr(sa, addrlo, addrhi); 1860 if (error != 0) 1861 return error; 1862 1863 error = ixl_add_multi(sc, addrlo, addrhi); 1864 if (error != 0 && error != ENETRESET) { 1865 ether_delmulti(sa, &sc->sc_ec); 1866 error = EIO; 1867 } 1868 } 1869 break; 1870 1871 case SIOCDELMULTI: 1872 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1873 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1874 error = ether_multiaddr(sa, addrlo, addrhi); 1875 if (error != 0) 1876 return error; 1877 1878 error = ixl_del_multi(sc, addrlo, addrhi); 1879 } 1880 break; 1881 1882 default: 1883 s = splnet(); 1884 error = ether_ioctl(ifp, cmd, data); 1885 splx(s); 1886 } 1887 1888 if (error == ENETRESET) 1889 error = ixl_iff(sc); 1890 1891 return error; 1892 } 1893 1894 static enum i40e_mac_type 1895 ixl_mactype(pci_product_id_t id) 1896 { 1897 1898 switch (id) { 1899 case PCI_PRODUCT_INTEL_XL710_SFP: 1900 case PCI_PRODUCT_INTEL_XL710_KX_B: 1901 case PCI_PRODUCT_INTEL_XL710_KX_C: 1902 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1903 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1904 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1905 case PCI_PRODUCT_INTEL_X710_10G_T: 1906 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1907 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1908 case PCI_PRODUCT_INTEL_X710_T4_10G: 1909 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1910 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1911 return I40E_MAC_XL710; 1912 1913 case PCI_PRODUCT_INTEL_X722_KX: 1914 case PCI_PRODUCT_INTEL_X722_QSFP: 1915 case PCI_PRODUCT_INTEL_X722_SFP: 1916 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1917 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1918 case PCI_PRODUCT_INTEL_X722_I_SFP: 1919 return I40E_MAC_X722; 1920 } 1921 1922 return I40E_MAC_GENERIC; 1923 } 1924 1925 static inline void * 1926 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1927 { 1928 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1929 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1930 1931 if (i >= e->hmc_count) 1932 return NULL; 1933 1934 kva += e->hmc_base; 1935 kva += i * e->hmc_size; 1936 1937 return kva; 1938 } 1939 1940 static inline size_t 1941 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1942 { 1943 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1944 1945 return e->hmc_size; 1946 } 1947 1948 static void 1949 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1950 { 1951 struct ixl_rx_ring *rxr = qp->qp_rxr; 1952 1953 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1954 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1955 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1956 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1957 ixl_flush(sc); 1958 } 1959 1960 static void 1961 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1962 { 1963 struct ixl_rx_ring *rxr = qp->qp_rxr; 1964 1965 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1966 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1967 ixl_flush(sc); 1968 } 1969 1970 static void 1971 ixl_enable_other_intr(struct ixl_softc *sc) 1972 { 1973 1974 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1975 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1976 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1977 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1978 ixl_flush(sc); 1979 } 1980 1981 static void 1982 ixl_disable_other_intr(struct ixl_softc *sc) 1983 { 1984 1985 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1986 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1987 ixl_flush(sc); 1988 } 1989 1990 static int 1991 ixl_reinit(struct ixl_softc *sc) 1992 { 1993 struct ixl_rx_ring *rxr; 1994 struct ixl_tx_ring *txr; 1995 unsigned int i; 1996 uint32_t reg; 1997 1998 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1999 2000 if (ixl_get_vsi(sc) != 0) 2001 return EIO; 2002 2003 if (ixl_set_vsi(sc) != 0) 2004 return EIO; 2005 2006 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2007 txr = sc->sc_qps[i].qp_txr; 2008 rxr = sc->sc_qps[i].qp_rxr; 2009 2010 txr->txr_cons = txr->txr_prod = 0; 2011 rxr->rxr_cons = rxr->rxr_prod = 0; 2012 2013 ixl_txr_config(sc, txr); 2014 ixl_rxr_config(sc, rxr); 2015 } 2016 2017 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2018 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 2019 2020 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2021 txr = sc->sc_qps[i].qp_txr; 2022 rxr = sc->sc_qps[i].qp_rxr; 2023 2024 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 2025 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 2026 ixl_flush(sc); 2027 2028 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 2029 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 2030 2031 /* ixl_rxfill() needs lock held */ 2032 mutex_enter(&rxr->rxr_lock); 2033 ixl_rxfill(sc, rxr); 2034 mutex_exit(&rxr->rxr_lock); 2035 2036 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2037 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2038 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2039 if (ixl_rxr_enabled(sc, rxr) != 0) 2040 goto stop; 2041 2042 ixl_txr_qdis(sc, txr, 1); 2043 2044 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2045 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2046 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2047 2048 if (ixl_txr_enabled(sc, txr) != 0) 2049 goto stop; 2050 } 2051 2052 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2053 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2054 2055 return 0; 2056 2057 stop: 2058 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2059 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2060 2061 return ETIMEDOUT; 2062 } 2063 2064 static int 2065 ixl_init_locked(struct ixl_softc *sc) 2066 { 2067 struct ifnet *ifp = &sc->sc_ec.ec_if; 2068 unsigned int i; 2069 int error, eccap_change; 2070 2071 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2072 2073 if (ISSET(ifp->if_flags, IFF_RUNNING)) 2074 ixl_stop_locked(sc); 2075 2076 if (sc->sc_dead) { 2077 return ENXIO; 2078 } 2079 2080 eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable; 2081 if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING)) 2082 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 2083 2084 if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) { 2085 if (ixl_update_macvlan(sc) == 0) { 2086 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 2087 } else { 2088 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 2089 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 2090 } 2091 } 2092 2093 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 2094 sc->sc_nqueue_pairs = 1; 2095 else 2096 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 2097 2098 error = ixl_reinit(sc); 2099 if (error) { 2100 ixl_stop_locked(sc); 2101 return error; 2102 } 2103 2104 SET(ifp->if_flags, IFF_RUNNING); 2105 CLR(ifp->if_flags, IFF_OACTIVE); 2106 2107 (void)ixl_get_link_status(sc); 2108 2109 ixl_config_rss(sc); 2110 ixl_config_queue_intr(sc); 2111 2112 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2113 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 2114 } 2115 2116 error = ixl_iff(sc); 2117 if (error) { 2118 ixl_stop_locked(sc); 2119 return error; 2120 } 2121 2122 return 0; 2123 } 2124 2125 static int 2126 ixl_init(struct ifnet *ifp) 2127 { 2128 struct ixl_softc *sc = ifp->if_softc; 2129 int error; 2130 2131 mutex_enter(&sc->sc_cfg_lock); 2132 error = ixl_init_locked(sc); 2133 mutex_exit(&sc->sc_cfg_lock); 2134 2135 return error; 2136 } 2137 2138 static int 2139 ixl_iff(struct ixl_softc *sc) 2140 { 2141 struct ifnet *ifp = &sc->sc_ec.ec_if; 2142 struct ixl_atq iatq; 2143 struct ixl_aq_desc *iaq; 2144 struct ixl_aq_vsi_promisc_param *param; 2145 uint16_t flag_add, flag_del; 2146 int error; 2147 2148 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2149 return 0; 2150 2151 memset(&iatq, 0, sizeof(iatq)); 2152 2153 iaq = &iatq.iatq_desc; 2154 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 2155 2156 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 2157 param->flags = htole16(0); 2158 2159 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER) 2160 || ISSET(ifp->if_flags, IFF_PROMISC)) { 2161 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2162 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2163 } 2164 2165 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 2166 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2167 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2168 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 2169 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2170 } 2171 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2172 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2173 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2174 param->seid = sc->sc_seid; 2175 2176 error = ixl_atq_exec(sc, &iatq); 2177 if (error) 2178 return error; 2179 2180 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 2181 return EIO; 2182 2183 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 2184 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 2185 flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH; 2186 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH; 2187 } else { 2188 flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN; 2189 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN; 2190 } 2191 2192 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del); 2193 2194 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2195 ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add); 2196 } 2197 return 0; 2198 } 2199 2200 static void 2201 ixl_stop_rendezvous(struct ixl_softc *sc) 2202 { 2203 struct ixl_tx_ring *txr; 2204 struct ixl_rx_ring *rxr; 2205 unsigned int i; 2206 2207 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2208 txr = sc->sc_qps[i].qp_txr; 2209 rxr = sc->sc_qps[i].qp_rxr; 2210 2211 mutex_enter(&txr->txr_lock); 2212 mutex_exit(&txr->txr_lock); 2213 2214 mutex_enter(&rxr->rxr_lock); 2215 mutex_exit(&rxr->rxr_lock); 2216 2217 ixl_work_wait(sc->sc_workq_txrx, 2218 &sc->sc_qps[i].qp_task); 2219 } 2220 } 2221 2222 static void 2223 ixl_stop_locked(struct ixl_softc *sc) 2224 { 2225 struct ifnet *ifp = &sc->sc_ec.ec_if; 2226 struct ixl_rx_ring *rxr; 2227 struct ixl_tx_ring *txr; 2228 unsigned int i; 2229 uint32_t reg; 2230 2231 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2232 2233 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 2234 2235 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2236 txr = sc->sc_qps[i].qp_txr; 2237 rxr = sc->sc_qps[i].qp_rxr; 2238 2239 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 2240 2241 mutex_enter(&txr->txr_lock); 2242 ixl_txr_qdis(sc, txr, 0); 2243 /* XXX wait at least 400 usec for all tx queues in one go */ 2244 ixl_flush(sc); 2245 DELAY(500); 2246 2247 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2248 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2249 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2250 /* XXX wait 50ms from completaion of the TX queue disable*/ 2251 ixl_flush(sc); 2252 DELAY(50); 2253 2254 if (ixl_txr_disabled(sc, txr) != 0) { 2255 mutex_exit(&txr->txr_lock); 2256 goto die; 2257 } 2258 mutex_exit(&txr->txr_lock); 2259 2260 mutex_enter(&rxr->rxr_lock); 2261 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2262 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2263 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2264 /* XXX wait 50ms from completion of the RX queue disable */ 2265 ixl_flush(sc); 2266 DELAY(50); 2267 2268 if (ixl_rxr_disabled(sc, rxr) != 0) { 2269 mutex_exit(&rxr->rxr_lock); 2270 goto die; 2271 } 2272 mutex_exit(&rxr->rxr_lock); 2273 } 2274 2275 ixl_stop_rendezvous(sc); 2276 2277 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2278 txr = sc->sc_qps[i].qp_txr; 2279 rxr = sc->sc_qps[i].qp_rxr; 2280 2281 ixl_txr_unconfig(sc, txr); 2282 ixl_rxr_unconfig(sc, rxr); 2283 2284 ixl_txr_clean(sc, txr); 2285 ixl_rxr_clean(sc, rxr); 2286 } 2287 2288 return; 2289 die: 2290 sc->sc_dead = true; 2291 log(LOG_CRIT, "%s: failed to shut down rings", 2292 device_xname(sc->sc_dev)); 2293 return; 2294 } 2295 2296 static void 2297 ixl_stop(struct ifnet *ifp, int disable) 2298 { 2299 struct ixl_softc *sc = ifp->if_softc; 2300 2301 mutex_enter(&sc->sc_cfg_lock); 2302 ixl_stop_locked(sc); 2303 mutex_exit(&sc->sc_cfg_lock); 2304 } 2305 2306 static int 2307 ixl_queue_pairs_alloc(struct ixl_softc *sc) 2308 { 2309 struct ixl_queue_pair *qp; 2310 unsigned int i; 2311 size_t sz; 2312 2313 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2314 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 2315 2316 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2317 qp = &sc->sc_qps[i]; 2318 2319 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2320 ixl_handle_queue, qp); 2321 if (qp->qp_si == NULL) 2322 goto free; 2323 2324 qp->qp_txr = ixl_txr_alloc(sc, i); 2325 if (qp->qp_txr == NULL) 2326 goto free; 2327 2328 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2329 if (qp->qp_rxr == NULL) 2330 goto free; 2331 2332 qp->qp_sc = sc; 2333 ixl_work_set(&qp->qp_task, ixl_handle_queue, qp); 2334 snprintf(qp->qp_name, sizeof(qp->qp_name), 2335 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2336 } 2337 2338 return 0; 2339 free: 2340 if (sc->sc_qps != NULL) { 2341 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2342 qp = &sc->sc_qps[i]; 2343 2344 if (qp->qp_txr != NULL) 2345 ixl_txr_free(sc, qp->qp_txr); 2346 if (qp->qp_rxr != NULL) 2347 ixl_rxr_free(sc, qp->qp_rxr); 2348 if (qp->qp_si != NULL) 2349 softint_disestablish(qp->qp_si); 2350 } 2351 2352 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2353 kmem_free(sc->sc_qps, sz); 2354 sc->sc_qps = NULL; 2355 } 2356 2357 return -1; 2358 } 2359 2360 static void 2361 ixl_queue_pairs_free(struct ixl_softc *sc) 2362 { 2363 struct ixl_queue_pair *qp; 2364 unsigned int i; 2365 size_t sz; 2366 2367 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2368 qp = &sc->sc_qps[i]; 2369 ixl_txr_free(sc, qp->qp_txr); 2370 ixl_rxr_free(sc, qp->qp_rxr); 2371 softint_disestablish(qp->qp_si); 2372 } 2373 2374 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2375 kmem_free(sc->sc_qps, sz); 2376 sc->sc_qps = NULL; 2377 } 2378 2379 static struct ixl_tx_ring * 2380 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2381 { 2382 struct ixl_tx_ring *txr = NULL; 2383 struct ixl_tx_map *maps = NULL, *txm; 2384 unsigned int i; 2385 2386 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2387 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2388 KM_SLEEP); 2389 2390 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2391 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2392 IXL_TX_QUEUE_ALIGN) != 0) 2393 goto free; 2394 2395 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2396 txm = &maps[i]; 2397 2398 if (bus_dmamap_create(sc->sc_dmat, IXL_TX_PKT_MAXSIZE, 2399 IXL_TX_PKT_DESCS, IXL_TX_PKT_MAXSIZE, 0, 2400 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2401 goto uncreate; 2402 2403 txm->txm_eop = -1; 2404 txm->txm_m = NULL; 2405 } 2406 2407 txr->txr_cons = txr->txr_prod = 0; 2408 txr->txr_maps = maps; 2409 2410 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2411 if (txr->txr_intrq == NULL) 2412 goto uncreate; 2413 2414 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2415 ixl_deferred_transmit, txr); 2416 if (txr->txr_si == NULL) 2417 goto destroy_pcq; 2418 2419 txr->txr_tail = I40E_QTX_TAIL(qid); 2420 txr->txr_qid = qid; 2421 txr->txr_sc = sc; 2422 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2423 2424 return txr; 2425 2426 destroy_pcq: 2427 pcq_destroy(txr->txr_intrq); 2428 uncreate: 2429 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2430 txm = &maps[i]; 2431 2432 if (txm->txm_map == NULL) 2433 continue; 2434 2435 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2436 } 2437 2438 ixl_dmamem_free(sc, &txr->txr_mem); 2439 free: 2440 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2441 kmem_free(txr, sizeof(*txr)); 2442 2443 return NULL; 2444 } 2445 2446 static void 2447 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2448 { 2449 unsigned int qid; 2450 bus_size_t reg; 2451 uint32_t r; 2452 2453 qid = txr->txr_qid + sc->sc_base_queue; 2454 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2455 qid %= 128; 2456 2457 r = ixl_rd(sc, reg); 2458 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2459 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2460 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2461 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2462 ixl_wr(sc, reg, r); 2463 } 2464 2465 static void 2466 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2467 { 2468 struct ixl_hmc_txq txq; 2469 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2470 void *hmc; 2471 2472 memset(&txq, 0, sizeof(txq)); 2473 txq.head = htole16(txr->txr_cons); 2474 txq.new_context = 1; 2475 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2476 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2477 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2478 txq.tphrdesc_ena = 0; 2479 txq.tphrpacket_ena = 0; 2480 txq.tphwdesc_ena = 0; 2481 txq.rdylist = data->qs_handle[0]; 2482 2483 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2484 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2485 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2486 __arraycount(ixl_hmc_pack_txq)); 2487 } 2488 2489 static void 2490 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2491 { 2492 void *hmc; 2493 2494 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2495 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2496 } 2497 2498 static void 2499 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2500 { 2501 struct ixl_tx_map *maps, *txm; 2502 bus_dmamap_t map; 2503 unsigned int i; 2504 2505 maps = txr->txr_maps; 2506 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2507 txm = &maps[i]; 2508 2509 if (txm->txm_m == NULL) 2510 continue; 2511 2512 map = txm->txm_map; 2513 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2514 BUS_DMASYNC_POSTWRITE); 2515 bus_dmamap_unload(sc->sc_dmat, map); 2516 2517 m_freem(txm->txm_m); 2518 txm->txm_m = NULL; 2519 } 2520 } 2521 2522 static int 2523 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2524 { 2525 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2526 uint32_t reg; 2527 int i; 2528 2529 for (i = 0; i < 10; i++) { 2530 reg = ixl_rd(sc, ena); 2531 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2532 return 0; 2533 2534 delaymsec(10); 2535 } 2536 2537 return ETIMEDOUT; 2538 } 2539 2540 static int 2541 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2542 { 2543 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2544 uint32_t reg; 2545 int i; 2546 2547 KASSERT(mutex_owned(&txr->txr_lock)); 2548 2549 for (i = 0; i < 20; i++) { 2550 reg = ixl_rd(sc, ena); 2551 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2552 return 0; 2553 2554 delaymsec(10); 2555 } 2556 2557 return ETIMEDOUT; 2558 } 2559 2560 static void 2561 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2562 { 2563 struct ixl_tx_map *maps, *txm; 2564 struct mbuf *m; 2565 unsigned int i; 2566 2567 softint_disestablish(txr->txr_si); 2568 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2569 m_freem(m); 2570 pcq_destroy(txr->txr_intrq); 2571 2572 maps = txr->txr_maps; 2573 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2574 txm = &maps[i]; 2575 2576 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2577 } 2578 2579 ixl_dmamem_free(sc, &txr->txr_mem); 2580 mutex_destroy(&txr->txr_lock); 2581 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2582 kmem_free(txr, sizeof(*txr)); 2583 } 2584 2585 static inline int 2586 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2587 struct ixl_tx_ring *txr) 2588 { 2589 struct mbuf *m; 2590 int error; 2591 2592 KASSERT(mutex_owned(&txr->txr_lock)); 2593 2594 m = *m0; 2595 2596 error = bus_dmamap_load_mbuf(dmat, map, m, 2597 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2598 if (error != EFBIG) 2599 return error; 2600 2601 m = m_defrag(m, M_DONTWAIT); 2602 if (m != NULL) { 2603 *m0 = m; 2604 txr->txr_defragged.ev_count++; 2605 2606 error = bus_dmamap_load_mbuf(dmat, map, m, 2607 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2608 } else { 2609 txr->txr_defrag_failed.ev_count++; 2610 error = ENOBUFS; 2611 } 2612 2613 return error; 2614 } 2615 2616 static inline int 2617 ixl_tx_setup_offloads(struct mbuf *m, uint64_t *cmd_txd) 2618 { 2619 struct ether_header *eh; 2620 size_t len; 2621 uint64_t cmd; 2622 2623 cmd = 0; 2624 2625 eh = mtod(m, struct ether_header *); 2626 switch (htons(eh->ether_type)) { 2627 case ETHERTYPE_IP: 2628 case ETHERTYPE_IPV6: 2629 len = ETHER_HDR_LEN; 2630 break; 2631 case ETHERTYPE_VLAN: 2632 len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2633 break; 2634 default: 2635 len = 0; 2636 } 2637 cmd |= ((len >> 1) << IXL_TX_DESC_MACLEN_SHIFT); 2638 2639 if (m->m_pkthdr.csum_flags & 2640 (M_CSUM_TSOv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) { 2641 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4; 2642 } 2643 if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) { 2644 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4_CSUM; 2645 } 2646 2647 if (m->m_pkthdr.csum_flags & 2648 (M_CSUM_TSOv6 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) { 2649 cmd |= IXL_TX_DESC_CMD_IIPT_IPV6; 2650 } 2651 2652 switch (cmd & IXL_TX_DESC_CMD_IIPT_MASK) { 2653 case IXL_TX_DESC_CMD_IIPT_IPV4: 2654 case IXL_TX_DESC_CMD_IIPT_IPV4_CSUM: 2655 len = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); 2656 break; 2657 case IXL_TX_DESC_CMD_IIPT_IPV6: 2658 len = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); 2659 break; 2660 default: 2661 len = 0; 2662 } 2663 cmd |= ((len >> 2) << IXL_TX_DESC_IPLEN_SHIFT); 2664 2665 if (m->m_pkthdr.csum_flags & 2666 (M_CSUM_TSOv4 | M_CSUM_TSOv6 | M_CSUM_TCPv4 | M_CSUM_TCPv6)) { 2667 len = sizeof(struct tcphdr); 2668 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_TCP; 2669 } else if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) { 2670 len = sizeof(struct udphdr); 2671 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_UDP; 2672 } else { 2673 len = 0; 2674 } 2675 cmd |= ((len >> 2) << IXL_TX_DESC_L4LEN_SHIFT); 2676 2677 *cmd_txd |= cmd; 2678 return 0; 2679 } 2680 2681 static void 2682 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2683 bool is_transmit) 2684 { 2685 struct ixl_softc *sc = ifp->if_softc; 2686 struct ixl_tx_desc *ring, *txd; 2687 struct ixl_tx_map *txm; 2688 bus_dmamap_t map; 2689 struct mbuf *m; 2690 uint64_t cmd, cmd_txd; 2691 unsigned int prod, free, last, i; 2692 unsigned int mask; 2693 int post = 0; 2694 2695 KASSERT(mutex_owned(&txr->txr_lock)); 2696 2697 if (ifp->if_link_state != LINK_STATE_UP 2698 || !ISSET(ifp->if_flags, IFF_RUNNING) 2699 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2700 if (!is_transmit) 2701 IFQ_PURGE(&ifp->if_snd); 2702 return; 2703 } 2704 2705 prod = txr->txr_prod; 2706 free = txr->txr_cons; 2707 if (free <= prod) 2708 free += sc->sc_tx_ring_ndescs; 2709 free -= prod; 2710 2711 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2712 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2713 2714 ring = IXL_DMA_KVA(&txr->txr_mem); 2715 mask = sc->sc_tx_ring_ndescs - 1; 2716 last = prod; 2717 cmd = 0; 2718 txd = NULL; 2719 2720 for (;;) { 2721 if (free <= IXL_TX_PKT_DESCS) { 2722 if (!is_transmit) 2723 SET(ifp->if_flags, IFF_OACTIVE); 2724 break; 2725 } 2726 2727 if (is_transmit) 2728 m = pcq_get(txr->txr_intrq); 2729 else 2730 IFQ_DEQUEUE(&ifp->if_snd, m); 2731 2732 if (m == NULL) 2733 break; 2734 2735 txm = &txr->txr_maps[prod]; 2736 map = txm->txm_map; 2737 2738 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2739 if_statinc(ifp, if_oerrors); 2740 m_freem(m); 2741 continue; 2742 } 2743 2744 cmd_txd = 0; 2745 if (m->m_pkthdr.csum_flags & IXL_CSUM_ALL_OFFLOAD) { 2746 ixl_tx_setup_offloads(m, &cmd_txd); 2747 } 2748 2749 if (vlan_has_tag(m)) { 2750 cmd_txd |= (uint64_t)vlan_get_tag(m) << 2751 IXL_TX_DESC_L2TAG1_SHIFT; 2752 cmd_txd |= IXL_TX_DESC_CMD_IL2TAG1; 2753 } 2754 2755 bus_dmamap_sync(sc->sc_dmat, map, 0, 2756 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2757 2758 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2759 txd = &ring[prod]; 2760 2761 cmd = (uint64_t)map->dm_segs[i].ds_len << 2762 IXL_TX_DESC_BSIZE_SHIFT; 2763 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2764 cmd |= cmd_txd; 2765 2766 txd->addr = htole64(map->dm_segs[i].ds_addr); 2767 txd->cmd = htole64(cmd); 2768 2769 last = prod; 2770 2771 prod++; 2772 prod &= mask; 2773 } 2774 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2775 txd->cmd = htole64(cmd); 2776 2777 txm->txm_m = m; 2778 txm->txm_eop = last; 2779 2780 bpf_mtap(ifp, m, BPF_D_OUT); 2781 2782 free -= i; 2783 post = 1; 2784 } 2785 2786 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2787 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2788 2789 if (post) { 2790 txr->txr_prod = prod; 2791 ixl_wr(sc, txr->txr_tail, prod); 2792 } 2793 } 2794 2795 static int 2796 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2797 { 2798 struct ifnet *ifp = &sc->sc_ec.ec_if; 2799 struct ixl_tx_desc *ring, *txd; 2800 struct ixl_tx_map *txm; 2801 struct mbuf *m; 2802 bus_dmamap_t map; 2803 unsigned int cons, prod, last; 2804 unsigned int mask; 2805 uint64_t dtype; 2806 int done = 0, more = 0; 2807 2808 KASSERT(mutex_owned(&txr->txr_lock)); 2809 2810 prod = txr->txr_prod; 2811 cons = txr->txr_cons; 2812 2813 if (cons == prod) 2814 return 0; 2815 2816 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2817 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2818 2819 ring = IXL_DMA_KVA(&txr->txr_mem); 2820 mask = sc->sc_tx_ring_ndescs - 1; 2821 2822 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2823 2824 do { 2825 if (txlimit-- <= 0) { 2826 more = 1; 2827 break; 2828 } 2829 2830 txm = &txr->txr_maps[cons]; 2831 last = txm->txm_eop; 2832 txd = &ring[last]; 2833 2834 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2835 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2836 break; 2837 2838 map = txm->txm_map; 2839 2840 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2841 BUS_DMASYNC_POSTWRITE); 2842 bus_dmamap_unload(sc->sc_dmat, map); 2843 2844 m = txm->txm_m; 2845 if (m != NULL) { 2846 if_statinc_ref(nsr, if_opackets); 2847 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 2848 if (ISSET(m->m_flags, M_MCAST)) 2849 if_statinc_ref(nsr, if_omcasts); 2850 m_freem(m); 2851 } 2852 2853 txm->txm_m = NULL; 2854 txm->txm_eop = -1; 2855 2856 cons = last + 1; 2857 cons &= mask; 2858 done = 1; 2859 } while (cons != prod); 2860 2861 IF_STAT_PUTREF(ifp); 2862 2863 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2864 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2865 2866 txr->txr_cons = cons; 2867 2868 if (done) { 2869 softint_schedule(txr->txr_si); 2870 if (txr->txr_qid == 0) { 2871 CLR(ifp->if_flags, IFF_OACTIVE); 2872 if_schedule_deferred_start(ifp); 2873 } 2874 } 2875 2876 return more; 2877 } 2878 2879 static void 2880 ixl_start(struct ifnet *ifp) 2881 { 2882 struct ixl_softc *sc; 2883 struct ixl_tx_ring *txr; 2884 2885 sc = ifp->if_softc; 2886 txr = sc->sc_qps[0].qp_txr; 2887 2888 mutex_enter(&txr->txr_lock); 2889 ixl_tx_common_locked(ifp, txr, false); 2890 mutex_exit(&txr->txr_lock); 2891 } 2892 2893 static inline unsigned int 2894 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2895 { 2896 u_int cpuid; 2897 2898 cpuid = cpu_index(curcpu()); 2899 2900 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2901 } 2902 2903 static int 2904 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2905 { 2906 struct ixl_softc *sc; 2907 struct ixl_tx_ring *txr; 2908 unsigned int qid; 2909 2910 sc = ifp->if_softc; 2911 qid = ixl_select_txqueue(sc, m); 2912 2913 txr = sc->sc_qps[qid].qp_txr; 2914 2915 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2916 mutex_enter(&txr->txr_lock); 2917 txr->txr_pcqdrop.ev_count++; 2918 mutex_exit(&txr->txr_lock); 2919 2920 m_freem(m); 2921 return ENOBUFS; 2922 } 2923 2924 if (mutex_tryenter(&txr->txr_lock)) { 2925 ixl_tx_common_locked(ifp, txr, true); 2926 mutex_exit(&txr->txr_lock); 2927 } else { 2928 kpreempt_disable(); 2929 softint_schedule(txr->txr_si); 2930 kpreempt_enable(); 2931 } 2932 2933 return 0; 2934 } 2935 2936 static void 2937 ixl_deferred_transmit(void *xtxr) 2938 { 2939 struct ixl_tx_ring *txr = xtxr; 2940 struct ixl_softc *sc = txr->txr_sc; 2941 struct ifnet *ifp = &sc->sc_ec.ec_if; 2942 2943 mutex_enter(&txr->txr_lock); 2944 txr->txr_transmitdef.ev_count++; 2945 if (pcq_peek(txr->txr_intrq) != NULL) 2946 ixl_tx_common_locked(ifp, txr, true); 2947 mutex_exit(&txr->txr_lock); 2948 } 2949 2950 static struct ixl_rx_ring * 2951 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2952 { 2953 struct ixl_rx_ring *rxr = NULL; 2954 struct ixl_rx_map *maps = NULL, *rxm; 2955 unsigned int i; 2956 2957 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2958 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2959 KM_SLEEP); 2960 2961 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2962 sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs, 2963 IXL_RX_QUEUE_ALIGN) != 0) 2964 goto free; 2965 2966 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2967 rxm = &maps[i]; 2968 2969 if (bus_dmamap_create(sc->sc_dmat, 2970 IXL_MCLBYTES, 1, IXL_MCLBYTES, 0, 2971 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2972 goto uncreate; 2973 2974 rxm->rxm_m = NULL; 2975 } 2976 2977 rxr->rxr_cons = rxr->rxr_prod = 0; 2978 rxr->rxr_m_head = NULL; 2979 rxr->rxr_m_tail = &rxr->rxr_m_head; 2980 rxr->rxr_maps = maps; 2981 2982 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2983 rxr->rxr_qid = qid; 2984 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2985 2986 return rxr; 2987 2988 uncreate: 2989 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2990 rxm = &maps[i]; 2991 2992 if (rxm->rxm_map == NULL) 2993 continue; 2994 2995 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2996 } 2997 2998 ixl_dmamem_free(sc, &rxr->rxr_mem); 2999 free: 3000 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3001 kmem_free(rxr, sizeof(*rxr)); 3002 3003 return NULL; 3004 } 3005 3006 static void 3007 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3008 { 3009 struct ixl_rx_map *maps, *rxm; 3010 bus_dmamap_t map; 3011 unsigned int i; 3012 3013 maps = rxr->rxr_maps; 3014 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3015 rxm = &maps[i]; 3016 3017 if (rxm->rxm_m == NULL) 3018 continue; 3019 3020 map = rxm->rxm_map; 3021 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3022 BUS_DMASYNC_POSTWRITE); 3023 bus_dmamap_unload(sc->sc_dmat, map); 3024 3025 m_freem(rxm->rxm_m); 3026 rxm->rxm_m = NULL; 3027 } 3028 3029 m_freem(rxr->rxr_m_head); 3030 rxr->rxr_m_head = NULL; 3031 rxr->rxr_m_tail = &rxr->rxr_m_head; 3032 3033 rxr->rxr_prod = rxr->rxr_cons = 0; 3034 } 3035 3036 static int 3037 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3038 { 3039 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3040 uint32_t reg; 3041 int i; 3042 3043 for (i = 0; i < 10; i++) { 3044 reg = ixl_rd(sc, ena); 3045 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 3046 return 0; 3047 3048 delaymsec(10); 3049 } 3050 3051 return ETIMEDOUT; 3052 } 3053 3054 static int 3055 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3056 { 3057 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3058 uint32_t reg; 3059 int i; 3060 3061 KASSERT(mutex_owned(&rxr->rxr_lock)); 3062 3063 for (i = 0; i < 20; i++) { 3064 reg = ixl_rd(sc, ena); 3065 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 3066 return 0; 3067 3068 delaymsec(10); 3069 } 3070 3071 return ETIMEDOUT; 3072 } 3073 3074 static void 3075 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3076 { 3077 struct ixl_hmc_rxq rxq; 3078 struct ifnet *ifp = &sc->sc_ec.ec_if; 3079 uint16_t rxmax; 3080 void *hmc; 3081 3082 memset(&rxq, 0, sizeof(rxq)); 3083 rxmax = ifp->if_mtu + IXL_MTU_ETHERLEN; 3084 3085 rxq.head = htole16(rxr->rxr_cons); 3086 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 3087 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 3088 rxq.dbuff = htole16(IXL_MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 3089 rxq.hbuff = 0; 3090 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 3091 rxq.dsize = IXL_HMC_RXQ_DSIZE_32; 3092 rxq.crcstrip = 1; 3093 rxq.l2sel = 1; 3094 rxq.showiv = 1; 3095 rxq.rxmax = htole16(rxmax); 3096 rxq.tphrdesc_ena = 0; 3097 rxq.tphwdesc_ena = 0; 3098 rxq.tphdata_ena = 0; 3099 rxq.tphhead_ena = 0; 3100 rxq.lrxqthresh = 0; 3101 rxq.prefena = 1; 3102 3103 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3104 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3105 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 3106 __arraycount(ixl_hmc_pack_rxq)); 3107 } 3108 3109 static void 3110 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3111 { 3112 void *hmc; 3113 3114 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3115 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3116 } 3117 3118 static void 3119 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3120 { 3121 struct ixl_rx_map *maps, *rxm; 3122 unsigned int i; 3123 3124 maps = rxr->rxr_maps; 3125 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3126 rxm = &maps[i]; 3127 3128 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 3129 } 3130 3131 ixl_dmamem_free(sc, &rxr->rxr_mem); 3132 mutex_destroy(&rxr->rxr_lock); 3133 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3134 kmem_free(rxr, sizeof(*rxr)); 3135 } 3136 3137 static inline void 3138 ixl_rx_csum(struct mbuf *m, uint64_t qword) 3139 { 3140 int flags_mask; 3141 3142 if (!ISSET(qword, IXL_RX_DESC_L3L4P)) { 3143 /* No L3 or L4 checksum was calculated */ 3144 return; 3145 } 3146 3147 switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) { 3148 case IXL_RX_DESC_PTYPE_IPV4FRAG: 3149 case IXL_RX_DESC_PTYPE_IPV4: 3150 case IXL_RX_DESC_PTYPE_SCTPV4: 3151 case IXL_RX_DESC_PTYPE_ICMPV4: 3152 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3153 break; 3154 case IXL_RX_DESC_PTYPE_TCPV4: 3155 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3156 flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD; 3157 break; 3158 case IXL_RX_DESC_PTYPE_UDPV4: 3159 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3160 flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD; 3161 break; 3162 case IXL_RX_DESC_PTYPE_TCPV6: 3163 flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD; 3164 break; 3165 case IXL_RX_DESC_PTYPE_UDPV6: 3166 flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD; 3167 break; 3168 default: 3169 flags_mask = 0; 3170 } 3171 3172 m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 | 3173 M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6)); 3174 3175 if (ISSET(qword, IXL_RX_DESC_IPE)) { 3176 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD); 3177 } 3178 3179 if (ISSET(qword, IXL_RX_DESC_L4E)) { 3180 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD); 3181 } 3182 } 3183 3184 static int 3185 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 3186 { 3187 struct ifnet *ifp = &sc->sc_ec.ec_if; 3188 struct ixl_rx_wb_desc_32 *ring, *rxd; 3189 struct ixl_rx_map *rxm; 3190 bus_dmamap_t map; 3191 unsigned int cons, prod; 3192 struct mbuf *m; 3193 uint64_t word, word0; 3194 unsigned int len; 3195 unsigned int mask; 3196 int done = 0, more = 0; 3197 3198 KASSERT(mutex_owned(&rxr->rxr_lock)); 3199 3200 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3201 return 0; 3202 3203 prod = rxr->rxr_prod; 3204 cons = rxr->rxr_cons; 3205 3206 if (cons == prod) 3207 return 0; 3208 3209 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3210 0, IXL_DMA_LEN(&rxr->rxr_mem), 3211 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3212 3213 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3214 mask = sc->sc_rx_ring_ndescs - 1; 3215 3216 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 3217 3218 do { 3219 if (rxlimit-- <= 0) { 3220 more = 1; 3221 break; 3222 } 3223 3224 rxd = &ring[cons]; 3225 3226 word = le64toh(rxd->qword1); 3227 3228 if (!ISSET(word, IXL_RX_DESC_DD)) 3229 break; 3230 3231 rxm = &rxr->rxr_maps[cons]; 3232 3233 map = rxm->rxm_map; 3234 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3235 BUS_DMASYNC_POSTREAD); 3236 bus_dmamap_unload(sc->sc_dmat, map); 3237 3238 m = rxm->rxm_m; 3239 rxm->rxm_m = NULL; 3240 3241 KASSERT(m != NULL); 3242 3243 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 3244 m->m_len = len; 3245 m->m_pkthdr.len = 0; 3246 3247 m->m_next = NULL; 3248 *rxr->rxr_m_tail = m; 3249 rxr->rxr_m_tail = &m->m_next; 3250 3251 m = rxr->rxr_m_head; 3252 m->m_pkthdr.len += len; 3253 3254 if (ISSET(word, IXL_RX_DESC_EOP)) { 3255 word0 = le64toh(rxd->qword0); 3256 3257 if (ISSET(word, IXL_RX_DESC_L2TAG1P)) { 3258 vlan_set_tag(m, 3259 __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK)); 3260 } 3261 3262 if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0) 3263 ixl_rx_csum(m, word); 3264 3265 if (!ISSET(word, 3266 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 3267 m_set_rcvif(m, ifp); 3268 if_statinc_ref(nsr, if_ipackets); 3269 if_statadd_ref(nsr, if_ibytes, 3270 m->m_pkthdr.len); 3271 if_percpuq_enqueue(ifp->if_percpuq, m); 3272 } else { 3273 if_statinc_ref(nsr, if_ierrors); 3274 m_freem(m); 3275 } 3276 3277 rxr->rxr_m_head = NULL; 3278 rxr->rxr_m_tail = &rxr->rxr_m_head; 3279 } 3280 3281 cons++; 3282 cons &= mask; 3283 3284 done = 1; 3285 } while (cons != prod); 3286 3287 if (done) { 3288 rxr->rxr_cons = cons; 3289 if (ixl_rxfill(sc, rxr) == -1) 3290 if_statinc_ref(nsr, if_iqdrops); 3291 } 3292 3293 IF_STAT_PUTREF(ifp); 3294 3295 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3296 0, IXL_DMA_LEN(&rxr->rxr_mem), 3297 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3298 3299 return more; 3300 } 3301 3302 static int 3303 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3304 { 3305 struct ixl_rx_rd_desc_32 *ring, *rxd; 3306 struct ixl_rx_map *rxm; 3307 bus_dmamap_t map; 3308 struct mbuf *m; 3309 unsigned int prod; 3310 unsigned int slots; 3311 unsigned int mask; 3312 int post = 0, error = 0; 3313 3314 KASSERT(mutex_owned(&rxr->rxr_lock)); 3315 3316 prod = rxr->rxr_prod; 3317 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 3318 sc->sc_rx_ring_ndescs); 3319 3320 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3321 mask = sc->sc_rx_ring_ndescs - 1; 3322 3323 if (__predict_false(slots <= 0)) 3324 return -1; 3325 3326 do { 3327 rxm = &rxr->rxr_maps[prod]; 3328 3329 MGETHDR(m, M_DONTWAIT, MT_DATA); 3330 if (m == NULL) { 3331 rxr->rxr_mgethdr_failed.ev_count++; 3332 error = -1; 3333 break; 3334 } 3335 3336 MCLGET(m, M_DONTWAIT); 3337 if (!ISSET(m->m_flags, M_EXT)) { 3338 rxr->rxr_mgetcl_failed.ev_count++; 3339 error = -1; 3340 m_freem(m); 3341 break; 3342 } 3343 3344 m->m_len = m->m_pkthdr.len = MCLBYTES; 3345 m_adj(m, ETHER_ALIGN); 3346 3347 map = rxm->rxm_map; 3348 3349 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 3350 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 3351 rxr->rxr_mbuf_load_failed.ev_count++; 3352 error = -1; 3353 m_freem(m); 3354 break; 3355 } 3356 3357 rxm->rxm_m = m; 3358 3359 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3360 BUS_DMASYNC_PREREAD); 3361 3362 rxd = &ring[prod]; 3363 3364 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 3365 rxd->haddr = htole64(0); 3366 3367 prod++; 3368 prod &= mask; 3369 3370 post = 1; 3371 3372 } while (--slots); 3373 3374 if (post) { 3375 rxr->rxr_prod = prod; 3376 ixl_wr(sc, rxr->rxr_tail, prod); 3377 } 3378 3379 return error; 3380 } 3381 3382 static inline int 3383 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 3384 u_int txlimit, struct evcnt *txevcnt, 3385 u_int rxlimit, struct evcnt *rxevcnt) 3386 { 3387 struct ixl_tx_ring *txr = qp->qp_txr; 3388 struct ixl_rx_ring *rxr = qp->qp_rxr; 3389 int txmore, rxmore; 3390 int rv; 3391 3392 KASSERT(!mutex_owned(&txr->txr_lock)); 3393 KASSERT(!mutex_owned(&rxr->rxr_lock)); 3394 3395 mutex_enter(&txr->txr_lock); 3396 txevcnt->ev_count++; 3397 txmore = ixl_txeof(sc, txr, txlimit); 3398 mutex_exit(&txr->txr_lock); 3399 3400 mutex_enter(&rxr->rxr_lock); 3401 rxevcnt->ev_count++; 3402 rxmore = ixl_rxeof(sc, rxr, rxlimit); 3403 mutex_exit(&rxr->rxr_lock); 3404 3405 rv = txmore | (rxmore << 1); 3406 3407 return rv; 3408 } 3409 3410 static void 3411 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 3412 { 3413 3414 if (qp->qp_workqueue) 3415 ixl_work_add(sc->sc_workq_txrx, &qp->qp_task); 3416 else 3417 softint_schedule(qp->qp_si); 3418 } 3419 3420 static int 3421 ixl_intr(void *xsc) 3422 { 3423 struct ixl_softc *sc = xsc; 3424 struct ixl_tx_ring *txr; 3425 struct ixl_rx_ring *rxr; 3426 uint32_t icr, rxintr, txintr; 3427 int rv = 0; 3428 unsigned int i; 3429 3430 KASSERT(sc != NULL); 3431 3432 ixl_enable_other_intr(sc); 3433 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3434 3435 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3436 atomic_inc_64(&sc->sc_event_atq.ev_count); 3437 ixl_atq_done(sc); 3438 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3439 rv = 1; 3440 } 3441 3442 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3443 atomic_inc_64(&sc->sc_event_link.ev_count); 3444 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3445 rv = 1; 3446 } 3447 3448 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 3449 txintr = icr & I40E_INTR_NOTX_TX_MASK; 3450 3451 if (txintr || rxintr) { 3452 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 3453 txr = sc->sc_qps[i].qp_txr; 3454 rxr = sc->sc_qps[i].qp_rxr; 3455 3456 ixl_handle_queue_common(sc, &sc->sc_qps[i], 3457 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 3458 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 3459 } 3460 rv = 1; 3461 } 3462 3463 return rv; 3464 } 3465 3466 static int 3467 ixl_queue_intr(void *xqp) 3468 { 3469 struct ixl_queue_pair *qp = xqp; 3470 struct ixl_tx_ring *txr = qp->qp_txr; 3471 struct ixl_rx_ring *rxr = qp->qp_rxr; 3472 struct ixl_softc *sc = qp->qp_sc; 3473 u_int txlimit, rxlimit; 3474 int more; 3475 3476 txlimit = sc->sc_tx_intr_process_limit; 3477 rxlimit = sc->sc_rx_intr_process_limit; 3478 qp->qp_workqueue = sc->sc_txrx_workqueue; 3479 3480 more = ixl_handle_queue_common(sc, qp, 3481 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3482 3483 if (more != 0) { 3484 ixl_sched_handle_queue(sc, qp); 3485 } else { 3486 /* for ALTQ */ 3487 if (txr->txr_qid == 0) 3488 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3489 softint_schedule(txr->txr_si); 3490 3491 ixl_enable_queue_intr(sc, qp); 3492 } 3493 3494 return 1; 3495 } 3496 3497 static void 3498 ixl_handle_queue(void *xqp) 3499 { 3500 struct ixl_queue_pair *qp = xqp; 3501 struct ixl_softc *sc = qp->qp_sc; 3502 struct ixl_tx_ring *txr = qp->qp_txr; 3503 struct ixl_rx_ring *rxr = qp->qp_rxr; 3504 u_int txlimit, rxlimit; 3505 int more; 3506 3507 txlimit = sc->sc_tx_process_limit; 3508 rxlimit = sc->sc_rx_process_limit; 3509 3510 more = ixl_handle_queue_common(sc, qp, 3511 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3512 3513 if (more != 0) 3514 ixl_sched_handle_queue(sc, qp); 3515 else 3516 ixl_enable_queue_intr(sc, qp); 3517 } 3518 3519 static inline void 3520 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3521 { 3522 uint32_t hmc_idx, hmc_isvf; 3523 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3524 3525 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3526 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3527 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3528 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3529 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3530 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3531 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3532 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3533 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3534 3535 device_printf(sc->sc_dev, 3536 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3537 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3538 } 3539 3540 static int 3541 ixl_other_intr(void *xsc) 3542 { 3543 struct ixl_softc *sc = xsc; 3544 uint32_t icr, mask, reg; 3545 int rv; 3546 3547 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3548 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3549 3550 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3551 atomic_inc_64(&sc->sc_event_atq.ev_count); 3552 ixl_atq_done(sc); 3553 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3554 rv = 1; 3555 } 3556 3557 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3558 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3559 device_printf(sc->sc_dev, "link stat changed\n"); 3560 3561 atomic_inc_64(&sc->sc_event_link.ev_count); 3562 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3563 rv = 1; 3564 } 3565 3566 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3567 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3568 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3569 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3570 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3571 3572 device_printf(sc->sc_dev, "GRST: %s\n", 3573 reg == I40E_RESET_CORER ? "CORER" : 3574 reg == I40E_RESET_GLOBR ? "GLOBR" : 3575 reg == I40E_RESET_EMPR ? "EMPR" : 3576 "POR"); 3577 } 3578 3579 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3580 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3581 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3582 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3583 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3584 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3585 3586 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3587 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3588 device_printf(sc->sc_dev, "critical error\n"); 3589 } 3590 3591 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3592 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3593 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3594 ixl_print_hmc_error(sc, reg); 3595 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3596 } 3597 3598 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3599 ixl_flush(sc); 3600 ixl_enable_other_intr(sc); 3601 return rv; 3602 } 3603 3604 static void 3605 ixl_get_link_status_done(struct ixl_softc *sc, 3606 const struct ixl_aq_desc *iaq) 3607 { 3608 3609 ixl_link_state_update(sc, iaq); 3610 } 3611 3612 static void 3613 ixl_get_link_status(void *xsc) 3614 { 3615 struct ixl_softc *sc = xsc; 3616 struct ixl_aq_desc *iaq; 3617 struct ixl_aq_link_param *param; 3618 3619 memset(&sc->sc_link_state_atq, 0, sizeof(sc->sc_link_state_atq)); 3620 iaq = &sc->sc_link_state_atq.iatq_desc; 3621 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3622 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3623 param->notify = IXL_AQ_LINK_NOTIFY; 3624 3625 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 3626 (void)ixl_atq_post(sc, &sc->sc_link_state_atq); 3627 } 3628 3629 static void 3630 ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3631 { 3632 struct ifnet *ifp = &sc->sc_ec.ec_if; 3633 int link_state; 3634 3635 KASSERT(kpreempt_disabled()); 3636 3637 link_state = ixl_set_link_status(sc, iaq); 3638 3639 if (ifp->if_link_state != link_state) 3640 if_link_state_change(ifp, link_state); 3641 3642 if (link_state != LINK_STATE_DOWN) { 3643 if_schedule_deferred_start(ifp); 3644 } 3645 } 3646 3647 static void 3648 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3649 const char *msg) 3650 { 3651 char buf[512]; 3652 size_t len; 3653 3654 len = sizeof(buf); 3655 buf[--len] = '\0'; 3656 3657 device_printf(sc->sc_dev, "%s\n", msg); 3658 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3659 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3660 buf, le16toh(iaq->iaq_opcode)); 3661 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3662 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3663 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3664 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3665 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3666 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3667 } 3668 3669 static void 3670 ixl_arq(void *xsc) 3671 { 3672 struct ixl_softc *sc = xsc; 3673 struct ixl_aq_desc *arq, *iaq; 3674 struct ixl_aq_buf *aqb; 3675 unsigned int cons = sc->sc_arq_cons; 3676 unsigned int prod; 3677 int done = 0; 3678 3679 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3680 sc->sc_aq_regs->arq_head_mask; 3681 3682 if (cons == prod) 3683 goto done; 3684 3685 arq = IXL_DMA_KVA(&sc->sc_arq); 3686 3687 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3688 0, IXL_DMA_LEN(&sc->sc_arq), 3689 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3690 3691 do { 3692 iaq = &arq[cons]; 3693 aqb = sc->sc_arq_live[cons]; 3694 3695 KASSERT(aqb != NULL); 3696 3697 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3698 BUS_DMASYNC_POSTREAD); 3699 3700 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3701 ixl_aq_dump(sc, iaq, "arq event"); 3702 3703 switch (iaq->iaq_opcode) { 3704 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3705 kpreempt_disable(); 3706 ixl_link_state_update(sc, iaq); 3707 kpreempt_enable(); 3708 break; 3709 } 3710 3711 memset(iaq, 0, sizeof(*iaq)); 3712 sc->sc_arq_live[cons] = NULL; 3713 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3714 3715 cons++; 3716 cons &= IXL_AQ_MASK; 3717 3718 done = 1; 3719 } while (cons != prod); 3720 3721 if (done) { 3722 sc->sc_arq_cons = cons; 3723 ixl_arq_fill(sc); 3724 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3725 0, IXL_DMA_LEN(&sc->sc_arq), 3726 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3727 } 3728 3729 done: 3730 ixl_enable_other_intr(sc); 3731 } 3732 3733 static void 3734 ixl_atq_set(struct ixl_atq *iatq, 3735 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3736 { 3737 3738 iatq->iatq_fn = fn; 3739 } 3740 3741 static int 3742 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3743 { 3744 struct ixl_aq_desc *atq, *slot; 3745 unsigned int prod, cons, prod_next; 3746 3747 /* assert locked */ 3748 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3749 3750 atq = IXL_DMA_KVA(&sc->sc_atq); 3751 prod = sc->sc_atq_prod; 3752 cons = sc->sc_atq_cons; 3753 prod_next = (prod +1) & IXL_AQ_MASK; 3754 3755 if (cons == prod_next) 3756 return ENOMEM; 3757 3758 slot = &atq[prod]; 3759 3760 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3761 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3762 3763 *slot = iatq->iatq_desc; 3764 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3765 3766 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3767 ixl_aq_dump(sc, slot, "atq command"); 3768 3769 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3770 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3771 3772 sc->sc_atq_prod = prod_next; 3773 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3774 3775 return 0; 3776 } 3777 3778 static int 3779 ixl_atq_post(struct ixl_softc *sc, struct ixl_atq *iatq) 3780 { 3781 int rv; 3782 3783 mutex_enter(&sc->sc_atq_lock); 3784 rv = ixl_atq_post_locked(sc, iatq); 3785 mutex_exit(&sc->sc_atq_lock); 3786 3787 return rv; 3788 } 3789 3790 static void 3791 ixl_atq_done_locked(struct ixl_softc *sc) 3792 { 3793 struct ixl_aq_desc *atq, *slot; 3794 struct ixl_atq *iatq; 3795 unsigned int cons; 3796 unsigned int prod; 3797 3798 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3799 3800 prod = sc->sc_atq_prod; 3801 cons = sc->sc_atq_cons; 3802 3803 if (prod == cons) 3804 return; 3805 3806 atq = IXL_DMA_KVA(&sc->sc_atq); 3807 3808 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3809 0, IXL_DMA_LEN(&sc->sc_atq), 3810 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3811 3812 do { 3813 slot = &atq[cons]; 3814 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3815 break; 3816 3817 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3818 iatq->iatq_desc = *slot; 3819 3820 memset(slot, 0, sizeof(*slot)); 3821 3822 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3823 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3824 3825 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3826 3827 cons++; 3828 cons &= IXL_AQ_MASK; 3829 } while (cons != prod); 3830 3831 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3832 0, IXL_DMA_LEN(&sc->sc_atq), 3833 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3834 3835 sc->sc_atq_cons = cons; 3836 } 3837 3838 static void 3839 ixl_atq_done(struct ixl_softc *sc) 3840 { 3841 3842 mutex_enter(&sc->sc_atq_lock); 3843 ixl_atq_done_locked(sc); 3844 mutex_exit(&sc->sc_atq_lock); 3845 } 3846 3847 static void 3848 ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3849 { 3850 3851 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3852 3853 cv_signal(&sc->sc_atq_cv); 3854 } 3855 3856 static int 3857 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3858 { 3859 int error; 3860 3861 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3862 3863 ixl_atq_set(iatq, ixl_wakeup); 3864 3865 mutex_enter(&sc->sc_atq_lock); 3866 error = ixl_atq_post_locked(sc, iatq); 3867 if (error) { 3868 mutex_exit(&sc->sc_atq_lock); 3869 return error; 3870 } 3871 3872 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3873 IXL_ATQ_EXEC_TIMEOUT); 3874 mutex_exit(&sc->sc_atq_lock); 3875 3876 return error; 3877 } 3878 3879 static int 3880 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3881 { 3882 struct ixl_aq_desc *atq, *slot; 3883 unsigned int prod; 3884 unsigned int t = 0; 3885 3886 mutex_enter(&sc->sc_atq_lock); 3887 3888 atq = IXL_DMA_KVA(&sc->sc_atq); 3889 prod = sc->sc_atq_prod; 3890 slot = atq + prod; 3891 3892 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3893 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3894 3895 *slot = *iaq; 3896 slot->iaq_flags |= htole16(IXL_AQ_SI); 3897 3898 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3899 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3900 3901 prod++; 3902 prod &= IXL_AQ_MASK; 3903 sc->sc_atq_prod = prod; 3904 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3905 3906 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3907 delaymsec(1); 3908 3909 if (t++ > tm) { 3910 mutex_exit(&sc->sc_atq_lock); 3911 return ETIMEDOUT; 3912 } 3913 } 3914 3915 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3916 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3917 *iaq = *slot; 3918 memset(slot, 0, sizeof(*slot)); 3919 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3920 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3921 3922 sc->sc_atq_cons = prod; 3923 3924 mutex_exit(&sc->sc_atq_lock); 3925 3926 return 0; 3927 } 3928 3929 static int 3930 ixl_get_version(struct ixl_softc *sc) 3931 { 3932 struct ixl_aq_desc iaq; 3933 uint32_t fwbuild, fwver, apiver; 3934 uint16_t api_maj_ver, api_min_ver; 3935 3936 memset(&iaq, 0, sizeof(iaq)); 3937 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3938 3939 iaq.iaq_retval = le16toh(23); 3940 3941 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3942 return ETIMEDOUT; 3943 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3944 return EIO; 3945 3946 fwbuild = le32toh(iaq.iaq_param[1]); 3947 fwver = le32toh(iaq.iaq_param[2]); 3948 apiver = le32toh(iaq.iaq_param[3]); 3949 3950 api_maj_ver = (uint16_t)apiver; 3951 api_min_ver = (uint16_t)(apiver >> 16); 3952 3953 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 3954 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 3955 3956 if (sc->sc_mac_type == I40E_MAC_X722) { 3957 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK | 3958 IXL_SC_AQ_FLAG_NVMREAD); 3959 } 3960 3961 #define IXL_API_VER(maj, min) (((uint32_t)(maj) << 16) | (min)) 3962 if (IXL_API_VER(api_maj_ver, api_min_ver) >= IXL_API_VER(1, 5)) { 3963 if (sc->sc_mac_type == I40E_MAC_X722) { 3964 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 3965 } 3966 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK); 3967 } 3968 #undef IXL_API_VER 3969 3970 return 0; 3971 } 3972 3973 static int 3974 ixl_get_nvm_version(struct ixl_softc *sc) 3975 { 3976 uint16_t nvmver, cfg_ptr, eetrack_hi, eetrack_lo, oem_hi, oem_lo; 3977 uint32_t eetrack, oem; 3978 uint16_t nvm_maj_ver, nvm_min_ver, oem_build; 3979 uint8_t oem_ver, oem_patch; 3980 3981 nvmver = cfg_ptr = eetrack_hi = eetrack_lo = oem_hi = oem_lo = 0; 3982 ixl_rd16_nvm(sc, I40E_SR_NVM_DEV_STARTER_VERSION, &nvmver); 3983 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_HI, &eetrack_hi); 3984 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_LO, &eetrack_lo); 3985 ixl_rd16_nvm(sc, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr); 3986 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF, &oem_hi); 3987 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF + 1, &oem_lo); 3988 3989 nvm_maj_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_HI_MASK); 3990 nvm_min_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_LO_MASK); 3991 eetrack = ((uint32_t)eetrack_hi << 16) | eetrack_lo; 3992 oem = ((uint32_t)oem_hi << 16) | oem_lo; 3993 oem_ver = __SHIFTOUT(oem, IXL_NVM_OEMVERSION_MASK); 3994 oem_build = __SHIFTOUT(oem, IXL_NVM_OEMBUILD_MASK); 3995 oem_patch = __SHIFTOUT(oem, IXL_NVM_OEMPATCH_MASK); 3996 3997 aprint_normal(" nvm %x.%02x etid %08x oem %d.%d.%d", 3998 nvm_maj_ver, nvm_min_ver, eetrack, 3999 oem_ver, oem_build, oem_patch); 4000 4001 return 0; 4002 } 4003 4004 static int 4005 ixl_pxe_clear(struct ixl_softc *sc) 4006 { 4007 struct ixl_aq_desc iaq; 4008 int rv; 4009 4010 memset(&iaq, 0, sizeof(iaq)); 4011 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 4012 iaq.iaq_param[0] = htole32(0x2); 4013 4014 rv = ixl_atq_poll(sc, &iaq, 250); 4015 4016 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 4017 4018 if (rv != 0) 4019 return ETIMEDOUT; 4020 4021 switch (iaq.iaq_retval) { 4022 case htole16(IXL_AQ_RC_OK): 4023 case htole16(IXL_AQ_RC_EEXIST): 4024 break; 4025 default: 4026 return EIO; 4027 } 4028 4029 return 0; 4030 } 4031 4032 static int 4033 ixl_lldp_shut(struct ixl_softc *sc) 4034 { 4035 struct ixl_aq_desc iaq; 4036 4037 memset(&iaq, 0, sizeof(iaq)); 4038 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 4039 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 4040 4041 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4042 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 4043 return -1; 4044 } 4045 4046 switch (iaq.iaq_retval) { 4047 case htole16(IXL_AQ_RC_EMODE): 4048 case htole16(IXL_AQ_RC_EPERM): 4049 /* ignore silently */ 4050 default: 4051 break; 4052 } 4053 4054 return 0; 4055 } 4056 4057 static void 4058 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 4059 { 4060 uint16_t id; 4061 uint32_t number, logical_id; 4062 4063 id = le16toh(cap->cap_id); 4064 number = le32toh(cap->number); 4065 logical_id = le32toh(cap->logical_id); 4066 4067 switch (id) { 4068 case IXL_AQ_CAP_RSS: 4069 sc->sc_rss_table_size = number; 4070 sc->sc_rss_table_entry_width = logical_id; 4071 break; 4072 case IXL_AQ_CAP_RXQ: 4073 case IXL_AQ_CAP_TXQ: 4074 sc->sc_nqueue_pairs_device = MIN(number, 4075 sc->sc_nqueue_pairs_device); 4076 break; 4077 } 4078 } 4079 4080 static int 4081 ixl_get_hw_capabilities(struct ixl_softc *sc) 4082 { 4083 struct ixl_dmamem idm; 4084 struct ixl_aq_desc iaq; 4085 struct ixl_aq_capability *caps; 4086 size_t i, ncaps; 4087 bus_size_t caps_size; 4088 uint16_t status; 4089 int rv; 4090 4091 caps_size = sizeof(caps[0]) * 40; 4092 memset(&iaq, 0, sizeof(iaq)); 4093 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 4094 4095 do { 4096 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 4097 return -1; 4098 } 4099 4100 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4101 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4102 iaq.iaq_datalen = htole16(caps_size); 4103 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4104 4105 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4106 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 4107 4108 rv = ixl_atq_poll(sc, &iaq, 250); 4109 4110 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4111 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 4112 4113 if (rv != 0) { 4114 aprint_error(", HW capabilities timeout\n"); 4115 goto done; 4116 } 4117 4118 status = le16toh(iaq.iaq_retval); 4119 4120 if (status == IXL_AQ_RC_ENOMEM) { 4121 caps_size = le16toh(iaq.iaq_datalen); 4122 ixl_dmamem_free(sc, &idm); 4123 } 4124 } while (status == IXL_AQ_RC_ENOMEM); 4125 4126 if (status != IXL_AQ_RC_OK) { 4127 aprint_error(", HW capabilities error\n"); 4128 goto done; 4129 } 4130 4131 caps = IXL_DMA_KVA(&idm); 4132 ncaps = le16toh(iaq.iaq_param[1]); 4133 4134 for (i = 0; i < ncaps; i++) { 4135 ixl_parse_hw_capability(sc, &caps[i]); 4136 } 4137 4138 done: 4139 ixl_dmamem_free(sc, &idm); 4140 return rv; 4141 } 4142 4143 static int 4144 ixl_get_mac(struct ixl_softc *sc) 4145 { 4146 struct ixl_dmamem idm; 4147 struct ixl_aq_desc iaq; 4148 struct ixl_aq_mac_addresses *addrs; 4149 int rv; 4150 4151 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 4152 aprint_error(", unable to allocate mac addresses\n"); 4153 return -1; 4154 } 4155 4156 memset(&iaq, 0, sizeof(iaq)); 4157 iaq.iaq_flags = htole16(IXL_AQ_BUF); 4158 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 4159 iaq.iaq_datalen = htole16(sizeof(*addrs)); 4160 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4161 4162 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4163 BUS_DMASYNC_PREREAD); 4164 4165 rv = ixl_atq_poll(sc, &iaq, 250); 4166 4167 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4168 BUS_DMASYNC_POSTREAD); 4169 4170 if (rv != 0) { 4171 aprint_error(", MAC ADDRESS READ timeout\n"); 4172 rv = -1; 4173 goto done; 4174 } 4175 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4176 aprint_error(", MAC ADDRESS READ error\n"); 4177 rv = -1; 4178 goto done; 4179 } 4180 4181 addrs = IXL_DMA_KVA(&idm); 4182 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 4183 printf(", port address is not valid\n"); 4184 goto done; 4185 } 4186 4187 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 4188 rv = 0; 4189 4190 done: 4191 ixl_dmamem_free(sc, &idm); 4192 return rv; 4193 } 4194 4195 static int 4196 ixl_get_switch_config(struct ixl_softc *sc) 4197 { 4198 struct ixl_dmamem idm; 4199 struct ixl_aq_desc iaq; 4200 struct ixl_aq_switch_config *hdr; 4201 struct ixl_aq_switch_config_element *elms, *elm; 4202 unsigned int nelm, i; 4203 int rv; 4204 4205 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4206 aprint_error_dev(sc->sc_dev, 4207 "unable to allocate switch config buffer\n"); 4208 return -1; 4209 } 4210 4211 memset(&iaq, 0, sizeof(iaq)); 4212 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4213 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4214 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 4215 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 4216 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4217 4218 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4219 BUS_DMASYNC_PREREAD); 4220 4221 rv = ixl_atq_poll(sc, &iaq, 250); 4222 4223 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4224 BUS_DMASYNC_POSTREAD); 4225 4226 if (rv != 0) { 4227 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 4228 rv = -1; 4229 goto done; 4230 } 4231 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4232 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 4233 rv = -1; 4234 goto done; 4235 } 4236 4237 hdr = IXL_DMA_KVA(&idm); 4238 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 4239 4240 nelm = le16toh(hdr->num_reported); 4241 if (nelm < 1) { 4242 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 4243 rv = -1; 4244 goto done; 4245 } 4246 4247 for (i = 0; i < nelm; i++) { 4248 elm = &elms[i]; 4249 4250 aprint_debug_dev(sc->sc_dev, 4251 "type %x revision %u seid %04x\n", 4252 elm->type, elm->revision, le16toh(elm->seid)); 4253 aprint_debug_dev(sc->sc_dev, 4254 "uplink %04x downlink %04x\n", 4255 le16toh(elm->uplink_seid), 4256 le16toh(elm->downlink_seid)); 4257 aprint_debug_dev(sc->sc_dev, 4258 "conntype %x scheduler %04x extra %04x\n", 4259 elm->connection_type, 4260 le16toh(elm->scheduler_id), 4261 le16toh(elm->element_info)); 4262 } 4263 4264 elm = &elms[0]; 4265 4266 sc->sc_uplink_seid = elm->uplink_seid; 4267 sc->sc_downlink_seid = elm->downlink_seid; 4268 sc->sc_seid = elm->seid; 4269 4270 if ((sc->sc_uplink_seid == htole16(0)) != 4271 (sc->sc_downlink_seid == htole16(0))) { 4272 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 4273 rv = -1; 4274 goto done; 4275 } 4276 4277 done: 4278 ixl_dmamem_free(sc, &idm); 4279 return rv; 4280 } 4281 4282 static int 4283 ixl_phy_mask_ints(struct ixl_softc *sc) 4284 { 4285 struct ixl_aq_desc iaq; 4286 4287 memset(&iaq, 0, sizeof(iaq)); 4288 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 4289 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 4290 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 4291 IXL_AQ_PHY_EV_MEDIA_NA)); 4292 4293 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4294 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 4295 return -1; 4296 } 4297 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4298 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 4299 return -1; 4300 } 4301 4302 return 0; 4303 } 4304 4305 static int 4306 ixl_get_phy_abilities(struct ixl_softc *sc,struct ixl_dmamem *idm) 4307 { 4308 struct ixl_aq_desc iaq; 4309 int rv; 4310 4311 memset(&iaq, 0, sizeof(iaq)); 4312 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4313 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4314 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 4315 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 4316 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 4317 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 4318 4319 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4320 BUS_DMASYNC_PREREAD); 4321 4322 rv = ixl_atq_poll(sc, &iaq, 250); 4323 4324 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4325 BUS_DMASYNC_POSTREAD); 4326 4327 if (rv != 0) 4328 return -1; 4329 4330 return le16toh(iaq.iaq_retval); 4331 } 4332 4333 static int 4334 ixl_get_phy_info(struct ixl_softc *sc) 4335 { 4336 struct ixl_dmamem idm; 4337 struct ixl_aq_phy_abilities *phy; 4338 int rv; 4339 4340 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4341 aprint_error_dev(sc->sc_dev, 4342 "unable to allocate switch config buffer\n"); 4343 return -1; 4344 } 4345 4346 rv = ixl_get_phy_abilities(sc, &idm); 4347 switch (rv) { 4348 case -1: 4349 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 4350 goto done; 4351 case IXL_AQ_RC_OK: 4352 break; 4353 case IXL_AQ_RC_EIO: 4354 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 4355 goto done; 4356 default: 4357 aprint_error_dev(sc->sc_dev, 4358 "GET PHY ABILITIIES error %u\n", rv); 4359 goto done; 4360 } 4361 4362 phy = IXL_DMA_KVA(&idm); 4363 4364 sc->sc_phy_types = le32toh(phy->phy_type); 4365 sc->sc_phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 4366 4367 sc->sc_phy_abilities = phy->abilities; 4368 sc->sc_phy_linkspeed = phy->link_speed; 4369 sc->sc_phy_fec_cfg = phy->fec_cfg_curr_mod_ext_info & 4370 (IXL_AQ_ENABLE_FEC_KR | IXL_AQ_ENABLE_FEC_RS | 4371 IXL_AQ_REQUEST_FEC_KR | IXL_AQ_REQUEST_FEC_RS); 4372 sc->sc_eee_cap = phy->eee_capability; 4373 sc->sc_eeer_val = phy->eeer_val; 4374 sc->sc_d3_lpan = phy->d3_lpan; 4375 4376 rv = 0; 4377 4378 done: 4379 ixl_dmamem_free(sc, &idm); 4380 return rv; 4381 } 4382 4383 static int 4384 ixl_set_phy_config(struct ixl_softc *sc, 4385 uint8_t link_speed, uint8_t abilities, bool polling) 4386 { 4387 struct ixl_aq_phy_param *param; 4388 struct ixl_atq iatq; 4389 struct ixl_aq_desc *iaq; 4390 int error; 4391 4392 memset(&iatq, 0, sizeof(iatq)); 4393 4394 iaq = &iatq.iatq_desc; 4395 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_CONFIG); 4396 param = (struct ixl_aq_phy_param *)&iaq->iaq_param; 4397 param->phy_types = htole32((uint32_t)sc->sc_phy_types); 4398 param->phy_type_ext = (uint8_t)(sc->sc_phy_types >> 32); 4399 param->link_speed = link_speed; 4400 param->abilities = abilities | IXL_AQ_PHY_ABILITY_AUTO_LINK; 4401 param->fec_cfg = sc->sc_phy_fec_cfg; 4402 param->eee_capability = sc->sc_eee_cap; 4403 param->eeer_val = sc->sc_eeer_val; 4404 param->d3_lpan = sc->sc_d3_lpan; 4405 4406 if (polling) 4407 error = ixl_atq_poll(sc, iaq, 250); 4408 else 4409 error = ixl_atq_exec(sc, &iatq); 4410 4411 if (error != 0) 4412 return error; 4413 4414 switch (le16toh(iaq->iaq_retval)) { 4415 case IXL_AQ_RC_OK: 4416 break; 4417 case IXL_AQ_RC_EPERM: 4418 return EPERM; 4419 default: 4420 return EIO; 4421 } 4422 4423 return 0; 4424 } 4425 4426 static int 4427 ixl_set_phy_autoselect(struct ixl_softc *sc) 4428 { 4429 uint8_t link_speed, abilities; 4430 4431 link_speed = sc->sc_phy_linkspeed; 4432 abilities = IXL_PHY_ABILITY_LINKUP | IXL_PHY_ABILITY_AUTONEGO; 4433 4434 return ixl_set_phy_config(sc, link_speed, abilities, true); 4435 } 4436 4437 static int 4438 ixl_get_link_status_poll(struct ixl_softc *sc) 4439 { 4440 struct ixl_aq_desc iaq; 4441 struct ixl_aq_link_param *param; 4442 int link; 4443 4444 memset(&iaq, 0, sizeof(iaq)); 4445 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 4446 param = (struct ixl_aq_link_param *)iaq.iaq_param; 4447 param->notify = IXL_AQ_LINK_NOTIFY; 4448 4449 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4450 return ETIMEDOUT; 4451 } 4452 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4453 return EIO; 4454 } 4455 4456 link = ixl_set_link_status(sc, &iaq); 4457 sc->sc_ec.ec_if.if_link_state = link; 4458 4459 return 0; 4460 } 4461 4462 static int 4463 ixl_get_vsi(struct ixl_softc *sc) 4464 { 4465 struct ixl_dmamem *vsi = &sc->sc_scratch; 4466 struct ixl_aq_desc iaq; 4467 struct ixl_aq_vsi_param *param; 4468 struct ixl_aq_vsi_reply *reply; 4469 struct ixl_aq_vsi_data *data; 4470 int rv; 4471 4472 /* grumble, vsi info isn't "known" at compile time */ 4473 4474 memset(&iaq, 0, sizeof(iaq)); 4475 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4476 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4477 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 4478 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4479 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4480 4481 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4482 param->uplink_seid = sc->sc_seid; 4483 4484 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4485 BUS_DMASYNC_PREREAD); 4486 4487 rv = ixl_atq_poll(sc, &iaq, 250); 4488 4489 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4490 BUS_DMASYNC_POSTREAD); 4491 4492 if (rv != 0) { 4493 return ETIMEDOUT; 4494 } 4495 4496 switch (le16toh(iaq.iaq_retval)) { 4497 case IXL_AQ_RC_OK: 4498 break; 4499 case IXL_AQ_RC_ENOENT: 4500 return ENOENT; 4501 case IXL_AQ_RC_EACCES: 4502 return EACCES; 4503 default: 4504 return EIO; 4505 } 4506 4507 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 4508 sc->sc_vsi_number = reply->vsi_number; 4509 data = IXL_DMA_KVA(vsi); 4510 sc->sc_vsi_stat_counter_idx = le16toh(data->stat_counter_idx); 4511 4512 return 0; 4513 } 4514 4515 static int 4516 ixl_set_vsi(struct ixl_softc *sc) 4517 { 4518 struct ixl_dmamem *vsi = &sc->sc_scratch; 4519 struct ixl_aq_desc iaq; 4520 struct ixl_aq_vsi_param *param; 4521 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 4522 unsigned int qnum; 4523 uint16_t val; 4524 int rv; 4525 4526 qnum = sc->sc_nqueue_pairs - 1; 4527 4528 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 4529 IXL_AQ_VSI_VALID_VLAN); 4530 4531 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 4532 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 4533 data->queue_mapping[0] = htole16(0); 4534 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 4535 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 4536 4537 val = le16toh(data->port_vlan_flags); 4538 CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK); 4539 SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL); 4540 4541 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { 4542 SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH); 4543 } else { 4544 SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING); 4545 } 4546 4547 data->port_vlan_flags = htole16(val); 4548 4549 /* grumble, vsi info isn't "known" at compile time */ 4550 4551 memset(&iaq, 0, sizeof(iaq)); 4552 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4553 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4554 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 4555 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4556 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4557 4558 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4559 param->uplink_seid = sc->sc_seid; 4560 4561 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4562 BUS_DMASYNC_PREWRITE); 4563 4564 rv = ixl_atq_poll(sc, &iaq, 250); 4565 4566 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4567 BUS_DMASYNC_POSTWRITE); 4568 4569 if (rv != 0) { 4570 return ETIMEDOUT; 4571 } 4572 4573 switch (le16toh(iaq.iaq_retval)) { 4574 case IXL_AQ_RC_OK: 4575 break; 4576 case IXL_AQ_RC_ENOENT: 4577 return ENOENT; 4578 case IXL_AQ_RC_EACCES: 4579 return EACCES; 4580 default: 4581 return EIO; 4582 } 4583 4584 return 0; 4585 } 4586 4587 static void 4588 ixl_set_filter_control(struct ixl_softc *sc) 4589 { 4590 uint32_t reg; 4591 4592 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4593 4594 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4595 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4596 4597 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4598 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4599 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4600 4601 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4602 } 4603 4604 static inline void 4605 ixl_get_default_rss_key(uint32_t *buf, size_t len) 4606 { 4607 size_t cplen; 4608 uint8_t rss_seed[RSS_KEYSIZE]; 4609 4610 rss_getkey(rss_seed); 4611 memset(buf, 0, len); 4612 4613 cplen = MIN(len, sizeof(rss_seed)); 4614 memcpy(buf, rss_seed, cplen); 4615 } 4616 4617 static void 4618 ixl_set_rss_key(struct ixl_softc *sc) 4619 { 4620 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4621 size_t i; 4622 4623 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4624 4625 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4626 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4627 } 4628 } 4629 4630 static void 4631 ixl_set_rss_pctype(struct ixl_softc *sc) 4632 { 4633 uint64_t set_hena = 0; 4634 uint32_t hena0, hena1; 4635 4636 if (sc->sc_mac_type == I40E_MAC_X722) 4637 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4638 else 4639 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4640 4641 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4642 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4643 4644 SET(hena0, set_hena); 4645 SET(hena1, set_hena >> 32); 4646 4647 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4648 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4649 } 4650 4651 static void 4652 ixl_set_rss_hlut(struct ixl_softc *sc) 4653 { 4654 unsigned int qid; 4655 uint8_t hlut_buf[512], lut_mask; 4656 uint32_t *hluts; 4657 size_t i, hluts_num; 4658 4659 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4660 4661 for (i = 0; i < sc->sc_rss_table_size; i++) { 4662 qid = i % sc->sc_nqueue_pairs; 4663 hlut_buf[i] = qid & lut_mask; 4664 } 4665 4666 hluts = (uint32_t *)hlut_buf; 4667 hluts_num = sc->sc_rss_table_size >> 2; 4668 for (i = 0; i < hluts_num; i++) { 4669 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4670 } 4671 ixl_flush(sc); 4672 } 4673 4674 static void 4675 ixl_config_rss(struct ixl_softc *sc) 4676 { 4677 4678 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4679 4680 ixl_set_rss_key(sc); 4681 ixl_set_rss_pctype(sc); 4682 ixl_set_rss_hlut(sc); 4683 } 4684 4685 static const struct ixl_phy_type * 4686 ixl_search_phy_type(uint8_t phy_type) 4687 { 4688 const struct ixl_phy_type *itype; 4689 uint64_t mask; 4690 unsigned int i; 4691 4692 if (phy_type >= 64) 4693 return NULL; 4694 4695 mask = 1ULL << phy_type; 4696 4697 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4698 itype = &ixl_phy_type_map[i]; 4699 4700 if (ISSET(itype->phy_type, mask)) 4701 return itype; 4702 } 4703 4704 return NULL; 4705 } 4706 4707 static uint64_t 4708 ixl_search_link_speed(uint8_t link_speed) 4709 { 4710 const struct ixl_speed_type *type; 4711 unsigned int i; 4712 4713 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4714 type = &ixl_speed_type_map[i]; 4715 4716 if (ISSET(type->dev_speed, link_speed)) 4717 return type->net_speed; 4718 } 4719 4720 return 0; 4721 } 4722 4723 static uint8_t 4724 ixl_search_baudrate(uint64_t baudrate) 4725 { 4726 const struct ixl_speed_type *type; 4727 unsigned int i; 4728 4729 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4730 type = &ixl_speed_type_map[i]; 4731 4732 if (type->net_speed == baudrate) { 4733 return type->dev_speed; 4734 } 4735 } 4736 4737 return 0; 4738 } 4739 4740 static int 4741 ixl_restart_an(struct ixl_softc *sc) 4742 { 4743 struct ixl_aq_desc iaq; 4744 4745 memset(&iaq, 0, sizeof(iaq)); 4746 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4747 iaq.iaq_param[0] = 4748 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4749 4750 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4751 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4752 return -1; 4753 } 4754 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4755 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4756 return -1; 4757 } 4758 4759 return 0; 4760 } 4761 4762 static int 4763 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4764 uint16_t vlan, uint16_t flags) 4765 { 4766 struct ixl_aq_desc iaq; 4767 struct ixl_aq_add_macvlan *param; 4768 struct ixl_aq_add_macvlan_elem *elem; 4769 4770 memset(&iaq, 0, sizeof(iaq)); 4771 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4772 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4773 iaq.iaq_datalen = htole16(sizeof(*elem)); 4774 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4775 4776 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4777 param->num_addrs = htole16(1); 4778 param->seid0 = htole16(0x8000) | sc->sc_seid; 4779 param->seid1 = 0; 4780 param->seid2 = 0; 4781 4782 elem = IXL_DMA_KVA(&sc->sc_scratch); 4783 memset(elem, 0, sizeof(*elem)); 4784 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4785 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4786 elem->vlan = htole16(vlan); 4787 4788 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4789 return IXL_AQ_RC_EINVAL; 4790 } 4791 4792 switch (le16toh(iaq.iaq_retval)) { 4793 case IXL_AQ_RC_OK: 4794 break; 4795 case IXL_AQ_RC_ENOSPC: 4796 return ENOSPC; 4797 case IXL_AQ_RC_ENOENT: 4798 return ENOENT; 4799 case IXL_AQ_RC_EACCES: 4800 return EACCES; 4801 case IXL_AQ_RC_EEXIST: 4802 return EEXIST; 4803 case IXL_AQ_RC_EINVAL: 4804 return EINVAL; 4805 default: 4806 return EIO; 4807 } 4808 4809 return 0; 4810 } 4811 4812 static int 4813 ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4814 uint16_t vlan, uint16_t flags) 4815 { 4816 struct ixl_aq_desc iaq; 4817 struct ixl_aq_remove_macvlan *param; 4818 struct ixl_aq_remove_macvlan_elem *elem; 4819 4820 memset(&iaq, 0, sizeof(iaq)); 4821 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4822 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 4823 iaq.iaq_datalen = htole16(sizeof(*elem)); 4824 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4825 4826 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 4827 param->num_addrs = htole16(1); 4828 param->seid0 = htole16(0x8000) | sc->sc_seid; 4829 param->seid1 = 0; 4830 param->seid2 = 0; 4831 4832 elem = IXL_DMA_KVA(&sc->sc_scratch); 4833 memset(elem, 0, sizeof(*elem)); 4834 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4835 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 4836 elem->vlan = htole16(vlan); 4837 4838 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4839 return EINVAL; 4840 } 4841 4842 switch (le16toh(iaq.iaq_retval)) { 4843 case IXL_AQ_RC_OK: 4844 break; 4845 case IXL_AQ_RC_ENOENT: 4846 return ENOENT; 4847 case IXL_AQ_RC_EACCES: 4848 return EACCES; 4849 case IXL_AQ_RC_EINVAL: 4850 return EINVAL; 4851 default: 4852 return EIO; 4853 } 4854 4855 return 0; 4856 } 4857 4858 static int 4859 ixl_hmc(struct ixl_softc *sc) 4860 { 4861 struct { 4862 uint32_t count; 4863 uint32_t minsize; 4864 bus_size_t objsiz; 4865 bus_size_t setoff; 4866 bus_size_t setcnt; 4867 } regs[] = { 4868 { 4869 0, 4870 IXL_HMC_TXQ_MINSIZE, 4871 I40E_GLHMC_LANTXOBJSZ, 4872 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 4873 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 4874 }, 4875 { 4876 0, 4877 IXL_HMC_RXQ_MINSIZE, 4878 I40E_GLHMC_LANRXOBJSZ, 4879 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 4880 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 4881 }, 4882 { 4883 0, 4884 0, 4885 I40E_GLHMC_FCOEDDPOBJSZ, 4886 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 4887 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 4888 }, 4889 { 4890 0, 4891 0, 4892 I40E_GLHMC_FCOEFOBJSZ, 4893 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 4894 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 4895 }, 4896 }; 4897 struct ixl_hmc_entry *e; 4898 uint64_t size, dva; 4899 uint8_t *kva; 4900 uint64_t *sdpage; 4901 unsigned int i; 4902 int npages, tables; 4903 uint32_t reg; 4904 4905 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 4906 4907 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 4908 ixl_rd(sc, I40E_GLHMC_LANQMAX); 4909 4910 size = 0; 4911 for (i = 0; i < __arraycount(regs); i++) { 4912 e = &sc->sc_hmc_entries[i]; 4913 4914 e->hmc_count = regs[i].count; 4915 reg = ixl_rd(sc, regs[i].objsiz); 4916 e->hmc_size = BIT_ULL(0x3F & reg); 4917 e->hmc_base = size; 4918 4919 if ((e->hmc_size * 8) < regs[i].minsize) { 4920 aprint_error_dev(sc->sc_dev, 4921 "kernel hmc entry is too big\n"); 4922 return -1; 4923 } 4924 4925 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 4926 } 4927 size = roundup(size, IXL_HMC_PGSIZE); 4928 npages = size / IXL_HMC_PGSIZE; 4929 4930 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 4931 4932 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 4933 aprint_error_dev(sc->sc_dev, 4934 "unable to allocate hmc pd memory\n"); 4935 return -1; 4936 } 4937 4938 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 4939 IXL_HMC_PGSIZE) != 0) { 4940 aprint_error_dev(sc->sc_dev, 4941 "unable to allocate hmc sd memory\n"); 4942 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 4943 return -1; 4944 } 4945 4946 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 4947 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 4948 4949 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 4950 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 4951 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4952 4953 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 4954 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 4955 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 4956 4957 for (i = 0; (int)i < npages; i++) { 4958 *sdpage = htole64(dva | IXL_HMC_PDVALID); 4959 sdpage++; 4960 4961 dva += IXL_HMC_PGSIZE; 4962 } 4963 4964 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 4965 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 4966 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4967 4968 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 4969 for (i = 0; (int)i < tables; i++) { 4970 uint32_t count; 4971 4972 KASSERT(npages >= 0); 4973 4974 count = ((unsigned int)npages > IXL_HMC_PGS) ? 4975 IXL_HMC_PGS : (unsigned int)npages; 4976 4977 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 4978 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 4979 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 4980 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 4981 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 4982 ixl_wr(sc, I40E_PFHMC_SDCMD, 4983 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 4984 4985 npages -= IXL_HMC_PGS; 4986 dva += IXL_HMC_PGSIZE; 4987 } 4988 4989 for (i = 0; i < __arraycount(regs); i++) { 4990 e = &sc->sc_hmc_entries[i]; 4991 4992 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 4993 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 4994 } 4995 4996 return 0; 4997 } 4998 4999 static void 5000 ixl_hmc_free(struct ixl_softc *sc) 5001 { 5002 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 5003 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5004 } 5005 5006 static void 5007 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 5008 unsigned int npacking) 5009 { 5010 uint8_t *dst = d; 5011 const uint8_t *src = s; 5012 unsigned int i; 5013 5014 for (i = 0; i < npacking; i++) { 5015 const struct ixl_hmc_pack *pack = &packing[i]; 5016 unsigned int offset = pack->lsb / 8; 5017 unsigned int align = pack->lsb % 8; 5018 const uint8_t *in = src + pack->offset; 5019 uint8_t *out = dst + offset; 5020 int width = pack->width; 5021 unsigned int inbits = 0; 5022 5023 if (align) { 5024 inbits = (*in++) << align; 5025 *out++ |= (inbits & 0xff); 5026 inbits >>= 8; 5027 5028 width -= 8 - align; 5029 } 5030 5031 while (width >= 8) { 5032 inbits |= (*in++) << align; 5033 *out++ = (inbits & 0xff); 5034 inbits >>= 8; 5035 5036 width -= 8; 5037 } 5038 5039 if (width > 0) { 5040 inbits |= (*in) << align; 5041 *out |= (inbits & ((1 << width) - 1)); 5042 } 5043 } 5044 } 5045 5046 static struct ixl_aq_buf * 5047 ixl_aqb_alloc(struct ixl_softc *sc) 5048 { 5049 struct ixl_aq_buf *aqb; 5050 5051 aqb = malloc(sizeof(*aqb), M_DEVBUF, M_WAITOK); 5052 if (aqb == NULL) 5053 return NULL; 5054 5055 aqb->aqb_size = IXL_AQ_BUFLEN; 5056 5057 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 5058 aqb->aqb_size, 0, 5059 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 5060 goto free; 5061 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 5062 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 5063 BUS_DMA_WAITOK) != 0) 5064 goto destroy; 5065 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 5066 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 5067 goto dma_free; 5068 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 5069 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 5070 goto unmap; 5071 5072 return aqb; 5073 unmap: 5074 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5075 dma_free: 5076 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5077 destroy: 5078 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5079 free: 5080 free(aqb, M_DEVBUF); 5081 5082 return NULL; 5083 } 5084 5085 static void 5086 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 5087 { 5088 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 5089 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5090 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5091 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5092 free(aqb, M_DEVBUF); 5093 } 5094 5095 static int 5096 ixl_arq_fill(struct ixl_softc *sc) 5097 { 5098 struct ixl_aq_buf *aqb; 5099 struct ixl_aq_desc *arq, *iaq; 5100 unsigned int prod = sc->sc_arq_prod; 5101 unsigned int n; 5102 int post = 0; 5103 5104 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 5105 IXL_AQ_NUM); 5106 arq = IXL_DMA_KVA(&sc->sc_arq); 5107 5108 if (__predict_false(n <= 0)) 5109 return 0; 5110 5111 do { 5112 aqb = sc->sc_arq_live[prod]; 5113 iaq = &arq[prod]; 5114 5115 if (aqb == NULL) { 5116 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 5117 if (aqb != NULL) { 5118 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5119 ixl_aq_buf, aqb_entry); 5120 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 5121 break; 5122 } 5123 5124 sc->sc_arq_live[prod] = aqb; 5125 memset(aqb->aqb_data, 0, aqb->aqb_size); 5126 5127 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 5128 aqb->aqb_size, BUS_DMASYNC_PREREAD); 5129 5130 iaq->iaq_flags = htole16(IXL_AQ_BUF | 5131 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 5132 IXL_AQ_LB : 0)); 5133 iaq->iaq_opcode = 0; 5134 iaq->iaq_datalen = htole16(aqb->aqb_size); 5135 iaq->iaq_retval = 0; 5136 iaq->iaq_cookie = 0; 5137 iaq->iaq_param[0] = 0; 5138 iaq->iaq_param[1] = 0; 5139 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 5140 } 5141 5142 prod++; 5143 prod &= IXL_AQ_MASK; 5144 5145 post = 1; 5146 5147 } while (--n); 5148 5149 if (post) { 5150 sc->sc_arq_prod = prod; 5151 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 5152 } 5153 5154 return post; 5155 } 5156 5157 static void 5158 ixl_arq_unfill(struct ixl_softc *sc) 5159 { 5160 struct ixl_aq_buf *aqb; 5161 unsigned int i; 5162 5163 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 5164 aqb = sc->sc_arq_live[i]; 5165 if (aqb == NULL) 5166 continue; 5167 5168 sc->sc_arq_live[i] = NULL; 5169 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 5170 BUS_DMASYNC_POSTREAD); 5171 ixl_aqb_free(sc, aqb); 5172 } 5173 5174 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 5175 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5176 ixl_aq_buf, aqb_entry); 5177 ixl_aqb_free(sc, aqb); 5178 } 5179 } 5180 5181 static void 5182 ixl_clear_hw(struct ixl_softc *sc) 5183 { 5184 uint32_t num_queues, base_queue; 5185 uint32_t num_pf_int; 5186 uint32_t num_vf_int; 5187 uint32_t num_vfs; 5188 uint32_t i, j; 5189 uint32_t val; 5190 uint32_t eol = 0x7ff; 5191 5192 /* get number of interrupts, queues, and vfs */ 5193 val = ixl_rd(sc, I40E_GLPCI_CNF2); 5194 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 5195 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 5196 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 5197 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 5198 5199 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 5200 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 5201 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 5202 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 5203 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 5204 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 5205 num_queues = (j - base_queue) + 1; 5206 else 5207 num_queues = 0; 5208 5209 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 5210 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 5211 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 5212 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 5213 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 5214 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 5215 num_vfs = (j - i) + 1; 5216 else 5217 num_vfs = 0; 5218 5219 /* stop all the interrupts */ 5220 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5221 ixl_flush(sc); 5222 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 5223 for (i = 0; i < num_pf_int - 2; i++) 5224 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 5225 ixl_flush(sc); 5226 5227 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 5228 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5229 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 5230 for (i = 0; i < num_pf_int - 2; i++) 5231 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 5232 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5233 for (i = 0; i < num_vfs; i++) 5234 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 5235 for (i = 0; i < num_vf_int - 2; i++) 5236 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 5237 5238 /* warn the HW of the coming Tx disables */ 5239 for (i = 0; i < num_queues; i++) { 5240 uint32_t abs_queue_idx = base_queue + i; 5241 uint32_t reg_block = 0; 5242 5243 if (abs_queue_idx >= 128) { 5244 reg_block = abs_queue_idx / 128; 5245 abs_queue_idx %= 128; 5246 } 5247 5248 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 5249 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 5250 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 5251 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 5252 5253 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 5254 } 5255 delaymsec(400); 5256 5257 /* stop all the queues */ 5258 for (i = 0; i < num_queues; i++) { 5259 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 5260 ixl_wr(sc, I40E_QTX_ENA(i), 0); 5261 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 5262 ixl_wr(sc, I40E_QRX_ENA(i), 0); 5263 } 5264 5265 /* short wait for all queue disables to settle */ 5266 delaymsec(50); 5267 } 5268 5269 static int 5270 ixl_pf_reset(struct ixl_softc *sc) 5271 { 5272 uint32_t cnt = 0; 5273 uint32_t cnt1 = 0; 5274 uint32_t reg = 0, reg0 = 0; 5275 uint32_t grst_del; 5276 5277 /* 5278 * Poll for Global Reset steady state in case of recent GRST. 5279 * The grst delay value is in 100ms units, and we'll wait a 5280 * couple counts longer to be sure we don't just miss the end. 5281 */ 5282 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 5283 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 5284 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 5285 5286 grst_del = grst_del * 20; 5287 5288 for (cnt = 0; cnt < grst_del; cnt++) { 5289 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 5290 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 5291 break; 5292 delaymsec(100); 5293 } 5294 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5295 aprint_error(", Global reset polling failed to complete\n"); 5296 return -1; 5297 } 5298 5299 /* Now Wait for the FW to be ready */ 5300 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 5301 reg = ixl_rd(sc, I40E_GLNVM_ULD); 5302 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5303 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 5304 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5305 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 5306 break; 5307 5308 delaymsec(10); 5309 } 5310 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5311 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 5312 aprint_error(", wait for FW Reset complete timed out " 5313 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 5314 return -1; 5315 } 5316 5317 /* 5318 * If there was a Global Reset in progress when we got here, 5319 * we don't need to do the PF Reset 5320 */ 5321 if (cnt == 0) { 5322 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5323 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 5324 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 5325 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5326 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 5327 break; 5328 delaymsec(1); 5329 5330 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 5331 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5332 aprint_error(", Core reset upcoming." 5333 " Skipping PF reset reset request\n"); 5334 return -1; 5335 } 5336 } 5337 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 5338 aprint_error(", PF reset polling failed to complete" 5339 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 5340 return -1; 5341 } 5342 } 5343 5344 return 0; 5345 } 5346 5347 static int 5348 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 5349 bus_size_t size, bus_size_t align) 5350 { 5351 ixm->ixm_size = size; 5352 5353 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 5354 ixm->ixm_size, 0, 5355 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 5356 &ixm->ixm_map) != 0) 5357 return 1; 5358 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 5359 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 5360 BUS_DMA_WAITOK) != 0) 5361 goto destroy; 5362 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 5363 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 5364 goto free; 5365 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 5366 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 5367 goto unmap; 5368 5369 memset(ixm->ixm_kva, 0, ixm->ixm_size); 5370 5371 return 0; 5372 unmap: 5373 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5374 free: 5375 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5376 destroy: 5377 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5378 return 1; 5379 } 5380 5381 static void 5382 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 5383 { 5384 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 5385 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5386 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5387 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5388 } 5389 5390 static int 5391 ixl_setup_vlan_hwfilter(struct ixl_softc *sc) 5392 { 5393 struct ethercom *ec = &sc->sc_ec; 5394 struct vlanid_list *vlanidp; 5395 int rv; 5396 5397 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5398 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5399 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5400 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5401 5402 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5403 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5404 if (rv != 0) 5405 return rv; 5406 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5407 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5408 if (rv != 0) 5409 return rv; 5410 5411 ETHER_LOCK(ec); 5412 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5413 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 5414 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5415 if (rv != 0) 5416 break; 5417 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 5418 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5419 if (rv != 0) 5420 break; 5421 } 5422 ETHER_UNLOCK(ec); 5423 5424 return rv; 5425 } 5426 5427 static void 5428 ixl_teardown_vlan_hwfilter(struct ixl_softc *sc) 5429 { 5430 struct vlanid_list *vlanidp; 5431 struct ethercom *ec = &sc->sc_ec; 5432 5433 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5434 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5435 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5436 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5437 5438 ETHER_LOCK(ec); 5439 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5440 ixl_remove_macvlan(sc, sc->sc_enaddr, 5441 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5442 ixl_remove_macvlan(sc, etherbroadcastaddr, 5443 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5444 } 5445 ETHER_UNLOCK(ec); 5446 5447 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5448 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5449 ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5450 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5451 } 5452 5453 static int 5454 ixl_update_macvlan(struct ixl_softc *sc) 5455 { 5456 int rv = 0; 5457 int next_ec_capenable = sc->sc_ec.ec_capenable; 5458 5459 if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 5460 rv = ixl_setup_vlan_hwfilter(sc); 5461 if (rv != 0) 5462 ixl_teardown_vlan_hwfilter(sc); 5463 } else { 5464 ixl_teardown_vlan_hwfilter(sc); 5465 } 5466 5467 return rv; 5468 } 5469 5470 static int 5471 ixl_ifflags_cb(struct ethercom *ec) 5472 { 5473 struct ifnet *ifp = &ec->ec_if; 5474 struct ixl_softc *sc = ifp->if_softc; 5475 int rv, change; 5476 5477 mutex_enter(&sc->sc_cfg_lock); 5478 5479 change = ec->ec_capenable ^ sc->sc_cur_ec_capenable; 5480 5481 if (ISSET(change, ETHERCAP_VLAN_HWTAGGING)) { 5482 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 5483 rv = ENETRESET; 5484 goto out; 5485 } 5486 5487 if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) { 5488 rv = ixl_update_macvlan(sc); 5489 if (rv == 0) { 5490 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 5491 } else { 5492 CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER); 5493 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 5494 } 5495 } 5496 5497 rv = ixl_iff(sc); 5498 out: 5499 mutex_exit(&sc->sc_cfg_lock); 5500 5501 return rv; 5502 } 5503 5504 static int 5505 ixl_set_link_status(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 5506 { 5507 const struct ixl_aq_link_status *status; 5508 const struct ixl_phy_type *itype; 5509 5510 uint64_t ifm_active = IFM_ETHER; 5511 uint64_t ifm_status = IFM_AVALID; 5512 int link_state = LINK_STATE_DOWN; 5513 uint64_t baudrate = 0; 5514 5515 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 5516 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) { 5517 ifm_active |= IFM_NONE; 5518 goto done; 5519 } 5520 5521 ifm_active |= IFM_FDX; 5522 ifm_status |= IFM_ACTIVE; 5523 link_state = LINK_STATE_UP; 5524 5525 itype = ixl_search_phy_type(status->phy_type); 5526 if (itype != NULL) 5527 ifm_active |= itype->ifm_type; 5528 5529 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 5530 ifm_active |= IFM_ETH_TXPAUSE; 5531 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 5532 ifm_active |= IFM_ETH_RXPAUSE; 5533 5534 baudrate = ixl_search_link_speed(status->link_speed); 5535 5536 done: 5537 /* NET_ASSERT_LOCKED() except during attach */ 5538 sc->sc_media_active = ifm_active; 5539 sc->sc_media_status = ifm_status; 5540 5541 sc->sc_ec.ec_if.if_baudrate = baudrate; 5542 5543 return link_state; 5544 } 5545 5546 static int 5547 ixl_establish_intx(struct ixl_softc *sc) 5548 { 5549 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5550 pci_intr_handle_t *intr; 5551 char xnamebuf[32]; 5552 char intrbuf[PCI_INTRSTR_LEN]; 5553 char const *intrstr; 5554 5555 KASSERT(sc->sc_nintrs == 1); 5556 5557 intr = &sc->sc_ihp[0]; 5558 5559 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 5560 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 5561 device_xname(sc->sc_dev)); 5562 5563 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 5564 sc, xnamebuf); 5565 5566 if (sc->sc_ihs[0] == NULL) { 5567 aprint_error_dev(sc->sc_dev, 5568 "unable to establish interrupt at %s\n", intrstr); 5569 return -1; 5570 } 5571 5572 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 5573 return 0; 5574 } 5575 5576 static int 5577 ixl_establish_msix(struct ixl_softc *sc) 5578 { 5579 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5580 kcpuset_t *affinity; 5581 unsigned int vector = 0; 5582 unsigned int i; 5583 int affinity_to, r; 5584 char xnamebuf[32]; 5585 char intrbuf[PCI_INTRSTR_LEN]; 5586 char const *intrstr; 5587 5588 kcpuset_create(&affinity, false); 5589 5590 /* the "other" intr is mapped to vector 0 */ 5591 vector = 0; 5592 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5593 intrbuf, sizeof(intrbuf)); 5594 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 5595 device_xname(sc->sc_dev)); 5596 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5597 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 5598 sc, xnamebuf); 5599 if (sc->sc_ihs[vector] == NULL) { 5600 aprint_error_dev(sc->sc_dev, 5601 "unable to establish interrupt at %s\n", intrstr); 5602 goto fail; 5603 } 5604 5605 aprint_normal_dev(sc->sc_dev, "other interrupt at %s", intrstr); 5606 5607 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5608 affinity_to = (affinity_to + sc->sc_nqueue_pairs_max) % ncpu; 5609 5610 kcpuset_zero(affinity); 5611 kcpuset_set(affinity, affinity_to); 5612 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5613 if (r == 0) { 5614 aprint_normal(", affinity to %u", affinity_to); 5615 } 5616 aprint_normal("\n"); 5617 vector++; 5618 5619 sc->sc_msix_vector_queue = vector; 5620 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5621 5622 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5623 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5624 intrbuf, sizeof(intrbuf)); 5625 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 5626 device_xname(sc->sc_dev), i); 5627 5628 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5629 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 5630 (void *)&sc->sc_qps[i], xnamebuf); 5631 5632 if (sc->sc_ihs[vector] == NULL) { 5633 aprint_error_dev(sc->sc_dev, 5634 "unable to establish interrupt at %s\n", intrstr); 5635 goto fail; 5636 } 5637 5638 aprint_normal_dev(sc->sc_dev, 5639 "for TXRX%d interrupt at %s",i , intrstr); 5640 5641 kcpuset_zero(affinity); 5642 kcpuset_set(affinity, affinity_to); 5643 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5644 if (r == 0) { 5645 aprint_normal(", affinity to %u", affinity_to); 5646 affinity_to = (affinity_to + 1) % ncpu; 5647 } 5648 aprint_normal("\n"); 5649 vector++; 5650 } 5651 5652 kcpuset_destroy(affinity); 5653 5654 return 0; 5655 fail: 5656 for (i = 0; i < vector; i++) { 5657 pci_intr_disestablish(pc, sc->sc_ihs[i]); 5658 } 5659 5660 sc->sc_msix_vector_queue = 0; 5661 sc->sc_msix_vector_queue = 0; 5662 kcpuset_destroy(affinity); 5663 5664 return -1; 5665 } 5666 5667 static void 5668 ixl_config_queue_intr(struct ixl_softc *sc) 5669 { 5670 unsigned int i, vector; 5671 5672 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5673 vector = sc->sc_msix_vector_queue; 5674 } else { 5675 vector = I40E_INTR_NOTX_INTR; 5676 5677 ixl_wr(sc, I40E_PFINT_LNKLST0, 5678 (I40E_INTR_NOTX_QUEUE << 5679 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5680 (I40E_QUEUE_TYPE_RX << 5681 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5682 } 5683 5684 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5685 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5686 ixl_flush(sc); 5687 5688 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5689 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5690 (I40E_QUEUE_TYPE_RX << 5691 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5692 5693 ixl_wr(sc, I40E_QINT_RQCTL(i), 5694 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5695 (I40E_ITR_INDEX_RX << 5696 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5697 (I40E_INTR_NOTX_RX_QUEUE << 5698 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5699 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5700 (I40E_QUEUE_TYPE_TX << 5701 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5702 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5703 5704 ixl_wr(sc, I40E_QINT_TQCTL(i), 5705 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5706 (I40E_ITR_INDEX_TX << 5707 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5708 (I40E_INTR_NOTX_TX_QUEUE << 5709 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5710 (I40E_QUEUE_TYPE_EOL << 5711 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5712 (I40E_QUEUE_TYPE_RX << 5713 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5714 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5715 5716 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) 5717 vector++; 5718 } 5719 ixl_flush(sc); 5720 5721 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), 0x7a); 5722 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), 0x7a); 5723 ixl_flush(sc); 5724 } 5725 5726 static void 5727 ixl_config_other_intr(struct ixl_softc *sc) 5728 { 5729 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5730 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5731 5732 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5733 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5734 I40E_PFINT_ICR0_ENA_GRST_MASK | 5735 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5736 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5737 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5738 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5739 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5740 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5741 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5742 5743 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5744 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5745 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5746 (I40E_ITR_INDEX_OTHER << 5747 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5748 ixl_flush(sc); 5749 } 5750 5751 static int 5752 ixl_setup_interrupts(struct ixl_softc *sc) 5753 { 5754 struct pci_attach_args *pa = &sc->sc_pa; 5755 pci_intr_type_t max_type, intr_type; 5756 int counts[PCI_INTR_TYPE_SIZE]; 5757 int error; 5758 unsigned int i; 5759 bool retry; 5760 5761 memset(counts, 0, sizeof(counts)); 5762 max_type = PCI_INTR_TYPE_MSIX; 5763 /* QPs + other interrupt */ 5764 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5765 counts[PCI_INTR_TYPE_INTX] = 1; 5766 5767 if (ixl_param_nomsix) 5768 counts[PCI_INTR_TYPE_MSIX] = 0; 5769 5770 do { 5771 retry = false; 5772 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5773 if (error != 0) { 5774 aprint_error_dev(sc->sc_dev, 5775 "couldn't map interrupt\n"); 5776 break; 5777 } 5778 5779 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5780 sc->sc_nintrs = counts[intr_type]; 5781 KASSERT(sc->sc_nintrs > 0); 5782 5783 for (i = 0; i < sc->sc_nintrs; i++) { 5784 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5785 PCI_INTR_MPSAFE, true); 5786 } 5787 5788 sc->sc_ihs = kmem_alloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5789 KM_SLEEP); 5790 5791 if (intr_type == PCI_INTR_TYPE_MSIX) { 5792 error = ixl_establish_msix(sc); 5793 if (error) { 5794 counts[PCI_INTR_TYPE_MSIX] = 0; 5795 retry = true; 5796 } 5797 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5798 error = ixl_establish_intx(sc); 5799 } else { 5800 error = -1; 5801 } 5802 5803 if (error) { 5804 kmem_free(sc->sc_ihs, 5805 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5806 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5807 } else { 5808 sc->sc_intrtype = intr_type; 5809 } 5810 } while (retry); 5811 5812 return error; 5813 } 5814 5815 static void 5816 ixl_teardown_interrupts(struct ixl_softc *sc) 5817 { 5818 struct pci_attach_args *pa = &sc->sc_pa; 5819 unsigned int i; 5820 5821 for (i = 0; i < sc->sc_nintrs; i++) { 5822 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 5823 } 5824 5825 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5826 5827 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5828 sc->sc_ihs = NULL; 5829 sc->sc_nintrs = 0; 5830 } 5831 5832 static int 5833 ixl_setup_stats(struct ixl_softc *sc) 5834 { 5835 struct ixl_queue_pair *qp; 5836 struct ixl_tx_ring *txr; 5837 struct ixl_rx_ring *rxr; 5838 struct ixl_stats_counters *isc; 5839 unsigned int i; 5840 5841 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5842 qp = &sc->sc_qps[i]; 5843 txr = qp->qp_txr; 5844 rxr = qp->qp_rxr; 5845 5846 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 5847 NULL, qp->qp_name, "m_defrag successed"); 5848 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 5849 NULL, qp->qp_name, "m_defrag_failed"); 5850 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 5851 NULL, qp->qp_name, "Dropped in pcq"); 5852 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 5853 NULL, qp->qp_name, "Deferred transmit"); 5854 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 5855 NULL, qp->qp_name, "Interrupt on queue"); 5856 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 5857 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5858 5859 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 5860 NULL, qp->qp_name, "MGETHDR failed"); 5861 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 5862 NULL, qp->qp_name, "MCLGET failed"); 5863 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 5864 EVCNT_TYPE_MISC, NULL, qp->qp_name, 5865 "bus_dmamap_load_mbuf failed"); 5866 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 5867 NULL, qp->qp_name, "Interrupt on queue"); 5868 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 5869 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5870 } 5871 5872 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 5873 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 5874 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 5875 NULL, device_xname(sc->sc_dev), "Link status event"); 5876 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 5877 NULL, device_xname(sc->sc_dev), "ECC error"); 5878 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 5879 NULL, device_xname(sc->sc_dev), "PCI exception"); 5880 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 5881 NULL, device_xname(sc->sc_dev), "Critical error"); 5882 5883 isc = &sc->sc_stats_counters; 5884 evcnt_attach_dynamic(&isc->isc_crc_errors, EVCNT_TYPE_MISC, 5885 NULL, device_xname(sc->sc_dev), "CRC errors"); 5886 evcnt_attach_dynamic(&isc->isc_illegal_bytes, EVCNT_TYPE_MISC, 5887 NULL, device_xname(sc->sc_dev), "Illegal bytes"); 5888 evcnt_attach_dynamic(&isc->isc_mac_local_faults, EVCNT_TYPE_MISC, 5889 NULL, device_xname(sc->sc_dev), "Mac local faults"); 5890 evcnt_attach_dynamic(&isc->isc_mac_remote_faults, EVCNT_TYPE_MISC, 5891 NULL, device_xname(sc->sc_dev), "Mac remote faults"); 5892 evcnt_attach_dynamic(&isc->isc_link_xon_rx, EVCNT_TYPE_MISC, 5893 NULL, device_xname(sc->sc_dev), "Rx xon"); 5894 evcnt_attach_dynamic(&isc->isc_link_xon_tx, EVCNT_TYPE_MISC, 5895 NULL, device_xname(sc->sc_dev), "Tx xon"); 5896 evcnt_attach_dynamic(&isc->isc_link_xoff_rx, EVCNT_TYPE_MISC, 5897 NULL, device_xname(sc->sc_dev), "Rx xoff"); 5898 evcnt_attach_dynamic(&isc->isc_link_xoff_tx, EVCNT_TYPE_MISC, 5899 NULL, device_xname(sc->sc_dev), "Tx xoff"); 5900 evcnt_attach_dynamic(&isc->isc_rx_fragments, EVCNT_TYPE_MISC, 5901 NULL, device_xname(sc->sc_dev), "Rx fragments"); 5902 evcnt_attach_dynamic(&isc->isc_rx_jabber, EVCNT_TYPE_MISC, 5903 NULL, device_xname(sc->sc_dev), "Rx jabber"); 5904 5905 evcnt_attach_dynamic(&isc->isc_rx_size_64, EVCNT_TYPE_MISC, 5906 NULL, device_xname(sc->sc_dev), "Rx size 64"); 5907 evcnt_attach_dynamic(&isc->isc_rx_size_127, EVCNT_TYPE_MISC, 5908 NULL, device_xname(sc->sc_dev), "Rx size 127"); 5909 evcnt_attach_dynamic(&isc->isc_rx_size_255, EVCNT_TYPE_MISC, 5910 NULL, device_xname(sc->sc_dev), "Rx size 255"); 5911 evcnt_attach_dynamic(&isc->isc_rx_size_511, EVCNT_TYPE_MISC, 5912 NULL, device_xname(sc->sc_dev), "Rx size 511"); 5913 evcnt_attach_dynamic(&isc->isc_rx_size_1023, EVCNT_TYPE_MISC, 5914 NULL, device_xname(sc->sc_dev), "Rx size 1023"); 5915 evcnt_attach_dynamic(&isc->isc_rx_size_1522, EVCNT_TYPE_MISC, 5916 NULL, device_xname(sc->sc_dev), "Rx size 1522"); 5917 evcnt_attach_dynamic(&isc->isc_rx_size_big, EVCNT_TYPE_MISC, 5918 NULL, device_xname(sc->sc_dev), "Rx jumbo packets"); 5919 evcnt_attach_dynamic(&isc->isc_rx_undersize, EVCNT_TYPE_MISC, 5920 NULL, device_xname(sc->sc_dev), "Rx under size"); 5921 evcnt_attach_dynamic(&isc->isc_rx_oversize, EVCNT_TYPE_MISC, 5922 NULL, device_xname(sc->sc_dev), "Rx over size"); 5923 5924 evcnt_attach_dynamic(&isc->isc_rx_bytes, EVCNT_TYPE_MISC, 5925 NULL, device_xname(sc->sc_dev), "Rx bytes / port"); 5926 evcnt_attach_dynamic(&isc->isc_rx_discards, EVCNT_TYPE_MISC, 5927 NULL, device_xname(sc->sc_dev), "Rx discards / port"); 5928 evcnt_attach_dynamic(&isc->isc_rx_unicast, EVCNT_TYPE_MISC, 5929 NULL, device_xname(sc->sc_dev), "Rx unicast / port"); 5930 evcnt_attach_dynamic(&isc->isc_rx_multicast, EVCNT_TYPE_MISC, 5931 NULL, device_xname(sc->sc_dev), "Rx multicast / port"); 5932 evcnt_attach_dynamic(&isc->isc_rx_broadcast, EVCNT_TYPE_MISC, 5933 NULL, device_xname(sc->sc_dev), "Rx broadcast / port"); 5934 5935 evcnt_attach_dynamic(&isc->isc_vsi_rx_bytes, EVCNT_TYPE_MISC, 5936 NULL, device_xname(sc->sc_dev), "Rx bytes / vsi"); 5937 evcnt_attach_dynamic(&isc->isc_vsi_rx_discards, EVCNT_TYPE_MISC, 5938 NULL, device_xname(sc->sc_dev), "Rx discard / vsi"); 5939 evcnt_attach_dynamic(&isc->isc_vsi_rx_unicast, EVCNT_TYPE_MISC, 5940 NULL, device_xname(sc->sc_dev), "Rx unicast / vsi"); 5941 evcnt_attach_dynamic(&isc->isc_vsi_rx_multicast, EVCNT_TYPE_MISC, 5942 NULL, device_xname(sc->sc_dev), "Rx multicast / vsi"); 5943 evcnt_attach_dynamic(&isc->isc_vsi_rx_broadcast, EVCNT_TYPE_MISC, 5944 NULL, device_xname(sc->sc_dev), "Rx broadcast / vsi"); 5945 5946 evcnt_attach_dynamic(&isc->isc_tx_size_64, EVCNT_TYPE_MISC, 5947 NULL, device_xname(sc->sc_dev), "Tx size 64"); 5948 evcnt_attach_dynamic(&isc->isc_tx_size_127, EVCNT_TYPE_MISC, 5949 NULL, device_xname(sc->sc_dev), "Tx size 127"); 5950 evcnt_attach_dynamic(&isc->isc_tx_size_255, EVCNT_TYPE_MISC, 5951 NULL, device_xname(sc->sc_dev), "Tx size 255"); 5952 evcnt_attach_dynamic(&isc->isc_tx_size_511, EVCNT_TYPE_MISC, 5953 NULL, device_xname(sc->sc_dev), "Tx size 511"); 5954 evcnt_attach_dynamic(&isc->isc_tx_size_1023, EVCNT_TYPE_MISC, 5955 NULL, device_xname(sc->sc_dev), "Tx size 1023"); 5956 evcnt_attach_dynamic(&isc->isc_tx_size_1522, EVCNT_TYPE_MISC, 5957 NULL, device_xname(sc->sc_dev), "Tx size 1522"); 5958 evcnt_attach_dynamic(&isc->isc_tx_size_big, EVCNT_TYPE_MISC, 5959 NULL, device_xname(sc->sc_dev), "Tx jumbo packets"); 5960 5961 evcnt_attach_dynamic(&isc->isc_tx_bytes, EVCNT_TYPE_MISC, 5962 NULL, device_xname(sc->sc_dev), "Tx bytes / port"); 5963 evcnt_attach_dynamic(&isc->isc_tx_dropped_link_down, EVCNT_TYPE_MISC, 5964 NULL, device_xname(sc->sc_dev), 5965 "Tx dropped due to link down / port"); 5966 evcnt_attach_dynamic(&isc->isc_tx_unicast, EVCNT_TYPE_MISC, 5967 NULL, device_xname(sc->sc_dev), "Tx unicast / port"); 5968 evcnt_attach_dynamic(&isc->isc_tx_multicast, EVCNT_TYPE_MISC, 5969 NULL, device_xname(sc->sc_dev), "Tx multicast / port"); 5970 evcnt_attach_dynamic(&isc->isc_tx_broadcast, EVCNT_TYPE_MISC, 5971 NULL, device_xname(sc->sc_dev), "Tx broadcast / port"); 5972 5973 evcnt_attach_dynamic(&isc->isc_vsi_tx_bytes, EVCNT_TYPE_MISC, 5974 NULL, device_xname(sc->sc_dev), "Tx bytes / vsi"); 5975 evcnt_attach_dynamic(&isc->isc_vsi_tx_errors, EVCNT_TYPE_MISC, 5976 NULL, device_xname(sc->sc_dev), "Tx errors / vsi"); 5977 evcnt_attach_dynamic(&isc->isc_vsi_tx_unicast, EVCNT_TYPE_MISC, 5978 NULL, device_xname(sc->sc_dev), "Tx unicast / vsi"); 5979 evcnt_attach_dynamic(&isc->isc_vsi_tx_multicast, EVCNT_TYPE_MISC, 5980 NULL, device_xname(sc->sc_dev), "Tx multicast / vsi"); 5981 evcnt_attach_dynamic(&isc->isc_vsi_tx_broadcast, EVCNT_TYPE_MISC, 5982 NULL, device_xname(sc->sc_dev), "Tx broadcast / vsi"); 5983 5984 sc->sc_stats_intval = ixl_param_stats_interval; 5985 callout_init(&sc->sc_stats_callout, CALLOUT_MPSAFE); 5986 callout_setfunc(&sc->sc_stats_callout, ixl_stats_callout, sc); 5987 ixl_work_set(&sc->sc_stats_task, ixl_stats_update, sc); 5988 5989 return 0; 5990 } 5991 5992 static void 5993 ixl_teardown_stats(struct ixl_softc *sc) 5994 { 5995 struct ixl_tx_ring *txr; 5996 struct ixl_rx_ring *rxr; 5997 struct ixl_stats_counters *isc; 5998 unsigned int i; 5999 6000 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6001 txr = sc->sc_qps[i].qp_txr; 6002 rxr = sc->sc_qps[i].qp_rxr; 6003 6004 evcnt_detach(&txr->txr_defragged); 6005 evcnt_detach(&txr->txr_defrag_failed); 6006 evcnt_detach(&txr->txr_pcqdrop); 6007 evcnt_detach(&txr->txr_transmitdef); 6008 evcnt_detach(&txr->txr_intr); 6009 evcnt_detach(&txr->txr_defer); 6010 6011 evcnt_detach(&rxr->rxr_mgethdr_failed); 6012 evcnt_detach(&rxr->rxr_mgetcl_failed); 6013 evcnt_detach(&rxr->rxr_mbuf_load_failed); 6014 evcnt_detach(&rxr->rxr_intr); 6015 evcnt_detach(&rxr->rxr_defer); 6016 } 6017 6018 isc = &sc->sc_stats_counters; 6019 evcnt_detach(&isc->isc_crc_errors); 6020 evcnt_detach(&isc->isc_illegal_bytes); 6021 evcnt_detach(&isc->isc_mac_local_faults); 6022 evcnt_detach(&isc->isc_mac_remote_faults); 6023 evcnt_detach(&isc->isc_link_xon_rx); 6024 evcnt_detach(&isc->isc_link_xon_tx); 6025 evcnt_detach(&isc->isc_link_xoff_rx); 6026 evcnt_detach(&isc->isc_link_xoff_tx); 6027 evcnt_detach(&isc->isc_rx_fragments); 6028 evcnt_detach(&isc->isc_rx_jabber); 6029 evcnt_detach(&isc->isc_rx_bytes); 6030 evcnt_detach(&isc->isc_rx_discards); 6031 evcnt_detach(&isc->isc_rx_unicast); 6032 evcnt_detach(&isc->isc_rx_multicast); 6033 evcnt_detach(&isc->isc_rx_broadcast); 6034 evcnt_detach(&isc->isc_rx_size_64); 6035 evcnt_detach(&isc->isc_rx_size_127); 6036 evcnt_detach(&isc->isc_rx_size_255); 6037 evcnt_detach(&isc->isc_rx_size_511); 6038 evcnt_detach(&isc->isc_rx_size_1023); 6039 evcnt_detach(&isc->isc_rx_size_1522); 6040 evcnt_detach(&isc->isc_rx_size_big); 6041 evcnt_detach(&isc->isc_rx_undersize); 6042 evcnt_detach(&isc->isc_rx_oversize); 6043 evcnt_detach(&isc->isc_tx_bytes); 6044 evcnt_detach(&isc->isc_tx_dropped_link_down); 6045 evcnt_detach(&isc->isc_tx_unicast); 6046 evcnt_detach(&isc->isc_tx_multicast); 6047 evcnt_detach(&isc->isc_tx_broadcast); 6048 evcnt_detach(&isc->isc_tx_size_64); 6049 evcnt_detach(&isc->isc_tx_size_127); 6050 evcnt_detach(&isc->isc_tx_size_255); 6051 evcnt_detach(&isc->isc_tx_size_511); 6052 evcnt_detach(&isc->isc_tx_size_1023); 6053 evcnt_detach(&isc->isc_tx_size_1522); 6054 evcnt_detach(&isc->isc_tx_size_big); 6055 evcnt_detach(&isc->isc_vsi_rx_discards); 6056 evcnt_detach(&isc->isc_vsi_rx_bytes); 6057 evcnt_detach(&isc->isc_vsi_rx_unicast); 6058 evcnt_detach(&isc->isc_vsi_rx_multicast); 6059 evcnt_detach(&isc->isc_vsi_rx_broadcast); 6060 evcnt_detach(&isc->isc_vsi_tx_errors); 6061 evcnt_detach(&isc->isc_vsi_tx_bytes); 6062 evcnt_detach(&isc->isc_vsi_tx_unicast); 6063 evcnt_detach(&isc->isc_vsi_tx_multicast); 6064 evcnt_detach(&isc->isc_vsi_tx_broadcast); 6065 6066 evcnt_detach(&sc->sc_event_atq); 6067 evcnt_detach(&sc->sc_event_link); 6068 evcnt_detach(&sc->sc_event_ecc_err); 6069 evcnt_detach(&sc->sc_event_pci_exception); 6070 evcnt_detach(&sc->sc_event_crit_err); 6071 6072 callout_destroy(&sc->sc_stats_callout); 6073 } 6074 6075 static void 6076 ixl_stats_callout(void *xsc) 6077 { 6078 struct ixl_softc *sc = xsc; 6079 6080 ixl_work_add(sc->sc_workq, &sc->sc_stats_task); 6081 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 6082 } 6083 6084 static uint64_t 6085 ixl_stat_delta(struct ixl_softc *sc, uint32_t reg_hi, uint32_t reg_lo, 6086 uint64_t *offset, bool has_offset) 6087 { 6088 uint64_t value, delta; 6089 int bitwidth; 6090 6091 bitwidth = reg_hi == 0 ? 32 : 48; 6092 6093 value = ixl_rd(sc, reg_lo); 6094 6095 if (bitwidth > 32) { 6096 value |= ((uint64_t)ixl_rd(sc, reg_hi) << 32); 6097 } 6098 6099 if (__predict_true(has_offset)) { 6100 delta = value; 6101 if (value < *offset) 6102 delta += ((uint64_t)1 << bitwidth); 6103 delta -= *offset; 6104 } else { 6105 delta = 0; 6106 } 6107 atomic_swap_64(offset, value); 6108 6109 return delta; 6110 } 6111 6112 static void 6113 ixl_stats_update(void *xsc) 6114 { 6115 struct ixl_softc *sc = xsc; 6116 struct ixl_stats_counters *isc; 6117 uint64_t delta; 6118 6119 isc = &sc->sc_stats_counters; 6120 6121 /* errors */ 6122 delta = ixl_stat_delta(sc, 6123 0, I40E_GLPRT_CRCERRS(sc->sc_port), 6124 &isc->isc_crc_errors_offset, isc->isc_has_offset); 6125 atomic_add_64(&isc->isc_crc_errors.ev_count, delta); 6126 6127 delta = ixl_stat_delta(sc, 6128 0, I40E_GLPRT_ILLERRC(sc->sc_port), 6129 &isc->isc_illegal_bytes_offset, isc->isc_has_offset); 6130 atomic_add_64(&isc->isc_illegal_bytes.ev_count, delta); 6131 6132 /* rx */ 6133 delta = ixl_stat_delta(sc, 6134 I40E_GLPRT_GORCH(sc->sc_port), I40E_GLPRT_GORCL(sc->sc_port), 6135 &isc->isc_rx_bytes_offset, isc->isc_has_offset); 6136 atomic_add_64(&isc->isc_rx_bytes.ev_count, delta); 6137 6138 delta = ixl_stat_delta(sc, 6139 0, I40E_GLPRT_RDPC(sc->sc_port), 6140 &isc->isc_rx_discards_offset, isc->isc_has_offset); 6141 atomic_add_64(&isc->isc_rx_discards.ev_count, delta); 6142 6143 delta = ixl_stat_delta(sc, 6144 I40E_GLPRT_UPRCH(sc->sc_port), I40E_GLPRT_UPRCL(sc->sc_port), 6145 &isc->isc_rx_unicast_offset, isc->isc_has_offset); 6146 atomic_add_64(&isc->isc_rx_unicast.ev_count, delta); 6147 6148 delta = ixl_stat_delta(sc, 6149 I40E_GLPRT_MPRCH(sc->sc_port), I40E_GLPRT_MPRCL(sc->sc_port), 6150 &isc->isc_rx_multicast_offset, isc->isc_has_offset); 6151 atomic_add_64(&isc->isc_rx_multicast.ev_count, delta); 6152 6153 delta = ixl_stat_delta(sc, 6154 I40E_GLPRT_BPRCH(sc->sc_port), I40E_GLPRT_BPRCL(sc->sc_port), 6155 &isc->isc_rx_broadcast_offset, isc->isc_has_offset); 6156 atomic_add_64(&isc->isc_rx_broadcast.ev_count, delta); 6157 6158 /* Packet size stats rx */ 6159 delta = ixl_stat_delta(sc, 6160 I40E_GLPRT_PRC64H(sc->sc_port), I40E_GLPRT_PRC64L(sc->sc_port), 6161 &isc->isc_rx_size_64_offset, isc->isc_has_offset); 6162 atomic_add_64(&isc->isc_rx_size_64.ev_count, delta); 6163 6164 delta = ixl_stat_delta(sc, 6165 I40E_GLPRT_PRC127H(sc->sc_port), I40E_GLPRT_PRC127L(sc->sc_port), 6166 &isc->isc_rx_size_127_offset, isc->isc_has_offset); 6167 atomic_add_64(&isc->isc_rx_size_127.ev_count, delta); 6168 6169 delta = ixl_stat_delta(sc, 6170 I40E_GLPRT_PRC255H(sc->sc_port), I40E_GLPRT_PRC255L(sc->sc_port), 6171 &isc->isc_rx_size_255_offset, isc->isc_has_offset); 6172 atomic_add_64(&isc->isc_rx_size_255.ev_count, delta); 6173 6174 delta = ixl_stat_delta(sc, 6175 I40E_GLPRT_PRC511H(sc->sc_port), I40E_GLPRT_PRC511L(sc->sc_port), 6176 &isc->isc_rx_size_511_offset, isc->isc_has_offset); 6177 atomic_add_64(&isc->isc_rx_size_511.ev_count, delta); 6178 6179 delta = ixl_stat_delta(sc, 6180 I40E_GLPRT_PRC1023H(sc->sc_port), I40E_GLPRT_PRC1023L(sc->sc_port), 6181 &isc->isc_rx_size_1023_offset, isc->isc_has_offset); 6182 atomic_add_64(&isc->isc_rx_size_1023.ev_count, delta); 6183 6184 delta = ixl_stat_delta(sc, 6185 I40E_GLPRT_PRC1522H(sc->sc_port), I40E_GLPRT_PRC1522L(sc->sc_port), 6186 &isc->isc_rx_size_1522_offset, isc->isc_has_offset); 6187 atomic_add_64(&isc->isc_rx_size_1522.ev_count, delta); 6188 6189 delta = ixl_stat_delta(sc, 6190 I40E_GLPRT_PRC9522H(sc->sc_port), I40E_GLPRT_PRC9522L(sc->sc_port), 6191 &isc->isc_rx_size_big_offset, isc->isc_has_offset); 6192 atomic_add_64(&isc->isc_rx_size_big.ev_count, delta); 6193 6194 delta = ixl_stat_delta(sc, 6195 0, I40E_GLPRT_RUC(sc->sc_port), 6196 &isc->isc_rx_undersize_offset, isc->isc_has_offset); 6197 atomic_add_64(&isc->isc_rx_undersize.ev_count, delta); 6198 6199 delta = ixl_stat_delta(sc, 6200 0, I40E_GLPRT_ROC(sc->sc_port), 6201 &isc->isc_rx_oversize_offset, isc->isc_has_offset); 6202 atomic_add_64(&isc->isc_rx_oversize.ev_count, delta); 6203 6204 /* tx */ 6205 delta = ixl_stat_delta(sc, 6206 I40E_GLPRT_GOTCH(sc->sc_port), I40E_GLPRT_GOTCL(sc->sc_port), 6207 &isc->isc_tx_bytes_offset, isc->isc_has_offset); 6208 atomic_add_64(&isc->isc_tx_bytes.ev_count, delta); 6209 6210 delta = ixl_stat_delta(sc, 6211 0, I40E_GLPRT_TDOLD(sc->sc_port), 6212 &isc->isc_tx_dropped_link_down_offset, isc->isc_has_offset); 6213 atomic_add_64(&isc->isc_tx_dropped_link_down.ev_count, delta); 6214 6215 delta = ixl_stat_delta(sc, 6216 I40E_GLPRT_UPTCH(sc->sc_port), I40E_GLPRT_UPTCL(sc->sc_port), 6217 &isc->isc_tx_unicast_offset, isc->isc_has_offset); 6218 atomic_add_64(&isc->isc_tx_unicast.ev_count, delta); 6219 6220 delta = ixl_stat_delta(sc, 6221 I40E_GLPRT_MPTCH(sc->sc_port), I40E_GLPRT_MPTCL(sc->sc_port), 6222 &isc->isc_tx_multicast_offset, isc->isc_has_offset); 6223 atomic_add_64(&isc->isc_tx_multicast.ev_count, delta); 6224 6225 delta = ixl_stat_delta(sc, 6226 I40E_GLPRT_BPTCH(sc->sc_port), I40E_GLPRT_BPTCL(sc->sc_port), 6227 &isc->isc_tx_broadcast_offset, isc->isc_has_offset); 6228 atomic_add_64(&isc->isc_tx_broadcast.ev_count, delta); 6229 6230 /* Packet size stats tx */ 6231 delta = ixl_stat_delta(sc, 6232 I40E_GLPRT_PTC64L(sc->sc_port), I40E_GLPRT_PTC64L(sc->sc_port), 6233 &isc->isc_tx_size_64_offset, isc->isc_has_offset); 6234 atomic_add_64(&isc->isc_tx_size_64.ev_count, delta); 6235 6236 delta = ixl_stat_delta(sc, 6237 I40E_GLPRT_PTC127H(sc->sc_port), I40E_GLPRT_PTC127L(sc->sc_port), 6238 &isc->isc_tx_size_127_offset, isc->isc_has_offset); 6239 atomic_add_64(&isc->isc_tx_size_127.ev_count, delta); 6240 6241 delta = ixl_stat_delta(sc, 6242 I40E_GLPRT_PTC255H(sc->sc_port), I40E_GLPRT_PTC255L(sc->sc_port), 6243 &isc->isc_tx_size_255_offset, isc->isc_has_offset); 6244 atomic_add_64(&isc->isc_tx_size_255.ev_count, delta); 6245 6246 delta = ixl_stat_delta(sc, 6247 I40E_GLPRT_PTC511H(sc->sc_port), I40E_GLPRT_PTC511L(sc->sc_port), 6248 &isc->isc_tx_size_511_offset, isc->isc_has_offset); 6249 atomic_add_64(&isc->isc_tx_size_511.ev_count, delta); 6250 6251 delta = ixl_stat_delta(sc, 6252 I40E_GLPRT_PTC1023H(sc->sc_port), I40E_GLPRT_PTC1023L(sc->sc_port), 6253 &isc->isc_tx_size_1023_offset, isc->isc_has_offset); 6254 atomic_add_64(&isc->isc_tx_size_1023.ev_count, delta); 6255 6256 delta = ixl_stat_delta(sc, 6257 I40E_GLPRT_PTC1522H(sc->sc_port), I40E_GLPRT_PTC1522L(sc->sc_port), 6258 &isc->isc_tx_size_1522_offset, isc->isc_has_offset); 6259 atomic_add_64(&isc->isc_tx_size_1522.ev_count, delta); 6260 6261 delta = ixl_stat_delta(sc, 6262 I40E_GLPRT_PTC9522H(sc->sc_port), I40E_GLPRT_PTC9522L(sc->sc_port), 6263 &isc->isc_tx_size_big_offset, isc->isc_has_offset); 6264 atomic_add_64(&isc->isc_tx_size_big.ev_count, delta); 6265 6266 /* mac faults */ 6267 delta = ixl_stat_delta(sc, 6268 0, I40E_GLPRT_MLFC(sc->sc_port), 6269 &isc->isc_mac_local_faults_offset, isc->isc_has_offset); 6270 atomic_add_64(&isc->isc_mac_local_faults.ev_count, delta); 6271 6272 delta = ixl_stat_delta(sc, 6273 0, I40E_GLPRT_MRFC(sc->sc_port), 6274 &isc->isc_mac_remote_faults_offset, isc->isc_has_offset); 6275 atomic_add_64(&isc->isc_mac_remote_faults.ev_count, delta); 6276 6277 /* Flow control (LFC) stats */ 6278 delta = ixl_stat_delta(sc, 6279 0, I40E_GLPRT_LXONRXC(sc->sc_port), 6280 &isc->isc_link_xon_rx_offset, isc->isc_has_offset); 6281 atomic_add_64(&isc->isc_link_xon_rx.ev_count, delta); 6282 6283 delta = ixl_stat_delta(sc, 6284 0, I40E_GLPRT_LXONTXC(sc->sc_port), 6285 &isc->isc_link_xon_tx_offset, isc->isc_has_offset); 6286 atomic_add_64(&isc->isc_link_xon_tx.ev_count, delta); 6287 6288 delta = ixl_stat_delta(sc, 6289 0, I40E_GLPRT_LXOFFRXC(sc->sc_port), 6290 &isc->isc_link_xoff_rx_offset, isc->isc_has_offset); 6291 atomic_add_64(&isc->isc_link_xoff_rx.ev_count, delta); 6292 6293 delta = ixl_stat_delta(sc, 6294 0, I40E_GLPRT_LXOFFTXC(sc->sc_port), 6295 &isc->isc_link_xoff_tx_offset, isc->isc_has_offset); 6296 atomic_add_64(&isc->isc_link_xoff_tx.ev_count, delta); 6297 6298 /* fragments */ 6299 delta = ixl_stat_delta(sc, 6300 0, I40E_GLPRT_RFC(sc->sc_port), 6301 &isc->isc_rx_fragments_offset, isc->isc_has_offset); 6302 atomic_add_64(&isc->isc_rx_fragments.ev_count, delta); 6303 6304 delta = ixl_stat_delta(sc, 6305 0, I40E_GLPRT_RJC(sc->sc_port), 6306 &isc->isc_rx_jabber_offset, isc->isc_has_offset); 6307 atomic_add_64(&isc->isc_rx_jabber.ev_count, delta); 6308 6309 /* VSI rx counters */ 6310 delta = ixl_stat_delta(sc, 6311 0, I40E_GLV_RDPC(sc->sc_vsi_stat_counter_idx), 6312 &isc->isc_vsi_rx_discards_offset, isc->isc_has_offset); 6313 atomic_add_64(&isc->isc_vsi_rx_discards.ev_count, delta); 6314 6315 delta = ixl_stat_delta(sc, 6316 I40E_GLV_GORCH(sc->sc_vsi_stat_counter_idx), 6317 I40E_GLV_GORCL(sc->sc_vsi_stat_counter_idx), 6318 &isc->isc_vsi_rx_bytes_offset, isc->isc_has_offset); 6319 atomic_add_64(&isc->isc_vsi_rx_bytes.ev_count, delta); 6320 6321 delta = ixl_stat_delta(sc, 6322 I40E_GLV_UPRCH(sc->sc_vsi_stat_counter_idx), 6323 I40E_GLV_UPRCL(sc->sc_vsi_stat_counter_idx), 6324 &isc->isc_vsi_rx_unicast_offset, isc->isc_has_offset); 6325 atomic_add_64(&isc->isc_vsi_rx_unicast.ev_count, delta); 6326 6327 delta = ixl_stat_delta(sc, 6328 I40E_GLV_MPRCH(sc->sc_vsi_stat_counter_idx), 6329 I40E_GLV_MPRCL(sc->sc_vsi_stat_counter_idx), 6330 &isc->isc_vsi_rx_multicast_offset, isc->isc_has_offset); 6331 atomic_add_64(&isc->isc_vsi_rx_multicast.ev_count, delta); 6332 6333 delta = ixl_stat_delta(sc, 6334 I40E_GLV_BPRCH(sc->sc_vsi_stat_counter_idx), 6335 I40E_GLV_BPRCL(sc->sc_vsi_stat_counter_idx), 6336 &isc->isc_vsi_rx_broadcast_offset, isc->isc_has_offset); 6337 atomic_add_64(&isc->isc_vsi_rx_broadcast.ev_count, delta); 6338 6339 /* VSI tx counters */ 6340 delta = ixl_stat_delta(sc, 6341 0, I40E_GLV_TEPC(sc->sc_vsi_stat_counter_idx), 6342 &isc->isc_vsi_tx_errors_offset, isc->isc_has_offset); 6343 atomic_add_64(&isc->isc_vsi_tx_errors.ev_count, delta); 6344 6345 delta = ixl_stat_delta(sc, 6346 I40E_GLV_GOTCH(sc->sc_vsi_stat_counter_idx), 6347 I40E_GLV_GOTCL(sc->sc_vsi_stat_counter_idx), 6348 &isc->isc_vsi_tx_bytes_offset, isc->isc_has_offset); 6349 atomic_add_64(&isc->isc_vsi_tx_bytes.ev_count, delta); 6350 6351 delta = ixl_stat_delta(sc, 6352 I40E_GLV_UPTCH(sc->sc_vsi_stat_counter_idx), 6353 I40E_GLV_UPTCL(sc->sc_vsi_stat_counter_idx), 6354 &isc->isc_vsi_tx_unicast_offset, isc->isc_has_offset); 6355 atomic_add_64(&isc->isc_vsi_tx_unicast.ev_count, delta); 6356 6357 delta = ixl_stat_delta(sc, 6358 I40E_GLV_MPTCH(sc->sc_vsi_stat_counter_idx), 6359 I40E_GLV_MPTCL(sc->sc_vsi_stat_counter_idx), 6360 &isc->isc_vsi_tx_multicast_offset, isc->isc_has_offset); 6361 atomic_add_64(&isc->isc_vsi_tx_multicast.ev_count, delta); 6362 6363 delta = ixl_stat_delta(sc, 6364 I40E_GLV_BPTCH(sc->sc_vsi_stat_counter_idx), 6365 I40E_GLV_BPTCL(sc->sc_vsi_stat_counter_idx), 6366 &isc->isc_vsi_tx_broadcast_offset, isc->isc_has_offset); 6367 atomic_add_64(&isc->isc_vsi_tx_broadcast.ev_count, delta); 6368 } 6369 6370 static int 6371 ixl_setup_sysctls(struct ixl_softc *sc) 6372 { 6373 const char *devname; 6374 struct sysctllog **log; 6375 const struct sysctlnode *rnode, *rxnode, *txnode; 6376 int error; 6377 6378 log = &sc->sc_sysctllog; 6379 devname = device_xname(sc->sc_dev); 6380 6381 error = sysctl_createv(log, 0, NULL, &rnode, 6382 0, CTLTYPE_NODE, devname, 6383 SYSCTL_DESCR("ixl information and settings"), 6384 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 6385 if (error) 6386 goto out; 6387 6388 error = sysctl_createv(log, 0, &rnode, NULL, 6389 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 6390 SYSCTL_DESCR("Use workqueue for packet processing"), 6391 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 6392 if (error) 6393 goto out; 6394 6395 error = sysctl_createv(log, 0, &rnode, NULL, 6396 CTLFLAG_READONLY, CTLTYPE_INT, "stats_interval", 6397 SYSCTL_DESCR("Statistics collection interval in milliseconds"), 6398 NULL, 0, &sc->sc_stats_intval, 0, CTL_CREATE, CTL_EOL); 6399 6400 error = sysctl_createv(log, 0, &rnode, &rxnode, 6401 0, CTLTYPE_NODE, "rx", 6402 SYSCTL_DESCR("ixl information and settings for Rx"), 6403 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6404 if (error) 6405 goto out; 6406 6407 error = sysctl_createv(log, 0, &rxnode, NULL, 6408 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6409 SYSCTL_DESCR("max number of Rx packets" 6410 " to process for interrupt processing"), 6411 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6412 if (error) 6413 goto out; 6414 6415 error = sysctl_createv(log, 0, &rxnode, NULL, 6416 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6417 SYSCTL_DESCR("max number of Rx packets" 6418 " to process for deferred processing"), 6419 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 6420 if (error) 6421 goto out; 6422 6423 error = sysctl_createv(log, 0, &rnode, &txnode, 6424 0, CTLTYPE_NODE, "tx", 6425 SYSCTL_DESCR("ixl information and settings for Tx"), 6426 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6427 if (error) 6428 goto out; 6429 6430 error = sysctl_createv(log, 0, &txnode, NULL, 6431 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6432 SYSCTL_DESCR("max number of Tx packets" 6433 " to process for interrupt processing"), 6434 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6435 if (error) 6436 goto out; 6437 6438 error = sysctl_createv(log, 0, &txnode, NULL, 6439 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6440 SYSCTL_DESCR("max number of Tx packets" 6441 " to process for deferred processing"), 6442 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 6443 if (error) 6444 goto out; 6445 6446 out: 6447 if (error) { 6448 aprint_error_dev(sc->sc_dev, 6449 "unable to create sysctl node\n"); 6450 sysctl_teardown(log); 6451 } 6452 6453 return error; 6454 } 6455 6456 static void 6457 ixl_teardown_sysctls(struct ixl_softc *sc) 6458 { 6459 6460 sysctl_teardown(&sc->sc_sysctllog); 6461 } 6462 6463 static struct workqueue * 6464 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 6465 { 6466 struct workqueue *wq; 6467 int error; 6468 6469 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 6470 prio, ipl, flags); 6471 6472 if (error) 6473 return NULL; 6474 6475 return wq; 6476 } 6477 6478 static void 6479 ixl_workq_destroy(struct workqueue *wq) 6480 { 6481 6482 workqueue_destroy(wq); 6483 } 6484 6485 static void 6486 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 6487 { 6488 6489 memset(work, 0, sizeof(*work)); 6490 work->ixw_func = func; 6491 work->ixw_arg = arg; 6492 } 6493 6494 static void 6495 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 6496 { 6497 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 6498 return; 6499 6500 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 6501 } 6502 6503 static void 6504 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 6505 { 6506 6507 workqueue_wait(wq, &work->ixw_cookie); 6508 } 6509 6510 static void 6511 ixl_workq_work(struct work *wk, void *context) 6512 { 6513 struct ixl_work *work; 6514 6515 work = container_of(wk, struct ixl_work, ixw_cookie); 6516 6517 atomic_swap_uint(&work->ixw_added, 0); 6518 work->ixw_func(work->ixw_arg); 6519 } 6520 6521 static int 6522 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 6523 { 6524 struct ixl_aq_desc iaq; 6525 6526 memset(&iaq, 0, sizeof(iaq)); 6527 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 6528 iaq.iaq_param[1] = htole32(reg); 6529 6530 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6531 return ETIMEDOUT; 6532 6533 switch (htole16(iaq.iaq_retval)) { 6534 case IXL_AQ_RC_OK: 6535 /* success */ 6536 break; 6537 case IXL_AQ_RC_EACCES: 6538 return EPERM; 6539 case IXL_AQ_RC_EAGAIN: 6540 return EAGAIN; 6541 default: 6542 return EIO; 6543 } 6544 6545 *rv = htole32(iaq.iaq_param[3]); 6546 return 0; 6547 } 6548 6549 static uint32_t 6550 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 6551 { 6552 uint32_t val; 6553 int rv, retry, retry_limit; 6554 6555 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6556 retry_limit = 5; 6557 } else { 6558 retry_limit = 0; 6559 } 6560 6561 for (retry = 0; retry < retry_limit; retry++) { 6562 rv = ixl_rx_ctl_read(sc, reg, &val); 6563 if (rv == 0) 6564 return val; 6565 else if (rv == EAGAIN) 6566 delaymsec(1); 6567 else 6568 break; 6569 } 6570 6571 val = ixl_rd(sc, reg); 6572 6573 return val; 6574 } 6575 6576 static int 6577 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6578 { 6579 struct ixl_aq_desc iaq; 6580 6581 memset(&iaq, 0, sizeof(iaq)); 6582 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 6583 iaq.iaq_param[1] = htole32(reg); 6584 iaq.iaq_param[3] = htole32(value); 6585 6586 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6587 return ETIMEDOUT; 6588 6589 switch (htole16(iaq.iaq_retval)) { 6590 case IXL_AQ_RC_OK: 6591 /* success */ 6592 break; 6593 case IXL_AQ_RC_EACCES: 6594 return EPERM; 6595 case IXL_AQ_RC_EAGAIN: 6596 return EAGAIN; 6597 default: 6598 return EIO; 6599 } 6600 6601 return 0; 6602 } 6603 6604 static void 6605 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6606 { 6607 int rv, retry, retry_limit; 6608 6609 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6610 retry_limit = 5; 6611 } else { 6612 retry_limit = 0; 6613 } 6614 6615 for (retry = 0; retry < retry_limit; retry++) { 6616 rv = ixl_rx_ctl_write(sc, reg, value); 6617 if (rv == 0) 6618 return; 6619 else if (rv == EAGAIN) 6620 delaymsec(1); 6621 else 6622 break; 6623 } 6624 6625 ixl_wr(sc, reg, value); 6626 } 6627 6628 static int 6629 ixl_nvm_lock(struct ixl_softc *sc, char rw) 6630 { 6631 struct ixl_aq_desc iaq; 6632 struct ixl_aq_req_resource_param *param; 6633 int rv; 6634 6635 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6636 return 0; 6637 6638 memset(&iaq, 0, sizeof(iaq)); 6639 iaq.iaq_opcode = htole16(IXL_AQ_OP_REQUEST_RESOURCE); 6640 6641 param = (struct ixl_aq_req_resource_param *)&iaq.iaq_param; 6642 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6643 if (rw == 'R') { 6644 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_READ); 6645 } else { 6646 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_WRITE); 6647 } 6648 6649 rv = ixl_atq_poll(sc, &iaq, 250); 6650 6651 if (rv != 0) 6652 return ETIMEDOUT; 6653 6654 switch (le16toh(iaq.iaq_retval)) { 6655 case IXL_AQ_RC_OK: 6656 break; 6657 case IXL_AQ_RC_EACCES: 6658 return EACCES; 6659 case IXL_AQ_RC_EBUSY: 6660 return EBUSY; 6661 case IXL_AQ_RC_EPERM: 6662 return EPERM; 6663 } 6664 6665 return 0; 6666 } 6667 6668 static int 6669 ixl_nvm_unlock(struct ixl_softc *sc) 6670 { 6671 struct ixl_aq_desc iaq; 6672 struct ixl_aq_rel_resource_param *param; 6673 int rv; 6674 6675 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6676 return 0; 6677 6678 memset(&iaq, 0, sizeof(iaq)); 6679 iaq.iaq_opcode = htole16(IXL_AQ_OP_RELEASE_RESOURCE); 6680 6681 param = (struct ixl_aq_rel_resource_param *)&iaq.iaq_param; 6682 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6683 6684 rv = ixl_atq_poll(sc, &iaq, 250); 6685 6686 if (rv != 0) 6687 return ETIMEDOUT; 6688 6689 switch (le16toh(iaq.iaq_retval)) { 6690 case IXL_AQ_RC_OK: 6691 break; 6692 default: 6693 return EIO; 6694 } 6695 return 0; 6696 } 6697 6698 static int 6699 ixl_srdone_poll(struct ixl_softc *sc) 6700 { 6701 int wait_count; 6702 uint32_t reg; 6703 6704 for (wait_count = 0; wait_count < IXL_SRRD_SRCTL_ATTEMPTS; 6705 wait_count++) { 6706 reg = ixl_rd(sc, I40E_GLNVM_SRCTL); 6707 if (ISSET(reg, I40E_GLNVM_SRCTL_DONE_MASK)) 6708 break; 6709 6710 delaymsec(5); 6711 } 6712 6713 if (wait_count == IXL_SRRD_SRCTL_ATTEMPTS) 6714 return -1; 6715 6716 return 0; 6717 } 6718 6719 static int 6720 ixl_nvm_read_srctl(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 6721 { 6722 uint32_t reg; 6723 6724 if (ixl_srdone_poll(sc) != 0) 6725 return ETIMEDOUT; 6726 6727 reg = ((uint32_t)offset << I40E_GLNVM_SRCTL_ADDR_SHIFT) | 6728 __BIT(I40E_GLNVM_SRCTL_START_SHIFT); 6729 ixl_wr(sc, I40E_GLNVM_SRCTL, reg); 6730 6731 if (ixl_srdone_poll(sc) != 0) { 6732 aprint_debug("NVM read error: couldn't access " 6733 "Shadow RAM address: 0x%x\n", offset); 6734 return ETIMEDOUT; 6735 } 6736 6737 reg = ixl_rd(sc, I40E_GLNVM_SRDATA); 6738 *data = (uint16_t)__SHIFTOUT(reg, I40E_GLNVM_SRDATA_RDDATA_MASK); 6739 6740 return 0; 6741 } 6742 6743 static int 6744 ixl_nvm_read_aq(struct ixl_softc *sc, uint16_t offset_word, 6745 void *data, size_t len) 6746 { 6747 struct ixl_dmamem *idm; 6748 struct ixl_aq_desc iaq; 6749 struct ixl_aq_nvm_param *param; 6750 uint32_t offset_bytes; 6751 int rv; 6752 6753 idm = &sc->sc_aqbuf; 6754 if (len > IXL_DMA_LEN(idm)) 6755 return ENOMEM; 6756 6757 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 6758 memset(&iaq, 0, sizeof(iaq)); 6759 iaq.iaq_opcode = htole16(IXL_AQ_OP_NVM_READ); 6760 iaq.iaq_flags = htole16(IXL_AQ_BUF | 6761 ((len > I40E_AQ_LARGE_BUF) ? IXL_AQ_LB : 0)); 6762 iaq.iaq_datalen = htole16(len); 6763 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 6764 6765 param = (struct ixl_aq_nvm_param *)iaq.iaq_param; 6766 param->command_flags = IXL_AQ_NVM_LAST_CMD; 6767 param->module_pointer = 0; 6768 param->length = htole16(len); 6769 offset_bytes = (uint32_t)offset_word * 2; 6770 offset_bytes &= 0x00FFFFFF; 6771 param->offset = htole32(offset_bytes); 6772 6773 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 6774 BUS_DMASYNC_PREREAD); 6775 6776 rv = ixl_atq_poll(sc, &iaq, 250); 6777 6778 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 6779 BUS_DMASYNC_POSTREAD); 6780 6781 if (rv != 0) { 6782 return ETIMEDOUT; 6783 } 6784 6785 switch (le16toh(iaq.iaq_retval)) { 6786 case IXL_AQ_RC_OK: 6787 break; 6788 case IXL_AQ_RC_EPERM: 6789 return EPERM; 6790 case IXL_AQ_RC_EINVAL: 6791 return EINVAL; 6792 case IXL_AQ_RC_EBUSY: 6793 return EBUSY; 6794 case IXL_AQ_RC_EIO: 6795 default: 6796 return EIO; 6797 } 6798 6799 memcpy(data, IXL_DMA_KVA(idm), len); 6800 6801 return 0; 6802 } 6803 6804 static int 6805 ixl_rd16_nvm(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 6806 { 6807 int error; 6808 uint16_t buf; 6809 6810 error = ixl_nvm_lock(sc, 'R'); 6811 if (error) 6812 return error; 6813 6814 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMREAD)) { 6815 error = ixl_nvm_read_aq(sc, offset, 6816 &buf, sizeof(buf)); 6817 if (error == 0) 6818 *data = le16toh(buf); 6819 } else { 6820 error = ixl_nvm_read_srctl(sc, offset, &buf); 6821 if (error == 0) 6822 *data = buf; 6823 } 6824 6825 ixl_nvm_unlock(sc); 6826 6827 return error; 6828 } 6829 6830 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 6831 6832 #ifdef _MODULE 6833 #include "ioconf.c" 6834 #endif 6835 6836 #ifdef _MODULE 6837 static void 6838 ixl_parse_modprop(prop_dictionary_t dict) 6839 { 6840 prop_object_t obj; 6841 int64_t val; 6842 uint64_t uval; 6843 6844 if (dict == NULL) 6845 return; 6846 6847 obj = prop_dictionary_get(dict, "nomsix"); 6848 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_BOOL) { 6849 ixl_param_nomsix = prop_bool_true((prop_bool_t)obj); 6850 } 6851 6852 obj = prop_dictionary_get(dict, "stats_interval"); 6853 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 6854 val = prop_number_integer_value((prop_number_t)obj); 6855 6856 /* the range has no reason */ 6857 if (100 < val && val < 180000) { 6858 ixl_param_stats_interval = val; 6859 } 6860 } 6861 6862 obj = prop_dictionary_get(dict, "nqps_limit"); 6863 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 6864 val = prop_number_integer_value((prop_number_t)obj); 6865 6866 if (val <= INT32_MAX) 6867 ixl_param_nqps_limit = val; 6868 } 6869 6870 obj = prop_dictionary_get(dict, "rx_ndescs"); 6871 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 6872 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 6873 6874 if (uval > 8) 6875 ixl_param_rx_ndescs = uval; 6876 } 6877 6878 obj = prop_dictionary_get(dict, "tx_ndescs"); 6879 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 6880 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 6881 6882 if (uval > IXL_TX_PKT_DESCS) 6883 ixl_param_tx_ndescs = uval; 6884 } 6885 6886 } 6887 #endif 6888 6889 static int 6890 if_ixl_modcmd(modcmd_t cmd, void *opaque) 6891 { 6892 int error = 0; 6893 6894 #ifdef _MODULE 6895 switch (cmd) { 6896 case MODULE_CMD_INIT: 6897 ixl_parse_modprop((prop_dictionary_t)opaque); 6898 error = config_init_component(cfdriver_ioconf_if_ixl, 6899 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 6900 break; 6901 case MODULE_CMD_FINI: 6902 error = config_fini_component(cfdriver_ioconf_if_ixl, 6903 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 6904 break; 6905 default: 6906 error = ENOTTY; 6907 break; 6908 } 6909 #endif 6910 6911 return error; 6912 } 6913
Indexes created Mon Sep 22 05:09:51 GMT 2025