if_ixl.c revision 1.56
1 /* $NetBSD: if_ixl.c,v 1.56 2020/02/25 08:05:24 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 __KERNEL_RCSID(0, "$NetBSD: if_ixl.c,v 1.56 2020/02/25 08:05:24 yamaguchi Exp $"); 78 79 #ifdef _KERNEL_OPT 80 #include "opt_net_mpsafe.h" 81 #include "opt_if_ixl.h" 82 #endif 83 84 #include <sys/param.h> 85 #include <sys/types.h> 86 87 #include <sys/cpu.h> 88 #include <sys/device.h> 89 #include <sys/evcnt.h> 90 #include <sys/interrupt.h> 91 #include <sys/kmem.h> 92 #include <sys/module.h> 93 #include <sys/mutex.h> 94 #include <sys/pcq.h> 95 #include <sys/syslog.h> 96 #include <sys/workqueue.h> 97 98 #include <sys/bus.h> 99 100 #include <net/bpf.h> 101 #include <net/if.h> 102 #include <net/if_dl.h> 103 #include <net/if_media.h> 104 #include <net/if_ether.h> 105 #include <net/rss_config.h> 106 107 #include <netinet/tcp.h> /* for struct tcphdr */ 108 #include <netinet/udp.h> /* for struct udphdr */ 109 110 #include <dev/pci/pcivar.h> 111 #include <dev/pci/pcidevs.h> 112 113 #include <dev/pci/if_ixlreg.h> 114 #include <dev/pci/if_ixlvar.h> 115 116 #include <prop/proplib.h> 117 118 struct ixl_softc; /* defined */ 119 120 #define I40E_PF_RESET_WAIT_COUNT 200 121 #define I40E_AQ_LARGE_BUF 512 122 123 /* bitfields for Tx queue mapping in QTX_CTL */ 124 #define I40E_QTX_CTL_VF_QUEUE 0x0 125 #define I40E_QTX_CTL_VM_QUEUE 0x1 126 #define I40E_QTX_CTL_PF_QUEUE 0x2 127 128 #define I40E_QUEUE_TYPE_EOL 0x7ff 129 #define I40E_INTR_NOTX_QUEUE 0 130 131 #define I40E_QUEUE_TYPE_RX 0x0 132 #define I40E_QUEUE_TYPE_TX 0x1 133 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 134 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 135 136 #define I40E_ITR_INDEX_RX 0x0 137 #define I40E_ITR_INDEX_TX 0x1 138 #define I40E_ITR_INDEX_OTHER 0x2 139 #define I40E_ITR_INDEX_NONE 0x3 140 141 #define I40E_INTR_NOTX_QUEUE 0 142 #define I40E_INTR_NOTX_INTR 0 143 #define I40E_INTR_NOTX_RX_QUEUE 0 144 #define I40E_INTR_NOTX_TX_QUEUE 1 145 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 146 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 147 148 #define BIT_ULL(a) (1ULL << (a)) 149 #define IXL_RSS_HENA_DEFAULT_BASE \ 150 (BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 151 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 152 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 153 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 154 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 155 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 156 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 157 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 158 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 159 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 160 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 161 #define IXL_RSS_HENA_DEFAULT_XL710 IXL_RSS_HENA_DEFAULT_BASE 162 #define IXL_RSS_HENA_DEFAULT_X722 (IXL_RSS_HENA_DEFAULT_XL710 | \ 163 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 164 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 165 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 166 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 167 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 168 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 169 #define I40E_HASH_LUT_SIZE_128 0 170 #define IXL_RSS_KEY_SIZE_REG 13 171 172 #define IXL_ICR0_CRIT_ERR_MASK \ 173 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 174 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 175 I40E_PFINT_ICR0_PE_CRITERR_MASK) 176 177 #define IXL_QUEUE_MAX_XL710 64 178 #define IXL_QUEUE_MAX_X722 128 179 180 #define IXL_TX_PKT_DESCS 8 181 #define IXL_TX_PKT_MAXSIZE (MCLBYTES * IXL_TX_PKT_DESCS) 182 #define IXL_TX_QUEUE_ALIGN 128 183 #define IXL_RX_QUEUE_ALIGN 128 184 185 #define IXL_MCLBYTES (MCLBYTES - ETHER_ALIGN) 186 #define IXL_MTU_ETHERLEN ETHER_HDR_LEN \ 187 + ETHER_CRC_LEN 188 #if 0 189 #define IXL_MAX_MTU (9728 - IXL_MTU_ETHERLEN) 190 #else 191 /* (dbuff * 5) - ETHER_HDR_LEN - ETHER_CRC_LEN */ 192 #define IXL_MAX_MTU (9600 - IXL_MTU_ETHERLEN) 193 #endif 194 #define IXL_MIN_MTU (ETHER_MIN_LEN - ETHER_CRC_LEN) 195 196 #define IXL_PCIREG PCI_MAPREG_START 197 198 #define IXL_ITR0 0x0 199 #define IXL_ITR1 0x1 200 #define IXL_ITR2 0x2 201 #define IXL_NOITR 0x3 202 203 #define IXL_AQ_NUM 256 204 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 205 #define IXL_AQ_ALIGN 64 /* lol */ 206 #define IXL_AQ_BUFLEN 4096 207 208 #define IXL_HMC_ROUNDUP 512 209 #define IXL_HMC_PGSIZE 4096 210 #define IXL_HMC_DVASZ sizeof(uint64_t) 211 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 212 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 213 #define IXL_HMC_PDVALID 1ULL 214 215 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 216 217 #define IXL_SRRD_SRCTL_ATTEMPTS 100000 218 219 struct ixl_aq_regs { 220 bus_size_t atq_tail; 221 bus_size_t atq_head; 222 bus_size_t atq_len; 223 bus_size_t atq_bal; 224 bus_size_t atq_bah; 225 226 bus_size_t arq_tail; 227 bus_size_t arq_head; 228 bus_size_t arq_len; 229 bus_size_t arq_bal; 230 bus_size_t arq_bah; 231 232 uint32_t atq_len_enable; 233 uint32_t atq_tail_mask; 234 uint32_t atq_head_mask; 235 236 uint32_t arq_len_enable; 237 uint32_t arq_tail_mask; 238 uint32_t arq_head_mask; 239 }; 240 241 struct ixl_phy_type { 242 uint64_t phy_type; 243 uint64_t ifm_type; 244 }; 245 246 struct ixl_speed_type { 247 uint8_t dev_speed; 248 uint64_t net_speed; 249 }; 250 251 struct ixl_aq_buf { 252 SIMPLEQ_ENTRY(ixl_aq_buf) 253 aqb_entry; 254 void *aqb_data; 255 bus_dmamap_t aqb_map; 256 bus_dma_segment_t aqb_seg; 257 size_t aqb_size; 258 int aqb_nsegs; 259 }; 260 SIMPLEQ_HEAD(ixl_aq_bufs, ixl_aq_buf); 261 262 struct ixl_dmamem { 263 bus_dmamap_t ixm_map; 264 bus_dma_segment_t ixm_seg; 265 int ixm_nsegs; 266 size_t ixm_size; 267 void *ixm_kva; 268 }; 269 270 #define IXL_DMA_MAP(_ixm) ((_ixm)->ixm_map) 271 #define IXL_DMA_DVA(_ixm) ((_ixm)->ixm_map->dm_segs[0].ds_addr) 272 #define IXL_DMA_KVA(_ixm) ((void *)(_ixm)->ixm_kva) 273 #define IXL_DMA_LEN(_ixm) ((_ixm)->ixm_size) 274 275 struct ixl_hmc_entry { 276 uint64_t hmc_base; 277 uint32_t hmc_count; 278 uint64_t hmc_size; 279 }; 280 281 enum ixl_hmc_types { 282 IXL_HMC_LAN_TX = 0, 283 IXL_HMC_LAN_RX, 284 IXL_HMC_FCOE_CTX, 285 IXL_HMC_FCOE_FILTER, 286 IXL_HMC_COUNT 287 }; 288 289 struct ixl_hmc_pack { 290 uint16_t offset; 291 uint16_t width; 292 uint16_t lsb; 293 }; 294 295 /* 296 * these hmc objects have weird sizes and alignments, so these are abstract 297 * representations of them that are nice for c to populate. 298 * 299 * the packing code relies on little-endian values being stored in the fields, 300 * no high bits in the fields being set, and the fields must be packed in the 301 * same order as they are in the ctx structure. 302 */ 303 304 struct ixl_hmc_rxq { 305 uint16_t head; 306 uint8_t cpuid; 307 uint64_t base; 308 #define IXL_HMC_RXQ_BASE_UNIT 128 309 uint16_t qlen; 310 uint16_t dbuff; 311 #define IXL_HMC_RXQ_DBUFF_UNIT 128 312 uint8_t hbuff; 313 #define IXL_HMC_RXQ_HBUFF_UNIT 64 314 uint8_t dtype; 315 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 316 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 317 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 318 uint8_t dsize; 319 #define IXL_HMC_RXQ_DSIZE_16 0 320 #define IXL_HMC_RXQ_DSIZE_32 1 321 uint8_t crcstrip; 322 uint8_t fc_ena; 323 uint8_t l2sel; 324 uint8_t hsplit_0; 325 uint8_t hsplit_1; 326 uint8_t showiv; 327 uint16_t rxmax; 328 uint8_t tphrdesc_ena; 329 uint8_t tphwdesc_ena; 330 uint8_t tphdata_ena; 331 uint8_t tphhead_ena; 332 uint8_t lrxqthresh; 333 uint8_t prefena; 334 }; 335 336 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 337 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 338 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 339 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 340 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 341 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 342 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 343 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 344 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 345 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 346 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 347 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 348 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 349 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 350 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 351 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 352 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 353 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 354 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 355 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 356 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 357 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 358 }; 359 360 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 361 362 struct ixl_hmc_txq { 363 uint16_t head; 364 uint8_t new_context; 365 uint64_t base; 366 #define IXL_HMC_TXQ_BASE_UNIT 128 367 uint8_t fc_ena; 368 uint8_t timesync_ena; 369 uint8_t fd_ena; 370 uint8_t alt_vlan_ena; 371 uint8_t cpuid; 372 uint16_t thead_wb; 373 uint8_t head_wb_ena; 374 #define IXL_HMC_TXQ_DESC_WB 0 375 #define IXL_HMC_TXQ_HEAD_WB 1 376 uint16_t qlen; 377 uint8_t tphrdesc_ena; 378 uint8_t tphrpacket_ena; 379 uint8_t tphwdesc_ena; 380 uint64_t head_wb_addr; 381 uint32_t crc; 382 uint16_t rdylist; 383 uint8_t rdylist_act; 384 }; 385 386 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 387 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 388 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 389 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 390 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 391 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 392 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 393 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 394 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 395 /* line 1 */ 396 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 397 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 398 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 399 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 400 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 401 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 402 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 403 /* line 7 */ 404 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 405 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 406 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 407 }; 408 409 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 410 411 struct ixl_work { 412 struct work ixw_cookie; 413 void (*ixw_func)(void *); 414 void *ixw_arg; 415 unsigned int ixw_added; 416 }; 417 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 418 419 struct ixl_tx_map { 420 struct mbuf *txm_m; 421 bus_dmamap_t txm_map; 422 unsigned int txm_eop; 423 }; 424 425 struct ixl_tx_ring { 426 kmutex_t txr_lock; 427 struct ixl_softc *txr_sc; 428 429 unsigned int txr_prod; 430 unsigned int txr_cons; 431 432 struct ixl_tx_map *txr_maps; 433 struct ixl_dmamem txr_mem; 434 435 bus_size_t txr_tail; 436 unsigned int txr_qid; 437 pcq_t *txr_intrq; 438 void *txr_si; 439 440 struct evcnt txr_defragged; 441 struct evcnt txr_defrag_failed; 442 struct evcnt txr_pcqdrop; 443 struct evcnt txr_transmitdef; 444 struct evcnt txr_intr; 445 struct evcnt txr_defer; 446 }; 447 448 struct ixl_rx_map { 449 struct mbuf *rxm_m; 450 bus_dmamap_t rxm_map; 451 }; 452 453 struct ixl_rx_ring { 454 kmutex_t rxr_lock; 455 456 unsigned int rxr_prod; 457 unsigned int rxr_cons; 458 459 struct ixl_rx_map *rxr_maps; 460 struct ixl_dmamem rxr_mem; 461 462 struct mbuf *rxr_m_head; 463 struct mbuf **rxr_m_tail; 464 465 bus_size_t rxr_tail; 466 unsigned int rxr_qid; 467 468 struct evcnt rxr_mgethdr_failed; 469 struct evcnt rxr_mgetcl_failed; 470 struct evcnt rxr_mbuf_load_failed; 471 struct evcnt rxr_intr; 472 struct evcnt rxr_defer; 473 }; 474 475 struct ixl_queue_pair { 476 struct ixl_softc *qp_sc; 477 struct ixl_tx_ring *qp_txr; 478 struct ixl_rx_ring *qp_rxr; 479 480 char qp_name[16]; 481 482 void *qp_si; 483 struct work qp_work; 484 bool qp_workqueue; 485 }; 486 487 struct ixl_atq { 488 struct ixl_aq_desc iatq_desc; 489 void (*iatq_fn)(struct ixl_softc *, 490 const struct ixl_aq_desc *); 491 }; 492 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 493 494 struct ixl_product { 495 unsigned int vendor_id; 496 unsigned int product_id; 497 }; 498 499 struct ixl_stats_counters { 500 bool isc_has_offset; 501 struct evcnt isc_crc_errors; 502 uint64_t isc_crc_errors_offset; 503 struct evcnt isc_illegal_bytes; 504 uint64_t isc_illegal_bytes_offset; 505 struct evcnt isc_rx_bytes; 506 uint64_t isc_rx_bytes_offset; 507 struct evcnt isc_rx_discards; 508 uint64_t isc_rx_discards_offset; 509 struct evcnt isc_rx_unicast; 510 uint64_t isc_rx_unicast_offset; 511 struct evcnt isc_rx_multicast; 512 uint64_t isc_rx_multicast_offset; 513 struct evcnt isc_rx_broadcast; 514 uint64_t isc_rx_broadcast_offset; 515 struct evcnt isc_rx_size_64; 516 uint64_t isc_rx_size_64_offset; 517 struct evcnt isc_rx_size_127; 518 uint64_t isc_rx_size_127_offset; 519 struct evcnt isc_rx_size_255; 520 uint64_t isc_rx_size_255_offset; 521 struct evcnt isc_rx_size_511; 522 uint64_t isc_rx_size_511_offset; 523 struct evcnt isc_rx_size_1023; 524 uint64_t isc_rx_size_1023_offset; 525 struct evcnt isc_rx_size_1522; 526 uint64_t isc_rx_size_1522_offset; 527 struct evcnt isc_rx_size_big; 528 uint64_t isc_rx_size_big_offset; 529 struct evcnt isc_rx_undersize; 530 uint64_t isc_rx_undersize_offset; 531 struct evcnt isc_rx_oversize; 532 uint64_t isc_rx_oversize_offset; 533 struct evcnt isc_rx_fragments; 534 uint64_t isc_rx_fragments_offset; 535 struct evcnt isc_rx_jabber; 536 uint64_t isc_rx_jabber_offset; 537 struct evcnt isc_tx_bytes; 538 uint64_t isc_tx_bytes_offset; 539 struct evcnt isc_tx_dropped_link_down; 540 uint64_t isc_tx_dropped_link_down_offset; 541 struct evcnt isc_tx_unicast; 542 uint64_t isc_tx_unicast_offset; 543 struct evcnt isc_tx_multicast; 544 uint64_t isc_tx_multicast_offset; 545 struct evcnt isc_tx_broadcast; 546 uint64_t isc_tx_broadcast_offset; 547 struct evcnt isc_tx_size_64; 548 uint64_t isc_tx_size_64_offset; 549 struct evcnt isc_tx_size_127; 550 uint64_t isc_tx_size_127_offset; 551 struct evcnt isc_tx_size_255; 552 uint64_t isc_tx_size_255_offset; 553 struct evcnt isc_tx_size_511; 554 uint64_t isc_tx_size_511_offset; 555 struct evcnt isc_tx_size_1023; 556 uint64_t isc_tx_size_1023_offset; 557 struct evcnt isc_tx_size_1522; 558 uint64_t isc_tx_size_1522_offset; 559 struct evcnt isc_tx_size_big; 560 uint64_t isc_tx_size_big_offset; 561 struct evcnt isc_mac_local_faults; 562 uint64_t isc_mac_local_faults_offset; 563 struct evcnt isc_mac_remote_faults; 564 uint64_t isc_mac_remote_faults_offset; 565 struct evcnt isc_link_xon_rx; 566 uint64_t isc_link_xon_rx_offset; 567 struct evcnt isc_link_xon_tx; 568 uint64_t isc_link_xon_tx_offset; 569 struct evcnt isc_link_xoff_rx; 570 uint64_t isc_link_xoff_rx_offset; 571 struct evcnt isc_link_xoff_tx; 572 uint64_t isc_link_xoff_tx_offset; 573 struct evcnt isc_vsi_rx_discards; 574 uint64_t isc_vsi_rx_discards_offset; 575 struct evcnt isc_vsi_rx_bytes; 576 uint64_t isc_vsi_rx_bytes_offset; 577 struct evcnt isc_vsi_rx_unicast; 578 uint64_t isc_vsi_rx_unicast_offset; 579 struct evcnt isc_vsi_rx_multicast; 580 uint64_t isc_vsi_rx_multicast_offset; 581 struct evcnt isc_vsi_rx_broadcast; 582 uint64_t isc_vsi_rx_broadcast_offset; 583 struct evcnt isc_vsi_tx_errors; 584 uint64_t isc_vsi_tx_errors_offset; 585 struct evcnt isc_vsi_tx_bytes; 586 uint64_t isc_vsi_tx_bytes_offset; 587 struct evcnt isc_vsi_tx_unicast; 588 uint64_t isc_vsi_tx_unicast_offset; 589 struct evcnt isc_vsi_tx_multicast; 590 uint64_t isc_vsi_tx_multicast_offset; 591 struct evcnt isc_vsi_tx_broadcast; 592 uint64_t isc_vsi_tx_broadcast_offset; 593 }; 594 595 /* 596 * Locking notes: 597 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 598 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 599 * - more than one lock of them cannot be held at once. 600 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 601 * (a spin mutex). 602 * - the lock cannot held with txr_lock or rxr_lock. 603 * + a field named sc_arq_* is not protected by any lock. 604 * - operations for sc_arq_* is done in one context related to 605 * sc_arq_task. 606 * + other fields in ixl_softc is protected by sc_cfg_lock 607 * (an adaptive mutex) 608 * - It must be held before another lock is held, and It can be 609 * released after the other lock is released. 610 * */ 611 612 struct ixl_softc { 613 device_t sc_dev; 614 struct ethercom sc_ec; 615 bool sc_attached; 616 bool sc_dead; 617 uint32_t sc_port; 618 struct sysctllog *sc_sysctllog; 619 struct workqueue *sc_workq; 620 struct workqueue *sc_workq_txrx; 621 int sc_stats_intval; 622 callout_t sc_stats_callout; 623 struct ixl_work sc_stats_task; 624 struct ixl_stats_counters 625 sc_stats_counters; 626 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 627 struct ifmedia sc_media; 628 uint64_t sc_media_status; 629 uint64_t sc_media_active; 630 uint64_t sc_phy_types; 631 uint8_t sc_phy_abilities; 632 uint8_t sc_phy_linkspeed; 633 uint8_t sc_phy_fec_cfg; 634 uint16_t sc_eee_cap; 635 uint32_t sc_eeer_val; 636 uint8_t sc_d3_lpan; 637 kmutex_t sc_cfg_lock; 638 enum i40e_mac_type sc_mac_type; 639 uint32_t sc_rss_table_size; 640 uint32_t sc_rss_table_entry_width; 641 bool sc_txrx_workqueue; 642 u_int sc_tx_process_limit; 643 u_int sc_rx_process_limit; 644 u_int sc_tx_intr_process_limit; 645 u_int sc_rx_intr_process_limit; 646 647 int sc_cur_ec_capenable; 648 649 struct pci_attach_args sc_pa; 650 pci_intr_handle_t *sc_ihp; 651 void **sc_ihs; 652 unsigned int sc_nintrs; 653 654 bus_dma_tag_t sc_dmat; 655 bus_space_tag_t sc_memt; 656 bus_space_handle_t sc_memh; 657 bus_size_t sc_mems; 658 659 uint8_t sc_pf_id; 660 uint16_t sc_uplink_seid; /* le */ 661 uint16_t sc_downlink_seid; /* le */ 662 uint16_t sc_vsi_number; 663 uint16_t sc_vsi_stat_counter_idx; 664 uint16_t sc_seid; 665 unsigned int sc_base_queue; 666 667 pci_intr_type_t sc_intrtype; 668 unsigned int sc_msix_vector_queue; 669 670 struct ixl_dmamem sc_scratch; 671 struct ixl_dmamem sc_aqbuf; 672 673 const struct ixl_aq_regs * 674 sc_aq_regs; 675 uint32_t sc_aq_flags; 676 #define IXL_SC_AQ_FLAG_RXCTL __BIT(0) 677 #define IXL_SC_AQ_FLAG_NVMLOCK __BIT(1) 678 #define IXL_SC_AQ_FLAG_NVMREAD __BIT(2) 679 #define IXL_SC_AQ_FLAG_RSS __BIT(3) 680 681 kmutex_t sc_atq_lock; 682 kcondvar_t sc_atq_cv; 683 struct ixl_dmamem sc_atq; 684 unsigned int sc_atq_prod; 685 unsigned int sc_atq_cons; 686 687 struct ixl_dmamem sc_arq; 688 struct ixl_work sc_arq_task; 689 struct ixl_aq_bufs sc_arq_idle; 690 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 691 unsigned int sc_arq_prod; 692 unsigned int sc_arq_cons; 693 694 struct ixl_work sc_link_state_task; 695 struct ixl_atq sc_link_state_atq; 696 697 struct ixl_dmamem sc_hmc_sd; 698 struct ixl_dmamem sc_hmc_pd; 699 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 700 701 unsigned int sc_tx_ring_ndescs; 702 unsigned int sc_rx_ring_ndescs; 703 unsigned int sc_nqueue_pairs; 704 unsigned int sc_nqueue_pairs_max; 705 unsigned int sc_nqueue_pairs_device; 706 struct ixl_queue_pair *sc_qps; 707 708 struct evcnt sc_event_atq; 709 struct evcnt sc_event_link; 710 struct evcnt sc_event_ecc_err; 711 struct evcnt sc_event_pci_exception; 712 struct evcnt sc_event_crit_err; 713 }; 714 715 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 716 #define IXL_TX_PROCESS_LIMIT 256 717 #define IXL_RX_PROCESS_LIMIT 256 718 #define IXL_TX_INTR_PROCESS_LIMIT 256 719 #define IXL_RX_INTR_PROCESS_LIMIT 0U 720 721 #define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx | \ 722 IFCAP_CSUM_TCPv4_Rx | \ 723 IFCAP_CSUM_UDPv4_Rx | \ 724 IFCAP_CSUM_TCPv6_Rx | \ 725 IFCAP_CSUM_UDPv6_Rx) 726 #define IXL_IFCAP_TXCSUM (IFCAP_CSUM_IPv4_Tx | \ 727 IFCAP_CSUM_TCPv4_Tx | \ 728 IFCAP_CSUM_UDPv4_Tx | \ 729 IFCAP_CSUM_TCPv6_Tx | \ 730 IFCAP_CSUM_UDPv6_Tx) 731 #define IXL_CSUM_ALL_OFFLOAD (M_CSUM_IPv4 | \ 732 M_CSUM_TCPv4 | M_CSUM_TCPv6 | \ 733 M_CSUM_UDPv4 | M_CSUM_UDPv6) 734 735 #define delaymsec(_x) DELAY(1000 * (_x)) 736 #ifdef IXL_DEBUG 737 #define DDPRINTF(sc, fmt, args...) \ 738 do { \ 739 if ((sc) != NULL) { \ 740 device_printf( \ 741 ((struct ixl_softc *)(sc))->sc_dev, \ 742 ""); \ 743 } \ 744 printf("%s:\t" fmt, __func__, ##args); \ 745 } while (0) 746 #else 747 #define DDPRINTF(sc, fmt, args...) __nothing 748 #endif 749 #ifndef IXL_STATS_INTERVAL_MSEC 750 #define IXL_STATS_INTERVAL_MSEC 10000 751 #endif 752 #ifndef IXL_QUEUE_NUM 753 #define IXL_QUEUE_NUM 0 754 #endif 755 756 static bool ixl_param_nomsix = false; 757 static int ixl_param_stats_interval = IXL_STATS_INTERVAL_MSEC; 758 static int ixl_param_nqps_limit = IXL_QUEUE_NUM; 759 static unsigned int ixl_param_tx_ndescs = 1024; 760 static unsigned int ixl_param_rx_ndescs = 1024; 761 762 static enum i40e_mac_type 763 ixl_mactype(pci_product_id_t); 764 static void ixl_clear_hw(struct ixl_softc *); 765 static int ixl_pf_reset(struct ixl_softc *); 766 767 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 768 bus_size_t, bus_size_t); 769 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 770 771 static int ixl_arq_fill(struct ixl_softc *); 772 static void ixl_arq_unfill(struct ixl_softc *); 773 774 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 775 unsigned int); 776 static void ixl_atq_set(struct ixl_atq *, 777 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 778 static int ixl_atq_post(struct ixl_softc *, struct ixl_atq *); 779 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 780 static void ixl_atq_done(struct ixl_softc *); 781 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 782 static int ixl_get_version(struct ixl_softc *); 783 static int ixl_get_nvm_version(struct ixl_softc *); 784 static int ixl_get_hw_capabilities(struct ixl_softc *); 785 static int ixl_pxe_clear(struct ixl_softc *); 786 static int ixl_lldp_shut(struct ixl_softc *); 787 static int ixl_get_mac(struct ixl_softc *); 788 static int ixl_get_switch_config(struct ixl_softc *); 789 static int ixl_phy_mask_ints(struct ixl_softc *); 790 static int ixl_get_phy_info(struct ixl_softc *); 791 static int ixl_set_phy_config(struct ixl_softc *, uint8_t, uint8_t, bool); 792 static int ixl_set_phy_autoselect(struct ixl_softc *); 793 static int ixl_restart_an(struct ixl_softc *); 794 static int ixl_hmc(struct ixl_softc *); 795 static void ixl_hmc_free(struct ixl_softc *); 796 static int ixl_get_vsi(struct ixl_softc *); 797 static int ixl_set_vsi(struct ixl_softc *); 798 static void ixl_set_filter_control(struct ixl_softc *); 799 static void ixl_get_link_status(void *); 800 static int ixl_get_link_status_poll(struct ixl_softc *, int *); 801 static int ixl_set_link_status(struct ixl_softc *, 802 const struct ixl_aq_desc *); 803 static uint64_t ixl_search_link_speed(uint8_t); 804 static uint8_t ixl_search_baudrate(uint64_t); 805 static void ixl_config_rss(struct ixl_softc *); 806 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 807 uint16_t, uint16_t); 808 static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *, 809 uint16_t, uint16_t); 810 static void ixl_arq(void *); 811 static void ixl_hmc_pack(void *, const void *, 812 const struct ixl_hmc_pack *, unsigned int); 813 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 814 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 815 static int ixl_rd16_nvm(struct ixl_softc *, uint16_t, uint16_t *); 816 817 static int ixl_match(device_t, cfdata_t, void *); 818 static void ixl_attach(device_t, device_t, void *); 819 static int ixl_detach(device_t, int); 820 821 static void ixl_media_add(struct ixl_softc *); 822 static int ixl_media_change(struct ifnet *); 823 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 824 static void ixl_watchdog(struct ifnet *); 825 static int ixl_ioctl(struct ifnet *, u_long, void *); 826 static void ixl_start(struct ifnet *); 827 static int ixl_transmit(struct ifnet *, struct mbuf *); 828 static void ixl_deferred_transmit(void *); 829 static int ixl_intr(void *); 830 static int ixl_queue_intr(void *); 831 static int ixl_other_intr(void *); 832 static void ixl_handle_queue(void *); 833 static void ixl_handle_queue_wk(struct work *, void *); 834 static void ixl_sched_handle_queue(struct ixl_softc *, 835 struct ixl_queue_pair *); 836 static int ixl_init(struct ifnet *); 837 static int ixl_init_locked(struct ixl_softc *); 838 static void ixl_stop(struct ifnet *, int); 839 static void ixl_stop_locked(struct ixl_softc *); 840 static int ixl_iff(struct ixl_softc *); 841 static int ixl_ifflags_cb(struct ethercom *); 842 static int ixl_setup_interrupts(struct ixl_softc *); 843 static int ixl_establish_intx(struct ixl_softc *); 844 static int ixl_establish_msix(struct ixl_softc *); 845 static void ixl_enable_queue_intr(struct ixl_softc *, 846 struct ixl_queue_pair *); 847 static void ixl_disable_queue_intr(struct ixl_softc *, 848 struct ixl_queue_pair *); 849 static void ixl_enable_other_intr(struct ixl_softc *); 850 static void ixl_disable_other_intr(struct ixl_softc *); 851 static void ixl_config_queue_intr(struct ixl_softc *); 852 static void ixl_config_other_intr(struct ixl_softc *); 853 854 static struct ixl_tx_ring * 855 ixl_txr_alloc(struct ixl_softc *, unsigned int); 856 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 857 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 858 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 859 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 860 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 861 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 862 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 863 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 864 865 static struct ixl_rx_ring * 866 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 867 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 868 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 869 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 870 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 871 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 872 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 873 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 874 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 875 876 static struct workqueue * 877 ixl_workq_create(const char *, pri_t, int, int); 878 static void ixl_workq_destroy(struct workqueue *); 879 static int ixl_workqs_teardown(device_t); 880 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 881 static void ixl_work_add(struct workqueue *, struct ixl_work *); 882 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 883 static void ixl_workq_work(struct work *, void *); 884 static const struct ixl_product * 885 ixl_lookup(const struct pci_attach_args *pa); 886 static void ixl_link_state_update(struct ixl_softc *, 887 const struct ixl_aq_desc *); 888 static int ixl_vlan_cb(struct ethercom *, uint16_t, bool); 889 static int ixl_setup_vlan_hwfilter(struct ixl_softc *); 890 static void ixl_teardown_vlan_hwfilter(struct ixl_softc *); 891 static int ixl_update_macvlan(struct ixl_softc *); 892 static int ixl_setup_interrupts(struct ixl_softc *);; 893 static void ixl_teardown_interrupts(struct ixl_softc *); 894 static int ixl_setup_stats(struct ixl_softc *); 895 static void ixl_teardown_stats(struct ixl_softc *); 896 static void ixl_stats_callout(void *); 897 static void ixl_stats_update(void *); 898 static int ixl_setup_sysctls(struct ixl_softc *); 899 static void ixl_teardown_sysctls(struct ixl_softc *); 900 static int ixl_queue_pairs_alloc(struct ixl_softc *); 901 static void ixl_queue_pairs_free(struct ixl_softc *); 902 903 static const struct ixl_phy_type ixl_phy_type_map[] = { 904 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 905 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 906 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 907 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 908 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 909 { 1ULL << IXL_PHY_TYPE_XAUI | 910 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 911 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 912 { 1ULL << IXL_PHY_TYPE_XLAUI | 913 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 914 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 915 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 916 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 917 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 918 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 919 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 920 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 921 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 922 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 923 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 924 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 925 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 926 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 927 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 928 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 929 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 930 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 931 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 932 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 933 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 934 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 935 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 936 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 937 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_ACC }, 938 }; 939 940 static const struct ixl_speed_type ixl_speed_type_map[] = { 941 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 942 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 943 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 944 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 945 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 946 }; 947 948 static const struct ixl_aq_regs ixl_pf_aq_regs = { 949 .atq_tail = I40E_PF_ATQT, 950 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 951 .atq_head = I40E_PF_ATQH, 952 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 953 .atq_len = I40E_PF_ATQLEN, 954 .atq_bal = I40E_PF_ATQBAL, 955 .atq_bah = I40E_PF_ATQBAH, 956 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 957 958 .arq_tail = I40E_PF_ARQT, 959 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 960 .arq_head = I40E_PF_ARQH, 961 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 962 .arq_len = I40E_PF_ARQLEN, 963 .arq_bal = I40E_PF_ARQBAL, 964 .arq_bah = I40E_PF_ARQBAH, 965 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 966 }; 967 968 #define ixl_rd(_s, _r) \ 969 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 970 #define ixl_wr(_s, _r, _v) \ 971 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 972 #define ixl_barrier(_s, _r, _l, _o) \ 973 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 974 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 975 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 976 977 static inline uint32_t 978 ixl_dmamem_hi(struct ixl_dmamem *ixm) 979 { 980 uint32_t retval; 981 uint64_t val; 982 983 if (sizeof(IXL_DMA_DVA(ixm)) > 4) { 984 val = (intptr_t)IXL_DMA_DVA(ixm); 985 retval = (uint32_t)(val >> 32); 986 } else { 987 retval = 0; 988 } 989 990 return retval; 991 } 992 993 static inline uint32_t 994 ixl_dmamem_lo(struct ixl_dmamem *ixm) 995 { 996 997 return (uint32_t)IXL_DMA_DVA(ixm); 998 } 999 1000 static inline void 1001 ixl_aq_dva(struct ixl_aq_desc *iaq, bus_addr_t addr) 1002 { 1003 uint64_t val; 1004 1005 if (sizeof(addr) > 4) { 1006 val = (intptr_t)addr; 1007 iaq->iaq_param[2] = htole32(val >> 32); 1008 } else { 1009 iaq->iaq_param[2] = htole32(0); 1010 } 1011 1012 iaq->iaq_param[3] = htole32(addr); 1013 } 1014 1015 static inline unsigned int 1016 ixl_rxr_unrefreshed(unsigned int prod, unsigned int cons, unsigned int ndescs) 1017 { 1018 unsigned int num; 1019 1020 if (prod < cons) 1021 num = cons - prod; 1022 else 1023 num = (ndescs - prod) + cons; 1024 1025 if (__predict_true(num > 0)) { 1026 /* device cannot receive packets if all descripter is filled */ 1027 num -= 1; 1028 } 1029 1030 return num; 1031 } 1032 1033 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 1034 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 1035 DVF_DETACH_SHUTDOWN); 1036 1037 static const struct ixl_product ixl_products[] = { 1038 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 1039 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 1040 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 1041 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 1042 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 1043 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 1044 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T }, 1045 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 1046 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 1047 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 1048 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 1049 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 1050 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 1051 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 1052 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 1053 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 1054 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 1055 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 1056 /* required last entry */ 1057 {0, 0} 1058 }; 1059 1060 static const struct ixl_product * 1061 ixl_lookup(const struct pci_attach_args *pa) 1062 { 1063 const struct ixl_product *ixlp; 1064 1065 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 1066 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 1067 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 1068 return ixlp; 1069 } 1070 1071 return NULL; 1072 } 1073 1074 static int 1075 ixl_match(device_t parent, cfdata_t match, void *aux) 1076 { 1077 const struct pci_attach_args *pa = aux; 1078 1079 return (ixl_lookup(pa) != NULL) ? 1 : 0; 1080 } 1081 1082 static void 1083 ixl_attach(device_t parent, device_t self, void *aux) 1084 { 1085 struct ixl_softc *sc; 1086 struct pci_attach_args *pa = aux; 1087 struct ifnet *ifp; 1088 pcireg_t memtype; 1089 uint32_t firstq, port, ari, func; 1090 char xnamebuf[32]; 1091 int tries, rv, link; 1092 1093 sc = device_private(self); 1094 sc->sc_dev = self; 1095 ifp = &sc->sc_ec.ec_if; 1096 1097 sc->sc_pa = *pa; 1098 sc->sc_dmat = (pci_dma64_available(pa)) ? 1099 pa->pa_dmat64 : pa->pa_dmat; 1100 sc->sc_aq_regs = &ixl_pf_aq_regs; 1101 1102 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(pa->pa_id)); 1103 1104 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 1105 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 1106 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 1107 aprint_error(": unable to map registers\n"); 1108 return; 1109 } 1110 1111 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 1112 1113 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 1114 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 1115 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 1116 sc->sc_base_queue = firstq; 1117 1118 ixl_clear_hw(sc); 1119 if (ixl_pf_reset(sc) == -1) { 1120 /* error printed by ixl pf_reset */ 1121 goto unmap; 1122 } 1123 1124 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 1125 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 1126 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 1127 sc->sc_port = port; 1128 aprint_normal(": port %u", sc->sc_port); 1129 1130 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 1131 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 1132 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 1133 1134 func = ixl_rd(sc, I40E_PF_FUNC_RID); 1135 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 1136 1137 /* initialise the adminq */ 1138 1139 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 1140 1141 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 1142 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1143 aprint_error("\n" "%s: unable to allocate atq\n", 1144 device_xname(self)); 1145 goto unmap; 1146 } 1147 1148 SIMPLEQ_INIT(&sc->sc_arq_idle); 1149 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 1150 sc->sc_arq_cons = 0; 1151 sc->sc_arq_prod = 0; 1152 1153 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 1154 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1155 aprint_error("\n" "%s: unable to allocate arq\n", 1156 device_xname(self)); 1157 goto free_atq; 1158 } 1159 1160 if (!ixl_arq_fill(sc)) { 1161 aprint_error("\n" "%s: unable to fill arq descriptors\n", 1162 device_xname(self)); 1163 goto free_arq; 1164 } 1165 1166 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1167 0, IXL_DMA_LEN(&sc->sc_atq), 1168 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1169 1170 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1171 0, IXL_DMA_LEN(&sc->sc_arq), 1172 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1173 1174 for (tries = 0; tries < 10; tries++) { 1175 sc->sc_atq_cons = 0; 1176 sc->sc_atq_prod = 0; 1177 1178 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1179 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1180 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1181 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1182 1183 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1184 1185 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1186 ixl_dmamem_lo(&sc->sc_atq)); 1187 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1188 ixl_dmamem_hi(&sc->sc_atq)); 1189 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1190 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1191 1192 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1193 ixl_dmamem_lo(&sc->sc_arq)); 1194 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1195 ixl_dmamem_hi(&sc->sc_arq)); 1196 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1197 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1198 1199 rv = ixl_get_version(sc); 1200 if (rv == 0) 1201 break; 1202 if (rv != ETIMEDOUT) { 1203 aprint_error(", unable to get firmware version\n"); 1204 goto shutdown; 1205 } 1206 1207 delaymsec(100); 1208 } 1209 1210 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1211 1212 if (ixl_dmamem_alloc(sc, &sc->sc_aqbuf, IXL_AQ_BUFLEN, 0) != 0) { 1213 aprint_error_dev(self, ", unable to allocate nvm buffer\n"); 1214 goto shutdown; 1215 } 1216 1217 ixl_get_nvm_version(sc); 1218 1219 if (sc->sc_mac_type == I40E_MAC_X722) 1220 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_X722; 1221 else 1222 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_XL710; 1223 1224 rv = ixl_get_hw_capabilities(sc); 1225 if (rv != 0) { 1226 aprint_error(", GET HW CAPABILITIES %s\n", 1227 rv == ETIMEDOUT ? "timeout" : "error"); 1228 goto free_aqbuf; 1229 } 1230 1231 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1232 if (ixl_param_nqps_limit > 0) { 1233 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_max, 1234 ixl_param_nqps_limit); 1235 } 1236 1237 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1238 sc->sc_tx_ring_ndescs = ixl_param_tx_ndescs; 1239 sc->sc_rx_ring_ndescs = ixl_param_rx_ndescs; 1240 1241 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1242 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1243 1244 if (ixl_get_mac(sc) != 0) { 1245 /* error printed by ixl_get_mac */ 1246 goto free_aqbuf; 1247 } 1248 1249 aprint_normal("\n"); 1250 aprint_naive("\n"); 1251 1252 aprint_normal_dev(self, "Ethernet address %s\n", 1253 ether_sprintf(sc->sc_enaddr)); 1254 1255 rv = ixl_pxe_clear(sc); 1256 if (rv != 0) { 1257 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1258 rv == ETIMEDOUT ? "timeout" : "error"); 1259 } 1260 1261 ixl_set_filter_control(sc); 1262 1263 if (ixl_hmc(sc) != 0) { 1264 /* error printed by ixl_hmc */ 1265 goto free_aqbuf; 1266 } 1267 1268 if (ixl_lldp_shut(sc) != 0) { 1269 /* error printed by ixl_lldp_shut */ 1270 goto free_hmc; 1271 } 1272 1273 if (ixl_phy_mask_ints(sc) != 0) { 1274 /* error printed by ixl_phy_mask_ints */ 1275 goto free_hmc; 1276 } 1277 1278 if (ixl_restart_an(sc) != 0) { 1279 /* error printed by ixl_restart_an */ 1280 goto free_hmc; 1281 } 1282 1283 if (ixl_get_switch_config(sc) != 0) { 1284 /* error printed by ixl_get_switch_config */ 1285 goto free_hmc; 1286 } 1287 1288 rv = ixl_get_link_status_poll(sc, NULL); 1289 if (rv != 0) { 1290 aprint_error_dev(self, "GET LINK STATUS %s\n", 1291 rv == ETIMEDOUT ? "timeout" : "error"); 1292 goto free_hmc; 1293 } 1294 1295 /* 1296 * The FW often returns EIO in "Get PHY Abilities" command 1297 * if there is no delay 1298 */ 1299 DELAY(500); 1300 if (ixl_get_phy_info(sc) != 0) { 1301 /* error printed by ixl_get_phy_info */ 1302 goto free_hmc; 1303 } 1304 1305 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1306 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1307 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1308 goto free_hmc; 1309 } 1310 1311 rv = ixl_get_vsi(sc); 1312 if (rv != 0) { 1313 aprint_error_dev(self, "GET VSI %s %d\n", 1314 rv == ETIMEDOUT ? "timeout" : "error", rv); 1315 goto free_scratch; 1316 } 1317 1318 rv = ixl_set_vsi(sc); 1319 if (rv != 0) { 1320 aprint_error_dev(self, "UPDATE VSI error %s %d\n", 1321 rv == ETIMEDOUT ? "timeout" : "error", rv); 1322 goto free_scratch; 1323 } 1324 1325 if (ixl_queue_pairs_alloc(sc) != 0) { 1326 /* error printed by ixl_queue_pairs_alloc */ 1327 goto free_scratch; 1328 } 1329 1330 if (ixl_setup_interrupts(sc) != 0) { 1331 /* error printed by ixl_setup_interrupts */ 1332 goto free_queue_pairs; 1333 } 1334 1335 if (ixl_setup_stats(sc) != 0) { 1336 aprint_error_dev(self, "failed to setup event counters\n"); 1337 goto teardown_intrs; 1338 } 1339 1340 if (ixl_setup_sysctls(sc) != 0) { 1341 /* error printed by ixl_setup_sysctls */ 1342 goto teardown_stats; 1343 } 1344 1345 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1346 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1347 IPL_NET, WQ_MPSAFE); 1348 if (sc->sc_workq == NULL) 1349 goto teardown_sysctls; 1350 1351 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1352 rv = workqueue_create(&sc->sc_workq_txrx, xnamebuf, ixl_handle_queue_wk, 1353 sc, IXL_WORKQUEUE_PRI, IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1354 if (rv != 0) { 1355 sc->sc_workq_txrx = NULL; 1356 goto teardown_wqs; 1357 } 1358 1359 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1360 cv_init(&sc->sc_atq_cv, xnamebuf); 1361 1362 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1363 1364 ifp->if_softc = sc; 1365 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1366 ifp->if_extflags = IFEF_MPSAFE; 1367 ifp->if_ioctl = ixl_ioctl; 1368 ifp->if_start = ixl_start; 1369 ifp->if_transmit = ixl_transmit; 1370 ifp->if_watchdog = ixl_watchdog; 1371 ifp->if_init = ixl_init; 1372 ifp->if_stop = ixl_stop; 1373 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1374 IFQ_SET_READY(&ifp->if_snd); 1375 ifp->if_capabilities |= IXL_IFCAP_RXCSUM; 1376 ifp->if_capabilities |= IXL_IFCAP_TXCSUM; 1377 #if 0 1378 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1379 #endif 1380 ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb); 1381 sc->sc_ec.ec_capabilities |= ETHERCAP_JUMBO_MTU; 1382 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1383 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1384 1385 sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities; 1386 /* Disable VLAN_HWFILTER by default */ 1387 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1388 1389 sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable; 1390 1391 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1392 ifmedia_init(&sc->sc_media, IFM_IMASK, ixl_media_change, 1393 ixl_media_status); 1394 1395 ixl_media_add(sc); 1396 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1397 if (ISSET(sc->sc_phy_abilities, 1398 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1399 ifmedia_add(&sc->sc_media, 1400 IFM_ETHER | IFM_AUTO | IFM_FLOW, 0, NULL); 1401 } 1402 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_NONE, 0, NULL); 1403 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1404 1405 if_attach(ifp); 1406 if_deferred_start_init(ifp, NULL); 1407 ether_ifattach(ifp, sc->sc_enaddr); 1408 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1409 1410 rv = ixl_get_link_status_poll(sc, &link); 1411 if (rv != 0) 1412 link = LINK_STATE_UNKNOWN; 1413 if_link_state_change(ifp, link); 1414 1415 ixl_work_set(&sc->sc_link_state_task, ixl_get_link_status, sc); 1416 1417 ixl_config_other_intr(sc); 1418 ixl_enable_other_intr(sc); 1419 1420 ixl_set_phy_autoselect(sc); 1421 1422 /* remove default mac filter and replace it so we can see vlans */ 1423 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 1424 if (rv != ENOENT) { 1425 aprint_debug_dev(self, 1426 "unable to remove macvlan %u\n", rv); 1427 } 1428 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1429 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1430 if (rv != ENOENT) { 1431 aprint_debug_dev(self, 1432 "unable to remove macvlan, ignore vlan %u\n", rv); 1433 } 1434 1435 if (ixl_update_macvlan(sc) != 0) { 1436 aprint_debug_dev(self, 1437 "couldn't enable vlan hardware filter\n"); 1438 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1439 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1440 } 1441 1442 sc->sc_txrx_workqueue = true; 1443 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1444 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1445 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1446 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1447 1448 ixl_stats_update(sc); 1449 sc->sc_stats_counters.isc_has_offset = true; 1450 1451 if (pmf_device_register(self, NULL, NULL) != true) 1452 aprint_debug_dev(self, "couldn't establish power handler\n"); 1453 sc->sc_attached = true; 1454 return; 1455 1456 teardown_wqs: 1457 config_finalize_register(self, ixl_workqs_teardown); 1458 teardown_sysctls: 1459 ixl_teardown_sysctls(sc); 1460 teardown_stats: 1461 ixl_teardown_stats(sc); 1462 teardown_intrs: 1463 ixl_teardown_interrupts(sc); 1464 free_queue_pairs: 1465 ixl_queue_pairs_free(sc); 1466 free_scratch: 1467 ixl_dmamem_free(sc, &sc->sc_scratch); 1468 free_hmc: 1469 ixl_hmc_free(sc); 1470 free_aqbuf: 1471 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1472 shutdown: 1473 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1474 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1475 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1476 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1477 1478 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1479 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1480 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1481 1482 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1483 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1484 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1485 1486 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1487 0, IXL_DMA_LEN(&sc->sc_arq), 1488 BUS_DMASYNC_POSTREAD); 1489 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1490 0, IXL_DMA_LEN(&sc->sc_atq), 1491 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1492 1493 ixl_arq_unfill(sc); 1494 free_arq: 1495 ixl_dmamem_free(sc, &sc->sc_arq); 1496 free_atq: 1497 ixl_dmamem_free(sc, &sc->sc_atq); 1498 unmap: 1499 mutex_destroy(&sc->sc_atq_lock); 1500 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1501 mutex_destroy(&sc->sc_cfg_lock); 1502 sc->sc_mems = 0; 1503 1504 sc->sc_attached = false; 1505 } 1506 1507 static int 1508 ixl_detach(device_t self, int flags) 1509 { 1510 struct ixl_softc *sc = device_private(self); 1511 struct ifnet *ifp = &sc->sc_ec.ec_if; 1512 1513 if (!sc->sc_attached) 1514 return 0; 1515 1516 ixl_stop(ifp, 1); 1517 1518 ixl_disable_other_intr(sc); 1519 1520 callout_halt(&sc->sc_stats_callout, NULL); 1521 ixl_work_wait(sc->sc_workq, &sc->sc_stats_task); 1522 1523 /* wait for ATQ handler */ 1524 mutex_enter(&sc->sc_atq_lock); 1525 mutex_exit(&sc->sc_atq_lock); 1526 1527 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1528 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1529 1530 if (sc->sc_workq != NULL) { 1531 ixl_workq_destroy(sc->sc_workq); 1532 sc->sc_workq = NULL; 1533 } 1534 1535 if (sc->sc_workq_txrx != NULL) { 1536 workqueue_destroy(sc->sc_workq_txrx); 1537 sc->sc_workq_txrx = NULL; 1538 } 1539 1540 ether_ifdetach(ifp); 1541 if_detach(ifp); 1542 ifmedia_fini(&sc->sc_media); 1543 1544 ixl_teardown_interrupts(sc); 1545 ixl_teardown_stats(sc); 1546 ixl_teardown_sysctls(sc); 1547 1548 ixl_queue_pairs_free(sc); 1549 1550 ixl_dmamem_free(sc, &sc->sc_scratch); 1551 ixl_hmc_free(sc); 1552 1553 /* shutdown */ 1554 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1555 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1556 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1557 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1558 1559 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1560 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1561 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1562 1563 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1564 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1565 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1566 1567 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1568 0, IXL_DMA_LEN(&sc->sc_arq), 1569 BUS_DMASYNC_POSTREAD); 1570 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1571 0, IXL_DMA_LEN(&sc->sc_atq), 1572 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1573 1574 ixl_arq_unfill(sc); 1575 1576 ixl_dmamem_free(sc, &sc->sc_arq); 1577 ixl_dmamem_free(sc, &sc->sc_atq); 1578 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1579 1580 cv_destroy(&sc->sc_atq_cv); 1581 mutex_destroy(&sc->sc_atq_lock); 1582 1583 if (sc->sc_mems != 0) { 1584 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1585 sc->sc_mems = 0; 1586 } 1587 1588 mutex_destroy(&sc->sc_cfg_lock); 1589 1590 return 0; 1591 } 1592 1593 static int 1594 ixl_workqs_teardown(device_t self) 1595 { 1596 struct ixl_softc *sc = device_private(self); 1597 1598 if (sc->sc_workq != NULL) { 1599 ixl_workq_destroy(sc->sc_workq); 1600 sc->sc_workq = NULL; 1601 } 1602 1603 if (sc->sc_workq_txrx != NULL) { 1604 workqueue_destroy(sc->sc_workq_txrx); 1605 sc->sc_workq_txrx = NULL; 1606 } 1607 1608 return 0; 1609 } 1610 1611 static int 1612 ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 1613 { 1614 struct ifnet *ifp = &ec->ec_if; 1615 struct ixl_softc *sc = ifp->if_softc; 1616 int rv; 1617 1618 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1619 return 0; 1620 } 1621 1622 if (set) { 1623 rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid, 1624 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1625 if (rv == 0) { 1626 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 1627 vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1628 } 1629 } else { 1630 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid, 1631 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1632 (void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid, 1633 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1634 } 1635 1636 return rv; 1637 } 1638 1639 static void 1640 ixl_media_add(struct ixl_softc *sc) 1641 { 1642 struct ifmedia *ifm = &sc->sc_media; 1643 const struct ixl_phy_type *itype; 1644 unsigned int i; 1645 bool flow; 1646 1647 if (ISSET(sc->sc_phy_abilities, 1648 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1649 flow = true; 1650 } else { 1651 flow = false; 1652 } 1653 1654 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1655 itype = &ixl_phy_type_map[i]; 1656 1657 if (ISSET(sc->sc_phy_types, itype->phy_type)) { 1658 ifmedia_add(ifm, 1659 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1660 1661 if (flow) { 1662 ifmedia_add(ifm, 1663 IFM_ETHER | IFM_FDX | IFM_FLOW | 1664 itype->ifm_type, 0, NULL); 1665 } 1666 1667 if (itype->ifm_type != IFM_100_TX) 1668 continue; 1669 1670 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1671 0, NULL); 1672 if (flow) { 1673 ifmedia_add(ifm, 1674 IFM_ETHER | IFM_FLOW | itype->ifm_type, 1675 0, NULL); 1676 } 1677 } 1678 } 1679 } 1680 1681 static void 1682 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1683 { 1684 struct ixl_softc *sc = ifp->if_softc; 1685 1686 ifmr->ifm_status = sc->sc_media_status; 1687 ifmr->ifm_active = sc->sc_media_active; 1688 1689 mutex_enter(&sc->sc_cfg_lock); 1690 if (ifp->if_link_state == LINK_STATE_UP) 1691 SET(ifmr->ifm_status, IFM_ACTIVE); 1692 mutex_exit(&sc->sc_cfg_lock); 1693 } 1694 1695 static int 1696 ixl_media_change(struct ifnet *ifp) 1697 { 1698 struct ixl_softc *sc = ifp->if_softc; 1699 struct ifmedia *ifm = &sc->sc_media; 1700 uint64_t ifm_active = sc->sc_media_active; 1701 uint8_t link_speed, abilities; 1702 1703 switch (IFM_SUBTYPE(ifm_active)) { 1704 case IFM_1000_SGMII: 1705 case IFM_1000_KX: 1706 case IFM_10G_KX4: 1707 case IFM_10G_KR: 1708 case IFM_40G_KR4: 1709 case IFM_20G_KR2: 1710 case IFM_25G_KR: 1711 /* backplanes */ 1712 return EINVAL; 1713 } 1714 1715 abilities = IXL_PHY_ABILITY_AUTONEGO | IXL_PHY_ABILITY_LINKUP; 1716 1717 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1718 case IFM_AUTO: 1719 link_speed = sc->sc_phy_linkspeed; 1720 break; 1721 case IFM_NONE: 1722 link_speed = 0; 1723 CLR(abilities, IXL_PHY_ABILITY_LINKUP); 1724 break; 1725 default: 1726 link_speed = ixl_search_baudrate( 1727 ifmedia_baudrate(ifm->ifm_media)); 1728 } 1729 1730 if (ISSET(abilities, IXL_PHY_ABILITY_LINKUP)) { 1731 if (ISSET(link_speed, sc->sc_phy_linkspeed) == 0) 1732 return EINVAL; 1733 } 1734 1735 if (ifm->ifm_media & IFM_FLOW) { 1736 abilities |= sc->sc_phy_abilities & 1737 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX); 1738 } 1739 1740 return ixl_set_phy_config(sc, link_speed, abilities, false); 1741 } 1742 1743 static void 1744 ixl_watchdog(struct ifnet *ifp) 1745 { 1746 1747 } 1748 1749 static void 1750 ixl_del_all_multiaddr(struct ixl_softc *sc) 1751 { 1752 struct ethercom *ec = &sc->sc_ec; 1753 struct ether_multi *enm; 1754 struct ether_multistep step; 1755 1756 ETHER_LOCK(ec); 1757 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1758 ETHER_NEXT_MULTI(step, enm)) { 1759 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1760 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1761 } 1762 ETHER_UNLOCK(ec); 1763 } 1764 1765 static int 1766 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1767 { 1768 struct ifnet *ifp = &sc->sc_ec.ec_if; 1769 int rv; 1770 1771 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1772 return 0; 1773 1774 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1775 ixl_del_all_multiaddr(sc); 1776 SET(ifp->if_flags, IFF_ALLMULTI); 1777 return ENETRESET; 1778 } 1779 1780 /* multicast address can not use VLAN HWFILTER */ 1781 rv = ixl_add_macvlan(sc, addrlo, 0, 1782 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1783 1784 if (rv == ENOSPC) { 1785 ixl_del_all_multiaddr(sc); 1786 SET(ifp->if_flags, IFF_ALLMULTI); 1787 return ENETRESET; 1788 } 1789 1790 return rv; 1791 } 1792 1793 static int 1794 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1795 { 1796 struct ifnet *ifp = &sc->sc_ec.ec_if; 1797 struct ethercom *ec = &sc->sc_ec; 1798 struct ether_multi *enm, *enm_last; 1799 struct ether_multistep step; 1800 int error, rv = 0; 1801 1802 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1803 ixl_remove_macvlan(sc, addrlo, 0, 1804 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1805 return 0; 1806 } 1807 1808 ETHER_LOCK(ec); 1809 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1810 ETHER_NEXT_MULTI(step, enm)) { 1811 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1812 ETHER_ADDR_LEN) != 0) { 1813 goto out; 1814 } 1815 } 1816 1817 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1818 ETHER_NEXT_MULTI(step, enm)) { 1819 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1820 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1821 if (error != 0) 1822 break; 1823 } 1824 1825 if (enm != NULL) { 1826 enm_last = enm; 1827 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1828 ETHER_NEXT_MULTI(step, enm)) { 1829 if (enm == enm_last) 1830 break; 1831 1832 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1833 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1834 } 1835 } else { 1836 CLR(ifp->if_flags, IFF_ALLMULTI); 1837 rv = ENETRESET; 1838 } 1839 1840 out: 1841 ETHER_UNLOCK(ec); 1842 return rv; 1843 } 1844 1845 static int 1846 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1847 { 1848 struct ifreq *ifr = (struct ifreq *)data; 1849 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1850 const struct sockaddr *sa; 1851 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1852 int s, error = 0; 1853 unsigned int nmtu; 1854 1855 switch (cmd) { 1856 case SIOCSIFMTU: 1857 nmtu = ifr->ifr_mtu; 1858 1859 if (nmtu < IXL_MIN_MTU || nmtu > IXL_MAX_MTU) { 1860 error = EINVAL; 1861 break; 1862 } 1863 if (ifp->if_mtu != nmtu) { 1864 s = splnet(); 1865 error = ether_ioctl(ifp, cmd, data); 1866 splx(s); 1867 if (error == ENETRESET) 1868 error = ixl_init(ifp); 1869 } 1870 break; 1871 case SIOCADDMULTI: 1872 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1873 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1874 error = ether_multiaddr(sa, addrlo, addrhi); 1875 if (error != 0) 1876 return error; 1877 1878 error = ixl_add_multi(sc, addrlo, addrhi); 1879 if (error != 0 && error != ENETRESET) { 1880 ether_delmulti(sa, &sc->sc_ec); 1881 error = EIO; 1882 } 1883 } 1884 break; 1885 1886 case SIOCDELMULTI: 1887 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1888 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1889 error = ether_multiaddr(sa, addrlo, addrhi); 1890 if (error != 0) 1891 return error; 1892 1893 error = ixl_del_multi(sc, addrlo, addrhi); 1894 } 1895 break; 1896 1897 default: 1898 s = splnet(); 1899 error = ether_ioctl(ifp, cmd, data); 1900 splx(s); 1901 } 1902 1903 if (error == ENETRESET) 1904 error = ixl_iff(sc); 1905 1906 return error; 1907 } 1908 1909 static enum i40e_mac_type 1910 ixl_mactype(pci_product_id_t id) 1911 { 1912 1913 switch (id) { 1914 case PCI_PRODUCT_INTEL_XL710_SFP: 1915 case PCI_PRODUCT_INTEL_XL710_KX_B: 1916 case PCI_PRODUCT_INTEL_XL710_KX_C: 1917 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1918 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1919 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1920 case PCI_PRODUCT_INTEL_X710_10G_T: 1921 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1922 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1923 case PCI_PRODUCT_INTEL_X710_T4_10G: 1924 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1925 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1926 return I40E_MAC_XL710; 1927 1928 case PCI_PRODUCT_INTEL_X722_KX: 1929 case PCI_PRODUCT_INTEL_X722_QSFP: 1930 case PCI_PRODUCT_INTEL_X722_SFP: 1931 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1932 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1933 case PCI_PRODUCT_INTEL_X722_I_SFP: 1934 return I40E_MAC_X722; 1935 } 1936 1937 return I40E_MAC_GENERIC; 1938 } 1939 1940 static inline void * 1941 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1942 { 1943 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1944 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1945 1946 if (i >= e->hmc_count) 1947 return NULL; 1948 1949 kva += e->hmc_base; 1950 kva += i * e->hmc_size; 1951 1952 return kva; 1953 } 1954 1955 static inline size_t 1956 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1957 { 1958 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1959 1960 return e->hmc_size; 1961 } 1962 1963 static void 1964 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1965 { 1966 struct ixl_rx_ring *rxr = qp->qp_rxr; 1967 1968 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1969 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1970 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1971 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1972 ixl_flush(sc); 1973 } 1974 1975 static void 1976 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1977 { 1978 struct ixl_rx_ring *rxr = qp->qp_rxr; 1979 1980 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1981 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1982 ixl_flush(sc); 1983 } 1984 1985 static void 1986 ixl_enable_other_intr(struct ixl_softc *sc) 1987 { 1988 1989 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1990 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1991 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1992 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1993 ixl_flush(sc); 1994 } 1995 1996 static void 1997 ixl_disable_other_intr(struct ixl_softc *sc) 1998 { 1999 2000 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 2001 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 2002 ixl_flush(sc); 2003 } 2004 2005 static int 2006 ixl_reinit(struct ixl_softc *sc) 2007 { 2008 struct ixl_rx_ring *rxr; 2009 struct ixl_tx_ring *txr; 2010 unsigned int i; 2011 uint32_t reg; 2012 2013 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2014 2015 if (ixl_get_vsi(sc) != 0) 2016 return EIO; 2017 2018 if (ixl_set_vsi(sc) != 0) 2019 return EIO; 2020 2021 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2022 txr = sc->sc_qps[i].qp_txr; 2023 rxr = sc->sc_qps[i].qp_rxr; 2024 2025 ixl_txr_config(sc, txr); 2026 ixl_rxr_config(sc, rxr); 2027 } 2028 2029 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2030 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 2031 2032 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2033 txr = sc->sc_qps[i].qp_txr; 2034 rxr = sc->sc_qps[i].qp_rxr; 2035 2036 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 2037 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 2038 ixl_flush(sc); 2039 2040 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 2041 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 2042 2043 /* ixl_rxfill() needs lock held */ 2044 mutex_enter(&rxr->rxr_lock); 2045 ixl_rxfill(sc, rxr); 2046 mutex_exit(&rxr->rxr_lock); 2047 2048 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2049 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2050 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2051 if (ixl_rxr_enabled(sc, rxr) != 0) 2052 goto stop; 2053 2054 ixl_txr_qdis(sc, txr, 1); 2055 2056 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2057 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2058 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2059 2060 if (ixl_txr_enabled(sc, txr) != 0) 2061 goto stop; 2062 } 2063 2064 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2065 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2066 2067 return 0; 2068 2069 stop: 2070 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2071 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2072 2073 return ETIMEDOUT; 2074 } 2075 2076 static int 2077 ixl_init_locked(struct ixl_softc *sc) 2078 { 2079 struct ifnet *ifp = &sc->sc_ec.ec_if; 2080 unsigned int i; 2081 int error, eccap_change; 2082 2083 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2084 2085 if (ISSET(ifp->if_flags, IFF_RUNNING)) 2086 ixl_stop_locked(sc); 2087 2088 if (sc->sc_dead) { 2089 return ENXIO; 2090 } 2091 2092 eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable; 2093 if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING)) 2094 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 2095 2096 if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) { 2097 if (ixl_update_macvlan(sc) == 0) { 2098 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 2099 } else { 2100 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 2101 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 2102 } 2103 } 2104 2105 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 2106 sc->sc_nqueue_pairs = 1; 2107 else 2108 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 2109 2110 error = ixl_reinit(sc); 2111 if (error) { 2112 ixl_stop_locked(sc); 2113 return error; 2114 } 2115 2116 SET(ifp->if_flags, IFF_RUNNING); 2117 CLR(ifp->if_flags, IFF_OACTIVE); 2118 2119 (void)ixl_get_link_status(sc); 2120 2121 ixl_config_rss(sc); 2122 ixl_config_queue_intr(sc); 2123 2124 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2125 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 2126 } 2127 2128 error = ixl_iff(sc); 2129 if (error) { 2130 ixl_stop_locked(sc); 2131 return error; 2132 } 2133 2134 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 2135 2136 return 0; 2137 } 2138 2139 static int 2140 ixl_init(struct ifnet *ifp) 2141 { 2142 struct ixl_softc *sc = ifp->if_softc; 2143 int error; 2144 2145 mutex_enter(&sc->sc_cfg_lock); 2146 error = ixl_init_locked(sc); 2147 mutex_exit(&sc->sc_cfg_lock); 2148 2149 return error; 2150 } 2151 2152 static int 2153 ixl_iff(struct ixl_softc *sc) 2154 { 2155 struct ifnet *ifp = &sc->sc_ec.ec_if; 2156 struct ixl_atq iatq; 2157 struct ixl_aq_desc *iaq; 2158 struct ixl_aq_vsi_promisc_param *param; 2159 uint16_t flag_add, flag_del; 2160 int error; 2161 2162 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2163 return 0; 2164 2165 memset(&iatq, 0, sizeof(iatq)); 2166 2167 iaq = &iatq.iatq_desc; 2168 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 2169 2170 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 2171 param->flags = htole16(0); 2172 2173 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER) 2174 || ISSET(ifp->if_flags, IFF_PROMISC)) { 2175 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2176 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2177 } 2178 2179 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 2180 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2181 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2182 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 2183 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2184 } 2185 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2186 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2187 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2188 param->seid = sc->sc_seid; 2189 2190 error = ixl_atq_exec(sc, &iatq); 2191 if (error) 2192 return error; 2193 2194 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 2195 return EIO; 2196 2197 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 2198 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 2199 flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH; 2200 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH; 2201 } else { 2202 flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN; 2203 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN; 2204 } 2205 2206 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del); 2207 2208 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2209 ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add); 2210 } 2211 return 0; 2212 } 2213 2214 static void 2215 ixl_stop_rendezvous(struct ixl_softc *sc) 2216 { 2217 struct ixl_tx_ring *txr; 2218 struct ixl_rx_ring *rxr; 2219 unsigned int i; 2220 2221 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2222 txr = sc->sc_qps[i].qp_txr; 2223 rxr = sc->sc_qps[i].qp_rxr; 2224 2225 mutex_enter(&txr->txr_lock); 2226 mutex_exit(&txr->txr_lock); 2227 2228 mutex_enter(&rxr->rxr_lock); 2229 mutex_exit(&rxr->rxr_lock); 2230 2231 sc->sc_qps[i].qp_workqueue = false; 2232 workqueue_wait(sc->sc_workq_txrx, 2233 &sc->sc_qps[i].qp_work); 2234 } 2235 } 2236 2237 static void 2238 ixl_stop_locked(struct ixl_softc *sc) 2239 { 2240 struct ifnet *ifp = &sc->sc_ec.ec_if; 2241 struct ixl_rx_ring *rxr; 2242 struct ixl_tx_ring *txr; 2243 unsigned int i; 2244 uint32_t reg; 2245 2246 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2247 2248 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 2249 callout_stop(&sc->sc_stats_callout); 2250 2251 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2252 txr = sc->sc_qps[i].qp_txr; 2253 rxr = sc->sc_qps[i].qp_rxr; 2254 2255 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 2256 2257 mutex_enter(&txr->txr_lock); 2258 ixl_txr_qdis(sc, txr, 0); 2259 mutex_exit(&txr->txr_lock); 2260 } 2261 2262 /* XXX wait at least 400 usec for all tx queues in one go */ 2263 ixl_flush(sc); 2264 DELAY(500); 2265 2266 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2267 txr = sc->sc_qps[i].qp_txr; 2268 rxr = sc->sc_qps[i].qp_rxr; 2269 2270 mutex_enter(&txr->txr_lock); 2271 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2272 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2273 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2274 mutex_exit(&txr->txr_lock); 2275 2276 mutex_enter(&rxr->rxr_lock); 2277 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2278 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2279 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2280 mutex_exit(&rxr->rxr_lock); 2281 } 2282 2283 /* XXX short wait for all queue disables to settle */ 2284 ixl_flush(sc); 2285 DELAY(50); 2286 2287 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2288 txr = sc->sc_qps[i].qp_txr; 2289 rxr = sc->sc_qps[i].qp_rxr; 2290 2291 mutex_enter(&txr->txr_lock); 2292 if (ixl_txr_disabled(sc, txr) != 0) { 2293 mutex_exit(&txr->txr_lock); 2294 goto die; 2295 } 2296 mutex_exit(&txr->txr_lock); 2297 2298 mutex_enter(&rxr->rxr_lock); 2299 if (ixl_rxr_disabled(sc, rxr) != 0) { 2300 mutex_exit(&rxr->rxr_lock); 2301 goto die; 2302 } 2303 mutex_exit(&rxr->rxr_lock); 2304 } 2305 2306 ixl_stop_rendezvous(sc); 2307 2308 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2309 txr = sc->sc_qps[i].qp_txr; 2310 rxr = sc->sc_qps[i].qp_rxr; 2311 2312 mutex_enter(&txr->txr_lock); 2313 ixl_txr_unconfig(sc, txr); 2314 mutex_exit(&txr->txr_lock); 2315 2316 mutex_enter(&rxr->rxr_lock); 2317 ixl_rxr_unconfig(sc, rxr); 2318 mutex_exit(&rxr->rxr_lock); 2319 2320 ixl_txr_clean(sc, txr); 2321 ixl_rxr_clean(sc, rxr); 2322 } 2323 2324 return; 2325 die: 2326 sc->sc_dead = true; 2327 log(LOG_CRIT, "%s: failed to shut down rings", 2328 device_xname(sc->sc_dev)); 2329 return; 2330 } 2331 2332 static void 2333 ixl_stop(struct ifnet *ifp, int disable) 2334 { 2335 struct ixl_softc *sc = ifp->if_softc; 2336 2337 mutex_enter(&sc->sc_cfg_lock); 2338 ixl_stop_locked(sc); 2339 mutex_exit(&sc->sc_cfg_lock); 2340 } 2341 2342 static int 2343 ixl_queue_pairs_alloc(struct ixl_softc *sc) 2344 { 2345 struct ixl_queue_pair *qp; 2346 unsigned int i; 2347 size_t sz; 2348 2349 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2350 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 2351 2352 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2353 qp = &sc->sc_qps[i]; 2354 2355 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2356 ixl_handle_queue, qp); 2357 if (qp->qp_si == NULL) 2358 goto free; 2359 2360 qp->qp_txr = ixl_txr_alloc(sc, i); 2361 if (qp->qp_txr == NULL) 2362 goto free; 2363 2364 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2365 if (qp->qp_rxr == NULL) 2366 goto free; 2367 2368 qp->qp_sc = sc; 2369 snprintf(qp->qp_name, sizeof(qp->qp_name), 2370 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2371 } 2372 2373 return 0; 2374 free: 2375 if (sc->sc_qps != NULL) { 2376 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2377 qp = &sc->sc_qps[i]; 2378 2379 if (qp->qp_txr != NULL) 2380 ixl_txr_free(sc, qp->qp_txr); 2381 if (qp->qp_rxr != NULL) 2382 ixl_rxr_free(sc, qp->qp_rxr); 2383 if (qp->qp_si != NULL) 2384 softint_disestablish(qp->qp_si); 2385 } 2386 2387 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2388 kmem_free(sc->sc_qps, sz); 2389 sc->sc_qps = NULL; 2390 } 2391 2392 return -1; 2393 } 2394 2395 static void 2396 ixl_queue_pairs_free(struct ixl_softc *sc) 2397 { 2398 struct ixl_queue_pair *qp; 2399 unsigned int i; 2400 size_t sz; 2401 2402 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2403 qp = &sc->sc_qps[i]; 2404 ixl_txr_free(sc, qp->qp_txr); 2405 ixl_rxr_free(sc, qp->qp_rxr); 2406 softint_disestablish(qp->qp_si); 2407 } 2408 2409 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2410 kmem_free(sc->sc_qps, sz); 2411 sc->sc_qps = NULL; 2412 } 2413 2414 static struct ixl_tx_ring * 2415 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2416 { 2417 struct ixl_tx_ring *txr = NULL; 2418 struct ixl_tx_map *maps = NULL, *txm; 2419 unsigned int i; 2420 2421 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2422 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2423 KM_SLEEP); 2424 2425 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2426 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2427 IXL_TX_QUEUE_ALIGN) != 0) 2428 goto free; 2429 2430 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2431 txm = &maps[i]; 2432 2433 if (bus_dmamap_create(sc->sc_dmat, IXL_TX_PKT_MAXSIZE, 2434 IXL_TX_PKT_DESCS, IXL_TX_PKT_MAXSIZE, 0, 2435 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2436 goto uncreate; 2437 2438 txm->txm_eop = -1; 2439 txm->txm_m = NULL; 2440 } 2441 2442 txr->txr_cons = txr->txr_prod = 0; 2443 txr->txr_maps = maps; 2444 2445 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2446 if (txr->txr_intrq == NULL) 2447 goto uncreate; 2448 2449 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2450 ixl_deferred_transmit, txr); 2451 if (txr->txr_si == NULL) 2452 goto destroy_pcq; 2453 2454 txr->txr_tail = I40E_QTX_TAIL(qid); 2455 txr->txr_qid = qid; 2456 txr->txr_sc = sc; 2457 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2458 2459 return txr; 2460 2461 destroy_pcq: 2462 pcq_destroy(txr->txr_intrq); 2463 uncreate: 2464 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2465 txm = &maps[i]; 2466 2467 if (txm->txm_map == NULL) 2468 continue; 2469 2470 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2471 } 2472 2473 ixl_dmamem_free(sc, &txr->txr_mem); 2474 free: 2475 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2476 kmem_free(txr, sizeof(*txr)); 2477 2478 return NULL; 2479 } 2480 2481 static void 2482 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2483 { 2484 unsigned int qid; 2485 bus_size_t reg; 2486 uint32_t r; 2487 2488 qid = txr->txr_qid + sc->sc_base_queue; 2489 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2490 qid %= 128; 2491 2492 r = ixl_rd(sc, reg); 2493 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2494 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2495 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2496 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2497 ixl_wr(sc, reg, r); 2498 } 2499 2500 static void 2501 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2502 { 2503 struct ixl_hmc_txq txq; 2504 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2505 void *hmc; 2506 2507 memset(&txq, 0, sizeof(txq)); 2508 txq.head = htole16(txr->txr_cons); 2509 txq.new_context = 1; 2510 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2511 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2512 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2513 txq.tphrdesc_ena = 0; 2514 txq.tphrpacket_ena = 0; 2515 txq.tphwdesc_ena = 0; 2516 txq.rdylist = data->qs_handle[0]; 2517 2518 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2519 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2520 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2521 __arraycount(ixl_hmc_pack_txq)); 2522 } 2523 2524 static void 2525 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2526 { 2527 void *hmc; 2528 2529 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2530 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2531 txr->txr_cons = txr->txr_prod = 0; 2532 } 2533 2534 static void 2535 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2536 { 2537 struct ixl_tx_map *maps, *txm; 2538 bus_dmamap_t map; 2539 unsigned int i; 2540 2541 maps = txr->txr_maps; 2542 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2543 txm = &maps[i]; 2544 2545 if (txm->txm_m == NULL) 2546 continue; 2547 2548 map = txm->txm_map; 2549 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2550 BUS_DMASYNC_POSTWRITE); 2551 bus_dmamap_unload(sc->sc_dmat, map); 2552 2553 m_freem(txm->txm_m); 2554 txm->txm_m = NULL; 2555 } 2556 } 2557 2558 static int 2559 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2560 { 2561 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2562 uint32_t reg; 2563 int i; 2564 2565 for (i = 0; i < 10; i++) { 2566 reg = ixl_rd(sc, ena); 2567 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2568 return 0; 2569 2570 delaymsec(10); 2571 } 2572 2573 return ETIMEDOUT; 2574 } 2575 2576 static int 2577 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2578 { 2579 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2580 uint32_t reg; 2581 int i; 2582 2583 KASSERT(mutex_owned(&txr->txr_lock)); 2584 2585 for (i = 0; i < 10; i++) { 2586 reg = ixl_rd(sc, ena); 2587 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2588 return 0; 2589 2590 delaymsec(10); 2591 } 2592 2593 return ETIMEDOUT; 2594 } 2595 2596 static void 2597 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2598 { 2599 struct ixl_tx_map *maps, *txm; 2600 struct mbuf *m; 2601 unsigned int i; 2602 2603 softint_disestablish(txr->txr_si); 2604 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2605 m_freem(m); 2606 pcq_destroy(txr->txr_intrq); 2607 2608 maps = txr->txr_maps; 2609 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2610 txm = &maps[i]; 2611 2612 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2613 } 2614 2615 ixl_dmamem_free(sc, &txr->txr_mem); 2616 mutex_destroy(&txr->txr_lock); 2617 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2618 kmem_free(txr, sizeof(*txr)); 2619 } 2620 2621 static inline int 2622 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2623 struct ixl_tx_ring *txr) 2624 { 2625 struct mbuf *m; 2626 int error; 2627 2628 KASSERT(mutex_owned(&txr->txr_lock)); 2629 2630 m = *m0; 2631 2632 error = bus_dmamap_load_mbuf(dmat, map, m, 2633 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2634 if (error != EFBIG) 2635 return error; 2636 2637 m = m_defrag(m, M_DONTWAIT); 2638 if (m != NULL) { 2639 *m0 = m; 2640 txr->txr_defragged.ev_count++; 2641 2642 error = bus_dmamap_load_mbuf(dmat, map, m, 2643 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2644 } else { 2645 txr->txr_defrag_failed.ev_count++; 2646 error = ENOBUFS; 2647 } 2648 2649 return error; 2650 } 2651 2652 static inline int 2653 ixl_tx_setup_offloads(struct mbuf *m, uint64_t *cmd_txd) 2654 { 2655 struct ether_header *eh; 2656 size_t len; 2657 uint64_t cmd; 2658 2659 cmd = 0; 2660 2661 eh = mtod(m, struct ether_header *); 2662 switch (htons(eh->ether_type)) { 2663 case ETHERTYPE_IP: 2664 case ETHERTYPE_IPV6: 2665 len = ETHER_HDR_LEN; 2666 break; 2667 case ETHERTYPE_VLAN: 2668 len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2669 break; 2670 default: 2671 len = 0; 2672 } 2673 cmd |= ((len >> 1) << IXL_TX_DESC_MACLEN_SHIFT); 2674 2675 if (m->m_pkthdr.csum_flags & 2676 (M_CSUM_TSOv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) { 2677 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4; 2678 } 2679 if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) { 2680 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4_CSUM; 2681 } 2682 2683 if (m->m_pkthdr.csum_flags & 2684 (M_CSUM_TSOv6 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) { 2685 cmd |= IXL_TX_DESC_CMD_IIPT_IPV6; 2686 } 2687 2688 switch (cmd & IXL_TX_DESC_CMD_IIPT_MASK) { 2689 case IXL_TX_DESC_CMD_IIPT_IPV4: 2690 case IXL_TX_DESC_CMD_IIPT_IPV4_CSUM: 2691 len = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); 2692 break; 2693 case IXL_TX_DESC_CMD_IIPT_IPV6: 2694 len = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); 2695 break; 2696 default: 2697 len = 0; 2698 } 2699 cmd |= ((len >> 2) << IXL_TX_DESC_IPLEN_SHIFT); 2700 2701 if (m->m_pkthdr.csum_flags & 2702 (M_CSUM_TSOv4 | M_CSUM_TSOv6 | M_CSUM_TCPv4 | M_CSUM_TCPv6)) { 2703 len = sizeof(struct tcphdr); 2704 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_TCP; 2705 } else if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) { 2706 len = sizeof(struct udphdr); 2707 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_UDP; 2708 } else { 2709 len = 0; 2710 } 2711 cmd |= ((len >> 2) << IXL_TX_DESC_L4LEN_SHIFT); 2712 2713 *cmd_txd |= cmd; 2714 return 0; 2715 } 2716 2717 static void 2718 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2719 bool is_transmit) 2720 { 2721 struct ixl_softc *sc = ifp->if_softc; 2722 struct ixl_tx_desc *ring, *txd; 2723 struct ixl_tx_map *txm; 2724 bus_dmamap_t map; 2725 struct mbuf *m; 2726 uint64_t cmd, cmd_txd; 2727 unsigned int prod, free, last, i; 2728 unsigned int mask; 2729 int post = 0; 2730 2731 KASSERT(mutex_owned(&txr->txr_lock)); 2732 2733 if (ifp->if_link_state != LINK_STATE_UP 2734 || !ISSET(ifp->if_flags, IFF_RUNNING) 2735 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2736 if (!is_transmit) 2737 IFQ_PURGE(&ifp->if_snd); 2738 return; 2739 } 2740 2741 prod = txr->txr_prod; 2742 free = txr->txr_cons; 2743 if (free <= prod) 2744 free += sc->sc_tx_ring_ndescs; 2745 free -= prod; 2746 2747 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2748 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2749 2750 ring = IXL_DMA_KVA(&txr->txr_mem); 2751 mask = sc->sc_tx_ring_ndescs - 1; 2752 last = prod; 2753 cmd = 0; 2754 txd = NULL; 2755 2756 for (;;) { 2757 if (free <= IXL_TX_PKT_DESCS) { 2758 if (!is_transmit) 2759 SET(ifp->if_flags, IFF_OACTIVE); 2760 break; 2761 } 2762 2763 if (is_transmit) 2764 m = pcq_get(txr->txr_intrq); 2765 else 2766 IFQ_DEQUEUE(&ifp->if_snd, m); 2767 2768 if (m == NULL) 2769 break; 2770 2771 txm = &txr->txr_maps[prod]; 2772 map = txm->txm_map; 2773 2774 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2775 if_statinc(ifp, if_oerrors); 2776 m_freem(m); 2777 continue; 2778 } 2779 2780 cmd_txd = 0; 2781 if (m->m_pkthdr.csum_flags & IXL_CSUM_ALL_OFFLOAD) { 2782 ixl_tx_setup_offloads(m, &cmd_txd); 2783 } 2784 2785 if (vlan_has_tag(m)) { 2786 cmd_txd |= (uint64_t)vlan_get_tag(m) << 2787 IXL_TX_DESC_L2TAG1_SHIFT; 2788 cmd_txd |= IXL_TX_DESC_CMD_IL2TAG1; 2789 } 2790 2791 bus_dmamap_sync(sc->sc_dmat, map, 0, 2792 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2793 2794 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2795 txd = &ring[prod]; 2796 2797 cmd = (uint64_t)map->dm_segs[i].ds_len << 2798 IXL_TX_DESC_BSIZE_SHIFT; 2799 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2800 cmd |= cmd_txd; 2801 2802 txd->addr = htole64(map->dm_segs[i].ds_addr); 2803 txd->cmd = htole64(cmd); 2804 2805 last = prod; 2806 2807 prod++; 2808 prod &= mask; 2809 } 2810 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2811 txd->cmd = htole64(cmd); 2812 2813 txm->txm_m = m; 2814 txm->txm_eop = last; 2815 2816 bpf_mtap(ifp, m, BPF_D_OUT); 2817 2818 free -= i; 2819 post = 1; 2820 } 2821 2822 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2823 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2824 2825 if (post) { 2826 txr->txr_prod = prod; 2827 ixl_wr(sc, txr->txr_tail, prod); 2828 } 2829 } 2830 2831 static int 2832 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2833 { 2834 struct ifnet *ifp = &sc->sc_ec.ec_if; 2835 struct ixl_tx_desc *ring, *txd; 2836 struct ixl_tx_map *txm; 2837 struct mbuf *m; 2838 bus_dmamap_t map; 2839 unsigned int cons, prod, last; 2840 unsigned int mask; 2841 uint64_t dtype; 2842 int done = 0, more = 0; 2843 2844 KASSERT(mutex_owned(&txr->txr_lock)); 2845 2846 prod = txr->txr_prod; 2847 cons = txr->txr_cons; 2848 2849 if (cons == prod) 2850 return 0; 2851 2852 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2853 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2854 2855 ring = IXL_DMA_KVA(&txr->txr_mem); 2856 mask = sc->sc_tx_ring_ndescs - 1; 2857 2858 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2859 2860 do { 2861 if (txlimit-- <= 0) { 2862 more = 1; 2863 break; 2864 } 2865 2866 txm = &txr->txr_maps[cons]; 2867 last = txm->txm_eop; 2868 txd = &ring[last]; 2869 2870 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2871 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2872 break; 2873 2874 map = txm->txm_map; 2875 2876 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2877 BUS_DMASYNC_POSTWRITE); 2878 bus_dmamap_unload(sc->sc_dmat, map); 2879 2880 m = txm->txm_m; 2881 if (m != NULL) { 2882 if_statinc_ref(nsr, if_opackets); 2883 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 2884 if (ISSET(m->m_flags, M_MCAST)) 2885 if_statinc_ref(nsr, if_omcasts); 2886 m_freem(m); 2887 } 2888 2889 txm->txm_m = NULL; 2890 txm->txm_eop = -1; 2891 2892 cons = last + 1; 2893 cons &= mask; 2894 done = 1; 2895 } while (cons != prod); 2896 2897 IF_STAT_PUTREF(ifp); 2898 2899 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2900 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2901 2902 txr->txr_cons = cons; 2903 2904 if (done) { 2905 softint_schedule(txr->txr_si); 2906 if (txr->txr_qid == 0) { 2907 CLR(ifp->if_flags, IFF_OACTIVE); 2908 if_schedule_deferred_start(ifp); 2909 } 2910 } 2911 2912 return more; 2913 } 2914 2915 static void 2916 ixl_start(struct ifnet *ifp) 2917 { 2918 struct ixl_softc *sc; 2919 struct ixl_tx_ring *txr; 2920 2921 sc = ifp->if_softc; 2922 txr = sc->sc_qps[0].qp_txr; 2923 2924 mutex_enter(&txr->txr_lock); 2925 ixl_tx_common_locked(ifp, txr, false); 2926 mutex_exit(&txr->txr_lock); 2927 } 2928 2929 static inline unsigned int 2930 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2931 { 2932 u_int cpuid; 2933 2934 cpuid = cpu_index(curcpu()); 2935 2936 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2937 } 2938 2939 static int 2940 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2941 { 2942 struct ixl_softc *sc; 2943 struct ixl_tx_ring *txr; 2944 unsigned int qid; 2945 2946 sc = ifp->if_softc; 2947 qid = ixl_select_txqueue(sc, m); 2948 2949 txr = sc->sc_qps[qid].qp_txr; 2950 2951 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2952 mutex_enter(&txr->txr_lock); 2953 txr->txr_pcqdrop.ev_count++; 2954 mutex_exit(&txr->txr_lock); 2955 2956 m_freem(m); 2957 return ENOBUFS; 2958 } 2959 2960 if (mutex_tryenter(&txr->txr_lock)) { 2961 ixl_tx_common_locked(ifp, txr, true); 2962 mutex_exit(&txr->txr_lock); 2963 } else { 2964 kpreempt_disable(); 2965 softint_schedule(txr->txr_si); 2966 kpreempt_enable(); 2967 } 2968 2969 return 0; 2970 } 2971 2972 static void 2973 ixl_deferred_transmit(void *xtxr) 2974 { 2975 struct ixl_tx_ring *txr = xtxr; 2976 struct ixl_softc *sc = txr->txr_sc; 2977 struct ifnet *ifp = &sc->sc_ec.ec_if; 2978 2979 mutex_enter(&txr->txr_lock); 2980 txr->txr_transmitdef.ev_count++; 2981 if (pcq_peek(txr->txr_intrq) != NULL) 2982 ixl_tx_common_locked(ifp, txr, true); 2983 mutex_exit(&txr->txr_lock); 2984 } 2985 2986 static struct ixl_rx_ring * 2987 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2988 { 2989 struct ixl_rx_ring *rxr = NULL; 2990 struct ixl_rx_map *maps = NULL, *rxm; 2991 unsigned int i; 2992 2993 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2994 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2995 KM_SLEEP); 2996 2997 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2998 sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs, 2999 IXL_RX_QUEUE_ALIGN) != 0) 3000 goto free; 3001 3002 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3003 rxm = &maps[i]; 3004 3005 if (bus_dmamap_create(sc->sc_dmat, 3006 IXL_MCLBYTES, 1, IXL_MCLBYTES, 0, 3007 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 3008 goto uncreate; 3009 3010 rxm->rxm_m = NULL; 3011 } 3012 3013 rxr->rxr_cons = rxr->rxr_prod = 0; 3014 rxr->rxr_m_head = NULL; 3015 rxr->rxr_m_tail = &rxr->rxr_m_head; 3016 rxr->rxr_maps = maps; 3017 3018 rxr->rxr_tail = I40E_QRX_TAIL(qid); 3019 rxr->rxr_qid = qid; 3020 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 3021 3022 return rxr; 3023 3024 uncreate: 3025 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3026 rxm = &maps[i]; 3027 3028 if (rxm->rxm_map == NULL) 3029 continue; 3030 3031 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 3032 } 3033 3034 ixl_dmamem_free(sc, &rxr->rxr_mem); 3035 free: 3036 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3037 kmem_free(rxr, sizeof(*rxr)); 3038 3039 return NULL; 3040 } 3041 3042 static void 3043 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3044 { 3045 struct ixl_rx_map *maps, *rxm; 3046 bus_dmamap_t map; 3047 unsigned int i; 3048 3049 maps = rxr->rxr_maps; 3050 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3051 rxm = &maps[i]; 3052 3053 if (rxm->rxm_m == NULL) 3054 continue; 3055 3056 map = rxm->rxm_map; 3057 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3058 BUS_DMASYNC_POSTWRITE); 3059 bus_dmamap_unload(sc->sc_dmat, map); 3060 3061 m_freem(rxm->rxm_m); 3062 rxm->rxm_m = NULL; 3063 } 3064 3065 m_freem(rxr->rxr_m_head); 3066 rxr->rxr_m_head = NULL; 3067 rxr->rxr_m_tail = &rxr->rxr_m_head; 3068 3069 rxr->rxr_prod = rxr->rxr_cons = 0; 3070 } 3071 3072 static int 3073 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3074 { 3075 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3076 uint32_t reg; 3077 int i; 3078 3079 for (i = 0; i < 10; i++) { 3080 reg = ixl_rd(sc, ena); 3081 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 3082 return 0; 3083 3084 delaymsec(10); 3085 } 3086 3087 return ETIMEDOUT; 3088 } 3089 3090 static int 3091 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3092 { 3093 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3094 uint32_t reg; 3095 int i; 3096 3097 KASSERT(mutex_owned(&rxr->rxr_lock)); 3098 3099 for (i = 0; i < 10; i++) { 3100 reg = ixl_rd(sc, ena); 3101 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 3102 return 0; 3103 3104 delaymsec(10); 3105 } 3106 3107 return ETIMEDOUT; 3108 } 3109 3110 static void 3111 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3112 { 3113 struct ixl_hmc_rxq rxq; 3114 struct ifnet *ifp = &sc->sc_ec.ec_if; 3115 uint16_t rxmax; 3116 void *hmc; 3117 3118 memset(&rxq, 0, sizeof(rxq)); 3119 rxmax = ifp->if_mtu + IXL_MTU_ETHERLEN; 3120 3121 rxq.head = htole16(rxr->rxr_cons); 3122 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 3123 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 3124 rxq.dbuff = htole16(IXL_MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 3125 rxq.hbuff = 0; 3126 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 3127 rxq.dsize = IXL_HMC_RXQ_DSIZE_32; 3128 rxq.crcstrip = 1; 3129 rxq.l2sel = 1; 3130 rxq.showiv = 1; 3131 rxq.rxmax = htole16(rxmax); 3132 rxq.tphrdesc_ena = 0; 3133 rxq.tphwdesc_ena = 0; 3134 rxq.tphdata_ena = 0; 3135 rxq.tphhead_ena = 0; 3136 rxq.lrxqthresh = 0; 3137 rxq.prefena = 1; 3138 3139 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3140 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3141 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 3142 __arraycount(ixl_hmc_pack_rxq)); 3143 } 3144 3145 static void 3146 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3147 { 3148 void *hmc; 3149 3150 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3151 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3152 rxr->rxr_cons = rxr->rxr_prod = 0; 3153 } 3154 3155 static void 3156 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3157 { 3158 struct ixl_rx_map *maps, *rxm; 3159 unsigned int i; 3160 3161 maps = rxr->rxr_maps; 3162 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3163 rxm = &maps[i]; 3164 3165 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 3166 } 3167 3168 ixl_dmamem_free(sc, &rxr->rxr_mem); 3169 mutex_destroy(&rxr->rxr_lock); 3170 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3171 kmem_free(rxr, sizeof(*rxr)); 3172 } 3173 3174 static inline void 3175 ixl_rx_csum(struct mbuf *m, uint64_t qword) 3176 { 3177 int flags_mask; 3178 3179 if (!ISSET(qword, IXL_RX_DESC_L3L4P)) { 3180 /* No L3 or L4 checksum was calculated */ 3181 return; 3182 } 3183 3184 switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) { 3185 case IXL_RX_DESC_PTYPE_IPV4FRAG: 3186 case IXL_RX_DESC_PTYPE_IPV4: 3187 case IXL_RX_DESC_PTYPE_SCTPV4: 3188 case IXL_RX_DESC_PTYPE_ICMPV4: 3189 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3190 break; 3191 case IXL_RX_DESC_PTYPE_TCPV4: 3192 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3193 flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD; 3194 break; 3195 case IXL_RX_DESC_PTYPE_UDPV4: 3196 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3197 flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD; 3198 break; 3199 case IXL_RX_DESC_PTYPE_TCPV6: 3200 flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD; 3201 break; 3202 case IXL_RX_DESC_PTYPE_UDPV6: 3203 flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD; 3204 break; 3205 default: 3206 flags_mask = 0; 3207 } 3208 3209 m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 | 3210 M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6)); 3211 3212 if (ISSET(qword, IXL_RX_DESC_IPE)) { 3213 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD); 3214 } 3215 3216 if (ISSET(qword, IXL_RX_DESC_L4E)) { 3217 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD); 3218 } 3219 } 3220 3221 static int 3222 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 3223 { 3224 struct ifnet *ifp = &sc->sc_ec.ec_if; 3225 struct ixl_rx_wb_desc_32 *ring, *rxd; 3226 struct ixl_rx_map *rxm; 3227 bus_dmamap_t map; 3228 unsigned int cons, prod; 3229 struct mbuf *m; 3230 uint64_t word, word0; 3231 unsigned int len; 3232 unsigned int mask; 3233 int done = 0, more = 0; 3234 3235 KASSERT(mutex_owned(&rxr->rxr_lock)); 3236 3237 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3238 return 0; 3239 3240 prod = rxr->rxr_prod; 3241 cons = rxr->rxr_cons; 3242 3243 if (cons == prod) 3244 return 0; 3245 3246 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3247 0, IXL_DMA_LEN(&rxr->rxr_mem), 3248 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3249 3250 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3251 mask = sc->sc_rx_ring_ndescs - 1; 3252 3253 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 3254 3255 do { 3256 if (rxlimit-- <= 0) { 3257 more = 1; 3258 break; 3259 } 3260 3261 rxd = &ring[cons]; 3262 3263 word = le64toh(rxd->qword1); 3264 3265 if (!ISSET(word, IXL_RX_DESC_DD)) 3266 break; 3267 3268 rxm = &rxr->rxr_maps[cons]; 3269 3270 map = rxm->rxm_map; 3271 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3272 BUS_DMASYNC_POSTREAD); 3273 bus_dmamap_unload(sc->sc_dmat, map); 3274 3275 m = rxm->rxm_m; 3276 rxm->rxm_m = NULL; 3277 3278 KASSERT(m != NULL); 3279 3280 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 3281 m->m_len = len; 3282 m->m_pkthdr.len = 0; 3283 3284 m->m_next = NULL; 3285 *rxr->rxr_m_tail = m; 3286 rxr->rxr_m_tail = &m->m_next; 3287 3288 m = rxr->rxr_m_head; 3289 m->m_pkthdr.len += len; 3290 3291 if (ISSET(word, IXL_RX_DESC_EOP)) { 3292 word0 = le64toh(rxd->qword0); 3293 3294 if (ISSET(word, IXL_RX_DESC_L2TAG1P)) { 3295 vlan_set_tag(m, 3296 __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK)); 3297 } 3298 3299 if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0) 3300 ixl_rx_csum(m, word); 3301 3302 if (!ISSET(word, 3303 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 3304 m_set_rcvif(m, ifp); 3305 if_statinc_ref(nsr, if_ipackets); 3306 if_statadd_ref(nsr, if_ibytes, 3307 m->m_pkthdr.len); 3308 if_percpuq_enqueue(ifp->if_percpuq, m); 3309 } else { 3310 if_statinc_ref(nsr, if_ierrors); 3311 m_freem(m); 3312 } 3313 3314 rxr->rxr_m_head = NULL; 3315 rxr->rxr_m_tail = &rxr->rxr_m_head; 3316 } 3317 3318 cons++; 3319 cons &= mask; 3320 3321 done = 1; 3322 } while (cons != prod); 3323 3324 if (done) { 3325 rxr->rxr_cons = cons; 3326 if (ixl_rxfill(sc, rxr) == -1) 3327 if_statinc_ref(nsr, if_iqdrops); 3328 } 3329 3330 IF_STAT_PUTREF(ifp); 3331 3332 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3333 0, IXL_DMA_LEN(&rxr->rxr_mem), 3334 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3335 3336 return more; 3337 } 3338 3339 static int 3340 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3341 { 3342 struct ixl_rx_rd_desc_32 *ring, *rxd; 3343 struct ixl_rx_map *rxm; 3344 bus_dmamap_t map; 3345 struct mbuf *m; 3346 unsigned int prod; 3347 unsigned int slots; 3348 unsigned int mask; 3349 int post = 0, error = 0; 3350 3351 KASSERT(mutex_owned(&rxr->rxr_lock)); 3352 3353 prod = rxr->rxr_prod; 3354 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 3355 sc->sc_rx_ring_ndescs); 3356 3357 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3358 mask = sc->sc_rx_ring_ndescs - 1; 3359 3360 if (__predict_false(slots <= 0)) 3361 return -1; 3362 3363 do { 3364 rxm = &rxr->rxr_maps[prod]; 3365 3366 MGETHDR(m, M_DONTWAIT, MT_DATA); 3367 if (m == NULL) { 3368 rxr->rxr_mgethdr_failed.ev_count++; 3369 error = -1; 3370 break; 3371 } 3372 3373 MCLGET(m, M_DONTWAIT); 3374 if (!ISSET(m->m_flags, M_EXT)) { 3375 rxr->rxr_mgetcl_failed.ev_count++; 3376 error = -1; 3377 m_freem(m); 3378 break; 3379 } 3380 3381 m->m_len = m->m_pkthdr.len = MCLBYTES; 3382 m_adj(m, ETHER_ALIGN); 3383 3384 map = rxm->rxm_map; 3385 3386 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 3387 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 3388 rxr->rxr_mbuf_load_failed.ev_count++; 3389 error = -1; 3390 m_freem(m); 3391 break; 3392 } 3393 3394 rxm->rxm_m = m; 3395 3396 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3397 BUS_DMASYNC_PREREAD); 3398 3399 rxd = &ring[prod]; 3400 3401 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 3402 rxd->haddr = htole64(0); 3403 3404 prod++; 3405 prod &= mask; 3406 3407 post = 1; 3408 3409 } while (--slots); 3410 3411 if (post) { 3412 rxr->rxr_prod = prod; 3413 ixl_wr(sc, rxr->rxr_tail, prod); 3414 } 3415 3416 return error; 3417 } 3418 3419 static inline int 3420 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 3421 u_int txlimit, struct evcnt *txevcnt, 3422 u_int rxlimit, struct evcnt *rxevcnt) 3423 { 3424 struct ixl_tx_ring *txr = qp->qp_txr; 3425 struct ixl_rx_ring *rxr = qp->qp_rxr; 3426 int txmore, rxmore; 3427 int rv; 3428 3429 mutex_enter(&txr->txr_lock); 3430 txevcnt->ev_count++; 3431 txmore = ixl_txeof(sc, txr, txlimit); 3432 mutex_exit(&txr->txr_lock); 3433 3434 mutex_enter(&rxr->rxr_lock); 3435 rxevcnt->ev_count++; 3436 rxmore = ixl_rxeof(sc, rxr, rxlimit); 3437 mutex_exit(&rxr->rxr_lock); 3438 3439 rv = txmore | (rxmore << 1); 3440 3441 return rv; 3442 } 3443 3444 static void 3445 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 3446 { 3447 3448 if (qp->qp_workqueue) 3449 workqueue_enqueue(sc->sc_workq_txrx, &qp->qp_work, NULL); 3450 else 3451 softint_schedule(qp->qp_si); 3452 } 3453 3454 static int 3455 ixl_intr(void *xsc) 3456 { 3457 struct ixl_softc *sc = xsc; 3458 struct ixl_tx_ring *txr; 3459 struct ixl_rx_ring *rxr; 3460 uint32_t icr, rxintr, txintr; 3461 int rv = 0; 3462 unsigned int i; 3463 3464 KASSERT(sc != NULL); 3465 3466 ixl_enable_other_intr(sc); 3467 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3468 3469 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3470 atomic_inc_64(&sc->sc_event_atq.ev_count); 3471 ixl_atq_done(sc); 3472 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3473 rv = 1; 3474 } 3475 3476 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3477 atomic_inc_64(&sc->sc_event_link.ev_count); 3478 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3479 rv = 1; 3480 } 3481 3482 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 3483 txintr = icr & I40E_INTR_NOTX_TX_MASK; 3484 3485 if (txintr || rxintr) { 3486 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 3487 txr = sc->sc_qps[i].qp_txr; 3488 rxr = sc->sc_qps[i].qp_rxr; 3489 3490 ixl_handle_queue_common(sc, &sc->sc_qps[i], 3491 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 3492 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 3493 } 3494 rv = 1; 3495 } 3496 3497 return rv; 3498 } 3499 3500 static int 3501 ixl_queue_intr(void *xqp) 3502 { 3503 struct ixl_queue_pair *qp = xqp; 3504 struct ixl_tx_ring *txr = qp->qp_txr; 3505 struct ixl_rx_ring *rxr = qp->qp_rxr; 3506 struct ixl_softc *sc = qp->qp_sc; 3507 u_int txlimit, rxlimit; 3508 int more; 3509 3510 txlimit = sc->sc_tx_intr_process_limit; 3511 rxlimit = sc->sc_rx_intr_process_limit; 3512 qp->qp_workqueue = sc->sc_txrx_workqueue; 3513 3514 more = ixl_handle_queue_common(sc, qp, 3515 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3516 3517 if (more != 0) { 3518 ixl_sched_handle_queue(sc, qp); 3519 } else { 3520 /* for ALTQ */ 3521 if (txr->txr_qid == 0) 3522 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3523 softint_schedule(txr->txr_si); 3524 3525 ixl_enable_queue_intr(sc, qp); 3526 } 3527 3528 return 1; 3529 } 3530 3531 static void 3532 ixl_handle_queue_wk(struct work *wk, void *xsc) 3533 { 3534 struct ixl_queue_pair *qp; 3535 3536 qp = container_of(wk, struct ixl_queue_pair, qp_work); 3537 ixl_handle_queue(qp); 3538 } 3539 3540 static void 3541 ixl_handle_queue(void *xqp) 3542 { 3543 struct ixl_queue_pair *qp = xqp; 3544 struct ixl_softc *sc = qp->qp_sc; 3545 struct ixl_tx_ring *txr = qp->qp_txr; 3546 struct ixl_rx_ring *rxr = qp->qp_rxr; 3547 u_int txlimit, rxlimit; 3548 int more; 3549 3550 txlimit = sc->sc_tx_process_limit; 3551 rxlimit = sc->sc_rx_process_limit; 3552 3553 more = ixl_handle_queue_common(sc, qp, 3554 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3555 3556 if (more != 0) 3557 ixl_sched_handle_queue(sc, qp); 3558 else 3559 ixl_enable_queue_intr(sc, qp); 3560 } 3561 3562 static inline void 3563 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3564 { 3565 uint32_t hmc_idx, hmc_isvf; 3566 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3567 3568 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3569 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3570 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3571 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3572 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3573 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3574 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3575 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3576 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3577 3578 device_printf(sc->sc_dev, 3579 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3580 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3581 } 3582 3583 static int 3584 ixl_other_intr(void *xsc) 3585 { 3586 struct ixl_softc *sc = xsc; 3587 uint32_t icr, mask, reg; 3588 int rv; 3589 3590 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3591 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3592 3593 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3594 atomic_inc_64(&sc->sc_event_atq.ev_count); 3595 ixl_atq_done(sc); 3596 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3597 rv = 1; 3598 } 3599 3600 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3601 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3602 device_printf(sc->sc_dev, "link stat changed\n"); 3603 3604 atomic_inc_64(&sc->sc_event_link.ev_count); 3605 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3606 rv = 1; 3607 } 3608 3609 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3610 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3611 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3612 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3613 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3614 3615 device_printf(sc->sc_dev, "GRST: %s\n", 3616 reg == I40E_RESET_CORER ? "CORER" : 3617 reg == I40E_RESET_GLOBR ? "GLOBR" : 3618 reg == I40E_RESET_EMPR ? "EMPR" : 3619 "POR"); 3620 } 3621 3622 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3623 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3624 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3625 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3626 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3627 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3628 3629 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3630 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3631 device_printf(sc->sc_dev, "critical error\n"); 3632 } 3633 3634 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3635 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3636 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3637 ixl_print_hmc_error(sc, reg); 3638 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3639 } 3640 3641 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3642 ixl_flush(sc); 3643 ixl_enable_other_intr(sc); 3644 return rv; 3645 } 3646 3647 static void 3648 ixl_get_link_status_done(struct ixl_softc *sc, 3649 const struct ixl_aq_desc *iaq) 3650 { 3651 3652 ixl_link_state_update(sc, iaq); 3653 } 3654 3655 static void 3656 ixl_get_link_status(void *xsc) 3657 { 3658 struct ixl_softc *sc = xsc; 3659 struct ixl_aq_desc *iaq; 3660 struct ixl_aq_link_param *param; 3661 3662 memset(&sc->sc_link_state_atq, 0, sizeof(sc->sc_link_state_atq)); 3663 iaq = &sc->sc_link_state_atq.iatq_desc; 3664 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3665 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3666 param->notify = IXL_AQ_LINK_NOTIFY; 3667 3668 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 3669 (void)ixl_atq_post(sc, &sc->sc_link_state_atq); 3670 } 3671 3672 static void 3673 ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3674 { 3675 struct ifnet *ifp = &sc->sc_ec.ec_if; 3676 int link_state; 3677 3678 KASSERT(kpreempt_disabled()); 3679 3680 link_state = ixl_set_link_status(sc, iaq); 3681 3682 if (ifp->if_link_state != link_state) 3683 if_link_state_change(ifp, link_state); 3684 3685 if (link_state != LINK_STATE_DOWN) { 3686 if_schedule_deferred_start(ifp); 3687 } 3688 } 3689 3690 static void 3691 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3692 const char *msg) 3693 { 3694 char buf[512]; 3695 size_t len; 3696 3697 len = sizeof(buf); 3698 buf[--len] = '\0'; 3699 3700 device_printf(sc->sc_dev, "%s\n", msg); 3701 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3702 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3703 buf, le16toh(iaq->iaq_opcode)); 3704 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3705 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3706 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3707 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3708 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3709 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3710 } 3711 3712 static void 3713 ixl_arq(void *xsc) 3714 { 3715 struct ixl_softc *sc = xsc; 3716 struct ixl_aq_desc *arq, *iaq; 3717 struct ixl_aq_buf *aqb; 3718 unsigned int cons = sc->sc_arq_cons; 3719 unsigned int prod; 3720 int done = 0; 3721 3722 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3723 sc->sc_aq_regs->arq_head_mask; 3724 3725 if (cons == prod) 3726 goto done; 3727 3728 arq = IXL_DMA_KVA(&sc->sc_arq); 3729 3730 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3731 0, IXL_DMA_LEN(&sc->sc_arq), 3732 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3733 3734 do { 3735 iaq = &arq[cons]; 3736 aqb = sc->sc_arq_live[cons]; 3737 3738 KASSERT(aqb != NULL); 3739 3740 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3741 BUS_DMASYNC_POSTREAD); 3742 3743 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3744 ixl_aq_dump(sc, iaq, "arq event"); 3745 3746 switch (iaq->iaq_opcode) { 3747 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3748 kpreempt_disable(); 3749 ixl_link_state_update(sc, iaq); 3750 kpreempt_enable(); 3751 break; 3752 } 3753 3754 memset(iaq, 0, sizeof(*iaq)); 3755 sc->sc_arq_live[cons] = NULL; 3756 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3757 3758 cons++; 3759 cons &= IXL_AQ_MASK; 3760 3761 done = 1; 3762 } while (cons != prod); 3763 3764 if (done) { 3765 sc->sc_arq_cons = cons; 3766 ixl_arq_fill(sc); 3767 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3768 0, IXL_DMA_LEN(&sc->sc_arq), 3769 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3770 } 3771 3772 done: 3773 ixl_enable_other_intr(sc); 3774 } 3775 3776 static void 3777 ixl_atq_set(struct ixl_atq *iatq, 3778 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3779 { 3780 3781 iatq->iatq_fn = fn; 3782 } 3783 3784 static int 3785 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3786 { 3787 struct ixl_aq_desc *atq, *slot; 3788 unsigned int prod, cons, prod_next; 3789 3790 /* assert locked */ 3791 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3792 3793 atq = IXL_DMA_KVA(&sc->sc_atq); 3794 prod = sc->sc_atq_prod; 3795 cons = sc->sc_atq_cons; 3796 prod_next = (prod +1) & IXL_AQ_MASK; 3797 3798 if (cons == prod_next) 3799 return ENOMEM; 3800 3801 slot = &atq[prod]; 3802 3803 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3804 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3805 3806 *slot = iatq->iatq_desc; 3807 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3808 3809 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3810 ixl_aq_dump(sc, slot, "atq command"); 3811 3812 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3813 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3814 3815 sc->sc_atq_prod = prod_next; 3816 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3817 3818 return 0; 3819 } 3820 3821 static int 3822 ixl_atq_post(struct ixl_softc *sc, struct ixl_atq *iatq) 3823 { 3824 int rv; 3825 3826 mutex_enter(&sc->sc_atq_lock); 3827 rv = ixl_atq_post_locked(sc, iatq); 3828 mutex_exit(&sc->sc_atq_lock); 3829 3830 return rv; 3831 } 3832 3833 static void 3834 ixl_atq_done_locked(struct ixl_softc *sc) 3835 { 3836 struct ixl_aq_desc *atq, *slot; 3837 struct ixl_atq *iatq; 3838 unsigned int cons; 3839 unsigned int prod; 3840 3841 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3842 3843 prod = sc->sc_atq_prod; 3844 cons = sc->sc_atq_cons; 3845 3846 if (prod == cons) 3847 return; 3848 3849 atq = IXL_DMA_KVA(&sc->sc_atq); 3850 3851 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3852 0, IXL_DMA_LEN(&sc->sc_atq), 3853 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3854 3855 do { 3856 slot = &atq[cons]; 3857 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3858 break; 3859 3860 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3861 iatq->iatq_desc = *slot; 3862 3863 memset(slot, 0, sizeof(*slot)); 3864 3865 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3866 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3867 3868 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3869 3870 cons++; 3871 cons &= IXL_AQ_MASK; 3872 } while (cons != prod); 3873 3874 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3875 0, IXL_DMA_LEN(&sc->sc_atq), 3876 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3877 3878 sc->sc_atq_cons = cons; 3879 } 3880 3881 static void 3882 ixl_atq_done(struct ixl_softc *sc) 3883 { 3884 3885 mutex_enter(&sc->sc_atq_lock); 3886 ixl_atq_done_locked(sc); 3887 mutex_exit(&sc->sc_atq_lock); 3888 } 3889 3890 static void 3891 ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3892 { 3893 3894 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3895 3896 cv_signal(&sc->sc_atq_cv); 3897 } 3898 3899 static int 3900 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3901 { 3902 int error; 3903 3904 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3905 3906 ixl_atq_set(iatq, ixl_wakeup); 3907 3908 mutex_enter(&sc->sc_atq_lock); 3909 error = ixl_atq_post_locked(sc, iatq); 3910 if (error) { 3911 mutex_exit(&sc->sc_atq_lock); 3912 return error; 3913 } 3914 3915 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3916 IXL_ATQ_EXEC_TIMEOUT); 3917 mutex_exit(&sc->sc_atq_lock); 3918 3919 return error; 3920 } 3921 3922 static int 3923 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3924 { 3925 struct ixl_aq_desc *atq, *slot; 3926 unsigned int prod; 3927 unsigned int t = 0; 3928 3929 mutex_enter(&sc->sc_atq_lock); 3930 3931 atq = IXL_DMA_KVA(&sc->sc_atq); 3932 prod = sc->sc_atq_prod; 3933 slot = atq + prod; 3934 3935 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3936 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3937 3938 *slot = *iaq; 3939 slot->iaq_flags |= htole16(IXL_AQ_SI); 3940 3941 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3942 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3943 3944 prod++; 3945 prod &= IXL_AQ_MASK; 3946 sc->sc_atq_prod = prod; 3947 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3948 3949 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3950 delaymsec(1); 3951 3952 if (t++ > tm) { 3953 mutex_exit(&sc->sc_atq_lock); 3954 return ETIMEDOUT; 3955 } 3956 } 3957 3958 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3959 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3960 *iaq = *slot; 3961 memset(slot, 0, sizeof(*slot)); 3962 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3963 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3964 3965 sc->sc_atq_cons = prod; 3966 3967 mutex_exit(&sc->sc_atq_lock); 3968 3969 return 0; 3970 } 3971 3972 static int 3973 ixl_get_version(struct ixl_softc *sc) 3974 { 3975 struct ixl_aq_desc iaq; 3976 uint32_t fwbuild, fwver, apiver; 3977 uint16_t api_maj_ver, api_min_ver; 3978 3979 memset(&iaq, 0, sizeof(iaq)); 3980 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3981 3982 iaq.iaq_retval = le16toh(23); 3983 3984 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3985 return ETIMEDOUT; 3986 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3987 return EIO; 3988 3989 fwbuild = le32toh(iaq.iaq_param[1]); 3990 fwver = le32toh(iaq.iaq_param[2]); 3991 apiver = le32toh(iaq.iaq_param[3]); 3992 3993 api_maj_ver = (uint16_t)apiver; 3994 api_min_ver = (uint16_t)(apiver >> 16); 3995 3996 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 3997 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 3998 3999 if (sc->sc_mac_type == I40E_MAC_X722) { 4000 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK | 4001 IXL_SC_AQ_FLAG_NVMREAD); 4002 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 4003 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS); 4004 } 4005 4006 #define IXL_API_VER(maj, min) (((uint32_t)(maj) << 16) | (min)) 4007 if (IXL_API_VER(api_maj_ver, api_min_ver) >= IXL_API_VER(1, 5)) { 4008 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 4009 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK); 4010 } 4011 #undef IXL_API_VER 4012 4013 return 0; 4014 } 4015 4016 static int 4017 ixl_get_nvm_version(struct ixl_softc *sc) 4018 { 4019 uint16_t nvmver, cfg_ptr, eetrack_hi, eetrack_lo, oem_hi, oem_lo; 4020 uint32_t eetrack, oem; 4021 uint16_t nvm_maj_ver, nvm_min_ver, oem_build; 4022 uint8_t oem_ver, oem_patch; 4023 4024 nvmver = cfg_ptr = eetrack_hi = eetrack_lo = oem_hi = oem_lo = 0; 4025 ixl_rd16_nvm(sc, I40E_SR_NVM_DEV_STARTER_VERSION, &nvmver); 4026 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_HI, &eetrack_hi); 4027 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_LO, &eetrack_lo); 4028 ixl_rd16_nvm(sc, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr); 4029 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF, &oem_hi); 4030 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF + 1, &oem_lo); 4031 4032 nvm_maj_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_HI_MASK); 4033 nvm_min_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_LO_MASK); 4034 eetrack = ((uint32_t)eetrack_hi << 16) | eetrack_lo; 4035 oem = ((uint32_t)oem_hi << 16) | oem_lo; 4036 oem_ver = __SHIFTOUT(oem, IXL_NVM_OEMVERSION_MASK); 4037 oem_build = __SHIFTOUT(oem, IXL_NVM_OEMBUILD_MASK); 4038 oem_patch = __SHIFTOUT(oem, IXL_NVM_OEMPATCH_MASK); 4039 4040 aprint_normal(" nvm %x.%02x etid %08x oem %d.%d.%d", 4041 nvm_maj_ver, nvm_min_ver, eetrack, 4042 oem_ver, oem_build, oem_patch); 4043 4044 return 0; 4045 } 4046 4047 static int 4048 ixl_pxe_clear(struct ixl_softc *sc) 4049 { 4050 struct ixl_aq_desc iaq; 4051 int rv; 4052 4053 memset(&iaq, 0, sizeof(iaq)); 4054 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 4055 iaq.iaq_param[0] = htole32(0x2); 4056 4057 rv = ixl_atq_poll(sc, &iaq, 250); 4058 4059 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 4060 4061 if (rv != 0) 4062 return ETIMEDOUT; 4063 4064 switch (iaq.iaq_retval) { 4065 case htole16(IXL_AQ_RC_OK): 4066 case htole16(IXL_AQ_RC_EEXIST): 4067 break; 4068 default: 4069 return EIO; 4070 } 4071 4072 return 0; 4073 } 4074 4075 static int 4076 ixl_lldp_shut(struct ixl_softc *sc) 4077 { 4078 struct ixl_aq_desc iaq; 4079 4080 memset(&iaq, 0, sizeof(iaq)); 4081 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 4082 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 4083 4084 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4085 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 4086 return -1; 4087 } 4088 4089 switch (iaq.iaq_retval) { 4090 case htole16(IXL_AQ_RC_EMODE): 4091 case htole16(IXL_AQ_RC_EPERM): 4092 /* ignore silently */ 4093 default: 4094 break; 4095 } 4096 4097 return 0; 4098 } 4099 4100 static void 4101 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 4102 { 4103 uint16_t id; 4104 uint32_t number, logical_id; 4105 4106 id = le16toh(cap->cap_id); 4107 number = le32toh(cap->number); 4108 logical_id = le32toh(cap->logical_id); 4109 4110 switch (id) { 4111 case IXL_AQ_CAP_RSS: 4112 sc->sc_rss_table_size = number; 4113 sc->sc_rss_table_entry_width = logical_id; 4114 break; 4115 case IXL_AQ_CAP_RXQ: 4116 case IXL_AQ_CAP_TXQ: 4117 sc->sc_nqueue_pairs_device = MIN(number, 4118 sc->sc_nqueue_pairs_device); 4119 break; 4120 } 4121 } 4122 4123 static int 4124 ixl_get_hw_capabilities(struct ixl_softc *sc) 4125 { 4126 struct ixl_dmamem idm; 4127 struct ixl_aq_desc iaq; 4128 struct ixl_aq_capability *caps; 4129 size_t i, ncaps; 4130 bus_size_t caps_size; 4131 uint16_t status; 4132 int rv; 4133 4134 caps_size = sizeof(caps[0]) * 40; 4135 memset(&iaq, 0, sizeof(iaq)); 4136 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 4137 4138 do { 4139 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 4140 return -1; 4141 } 4142 4143 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4144 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4145 iaq.iaq_datalen = htole16(caps_size); 4146 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4147 4148 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4149 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 4150 4151 rv = ixl_atq_poll(sc, &iaq, 250); 4152 4153 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4154 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 4155 4156 if (rv != 0) { 4157 aprint_error(", HW capabilities timeout\n"); 4158 goto done; 4159 } 4160 4161 status = le16toh(iaq.iaq_retval); 4162 4163 if (status == IXL_AQ_RC_ENOMEM) { 4164 caps_size = le16toh(iaq.iaq_datalen); 4165 ixl_dmamem_free(sc, &idm); 4166 } 4167 } while (status == IXL_AQ_RC_ENOMEM); 4168 4169 if (status != IXL_AQ_RC_OK) { 4170 aprint_error(", HW capabilities error\n"); 4171 goto done; 4172 } 4173 4174 caps = IXL_DMA_KVA(&idm); 4175 ncaps = le16toh(iaq.iaq_param[1]); 4176 4177 for (i = 0; i < ncaps; i++) { 4178 ixl_parse_hw_capability(sc, &caps[i]); 4179 } 4180 4181 done: 4182 ixl_dmamem_free(sc, &idm); 4183 return rv; 4184 } 4185 4186 static int 4187 ixl_get_mac(struct ixl_softc *sc) 4188 { 4189 struct ixl_dmamem idm; 4190 struct ixl_aq_desc iaq; 4191 struct ixl_aq_mac_addresses *addrs; 4192 int rv; 4193 4194 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 4195 aprint_error(", unable to allocate mac addresses\n"); 4196 return -1; 4197 } 4198 4199 memset(&iaq, 0, sizeof(iaq)); 4200 iaq.iaq_flags = htole16(IXL_AQ_BUF); 4201 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 4202 iaq.iaq_datalen = htole16(sizeof(*addrs)); 4203 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4204 4205 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4206 BUS_DMASYNC_PREREAD); 4207 4208 rv = ixl_atq_poll(sc, &iaq, 250); 4209 4210 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4211 BUS_DMASYNC_POSTREAD); 4212 4213 if (rv != 0) { 4214 aprint_error(", MAC ADDRESS READ timeout\n"); 4215 rv = -1; 4216 goto done; 4217 } 4218 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4219 aprint_error(", MAC ADDRESS READ error\n"); 4220 rv = -1; 4221 goto done; 4222 } 4223 4224 addrs = IXL_DMA_KVA(&idm); 4225 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 4226 printf(", port address is not valid\n"); 4227 goto done; 4228 } 4229 4230 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 4231 rv = 0; 4232 4233 done: 4234 ixl_dmamem_free(sc, &idm); 4235 return rv; 4236 } 4237 4238 static int 4239 ixl_get_switch_config(struct ixl_softc *sc) 4240 { 4241 struct ixl_dmamem idm; 4242 struct ixl_aq_desc iaq; 4243 struct ixl_aq_switch_config *hdr; 4244 struct ixl_aq_switch_config_element *elms, *elm; 4245 unsigned int nelm, i; 4246 int rv; 4247 4248 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4249 aprint_error_dev(sc->sc_dev, 4250 "unable to allocate switch config buffer\n"); 4251 return -1; 4252 } 4253 4254 memset(&iaq, 0, sizeof(iaq)); 4255 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4256 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4257 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 4258 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 4259 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4260 4261 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4262 BUS_DMASYNC_PREREAD); 4263 4264 rv = ixl_atq_poll(sc, &iaq, 250); 4265 4266 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4267 BUS_DMASYNC_POSTREAD); 4268 4269 if (rv != 0) { 4270 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 4271 rv = -1; 4272 goto done; 4273 } 4274 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4275 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 4276 rv = -1; 4277 goto done; 4278 } 4279 4280 hdr = IXL_DMA_KVA(&idm); 4281 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 4282 4283 nelm = le16toh(hdr->num_reported); 4284 if (nelm < 1) { 4285 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 4286 rv = -1; 4287 goto done; 4288 } 4289 4290 for (i = 0; i < nelm; i++) { 4291 elm = &elms[i]; 4292 4293 aprint_debug_dev(sc->sc_dev, 4294 "type %x revision %u seid %04x\n", 4295 elm->type, elm->revision, le16toh(elm->seid)); 4296 aprint_debug_dev(sc->sc_dev, 4297 "uplink %04x downlink %04x\n", 4298 le16toh(elm->uplink_seid), 4299 le16toh(elm->downlink_seid)); 4300 aprint_debug_dev(sc->sc_dev, 4301 "conntype %x scheduler %04x extra %04x\n", 4302 elm->connection_type, 4303 le16toh(elm->scheduler_id), 4304 le16toh(elm->element_info)); 4305 } 4306 4307 elm = &elms[0]; 4308 4309 sc->sc_uplink_seid = elm->uplink_seid; 4310 sc->sc_downlink_seid = elm->downlink_seid; 4311 sc->sc_seid = elm->seid; 4312 4313 if ((sc->sc_uplink_seid == htole16(0)) != 4314 (sc->sc_downlink_seid == htole16(0))) { 4315 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 4316 rv = -1; 4317 goto done; 4318 } 4319 4320 done: 4321 ixl_dmamem_free(sc, &idm); 4322 return rv; 4323 } 4324 4325 static int 4326 ixl_phy_mask_ints(struct ixl_softc *sc) 4327 { 4328 struct ixl_aq_desc iaq; 4329 4330 memset(&iaq, 0, sizeof(iaq)); 4331 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 4332 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 4333 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 4334 IXL_AQ_PHY_EV_MEDIA_NA)); 4335 4336 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4337 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 4338 return -1; 4339 } 4340 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4341 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 4342 return -1; 4343 } 4344 4345 return 0; 4346 } 4347 4348 static int 4349 ixl_get_phy_abilities(struct ixl_softc *sc,struct ixl_dmamem *idm) 4350 { 4351 struct ixl_aq_desc iaq; 4352 int rv; 4353 4354 memset(&iaq, 0, sizeof(iaq)); 4355 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4356 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4357 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 4358 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 4359 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 4360 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 4361 4362 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4363 BUS_DMASYNC_PREREAD); 4364 4365 rv = ixl_atq_poll(sc, &iaq, 250); 4366 4367 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4368 BUS_DMASYNC_POSTREAD); 4369 4370 if (rv != 0) 4371 return -1; 4372 4373 return le16toh(iaq.iaq_retval); 4374 } 4375 4376 static int 4377 ixl_get_phy_info(struct ixl_softc *sc) 4378 { 4379 struct ixl_dmamem idm; 4380 struct ixl_aq_phy_abilities *phy; 4381 int rv; 4382 4383 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4384 aprint_error_dev(sc->sc_dev, 4385 "unable to allocate phy abilities buffer\n"); 4386 return -1; 4387 } 4388 4389 rv = ixl_get_phy_abilities(sc, &idm); 4390 switch (rv) { 4391 case -1: 4392 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 4393 goto done; 4394 case IXL_AQ_RC_OK: 4395 break; 4396 case IXL_AQ_RC_EIO: 4397 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 4398 goto done; 4399 default: 4400 aprint_error_dev(sc->sc_dev, 4401 "GET PHY ABILITIIES error %u\n", rv); 4402 goto done; 4403 } 4404 4405 phy = IXL_DMA_KVA(&idm); 4406 4407 sc->sc_phy_types = le32toh(phy->phy_type); 4408 sc->sc_phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 4409 4410 sc->sc_phy_abilities = phy->abilities; 4411 sc->sc_phy_linkspeed = phy->link_speed; 4412 sc->sc_phy_fec_cfg = phy->fec_cfg_curr_mod_ext_info & 4413 (IXL_AQ_ENABLE_FEC_KR | IXL_AQ_ENABLE_FEC_RS | 4414 IXL_AQ_REQUEST_FEC_KR | IXL_AQ_REQUEST_FEC_RS); 4415 sc->sc_eee_cap = phy->eee_capability; 4416 sc->sc_eeer_val = phy->eeer_val; 4417 sc->sc_d3_lpan = phy->d3_lpan; 4418 4419 rv = 0; 4420 4421 done: 4422 ixl_dmamem_free(sc, &idm); 4423 return rv; 4424 } 4425 4426 static int 4427 ixl_set_phy_config(struct ixl_softc *sc, 4428 uint8_t link_speed, uint8_t abilities, bool polling) 4429 { 4430 struct ixl_aq_phy_param *param; 4431 struct ixl_atq iatq; 4432 struct ixl_aq_desc *iaq; 4433 int error; 4434 4435 memset(&iatq, 0, sizeof(iatq)); 4436 4437 iaq = &iatq.iatq_desc; 4438 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_CONFIG); 4439 param = (struct ixl_aq_phy_param *)&iaq->iaq_param; 4440 param->phy_types = htole32((uint32_t)sc->sc_phy_types); 4441 param->phy_type_ext = (uint8_t)(sc->sc_phy_types >> 32); 4442 param->link_speed = link_speed; 4443 param->abilities = abilities | IXL_AQ_PHY_ABILITY_AUTO_LINK; 4444 param->fec_cfg = sc->sc_phy_fec_cfg; 4445 param->eee_capability = sc->sc_eee_cap; 4446 param->eeer_val = sc->sc_eeer_val; 4447 param->d3_lpan = sc->sc_d3_lpan; 4448 4449 if (polling) 4450 error = ixl_atq_poll(sc, iaq, 250); 4451 else 4452 error = ixl_atq_exec(sc, &iatq); 4453 4454 if (error != 0) 4455 return error; 4456 4457 switch (le16toh(iaq->iaq_retval)) { 4458 case IXL_AQ_RC_OK: 4459 break; 4460 case IXL_AQ_RC_EPERM: 4461 return EPERM; 4462 default: 4463 return EIO; 4464 } 4465 4466 return 0; 4467 } 4468 4469 static int 4470 ixl_set_phy_autoselect(struct ixl_softc *sc) 4471 { 4472 uint8_t link_speed, abilities; 4473 4474 link_speed = sc->sc_phy_linkspeed; 4475 abilities = IXL_PHY_ABILITY_LINKUP | IXL_PHY_ABILITY_AUTONEGO; 4476 4477 return ixl_set_phy_config(sc, link_speed, abilities, true); 4478 } 4479 4480 static int 4481 ixl_get_link_status_poll(struct ixl_softc *sc, int *l) 4482 { 4483 struct ixl_aq_desc iaq; 4484 struct ixl_aq_link_param *param; 4485 int link; 4486 4487 memset(&iaq, 0, sizeof(iaq)); 4488 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 4489 param = (struct ixl_aq_link_param *)iaq.iaq_param; 4490 param->notify = IXL_AQ_LINK_NOTIFY; 4491 4492 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4493 return ETIMEDOUT; 4494 } 4495 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4496 return EIO; 4497 } 4498 4499 link = ixl_set_link_status(sc, &iaq); 4500 4501 if (l != NULL) 4502 *l = link; 4503 4504 return 0; 4505 } 4506 4507 static int 4508 ixl_get_vsi(struct ixl_softc *sc) 4509 { 4510 struct ixl_dmamem *vsi = &sc->sc_scratch; 4511 struct ixl_aq_desc iaq; 4512 struct ixl_aq_vsi_param *param; 4513 struct ixl_aq_vsi_reply *reply; 4514 struct ixl_aq_vsi_data *data; 4515 int rv; 4516 4517 /* grumble, vsi info isn't "known" at compile time */ 4518 4519 memset(&iaq, 0, sizeof(iaq)); 4520 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4521 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4522 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 4523 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4524 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4525 4526 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4527 param->uplink_seid = sc->sc_seid; 4528 4529 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4530 BUS_DMASYNC_PREREAD); 4531 4532 rv = ixl_atq_poll(sc, &iaq, 250); 4533 4534 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4535 BUS_DMASYNC_POSTREAD); 4536 4537 if (rv != 0) { 4538 return ETIMEDOUT; 4539 } 4540 4541 switch (le16toh(iaq.iaq_retval)) { 4542 case IXL_AQ_RC_OK: 4543 break; 4544 case IXL_AQ_RC_ENOENT: 4545 return ENOENT; 4546 case IXL_AQ_RC_EACCES: 4547 return EACCES; 4548 default: 4549 return EIO; 4550 } 4551 4552 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 4553 sc->sc_vsi_number = le16toh(reply->vsi_number); 4554 data = IXL_DMA_KVA(vsi); 4555 sc->sc_vsi_stat_counter_idx = le16toh(data->stat_counter_idx); 4556 4557 return 0; 4558 } 4559 4560 static int 4561 ixl_set_vsi(struct ixl_softc *sc) 4562 { 4563 struct ixl_dmamem *vsi = &sc->sc_scratch; 4564 struct ixl_aq_desc iaq; 4565 struct ixl_aq_vsi_param *param; 4566 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 4567 unsigned int qnum; 4568 uint16_t val; 4569 int rv; 4570 4571 qnum = sc->sc_nqueue_pairs - 1; 4572 4573 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 4574 IXL_AQ_VSI_VALID_VLAN); 4575 4576 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 4577 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 4578 data->queue_mapping[0] = htole16(0); 4579 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 4580 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 4581 4582 val = le16toh(data->port_vlan_flags); 4583 CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK); 4584 SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL); 4585 4586 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { 4587 SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH); 4588 } else { 4589 SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING); 4590 } 4591 4592 data->port_vlan_flags = htole16(val); 4593 4594 /* grumble, vsi info isn't "known" at compile time */ 4595 4596 memset(&iaq, 0, sizeof(iaq)); 4597 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4598 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4599 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 4600 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4601 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4602 4603 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4604 param->uplink_seid = sc->sc_seid; 4605 4606 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4607 BUS_DMASYNC_PREWRITE); 4608 4609 rv = ixl_atq_poll(sc, &iaq, 250); 4610 4611 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4612 BUS_DMASYNC_POSTWRITE); 4613 4614 if (rv != 0) { 4615 return ETIMEDOUT; 4616 } 4617 4618 switch (le16toh(iaq.iaq_retval)) { 4619 case IXL_AQ_RC_OK: 4620 break; 4621 case IXL_AQ_RC_ENOENT: 4622 return ENOENT; 4623 case IXL_AQ_RC_EACCES: 4624 return EACCES; 4625 default: 4626 return EIO; 4627 } 4628 4629 return 0; 4630 } 4631 4632 static void 4633 ixl_set_filter_control(struct ixl_softc *sc) 4634 { 4635 uint32_t reg; 4636 4637 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4638 4639 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4640 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4641 4642 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4643 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4644 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4645 4646 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4647 } 4648 4649 static inline void 4650 ixl_get_default_rss_key(uint32_t *buf, size_t len) 4651 { 4652 size_t cplen; 4653 uint8_t rss_seed[RSS_KEYSIZE]; 4654 4655 rss_getkey(rss_seed); 4656 memset(buf, 0, len); 4657 4658 cplen = MIN(len, sizeof(rss_seed)); 4659 memcpy(buf, rss_seed, cplen); 4660 } 4661 4662 static int 4663 ixl_set_rss_key(struct ixl_softc *sc, uint8_t *key, size_t keylen) 4664 { 4665 struct ixl_dmamem *idm; 4666 struct ixl_atq iatq; 4667 struct ixl_aq_desc *iaq; 4668 struct ixl_aq_rss_key_param *param; 4669 struct ixl_aq_rss_key_data *data; 4670 size_t len, datalen, stdlen, extlen; 4671 uint16_t vsi_id; 4672 int rv; 4673 4674 memset(&iatq, 0, sizeof(iatq)); 4675 iaq = &iatq.iatq_desc; 4676 idm = &sc->sc_aqbuf; 4677 4678 datalen = sizeof(*data); 4679 4680 /*XXX The buf size has to be less than the size of the register */ 4681 datalen = MIN(IXL_RSS_KEY_SIZE_REG * sizeof(uint32_t), datalen); 4682 4683 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4684 (datalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4685 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_KEY); 4686 iaq->iaq_datalen = htole16(datalen); 4687 4688 param = (struct ixl_aq_rss_key_param *)iaq->iaq_param; 4689 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSKEY_VSI_ID_SHIFT) | 4690 IXL_AQ_RSSKEY_VSI_VALID; 4691 param->vsi_id = htole16(vsi_id); 4692 4693 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4694 data = IXL_DMA_KVA(idm); 4695 4696 len = MIN(keylen, datalen); 4697 stdlen = MIN(sizeof(data->standard_rss_key), len); 4698 memcpy(data->standard_rss_key, key, stdlen); 4699 len = (len > stdlen) ? (len - stdlen) : 0; 4700 4701 extlen = MIN(sizeof(data->extended_hash_key), len); 4702 extlen = (stdlen < keylen) ? 0 : keylen - stdlen; 4703 memcpy(data->extended_hash_key, key + stdlen, extlen); 4704 4705 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4706 4707 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4708 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4709 4710 rv = ixl_atq_exec(sc, &iatq); 4711 4712 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4713 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4714 4715 if (rv != 0) { 4716 return ETIMEDOUT; 4717 } 4718 4719 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4720 return EIO; 4721 } 4722 4723 return 0; 4724 } 4725 4726 static int 4727 ixl_set_rss_lut(struct ixl_softc *sc, uint8_t *lut, size_t lutlen) 4728 { 4729 struct ixl_dmamem *idm; 4730 struct ixl_atq iatq; 4731 struct ixl_aq_desc *iaq; 4732 struct ixl_aq_rss_lut_param *param; 4733 uint16_t vsi_id; 4734 uint8_t *data; 4735 size_t dmalen; 4736 int rv; 4737 4738 memset(&iatq, 0, sizeof(iatq)); 4739 iaq = &iatq.iatq_desc; 4740 idm = &sc->sc_aqbuf; 4741 4742 dmalen = MIN(lutlen, IXL_DMA_LEN(idm)); 4743 4744 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4745 (dmalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4746 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_LUT); 4747 iaq->iaq_datalen = htole16(dmalen); 4748 4749 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4750 data = IXL_DMA_KVA(idm); 4751 memcpy(data, lut, dmalen); 4752 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4753 4754 param = (struct ixl_aq_rss_lut_param *)iaq->iaq_param; 4755 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSLUT_VSI_ID_SHIFT) | 4756 IXL_AQ_RSSLUT_VSI_VALID; 4757 param->vsi_id = htole16(vsi_id); 4758 param->flags = htole16(IXL_AQ_RSSLUT_TABLE_TYPE_PF << 4759 IXL_AQ_RSSLUT_TABLE_TYPE_SHIFT); 4760 4761 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4762 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4763 4764 rv = ixl_atq_exec(sc, &iatq); 4765 4766 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4767 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4768 4769 if (rv != 0) { 4770 return ETIMEDOUT; 4771 } 4772 4773 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4774 return EIO; 4775 } 4776 4777 return 0; 4778 } 4779 4780 static int 4781 ixl_register_rss_key(struct ixl_softc *sc) 4782 { 4783 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4784 int rv; 4785 size_t i; 4786 4787 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4788 4789 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)){ 4790 rv = ixl_set_rss_key(sc, (uint8_t*)rss_seed, 4791 sizeof(rss_seed)); 4792 } else { 4793 rv = 0; 4794 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4795 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4796 } 4797 } 4798 4799 return rv; 4800 } 4801 4802 static void 4803 ixl_register_rss_pctype(struct ixl_softc *sc) 4804 { 4805 uint64_t set_hena = 0; 4806 uint32_t hena0, hena1; 4807 4808 /* 4809 * We use TCP/UDP with IPv4/IPv6 by default. 4810 * Note: the device can not use just IP header in each 4811 * TCP/UDP packets for the RSS hash calculation. 4812 */ 4813 if (sc->sc_mac_type == I40E_MAC_X722) 4814 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4815 else 4816 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4817 4818 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4819 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4820 4821 SET(hena0, set_hena); 4822 SET(hena1, set_hena >> 32); 4823 4824 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4825 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4826 } 4827 4828 static int 4829 ixl_register_rss_hlut(struct ixl_softc *sc) 4830 { 4831 unsigned int qid; 4832 uint8_t hlut_buf[512], lut_mask; 4833 uint32_t *hluts; 4834 size_t i, hluts_num; 4835 int rv; 4836 4837 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4838 4839 for (i = 0; i < sc->sc_rss_table_size; i++) { 4840 qid = i % sc->sc_nqueue_pairs; 4841 hlut_buf[i] = qid & lut_mask; 4842 } 4843 4844 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4845 rv = ixl_set_rss_lut(sc, hlut_buf, sizeof(hlut_buf)); 4846 } else { 4847 rv = 0; 4848 hluts = (uint32_t *)hlut_buf; 4849 hluts_num = sc->sc_rss_table_size >> 2; 4850 for (i = 0; i < hluts_num; i++) { 4851 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4852 } 4853 ixl_flush(sc); 4854 } 4855 4856 return rv; 4857 } 4858 4859 static void 4860 ixl_config_rss(struct ixl_softc *sc) 4861 { 4862 4863 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4864 4865 ixl_register_rss_key(sc); 4866 ixl_register_rss_pctype(sc); 4867 ixl_register_rss_hlut(sc); 4868 } 4869 4870 static const struct ixl_phy_type * 4871 ixl_search_phy_type(uint8_t phy_type) 4872 { 4873 const struct ixl_phy_type *itype; 4874 uint64_t mask; 4875 unsigned int i; 4876 4877 if (phy_type >= 64) 4878 return NULL; 4879 4880 mask = 1ULL << phy_type; 4881 4882 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4883 itype = &ixl_phy_type_map[i]; 4884 4885 if (ISSET(itype->phy_type, mask)) 4886 return itype; 4887 } 4888 4889 return NULL; 4890 } 4891 4892 static uint64_t 4893 ixl_search_link_speed(uint8_t link_speed) 4894 { 4895 const struct ixl_speed_type *type; 4896 unsigned int i; 4897 4898 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4899 type = &ixl_speed_type_map[i]; 4900 4901 if (ISSET(type->dev_speed, link_speed)) 4902 return type->net_speed; 4903 } 4904 4905 return 0; 4906 } 4907 4908 static uint8_t 4909 ixl_search_baudrate(uint64_t baudrate) 4910 { 4911 const struct ixl_speed_type *type; 4912 unsigned int i; 4913 4914 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4915 type = &ixl_speed_type_map[i]; 4916 4917 if (type->net_speed == baudrate) { 4918 return type->dev_speed; 4919 } 4920 } 4921 4922 return 0; 4923 } 4924 4925 static int 4926 ixl_restart_an(struct ixl_softc *sc) 4927 { 4928 struct ixl_aq_desc iaq; 4929 4930 memset(&iaq, 0, sizeof(iaq)); 4931 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4932 iaq.iaq_param[0] = 4933 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4934 4935 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4936 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4937 return -1; 4938 } 4939 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4940 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4941 return -1; 4942 } 4943 4944 return 0; 4945 } 4946 4947 static int 4948 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4949 uint16_t vlan, uint16_t flags) 4950 { 4951 struct ixl_aq_desc iaq; 4952 struct ixl_aq_add_macvlan *param; 4953 struct ixl_aq_add_macvlan_elem *elem; 4954 4955 memset(&iaq, 0, sizeof(iaq)); 4956 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4957 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4958 iaq.iaq_datalen = htole16(sizeof(*elem)); 4959 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4960 4961 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4962 param->num_addrs = htole16(1); 4963 param->seid0 = htole16(0x8000) | sc->sc_seid; 4964 param->seid1 = 0; 4965 param->seid2 = 0; 4966 4967 elem = IXL_DMA_KVA(&sc->sc_scratch); 4968 memset(elem, 0, sizeof(*elem)); 4969 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4970 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4971 elem->vlan = htole16(vlan); 4972 4973 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4974 return IXL_AQ_RC_EINVAL; 4975 } 4976 4977 switch (le16toh(iaq.iaq_retval)) { 4978 case IXL_AQ_RC_OK: 4979 break; 4980 case IXL_AQ_RC_ENOSPC: 4981 return ENOSPC; 4982 case IXL_AQ_RC_ENOENT: 4983 return ENOENT; 4984 case IXL_AQ_RC_EACCES: 4985 return EACCES; 4986 case IXL_AQ_RC_EEXIST: 4987 return EEXIST; 4988 case IXL_AQ_RC_EINVAL: 4989 return EINVAL; 4990 default: 4991 return EIO; 4992 } 4993 4994 return 0; 4995 } 4996 4997 static int 4998 ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4999 uint16_t vlan, uint16_t flags) 5000 { 5001 struct ixl_aq_desc iaq; 5002 struct ixl_aq_remove_macvlan *param; 5003 struct ixl_aq_remove_macvlan_elem *elem; 5004 5005 memset(&iaq, 0, sizeof(iaq)); 5006 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 5007 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 5008 iaq.iaq_datalen = htole16(sizeof(*elem)); 5009 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 5010 5011 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 5012 param->num_addrs = htole16(1); 5013 param->seid0 = htole16(0x8000) | sc->sc_seid; 5014 param->seid1 = 0; 5015 param->seid2 = 0; 5016 5017 elem = IXL_DMA_KVA(&sc->sc_scratch); 5018 memset(elem, 0, sizeof(*elem)); 5019 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 5020 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 5021 elem->vlan = htole16(vlan); 5022 5023 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 5024 return EINVAL; 5025 } 5026 5027 switch (le16toh(iaq.iaq_retval)) { 5028 case IXL_AQ_RC_OK: 5029 break; 5030 case IXL_AQ_RC_ENOENT: 5031 return ENOENT; 5032 case IXL_AQ_RC_EACCES: 5033 return EACCES; 5034 case IXL_AQ_RC_EINVAL: 5035 return EINVAL; 5036 default: 5037 return EIO; 5038 } 5039 5040 return 0; 5041 } 5042 5043 static int 5044 ixl_hmc(struct ixl_softc *sc) 5045 { 5046 struct { 5047 uint32_t count; 5048 uint32_t minsize; 5049 bus_size_t objsiz; 5050 bus_size_t setoff; 5051 bus_size_t setcnt; 5052 } regs[] = { 5053 { 5054 0, 5055 IXL_HMC_TXQ_MINSIZE, 5056 I40E_GLHMC_LANTXOBJSZ, 5057 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 5058 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 5059 }, 5060 { 5061 0, 5062 IXL_HMC_RXQ_MINSIZE, 5063 I40E_GLHMC_LANRXOBJSZ, 5064 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 5065 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 5066 }, 5067 { 5068 0, 5069 0, 5070 I40E_GLHMC_FCOEDDPOBJSZ, 5071 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 5072 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 5073 }, 5074 { 5075 0, 5076 0, 5077 I40E_GLHMC_FCOEFOBJSZ, 5078 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 5079 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 5080 }, 5081 }; 5082 struct ixl_hmc_entry *e; 5083 uint64_t size, dva; 5084 uint8_t *kva; 5085 uint64_t *sdpage; 5086 unsigned int i; 5087 int npages, tables; 5088 uint32_t reg; 5089 5090 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 5091 5092 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 5093 ixl_rd(sc, I40E_GLHMC_LANQMAX); 5094 5095 size = 0; 5096 for (i = 0; i < __arraycount(regs); i++) { 5097 e = &sc->sc_hmc_entries[i]; 5098 5099 e->hmc_count = regs[i].count; 5100 reg = ixl_rd(sc, regs[i].objsiz); 5101 e->hmc_size = BIT_ULL(0x3F & reg); 5102 e->hmc_base = size; 5103 5104 if ((e->hmc_size * 8) < regs[i].minsize) { 5105 aprint_error_dev(sc->sc_dev, 5106 "kernel hmc entry is too big\n"); 5107 return -1; 5108 } 5109 5110 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 5111 } 5112 size = roundup(size, IXL_HMC_PGSIZE); 5113 npages = size / IXL_HMC_PGSIZE; 5114 5115 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 5116 5117 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 5118 aprint_error_dev(sc->sc_dev, 5119 "unable to allocate hmc pd memory\n"); 5120 return -1; 5121 } 5122 5123 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 5124 IXL_HMC_PGSIZE) != 0) { 5125 aprint_error_dev(sc->sc_dev, 5126 "unable to allocate hmc sd memory\n"); 5127 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5128 return -1; 5129 } 5130 5131 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 5132 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 5133 5134 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 5135 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 5136 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5137 5138 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 5139 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 5140 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 5141 5142 for (i = 0; (int)i < npages; i++) { 5143 *sdpage = htole64(dva | IXL_HMC_PDVALID); 5144 sdpage++; 5145 5146 dva += IXL_HMC_PGSIZE; 5147 } 5148 5149 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 5150 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 5151 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5152 5153 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 5154 for (i = 0; (int)i < tables; i++) { 5155 uint32_t count; 5156 5157 KASSERT(npages >= 0); 5158 5159 count = ((unsigned int)npages > IXL_HMC_PGS) ? 5160 IXL_HMC_PGS : (unsigned int)npages; 5161 5162 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 5163 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 5164 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 5165 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 5166 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 5167 ixl_wr(sc, I40E_PFHMC_SDCMD, 5168 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 5169 5170 npages -= IXL_HMC_PGS; 5171 dva += IXL_HMC_PGSIZE; 5172 } 5173 5174 for (i = 0; i < __arraycount(regs); i++) { 5175 e = &sc->sc_hmc_entries[i]; 5176 5177 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 5178 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 5179 } 5180 5181 return 0; 5182 } 5183 5184 static void 5185 ixl_hmc_free(struct ixl_softc *sc) 5186 { 5187 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 5188 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5189 } 5190 5191 static void 5192 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 5193 unsigned int npacking) 5194 { 5195 uint8_t *dst = d; 5196 const uint8_t *src = s; 5197 unsigned int i; 5198 5199 for (i = 0; i < npacking; i++) { 5200 const struct ixl_hmc_pack *pack = &packing[i]; 5201 unsigned int offset = pack->lsb / 8; 5202 unsigned int align = pack->lsb % 8; 5203 const uint8_t *in = src + pack->offset; 5204 uint8_t *out = dst + offset; 5205 int width = pack->width; 5206 unsigned int inbits = 0; 5207 5208 if (align) { 5209 inbits = (*in++) << align; 5210 *out++ |= (inbits & 0xff); 5211 inbits >>= 8; 5212 5213 width -= 8 - align; 5214 } 5215 5216 while (width >= 8) { 5217 inbits |= (*in++) << align; 5218 *out++ = (inbits & 0xff); 5219 inbits >>= 8; 5220 5221 width -= 8; 5222 } 5223 5224 if (width > 0) { 5225 inbits |= (*in) << align; 5226 *out |= (inbits & ((1 << width) - 1)); 5227 } 5228 } 5229 } 5230 5231 static struct ixl_aq_buf * 5232 ixl_aqb_alloc(struct ixl_softc *sc) 5233 { 5234 struct ixl_aq_buf *aqb; 5235 5236 aqb = kmem_alloc(sizeof(*aqb), KM_SLEEP); 5237 5238 aqb->aqb_size = IXL_AQ_BUFLEN; 5239 5240 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 5241 aqb->aqb_size, 0, 5242 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 5243 goto free; 5244 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 5245 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 5246 BUS_DMA_WAITOK) != 0) 5247 goto destroy; 5248 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 5249 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 5250 goto dma_free; 5251 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 5252 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 5253 goto unmap; 5254 5255 return aqb; 5256 unmap: 5257 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5258 dma_free: 5259 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5260 destroy: 5261 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5262 free: 5263 kmem_free(aqb, sizeof(*aqb)); 5264 5265 return NULL; 5266 } 5267 5268 static void 5269 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 5270 { 5271 5272 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 5273 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5274 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5275 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5276 kmem_free(aqb, sizeof(*aqb)); 5277 } 5278 5279 static int 5280 ixl_arq_fill(struct ixl_softc *sc) 5281 { 5282 struct ixl_aq_buf *aqb; 5283 struct ixl_aq_desc *arq, *iaq; 5284 unsigned int prod = sc->sc_arq_prod; 5285 unsigned int n; 5286 int post = 0; 5287 5288 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 5289 IXL_AQ_NUM); 5290 arq = IXL_DMA_KVA(&sc->sc_arq); 5291 5292 if (__predict_false(n <= 0)) 5293 return 0; 5294 5295 do { 5296 aqb = sc->sc_arq_live[prod]; 5297 iaq = &arq[prod]; 5298 5299 if (aqb == NULL) { 5300 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 5301 if (aqb != NULL) { 5302 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5303 ixl_aq_buf, aqb_entry); 5304 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 5305 break; 5306 } 5307 5308 sc->sc_arq_live[prod] = aqb; 5309 memset(aqb->aqb_data, 0, aqb->aqb_size); 5310 5311 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 5312 aqb->aqb_size, BUS_DMASYNC_PREREAD); 5313 5314 iaq->iaq_flags = htole16(IXL_AQ_BUF | 5315 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 5316 IXL_AQ_LB : 0)); 5317 iaq->iaq_opcode = 0; 5318 iaq->iaq_datalen = htole16(aqb->aqb_size); 5319 iaq->iaq_retval = 0; 5320 iaq->iaq_cookie = 0; 5321 iaq->iaq_param[0] = 0; 5322 iaq->iaq_param[1] = 0; 5323 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 5324 } 5325 5326 prod++; 5327 prod &= IXL_AQ_MASK; 5328 5329 post = 1; 5330 5331 } while (--n); 5332 5333 if (post) { 5334 sc->sc_arq_prod = prod; 5335 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 5336 } 5337 5338 return post; 5339 } 5340 5341 static void 5342 ixl_arq_unfill(struct ixl_softc *sc) 5343 { 5344 struct ixl_aq_buf *aqb; 5345 unsigned int i; 5346 5347 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 5348 aqb = sc->sc_arq_live[i]; 5349 if (aqb == NULL) 5350 continue; 5351 5352 sc->sc_arq_live[i] = NULL; 5353 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 5354 BUS_DMASYNC_POSTREAD); 5355 ixl_aqb_free(sc, aqb); 5356 } 5357 5358 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 5359 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5360 ixl_aq_buf, aqb_entry); 5361 ixl_aqb_free(sc, aqb); 5362 } 5363 } 5364 5365 static void 5366 ixl_clear_hw(struct ixl_softc *sc) 5367 { 5368 uint32_t num_queues, base_queue; 5369 uint32_t num_pf_int; 5370 uint32_t num_vf_int; 5371 uint32_t num_vfs; 5372 uint32_t i, j; 5373 uint32_t val; 5374 uint32_t eol = 0x7ff; 5375 5376 /* get number of interrupts, queues, and vfs */ 5377 val = ixl_rd(sc, I40E_GLPCI_CNF2); 5378 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 5379 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 5380 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 5381 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 5382 5383 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 5384 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 5385 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 5386 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 5387 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 5388 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 5389 num_queues = (j - base_queue) + 1; 5390 else 5391 num_queues = 0; 5392 5393 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 5394 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 5395 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 5396 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 5397 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 5398 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 5399 num_vfs = (j - i) + 1; 5400 else 5401 num_vfs = 0; 5402 5403 /* stop all the interrupts */ 5404 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5405 ixl_flush(sc); 5406 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 5407 for (i = 0; i < num_pf_int - 2; i++) 5408 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 5409 ixl_flush(sc); 5410 5411 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 5412 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5413 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 5414 for (i = 0; i < num_pf_int - 2; i++) 5415 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 5416 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5417 for (i = 0; i < num_vfs; i++) 5418 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 5419 for (i = 0; i < num_vf_int - 2; i++) 5420 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 5421 5422 /* warn the HW of the coming Tx disables */ 5423 for (i = 0; i < num_queues; i++) { 5424 uint32_t abs_queue_idx = base_queue + i; 5425 uint32_t reg_block = 0; 5426 5427 if (abs_queue_idx >= 128) { 5428 reg_block = abs_queue_idx / 128; 5429 abs_queue_idx %= 128; 5430 } 5431 5432 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 5433 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 5434 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 5435 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 5436 5437 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 5438 } 5439 delaymsec(400); 5440 5441 /* stop all the queues */ 5442 for (i = 0; i < num_queues; i++) { 5443 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 5444 ixl_wr(sc, I40E_QTX_ENA(i), 0); 5445 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 5446 ixl_wr(sc, I40E_QRX_ENA(i), 0); 5447 } 5448 5449 /* short wait for all queue disables to settle */ 5450 delaymsec(50); 5451 } 5452 5453 static int 5454 ixl_pf_reset(struct ixl_softc *sc) 5455 { 5456 uint32_t cnt = 0; 5457 uint32_t cnt1 = 0; 5458 uint32_t reg = 0, reg0 = 0; 5459 uint32_t grst_del; 5460 5461 /* 5462 * Poll for Global Reset steady state in case of recent GRST. 5463 * The grst delay value is in 100ms units, and we'll wait a 5464 * couple counts longer to be sure we don't just miss the end. 5465 */ 5466 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 5467 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 5468 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 5469 5470 grst_del = grst_del * 20; 5471 5472 for (cnt = 0; cnt < grst_del; cnt++) { 5473 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 5474 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 5475 break; 5476 delaymsec(100); 5477 } 5478 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5479 aprint_error(", Global reset polling failed to complete\n"); 5480 return -1; 5481 } 5482 5483 /* Now Wait for the FW to be ready */ 5484 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 5485 reg = ixl_rd(sc, I40E_GLNVM_ULD); 5486 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5487 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 5488 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5489 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 5490 break; 5491 5492 delaymsec(10); 5493 } 5494 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5495 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 5496 aprint_error(", wait for FW Reset complete timed out " 5497 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 5498 return -1; 5499 } 5500 5501 /* 5502 * If there was a Global Reset in progress when we got here, 5503 * we don't need to do the PF Reset 5504 */ 5505 if (cnt == 0) { 5506 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5507 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 5508 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 5509 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5510 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 5511 break; 5512 delaymsec(1); 5513 5514 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 5515 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5516 aprint_error(", Core reset upcoming." 5517 " Skipping PF reset reset request\n"); 5518 return -1; 5519 } 5520 } 5521 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 5522 aprint_error(", PF reset polling failed to complete" 5523 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 5524 return -1; 5525 } 5526 } 5527 5528 return 0; 5529 } 5530 5531 static int 5532 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 5533 bus_size_t size, bus_size_t align) 5534 { 5535 ixm->ixm_size = size; 5536 5537 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 5538 ixm->ixm_size, 0, 5539 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 5540 &ixm->ixm_map) != 0) 5541 return 1; 5542 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 5543 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 5544 BUS_DMA_WAITOK) != 0) 5545 goto destroy; 5546 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 5547 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 5548 goto free; 5549 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 5550 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 5551 goto unmap; 5552 5553 memset(ixm->ixm_kva, 0, ixm->ixm_size); 5554 5555 return 0; 5556 unmap: 5557 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5558 free: 5559 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5560 destroy: 5561 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5562 return 1; 5563 } 5564 5565 static void 5566 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 5567 { 5568 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 5569 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5570 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5571 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5572 } 5573 5574 static int 5575 ixl_setup_vlan_hwfilter(struct ixl_softc *sc) 5576 { 5577 struct ethercom *ec = &sc->sc_ec; 5578 struct vlanid_list *vlanidp; 5579 int rv; 5580 5581 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5582 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5583 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5584 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5585 5586 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5587 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5588 if (rv != 0) 5589 return rv; 5590 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5591 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5592 if (rv != 0) 5593 return rv; 5594 5595 ETHER_LOCK(ec); 5596 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5597 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 5598 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5599 if (rv != 0) 5600 break; 5601 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 5602 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5603 if (rv != 0) 5604 break; 5605 } 5606 ETHER_UNLOCK(ec); 5607 5608 return rv; 5609 } 5610 5611 static void 5612 ixl_teardown_vlan_hwfilter(struct ixl_softc *sc) 5613 { 5614 struct vlanid_list *vlanidp; 5615 struct ethercom *ec = &sc->sc_ec; 5616 5617 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5618 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5619 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5620 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5621 5622 ETHER_LOCK(ec); 5623 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5624 ixl_remove_macvlan(sc, sc->sc_enaddr, 5625 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5626 ixl_remove_macvlan(sc, etherbroadcastaddr, 5627 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5628 } 5629 ETHER_UNLOCK(ec); 5630 5631 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5632 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5633 ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5634 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5635 } 5636 5637 static int 5638 ixl_update_macvlan(struct ixl_softc *sc) 5639 { 5640 int rv = 0; 5641 int next_ec_capenable = sc->sc_ec.ec_capenable; 5642 5643 if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 5644 rv = ixl_setup_vlan_hwfilter(sc); 5645 if (rv != 0) 5646 ixl_teardown_vlan_hwfilter(sc); 5647 } else { 5648 ixl_teardown_vlan_hwfilter(sc); 5649 } 5650 5651 return rv; 5652 } 5653 5654 static int 5655 ixl_ifflags_cb(struct ethercom *ec) 5656 { 5657 struct ifnet *ifp = &ec->ec_if; 5658 struct ixl_softc *sc = ifp->if_softc; 5659 int rv, change; 5660 5661 mutex_enter(&sc->sc_cfg_lock); 5662 5663 change = ec->ec_capenable ^ sc->sc_cur_ec_capenable; 5664 5665 if (ISSET(change, ETHERCAP_VLAN_HWTAGGING)) { 5666 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 5667 rv = ENETRESET; 5668 goto out; 5669 } 5670 5671 if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) { 5672 rv = ixl_update_macvlan(sc); 5673 if (rv == 0) { 5674 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 5675 } else { 5676 CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER); 5677 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 5678 } 5679 } 5680 5681 rv = ixl_iff(sc); 5682 out: 5683 mutex_exit(&sc->sc_cfg_lock); 5684 5685 return rv; 5686 } 5687 5688 static int 5689 ixl_set_link_status(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 5690 { 5691 const struct ixl_aq_link_status *status; 5692 const struct ixl_phy_type *itype; 5693 5694 uint64_t ifm_active = IFM_ETHER; 5695 uint64_t ifm_status = IFM_AVALID; 5696 int link_state = LINK_STATE_DOWN; 5697 uint64_t baudrate = 0; 5698 5699 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 5700 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) { 5701 ifm_active |= IFM_NONE; 5702 goto done; 5703 } 5704 5705 ifm_active |= IFM_FDX; 5706 ifm_status |= IFM_ACTIVE; 5707 link_state = LINK_STATE_UP; 5708 5709 itype = ixl_search_phy_type(status->phy_type); 5710 if (itype != NULL) 5711 ifm_active |= itype->ifm_type; 5712 5713 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 5714 ifm_active |= IFM_ETH_TXPAUSE; 5715 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 5716 ifm_active |= IFM_ETH_RXPAUSE; 5717 5718 baudrate = ixl_search_link_speed(status->link_speed); 5719 5720 done: 5721 sc->sc_media_active = ifm_active; 5722 sc->sc_media_status = ifm_status; 5723 5724 sc->sc_ec.ec_if.if_baudrate = baudrate; 5725 5726 return link_state; 5727 } 5728 5729 static int 5730 ixl_establish_intx(struct ixl_softc *sc) 5731 { 5732 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5733 pci_intr_handle_t *intr; 5734 char xnamebuf[32]; 5735 char intrbuf[PCI_INTRSTR_LEN]; 5736 char const *intrstr; 5737 5738 KASSERT(sc->sc_nintrs == 1); 5739 5740 intr = &sc->sc_ihp[0]; 5741 5742 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 5743 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 5744 device_xname(sc->sc_dev)); 5745 5746 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 5747 sc, xnamebuf); 5748 5749 if (sc->sc_ihs[0] == NULL) { 5750 aprint_error_dev(sc->sc_dev, 5751 "unable to establish interrupt at %s\n", intrstr); 5752 return -1; 5753 } 5754 5755 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 5756 return 0; 5757 } 5758 5759 static int 5760 ixl_establish_msix(struct ixl_softc *sc) 5761 { 5762 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5763 kcpuset_t *affinity; 5764 unsigned int vector = 0; 5765 unsigned int i; 5766 int affinity_to, r; 5767 char xnamebuf[32]; 5768 char intrbuf[PCI_INTRSTR_LEN]; 5769 char const *intrstr; 5770 5771 kcpuset_create(&affinity, false); 5772 5773 /* the "other" intr is mapped to vector 0 */ 5774 vector = 0; 5775 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5776 intrbuf, sizeof(intrbuf)); 5777 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 5778 device_xname(sc->sc_dev)); 5779 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5780 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 5781 sc, xnamebuf); 5782 if (sc->sc_ihs[vector] == NULL) { 5783 aprint_error_dev(sc->sc_dev, 5784 "unable to establish interrupt at %s\n", intrstr); 5785 goto fail; 5786 } 5787 5788 aprint_normal_dev(sc->sc_dev, "other interrupt at %s", intrstr); 5789 5790 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5791 affinity_to = (affinity_to + sc->sc_nqueue_pairs_max) % ncpu; 5792 5793 kcpuset_zero(affinity); 5794 kcpuset_set(affinity, affinity_to); 5795 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5796 if (r == 0) { 5797 aprint_normal(", affinity to %u", affinity_to); 5798 } 5799 aprint_normal("\n"); 5800 vector++; 5801 5802 sc->sc_msix_vector_queue = vector; 5803 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5804 5805 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5806 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5807 intrbuf, sizeof(intrbuf)); 5808 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 5809 device_xname(sc->sc_dev), i); 5810 5811 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5812 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 5813 (void *)&sc->sc_qps[i], xnamebuf); 5814 5815 if (sc->sc_ihs[vector] == NULL) { 5816 aprint_error_dev(sc->sc_dev, 5817 "unable to establish interrupt at %s\n", intrstr); 5818 goto fail; 5819 } 5820 5821 aprint_normal_dev(sc->sc_dev, 5822 "for TXRX%d interrupt at %s",i , intrstr); 5823 5824 kcpuset_zero(affinity); 5825 kcpuset_set(affinity, affinity_to); 5826 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5827 if (r == 0) { 5828 aprint_normal(", affinity to %u", affinity_to); 5829 affinity_to = (affinity_to + 1) % ncpu; 5830 } 5831 aprint_normal("\n"); 5832 vector++; 5833 } 5834 5835 kcpuset_destroy(affinity); 5836 5837 return 0; 5838 fail: 5839 for (i = 0; i < vector; i++) { 5840 pci_intr_disestablish(pc, sc->sc_ihs[i]); 5841 } 5842 5843 sc->sc_msix_vector_queue = 0; 5844 sc->sc_msix_vector_queue = 0; 5845 kcpuset_destroy(affinity); 5846 5847 return -1; 5848 } 5849 5850 static void 5851 ixl_config_queue_intr(struct ixl_softc *sc) 5852 { 5853 unsigned int i, vector; 5854 5855 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5856 vector = sc->sc_msix_vector_queue; 5857 } else { 5858 vector = I40E_INTR_NOTX_INTR; 5859 5860 ixl_wr(sc, I40E_PFINT_LNKLST0, 5861 (I40E_INTR_NOTX_QUEUE << 5862 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5863 (I40E_QUEUE_TYPE_RX << 5864 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5865 } 5866 5867 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5868 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5869 ixl_flush(sc); 5870 5871 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5872 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5873 (I40E_QUEUE_TYPE_RX << 5874 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5875 5876 ixl_wr(sc, I40E_QINT_RQCTL(i), 5877 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5878 (I40E_ITR_INDEX_RX << 5879 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5880 (I40E_INTR_NOTX_RX_QUEUE << 5881 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5882 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5883 (I40E_QUEUE_TYPE_TX << 5884 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5885 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5886 5887 ixl_wr(sc, I40E_QINT_TQCTL(i), 5888 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5889 (I40E_ITR_INDEX_TX << 5890 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5891 (I40E_INTR_NOTX_TX_QUEUE << 5892 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5893 (I40E_QUEUE_TYPE_EOL << 5894 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5895 (I40E_QUEUE_TYPE_RX << 5896 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5897 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5898 5899 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) 5900 vector++; 5901 } 5902 ixl_flush(sc); 5903 5904 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), 0x7a); 5905 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), 0x7a); 5906 ixl_flush(sc); 5907 } 5908 5909 static void 5910 ixl_config_other_intr(struct ixl_softc *sc) 5911 { 5912 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5913 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5914 5915 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5916 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5917 I40E_PFINT_ICR0_ENA_GRST_MASK | 5918 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5919 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5920 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5921 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5922 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5923 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5924 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5925 5926 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5927 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5928 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5929 (I40E_ITR_INDEX_OTHER << 5930 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5931 ixl_flush(sc); 5932 } 5933 5934 static int 5935 ixl_setup_interrupts(struct ixl_softc *sc) 5936 { 5937 struct pci_attach_args *pa = &sc->sc_pa; 5938 pci_intr_type_t max_type, intr_type; 5939 int counts[PCI_INTR_TYPE_SIZE]; 5940 int error; 5941 unsigned int i; 5942 bool retry; 5943 5944 memset(counts, 0, sizeof(counts)); 5945 max_type = PCI_INTR_TYPE_MSIX; 5946 /* QPs + other interrupt */ 5947 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5948 counts[PCI_INTR_TYPE_INTX] = 1; 5949 5950 if (ixl_param_nomsix) 5951 counts[PCI_INTR_TYPE_MSIX] = 0; 5952 5953 do { 5954 retry = false; 5955 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5956 if (error != 0) { 5957 aprint_error_dev(sc->sc_dev, 5958 "couldn't map interrupt\n"); 5959 break; 5960 } 5961 5962 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5963 sc->sc_nintrs = counts[intr_type]; 5964 KASSERT(sc->sc_nintrs > 0); 5965 5966 for (i = 0; i < sc->sc_nintrs; i++) { 5967 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5968 PCI_INTR_MPSAFE, true); 5969 } 5970 5971 sc->sc_ihs = kmem_alloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5972 KM_SLEEP); 5973 5974 if (intr_type == PCI_INTR_TYPE_MSIX) { 5975 error = ixl_establish_msix(sc); 5976 if (error) { 5977 counts[PCI_INTR_TYPE_MSIX] = 0; 5978 retry = true; 5979 } 5980 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5981 error = ixl_establish_intx(sc); 5982 } else { 5983 error = -1; 5984 } 5985 5986 if (error) { 5987 kmem_free(sc->sc_ihs, 5988 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5989 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5990 } else { 5991 sc->sc_intrtype = intr_type; 5992 } 5993 } while (retry); 5994 5995 return error; 5996 } 5997 5998 static void 5999 ixl_teardown_interrupts(struct ixl_softc *sc) 6000 { 6001 struct pci_attach_args *pa = &sc->sc_pa; 6002 unsigned int i; 6003 6004 for (i = 0; i < sc->sc_nintrs; i++) { 6005 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 6006 } 6007 6008 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 6009 6010 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 6011 sc->sc_ihs = NULL; 6012 sc->sc_nintrs = 0; 6013 } 6014 6015 static int 6016 ixl_setup_stats(struct ixl_softc *sc) 6017 { 6018 struct ixl_queue_pair *qp; 6019 struct ixl_tx_ring *txr; 6020 struct ixl_rx_ring *rxr; 6021 struct ixl_stats_counters *isc; 6022 unsigned int i; 6023 6024 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6025 qp = &sc->sc_qps[i]; 6026 txr = qp->qp_txr; 6027 rxr = qp->qp_rxr; 6028 6029 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 6030 NULL, qp->qp_name, "m_defrag successed"); 6031 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 6032 NULL, qp->qp_name, "m_defrag_failed"); 6033 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 6034 NULL, qp->qp_name, "Dropped in pcq"); 6035 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 6036 NULL, qp->qp_name, "Deferred transmit"); 6037 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 6038 NULL, qp->qp_name, "Interrupt on queue"); 6039 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 6040 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6041 6042 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 6043 NULL, qp->qp_name, "MGETHDR failed"); 6044 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 6045 NULL, qp->qp_name, "MCLGET failed"); 6046 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 6047 EVCNT_TYPE_MISC, NULL, qp->qp_name, 6048 "bus_dmamap_load_mbuf failed"); 6049 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 6050 NULL, qp->qp_name, "Interrupt on queue"); 6051 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 6052 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6053 } 6054 6055 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 6056 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 6057 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 6058 NULL, device_xname(sc->sc_dev), "Link status event"); 6059 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 6060 NULL, device_xname(sc->sc_dev), "ECC error"); 6061 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 6062 NULL, device_xname(sc->sc_dev), "PCI exception"); 6063 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 6064 NULL, device_xname(sc->sc_dev), "Critical error"); 6065 6066 isc = &sc->sc_stats_counters; 6067 evcnt_attach_dynamic(&isc->isc_crc_errors, EVCNT_TYPE_MISC, 6068 NULL, device_xname(sc->sc_dev), "CRC errors"); 6069 evcnt_attach_dynamic(&isc->isc_illegal_bytes, EVCNT_TYPE_MISC, 6070 NULL, device_xname(sc->sc_dev), "Illegal bytes"); 6071 evcnt_attach_dynamic(&isc->isc_mac_local_faults, EVCNT_TYPE_MISC, 6072 NULL, device_xname(sc->sc_dev), "Mac local faults"); 6073 evcnt_attach_dynamic(&isc->isc_mac_remote_faults, EVCNT_TYPE_MISC, 6074 NULL, device_xname(sc->sc_dev), "Mac remote faults"); 6075 evcnt_attach_dynamic(&isc->isc_link_xon_rx, EVCNT_TYPE_MISC, 6076 NULL, device_xname(sc->sc_dev), "Rx xon"); 6077 evcnt_attach_dynamic(&isc->isc_link_xon_tx, EVCNT_TYPE_MISC, 6078 NULL, device_xname(sc->sc_dev), "Tx xon"); 6079 evcnt_attach_dynamic(&isc->isc_link_xoff_rx, EVCNT_TYPE_MISC, 6080 NULL, device_xname(sc->sc_dev), "Rx xoff"); 6081 evcnt_attach_dynamic(&isc->isc_link_xoff_tx, EVCNT_TYPE_MISC, 6082 NULL, device_xname(sc->sc_dev), "Tx xoff"); 6083 evcnt_attach_dynamic(&isc->isc_rx_fragments, EVCNT_TYPE_MISC, 6084 NULL, device_xname(sc->sc_dev), "Rx fragments"); 6085 evcnt_attach_dynamic(&isc->isc_rx_jabber, EVCNT_TYPE_MISC, 6086 NULL, device_xname(sc->sc_dev), "Rx jabber"); 6087 6088 evcnt_attach_dynamic(&isc->isc_rx_size_64, EVCNT_TYPE_MISC, 6089 NULL, device_xname(sc->sc_dev), "Rx size 64"); 6090 evcnt_attach_dynamic(&isc->isc_rx_size_127, EVCNT_TYPE_MISC, 6091 NULL, device_xname(sc->sc_dev), "Rx size 127"); 6092 evcnt_attach_dynamic(&isc->isc_rx_size_255, EVCNT_TYPE_MISC, 6093 NULL, device_xname(sc->sc_dev), "Rx size 255"); 6094 evcnt_attach_dynamic(&isc->isc_rx_size_511, EVCNT_TYPE_MISC, 6095 NULL, device_xname(sc->sc_dev), "Rx size 511"); 6096 evcnt_attach_dynamic(&isc->isc_rx_size_1023, EVCNT_TYPE_MISC, 6097 NULL, device_xname(sc->sc_dev), "Rx size 1023"); 6098 evcnt_attach_dynamic(&isc->isc_rx_size_1522, EVCNT_TYPE_MISC, 6099 NULL, device_xname(sc->sc_dev), "Rx size 1522"); 6100 evcnt_attach_dynamic(&isc->isc_rx_size_big, EVCNT_TYPE_MISC, 6101 NULL, device_xname(sc->sc_dev), "Rx jumbo packets"); 6102 evcnt_attach_dynamic(&isc->isc_rx_undersize, EVCNT_TYPE_MISC, 6103 NULL, device_xname(sc->sc_dev), "Rx under size"); 6104 evcnt_attach_dynamic(&isc->isc_rx_oversize, EVCNT_TYPE_MISC, 6105 NULL, device_xname(sc->sc_dev), "Rx over size"); 6106 6107 evcnt_attach_dynamic(&isc->isc_rx_bytes, EVCNT_TYPE_MISC, 6108 NULL, device_xname(sc->sc_dev), "Rx bytes / port"); 6109 evcnt_attach_dynamic(&isc->isc_rx_discards, EVCNT_TYPE_MISC, 6110 NULL, device_xname(sc->sc_dev), "Rx discards / port"); 6111 evcnt_attach_dynamic(&isc->isc_rx_unicast, EVCNT_TYPE_MISC, 6112 NULL, device_xname(sc->sc_dev), "Rx unicast / port"); 6113 evcnt_attach_dynamic(&isc->isc_rx_multicast, EVCNT_TYPE_MISC, 6114 NULL, device_xname(sc->sc_dev), "Rx multicast / port"); 6115 evcnt_attach_dynamic(&isc->isc_rx_broadcast, EVCNT_TYPE_MISC, 6116 NULL, device_xname(sc->sc_dev), "Rx broadcast / port"); 6117 6118 evcnt_attach_dynamic(&isc->isc_vsi_rx_bytes, EVCNT_TYPE_MISC, 6119 NULL, device_xname(sc->sc_dev), "Rx bytes / vsi"); 6120 evcnt_attach_dynamic(&isc->isc_vsi_rx_discards, EVCNT_TYPE_MISC, 6121 NULL, device_xname(sc->sc_dev), "Rx discard / vsi"); 6122 evcnt_attach_dynamic(&isc->isc_vsi_rx_unicast, EVCNT_TYPE_MISC, 6123 NULL, device_xname(sc->sc_dev), "Rx unicast / vsi"); 6124 evcnt_attach_dynamic(&isc->isc_vsi_rx_multicast, EVCNT_TYPE_MISC, 6125 NULL, device_xname(sc->sc_dev), "Rx multicast / vsi"); 6126 evcnt_attach_dynamic(&isc->isc_vsi_rx_broadcast, EVCNT_TYPE_MISC, 6127 NULL, device_xname(sc->sc_dev), "Rx broadcast / vsi"); 6128 6129 evcnt_attach_dynamic(&isc->isc_tx_size_64, EVCNT_TYPE_MISC, 6130 NULL, device_xname(sc->sc_dev), "Tx size 64"); 6131 evcnt_attach_dynamic(&isc->isc_tx_size_127, EVCNT_TYPE_MISC, 6132 NULL, device_xname(sc->sc_dev), "Tx size 127"); 6133 evcnt_attach_dynamic(&isc->isc_tx_size_255, EVCNT_TYPE_MISC, 6134 NULL, device_xname(sc->sc_dev), "Tx size 255"); 6135 evcnt_attach_dynamic(&isc->isc_tx_size_511, EVCNT_TYPE_MISC, 6136 NULL, device_xname(sc->sc_dev), "Tx size 511"); 6137 evcnt_attach_dynamic(&isc->isc_tx_size_1023, EVCNT_TYPE_MISC, 6138 NULL, device_xname(sc->sc_dev), "Tx size 1023"); 6139 evcnt_attach_dynamic(&isc->isc_tx_size_1522, EVCNT_TYPE_MISC, 6140 NULL, device_xname(sc->sc_dev), "Tx size 1522"); 6141 evcnt_attach_dynamic(&isc->isc_tx_size_big, EVCNT_TYPE_MISC, 6142 NULL, device_xname(sc->sc_dev), "Tx jumbo packets"); 6143 6144 evcnt_attach_dynamic(&isc->isc_tx_bytes, EVCNT_TYPE_MISC, 6145 NULL, device_xname(sc->sc_dev), "Tx bytes / port"); 6146 evcnt_attach_dynamic(&isc->isc_tx_dropped_link_down, EVCNT_TYPE_MISC, 6147 NULL, device_xname(sc->sc_dev), 6148 "Tx dropped due to link down / port"); 6149 evcnt_attach_dynamic(&isc->isc_tx_unicast, EVCNT_TYPE_MISC, 6150 NULL, device_xname(sc->sc_dev), "Tx unicast / port"); 6151 evcnt_attach_dynamic(&isc->isc_tx_multicast, EVCNT_TYPE_MISC, 6152 NULL, device_xname(sc->sc_dev), "Tx multicast / port"); 6153 evcnt_attach_dynamic(&isc->isc_tx_broadcast, EVCNT_TYPE_MISC, 6154 NULL, device_xname(sc->sc_dev), "Tx broadcast / port"); 6155 6156 evcnt_attach_dynamic(&isc->isc_vsi_tx_bytes, EVCNT_TYPE_MISC, 6157 NULL, device_xname(sc->sc_dev), "Tx bytes / vsi"); 6158 evcnt_attach_dynamic(&isc->isc_vsi_tx_errors, EVCNT_TYPE_MISC, 6159 NULL, device_xname(sc->sc_dev), "Tx errors / vsi"); 6160 evcnt_attach_dynamic(&isc->isc_vsi_tx_unicast, EVCNT_TYPE_MISC, 6161 NULL, device_xname(sc->sc_dev), "Tx unicast / vsi"); 6162 evcnt_attach_dynamic(&isc->isc_vsi_tx_multicast, EVCNT_TYPE_MISC, 6163 NULL, device_xname(sc->sc_dev), "Tx multicast / vsi"); 6164 evcnt_attach_dynamic(&isc->isc_vsi_tx_broadcast, EVCNT_TYPE_MISC, 6165 NULL, device_xname(sc->sc_dev), "Tx broadcast / vsi"); 6166 6167 sc->sc_stats_intval = ixl_param_stats_interval; 6168 callout_init(&sc->sc_stats_callout, CALLOUT_MPSAFE); 6169 callout_setfunc(&sc->sc_stats_callout, ixl_stats_callout, sc); 6170 ixl_work_set(&sc->sc_stats_task, ixl_stats_update, sc); 6171 6172 return 0; 6173 } 6174 6175 static void 6176 ixl_teardown_stats(struct ixl_softc *sc) 6177 { 6178 struct ixl_tx_ring *txr; 6179 struct ixl_rx_ring *rxr; 6180 struct ixl_stats_counters *isc; 6181 unsigned int i; 6182 6183 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6184 txr = sc->sc_qps[i].qp_txr; 6185 rxr = sc->sc_qps[i].qp_rxr; 6186 6187 evcnt_detach(&txr->txr_defragged); 6188 evcnt_detach(&txr->txr_defrag_failed); 6189 evcnt_detach(&txr->txr_pcqdrop); 6190 evcnt_detach(&txr->txr_transmitdef); 6191 evcnt_detach(&txr->txr_intr); 6192 evcnt_detach(&txr->txr_defer); 6193 6194 evcnt_detach(&rxr->rxr_mgethdr_failed); 6195 evcnt_detach(&rxr->rxr_mgetcl_failed); 6196 evcnt_detach(&rxr->rxr_mbuf_load_failed); 6197 evcnt_detach(&rxr->rxr_intr); 6198 evcnt_detach(&rxr->rxr_defer); 6199 } 6200 6201 isc = &sc->sc_stats_counters; 6202 evcnt_detach(&isc->isc_crc_errors); 6203 evcnt_detach(&isc->isc_illegal_bytes); 6204 evcnt_detach(&isc->isc_mac_local_faults); 6205 evcnt_detach(&isc->isc_mac_remote_faults); 6206 evcnt_detach(&isc->isc_link_xon_rx); 6207 evcnt_detach(&isc->isc_link_xon_tx); 6208 evcnt_detach(&isc->isc_link_xoff_rx); 6209 evcnt_detach(&isc->isc_link_xoff_tx); 6210 evcnt_detach(&isc->isc_rx_fragments); 6211 evcnt_detach(&isc->isc_rx_jabber); 6212 evcnt_detach(&isc->isc_rx_bytes); 6213 evcnt_detach(&isc->isc_rx_discards); 6214 evcnt_detach(&isc->isc_rx_unicast); 6215 evcnt_detach(&isc->isc_rx_multicast); 6216 evcnt_detach(&isc->isc_rx_broadcast); 6217 evcnt_detach(&isc->isc_rx_size_64); 6218 evcnt_detach(&isc->isc_rx_size_127); 6219 evcnt_detach(&isc->isc_rx_size_255); 6220 evcnt_detach(&isc->isc_rx_size_511); 6221 evcnt_detach(&isc->isc_rx_size_1023); 6222 evcnt_detach(&isc->isc_rx_size_1522); 6223 evcnt_detach(&isc->isc_rx_size_big); 6224 evcnt_detach(&isc->isc_rx_undersize); 6225 evcnt_detach(&isc->isc_rx_oversize); 6226 evcnt_detach(&isc->isc_tx_bytes); 6227 evcnt_detach(&isc->isc_tx_dropped_link_down); 6228 evcnt_detach(&isc->isc_tx_unicast); 6229 evcnt_detach(&isc->isc_tx_multicast); 6230 evcnt_detach(&isc->isc_tx_broadcast); 6231 evcnt_detach(&isc->isc_tx_size_64); 6232 evcnt_detach(&isc->isc_tx_size_127); 6233 evcnt_detach(&isc->isc_tx_size_255); 6234 evcnt_detach(&isc->isc_tx_size_511); 6235 evcnt_detach(&isc->isc_tx_size_1023); 6236 evcnt_detach(&isc->isc_tx_size_1522); 6237 evcnt_detach(&isc->isc_tx_size_big); 6238 evcnt_detach(&isc->isc_vsi_rx_discards); 6239 evcnt_detach(&isc->isc_vsi_rx_bytes); 6240 evcnt_detach(&isc->isc_vsi_rx_unicast); 6241 evcnt_detach(&isc->isc_vsi_rx_multicast); 6242 evcnt_detach(&isc->isc_vsi_rx_broadcast); 6243 evcnt_detach(&isc->isc_vsi_tx_errors); 6244 evcnt_detach(&isc->isc_vsi_tx_bytes); 6245 evcnt_detach(&isc->isc_vsi_tx_unicast); 6246 evcnt_detach(&isc->isc_vsi_tx_multicast); 6247 evcnt_detach(&isc->isc_vsi_tx_broadcast); 6248 6249 evcnt_detach(&sc->sc_event_atq); 6250 evcnt_detach(&sc->sc_event_link); 6251 evcnt_detach(&sc->sc_event_ecc_err); 6252 evcnt_detach(&sc->sc_event_pci_exception); 6253 evcnt_detach(&sc->sc_event_crit_err); 6254 6255 callout_destroy(&sc->sc_stats_callout); 6256 } 6257 6258 static void 6259 ixl_stats_callout(void *xsc) 6260 { 6261 struct ixl_softc *sc = xsc; 6262 6263 ixl_work_add(sc->sc_workq, &sc->sc_stats_task); 6264 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 6265 } 6266 6267 static uint64_t 6268 ixl_stat_delta(struct ixl_softc *sc, uint32_t reg_hi, uint32_t reg_lo, 6269 uint64_t *offset, bool has_offset) 6270 { 6271 uint64_t value, delta; 6272 int bitwidth; 6273 6274 bitwidth = reg_hi == 0 ? 32 : 48; 6275 6276 value = ixl_rd(sc, reg_lo); 6277 6278 if (bitwidth > 32) { 6279 value |= ((uint64_t)ixl_rd(sc, reg_hi) << 32); 6280 } 6281 6282 if (__predict_true(has_offset)) { 6283 delta = value; 6284 if (value < *offset) 6285 delta += ((uint64_t)1 << bitwidth); 6286 delta -= *offset; 6287 } else { 6288 delta = 0; 6289 } 6290 atomic_swap_64(offset, value); 6291 6292 return delta; 6293 } 6294 6295 static void 6296 ixl_stats_update(void *xsc) 6297 { 6298 struct ixl_softc *sc = xsc; 6299 struct ixl_stats_counters *isc; 6300 uint64_t delta; 6301 6302 isc = &sc->sc_stats_counters; 6303 6304 /* errors */ 6305 delta = ixl_stat_delta(sc, 6306 0, I40E_GLPRT_CRCERRS(sc->sc_port), 6307 &isc->isc_crc_errors_offset, isc->isc_has_offset); 6308 atomic_add_64(&isc->isc_crc_errors.ev_count, delta); 6309 6310 delta = ixl_stat_delta(sc, 6311 0, I40E_GLPRT_ILLERRC(sc->sc_port), 6312 &isc->isc_illegal_bytes_offset, isc->isc_has_offset); 6313 atomic_add_64(&isc->isc_illegal_bytes.ev_count, delta); 6314 6315 /* rx */ 6316 delta = ixl_stat_delta(sc, 6317 I40E_GLPRT_GORCH(sc->sc_port), I40E_GLPRT_GORCL(sc->sc_port), 6318 &isc->isc_rx_bytes_offset, isc->isc_has_offset); 6319 atomic_add_64(&isc->isc_rx_bytes.ev_count, delta); 6320 6321 delta = ixl_stat_delta(sc, 6322 0, I40E_GLPRT_RDPC(sc->sc_port), 6323 &isc->isc_rx_discards_offset, isc->isc_has_offset); 6324 atomic_add_64(&isc->isc_rx_discards.ev_count, delta); 6325 6326 delta = ixl_stat_delta(sc, 6327 I40E_GLPRT_UPRCH(sc->sc_port), I40E_GLPRT_UPRCL(sc->sc_port), 6328 &isc->isc_rx_unicast_offset, isc->isc_has_offset); 6329 atomic_add_64(&isc->isc_rx_unicast.ev_count, delta); 6330 6331 delta = ixl_stat_delta(sc, 6332 I40E_GLPRT_MPRCH(sc->sc_port), I40E_GLPRT_MPRCL(sc->sc_port), 6333 &isc->isc_rx_multicast_offset, isc->isc_has_offset); 6334 atomic_add_64(&isc->isc_rx_multicast.ev_count, delta); 6335 6336 delta = ixl_stat_delta(sc, 6337 I40E_GLPRT_BPRCH(sc->sc_port), I40E_GLPRT_BPRCL(sc->sc_port), 6338 &isc->isc_rx_broadcast_offset, isc->isc_has_offset); 6339 atomic_add_64(&isc->isc_rx_broadcast.ev_count, delta); 6340 6341 /* Packet size stats rx */ 6342 delta = ixl_stat_delta(sc, 6343 I40E_GLPRT_PRC64H(sc->sc_port), I40E_GLPRT_PRC64L(sc->sc_port), 6344 &isc->isc_rx_size_64_offset, isc->isc_has_offset); 6345 atomic_add_64(&isc->isc_rx_size_64.ev_count, delta); 6346 6347 delta = ixl_stat_delta(sc, 6348 I40E_GLPRT_PRC127H(sc->sc_port), I40E_GLPRT_PRC127L(sc->sc_port), 6349 &isc->isc_rx_size_127_offset, isc->isc_has_offset); 6350 atomic_add_64(&isc->isc_rx_size_127.ev_count, delta); 6351 6352 delta = ixl_stat_delta(sc, 6353 I40E_GLPRT_PRC255H(sc->sc_port), I40E_GLPRT_PRC255L(sc->sc_port), 6354 &isc->isc_rx_size_255_offset, isc->isc_has_offset); 6355 atomic_add_64(&isc->isc_rx_size_255.ev_count, delta); 6356 6357 delta = ixl_stat_delta(sc, 6358 I40E_GLPRT_PRC511H(sc->sc_port), I40E_GLPRT_PRC511L(sc->sc_port), 6359 &isc->isc_rx_size_511_offset, isc->isc_has_offset); 6360 atomic_add_64(&isc->isc_rx_size_511.ev_count, delta); 6361 6362 delta = ixl_stat_delta(sc, 6363 I40E_GLPRT_PRC1023H(sc->sc_port), I40E_GLPRT_PRC1023L(sc->sc_port), 6364 &isc->isc_rx_size_1023_offset, isc->isc_has_offset); 6365 atomic_add_64(&isc->isc_rx_size_1023.ev_count, delta); 6366 6367 delta = ixl_stat_delta(sc, 6368 I40E_GLPRT_PRC1522H(sc->sc_port), I40E_GLPRT_PRC1522L(sc->sc_port), 6369 &isc->isc_rx_size_1522_offset, isc->isc_has_offset); 6370 atomic_add_64(&isc->isc_rx_size_1522.ev_count, delta); 6371 6372 delta = ixl_stat_delta(sc, 6373 I40E_GLPRT_PRC9522H(sc->sc_port), I40E_GLPRT_PRC9522L(sc->sc_port), 6374 &isc->isc_rx_size_big_offset, isc->isc_has_offset); 6375 atomic_add_64(&isc->isc_rx_size_big.ev_count, delta); 6376 6377 delta = ixl_stat_delta(sc, 6378 0, I40E_GLPRT_RUC(sc->sc_port), 6379 &isc->isc_rx_undersize_offset, isc->isc_has_offset); 6380 atomic_add_64(&isc->isc_rx_undersize.ev_count, delta); 6381 6382 delta = ixl_stat_delta(sc, 6383 0, I40E_GLPRT_ROC(sc->sc_port), 6384 &isc->isc_rx_oversize_offset, isc->isc_has_offset); 6385 atomic_add_64(&isc->isc_rx_oversize.ev_count, delta); 6386 6387 /* tx */ 6388 delta = ixl_stat_delta(sc, 6389 I40E_GLPRT_GOTCH(sc->sc_port), I40E_GLPRT_GOTCL(sc->sc_port), 6390 &isc->isc_tx_bytes_offset, isc->isc_has_offset); 6391 atomic_add_64(&isc->isc_tx_bytes.ev_count, delta); 6392 6393 delta = ixl_stat_delta(sc, 6394 0, I40E_GLPRT_TDOLD(sc->sc_port), 6395 &isc->isc_tx_dropped_link_down_offset, isc->isc_has_offset); 6396 atomic_add_64(&isc->isc_tx_dropped_link_down.ev_count, delta); 6397 6398 delta = ixl_stat_delta(sc, 6399 I40E_GLPRT_UPTCH(sc->sc_port), I40E_GLPRT_UPTCL(sc->sc_port), 6400 &isc->isc_tx_unicast_offset, isc->isc_has_offset); 6401 atomic_add_64(&isc->isc_tx_unicast.ev_count, delta); 6402 6403 delta = ixl_stat_delta(sc, 6404 I40E_GLPRT_MPTCH(sc->sc_port), I40E_GLPRT_MPTCL(sc->sc_port), 6405 &isc->isc_tx_multicast_offset, isc->isc_has_offset); 6406 atomic_add_64(&isc->isc_tx_multicast.ev_count, delta); 6407 6408 delta = ixl_stat_delta(sc, 6409 I40E_GLPRT_BPTCH(sc->sc_port), I40E_GLPRT_BPTCL(sc->sc_port), 6410 &isc->isc_tx_broadcast_offset, isc->isc_has_offset); 6411 atomic_add_64(&isc->isc_tx_broadcast.ev_count, delta); 6412 6413 /* Packet size stats tx */ 6414 delta = ixl_stat_delta(sc, 6415 I40E_GLPRT_PTC64L(sc->sc_port), I40E_GLPRT_PTC64L(sc->sc_port), 6416 &isc->isc_tx_size_64_offset, isc->isc_has_offset); 6417 atomic_add_64(&isc->isc_tx_size_64.ev_count, delta); 6418 6419 delta = ixl_stat_delta(sc, 6420 I40E_GLPRT_PTC127H(sc->sc_port), I40E_GLPRT_PTC127L(sc->sc_port), 6421 &isc->isc_tx_size_127_offset, isc->isc_has_offset); 6422 atomic_add_64(&isc->isc_tx_size_127.ev_count, delta); 6423 6424 delta = ixl_stat_delta(sc, 6425 I40E_GLPRT_PTC255H(sc->sc_port), I40E_GLPRT_PTC255L(sc->sc_port), 6426 &isc->isc_tx_size_255_offset, isc->isc_has_offset); 6427 atomic_add_64(&isc->isc_tx_size_255.ev_count, delta); 6428 6429 delta = ixl_stat_delta(sc, 6430 I40E_GLPRT_PTC511H(sc->sc_port), I40E_GLPRT_PTC511L(sc->sc_port), 6431 &isc->isc_tx_size_511_offset, isc->isc_has_offset); 6432 atomic_add_64(&isc->isc_tx_size_511.ev_count, delta); 6433 6434 delta = ixl_stat_delta(sc, 6435 I40E_GLPRT_PTC1023H(sc->sc_port), I40E_GLPRT_PTC1023L(sc->sc_port), 6436 &isc->isc_tx_size_1023_offset, isc->isc_has_offset); 6437 atomic_add_64(&isc->isc_tx_size_1023.ev_count, delta); 6438 6439 delta = ixl_stat_delta(sc, 6440 I40E_GLPRT_PTC1522H(sc->sc_port), I40E_GLPRT_PTC1522L(sc->sc_port), 6441 &isc->isc_tx_size_1522_offset, isc->isc_has_offset); 6442 atomic_add_64(&isc->isc_tx_size_1522.ev_count, delta); 6443 6444 delta = ixl_stat_delta(sc, 6445 I40E_GLPRT_PTC9522H(sc->sc_port), I40E_GLPRT_PTC9522L(sc->sc_port), 6446 &isc->isc_tx_size_big_offset, isc->isc_has_offset); 6447 atomic_add_64(&isc->isc_tx_size_big.ev_count, delta); 6448 6449 /* mac faults */ 6450 delta = ixl_stat_delta(sc, 6451 0, I40E_GLPRT_MLFC(sc->sc_port), 6452 &isc->isc_mac_local_faults_offset, isc->isc_has_offset); 6453 atomic_add_64(&isc->isc_mac_local_faults.ev_count, delta); 6454 6455 delta = ixl_stat_delta(sc, 6456 0, I40E_GLPRT_MRFC(sc->sc_port), 6457 &isc->isc_mac_remote_faults_offset, isc->isc_has_offset); 6458 atomic_add_64(&isc->isc_mac_remote_faults.ev_count, delta); 6459 6460 /* Flow control (LFC) stats */ 6461 delta = ixl_stat_delta(sc, 6462 0, I40E_GLPRT_LXONRXC(sc->sc_port), 6463 &isc->isc_link_xon_rx_offset, isc->isc_has_offset); 6464 atomic_add_64(&isc->isc_link_xon_rx.ev_count, delta); 6465 6466 delta = ixl_stat_delta(sc, 6467 0, I40E_GLPRT_LXONTXC(sc->sc_port), 6468 &isc->isc_link_xon_tx_offset, isc->isc_has_offset); 6469 atomic_add_64(&isc->isc_link_xon_tx.ev_count, delta); 6470 6471 delta = ixl_stat_delta(sc, 6472 0, I40E_GLPRT_LXOFFRXC(sc->sc_port), 6473 &isc->isc_link_xoff_rx_offset, isc->isc_has_offset); 6474 atomic_add_64(&isc->isc_link_xoff_rx.ev_count, delta); 6475 6476 delta = ixl_stat_delta(sc, 6477 0, I40E_GLPRT_LXOFFTXC(sc->sc_port), 6478 &isc->isc_link_xoff_tx_offset, isc->isc_has_offset); 6479 atomic_add_64(&isc->isc_link_xoff_tx.ev_count, delta); 6480 6481 /* fragments */ 6482 delta = ixl_stat_delta(sc, 6483 0, I40E_GLPRT_RFC(sc->sc_port), 6484 &isc->isc_rx_fragments_offset, isc->isc_has_offset); 6485 atomic_add_64(&isc->isc_rx_fragments.ev_count, delta); 6486 6487 delta = ixl_stat_delta(sc, 6488 0, I40E_GLPRT_RJC(sc->sc_port), 6489 &isc->isc_rx_jabber_offset, isc->isc_has_offset); 6490 atomic_add_64(&isc->isc_rx_jabber.ev_count, delta); 6491 6492 /* VSI rx counters */ 6493 delta = ixl_stat_delta(sc, 6494 0, I40E_GLV_RDPC(sc->sc_vsi_stat_counter_idx), 6495 &isc->isc_vsi_rx_discards_offset, isc->isc_has_offset); 6496 atomic_add_64(&isc->isc_vsi_rx_discards.ev_count, delta); 6497 6498 delta = ixl_stat_delta(sc, 6499 I40E_GLV_GORCH(sc->sc_vsi_stat_counter_idx), 6500 I40E_GLV_GORCL(sc->sc_vsi_stat_counter_idx), 6501 &isc->isc_vsi_rx_bytes_offset, isc->isc_has_offset); 6502 atomic_add_64(&isc->isc_vsi_rx_bytes.ev_count, delta); 6503 6504 delta = ixl_stat_delta(sc, 6505 I40E_GLV_UPRCH(sc->sc_vsi_stat_counter_idx), 6506 I40E_GLV_UPRCL(sc->sc_vsi_stat_counter_idx), 6507 &isc->isc_vsi_rx_unicast_offset, isc->isc_has_offset); 6508 atomic_add_64(&isc->isc_vsi_rx_unicast.ev_count, delta); 6509 6510 delta = ixl_stat_delta(sc, 6511 I40E_GLV_MPRCH(sc->sc_vsi_stat_counter_idx), 6512 I40E_GLV_MPRCL(sc->sc_vsi_stat_counter_idx), 6513 &isc->isc_vsi_rx_multicast_offset, isc->isc_has_offset); 6514 atomic_add_64(&isc->isc_vsi_rx_multicast.ev_count, delta); 6515 6516 delta = ixl_stat_delta(sc, 6517 I40E_GLV_BPRCH(sc->sc_vsi_stat_counter_idx), 6518 I40E_GLV_BPRCL(sc->sc_vsi_stat_counter_idx), 6519 &isc->isc_vsi_rx_broadcast_offset, isc->isc_has_offset); 6520 atomic_add_64(&isc->isc_vsi_rx_broadcast.ev_count, delta); 6521 6522 /* VSI tx counters */ 6523 delta = ixl_stat_delta(sc, 6524 0, I40E_GLV_TEPC(sc->sc_vsi_stat_counter_idx), 6525 &isc->isc_vsi_tx_errors_offset, isc->isc_has_offset); 6526 atomic_add_64(&isc->isc_vsi_tx_errors.ev_count, delta); 6527 6528 delta = ixl_stat_delta(sc, 6529 I40E_GLV_GOTCH(sc->sc_vsi_stat_counter_idx), 6530 I40E_GLV_GOTCL(sc->sc_vsi_stat_counter_idx), 6531 &isc->isc_vsi_tx_bytes_offset, isc->isc_has_offset); 6532 atomic_add_64(&isc->isc_vsi_tx_bytes.ev_count, delta); 6533 6534 delta = ixl_stat_delta(sc, 6535 I40E_GLV_UPTCH(sc->sc_vsi_stat_counter_idx), 6536 I40E_GLV_UPTCL(sc->sc_vsi_stat_counter_idx), 6537 &isc->isc_vsi_tx_unicast_offset, isc->isc_has_offset); 6538 atomic_add_64(&isc->isc_vsi_tx_unicast.ev_count, delta); 6539 6540 delta = ixl_stat_delta(sc, 6541 I40E_GLV_MPTCH(sc->sc_vsi_stat_counter_idx), 6542 I40E_GLV_MPTCL(sc->sc_vsi_stat_counter_idx), 6543 &isc->isc_vsi_tx_multicast_offset, isc->isc_has_offset); 6544 atomic_add_64(&isc->isc_vsi_tx_multicast.ev_count, delta); 6545 6546 delta = ixl_stat_delta(sc, 6547 I40E_GLV_BPTCH(sc->sc_vsi_stat_counter_idx), 6548 I40E_GLV_BPTCL(sc->sc_vsi_stat_counter_idx), 6549 &isc->isc_vsi_tx_broadcast_offset, isc->isc_has_offset); 6550 atomic_add_64(&isc->isc_vsi_tx_broadcast.ev_count, delta); 6551 } 6552 6553 static int 6554 ixl_setup_sysctls(struct ixl_softc *sc) 6555 { 6556 const char *devname; 6557 struct sysctllog **log; 6558 const struct sysctlnode *rnode, *rxnode, *txnode; 6559 int error; 6560 6561 log = &sc->sc_sysctllog; 6562 devname = device_xname(sc->sc_dev); 6563 6564 error = sysctl_createv(log, 0, NULL, &rnode, 6565 0, CTLTYPE_NODE, devname, 6566 SYSCTL_DESCR("ixl information and settings"), 6567 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 6568 if (error) 6569 goto out; 6570 6571 error = sysctl_createv(log, 0, &rnode, NULL, 6572 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 6573 SYSCTL_DESCR("Use workqueue for packet processing"), 6574 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 6575 if (error) 6576 goto out; 6577 6578 error = sysctl_createv(log, 0, &rnode, NULL, 6579 CTLFLAG_READONLY, CTLTYPE_INT, "stats_interval", 6580 SYSCTL_DESCR("Statistics collection interval in milliseconds"), 6581 NULL, 0, &sc->sc_stats_intval, 0, CTL_CREATE, CTL_EOL); 6582 6583 error = sysctl_createv(log, 0, &rnode, &rxnode, 6584 0, CTLTYPE_NODE, "rx", 6585 SYSCTL_DESCR("ixl information and settings for Rx"), 6586 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6587 if (error) 6588 goto out; 6589 6590 error = sysctl_createv(log, 0, &rxnode, NULL, 6591 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6592 SYSCTL_DESCR("max number of Rx packets" 6593 " to process for interrupt processing"), 6594 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6595 if (error) 6596 goto out; 6597 6598 error = sysctl_createv(log, 0, &rxnode, NULL, 6599 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6600 SYSCTL_DESCR("max number of Rx packets" 6601 " to process for deferred processing"), 6602 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 6603 if (error) 6604 goto out; 6605 6606 error = sysctl_createv(log, 0, &rnode, &txnode, 6607 0, CTLTYPE_NODE, "tx", 6608 SYSCTL_DESCR("ixl information and settings for Tx"), 6609 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6610 if (error) 6611 goto out; 6612 6613 error = sysctl_createv(log, 0, &txnode, NULL, 6614 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6615 SYSCTL_DESCR("max number of Tx packets" 6616 " to process for interrupt processing"), 6617 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6618 if (error) 6619 goto out; 6620 6621 error = sysctl_createv(log, 0, &txnode, NULL, 6622 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6623 SYSCTL_DESCR("max number of Tx packets" 6624 " to process for deferred processing"), 6625 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 6626 if (error) 6627 goto out; 6628 6629 out: 6630 if (error) { 6631 aprint_error_dev(sc->sc_dev, 6632 "unable to create sysctl node\n"); 6633 sysctl_teardown(log); 6634 } 6635 6636 return error; 6637 } 6638 6639 static void 6640 ixl_teardown_sysctls(struct ixl_softc *sc) 6641 { 6642 6643 sysctl_teardown(&sc->sc_sysctllog); 6644 } 6645 6646 static struct workqueue * 6647 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 6648 { 6649 struct workqueue *wq; 6650 int error; 6651 6652 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 6653 prio, ipl, flags); 6654 6655 if (error) 6656 return NULL; 6657 6658 return wq; 6659 } 6660 6661 static void 6662 ixl_workq_destroy(struct workqueue *wq) 6663 { 6664 6665 workqueue_destroy(wq); 6666 } 6667 6668 static void 6669 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 6670 { 6671 6672 memset(work, 0, sizeof(*work)); 6673 work->ixw_func = func; 6674 work->ixw_arg = arg; 6675 } 6676 6677 static void 6678 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 6679 { 6680 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 6681 return; 6682 6683 kpreempt_disable(); 6684 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 6685 kpreempt_enable(); 6686 } 6687 6688 static void 6689 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 6690 { 6691 6692 workqueue_wait(wq, &work->ixw_cookie); 6693 } 6694 6695 static void 6696 ixl_workq_work(struct work *wk, void *context) 6697 { 6698 struct ixl_work *work; 6699 6700 work = container_of(wk, struct ixl_work, ixw_cookie); 6701 6702 atomic_swap_uint(&work->ixw_added, 0); 6703 work->ixw_func(work->ixw_arg); 6704 } 6705 6706 static int 6707 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 6708 { 6709 struct ixl_aq_desc iaq; 6710 6711 memset(&iaq, 0, sizeof(iaq)); 6712 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 6713 iaq.iaq_param[1] = htole32(reg); 6714 6715 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6716 return ETIMEDOUT; 6717 6718 switch (htole16(iaq.iaq_retval)) { 6719 case IXL_AQ_RC_OK: 6720 /* success */ 6721 break; 6722 case IXL_AQ_RC_EACCES: 6723 return EPERM; 6724 case IXL_AQ_RC_EAGAIN: 6725 return EAGAIN; 6726 default: 6727 return EIO; 6728 } 6729 6730 *rv = htole32(iaq.iaq_param[3]); 6731 return 0; 6732 } 6733 6734 static uint32_t 6735 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 6736 { 6737 uint32_t val; 6738 int rv, retry, retry_limit; 6739 6740 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6741 retry_limit = 5; 6742 } else { 6743 retry_limit = 0; 6744 } 6745 6746 for (retry = 0; retry < retry_limit; retry++) { 6747 rv = ixl_rx_ctl_read(sc, reg, &val); 6748 if (rv == 0) 6749 return val; 6750 else if (rv == EAGAIN) 6751 delaymsec(1); 6752 else 6753 break; 6754 } 6755 6756 val = ixl_rd(sc, reg); 6757 6758 return val; 6759 } 6760 6761 static int 6762 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6763 { 6764 struct ixl_aq_desc iaq; 6765 6766 memset(&iaq, 0, sizeof(iaq)); 6767 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 6768 iaq.iaq_param[1] = htole32(reg); 6769 iaq.iaq_param[3] = htole32(value); 6770 6771 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6772 return ETIMEDOUT; 6773 6774 switch (htole16(iaq.iaq_retval)) { 6775 case IXL_AQ_RC_OK: 6776 /* success */ 6777 break; 6778 case IXL_AQ_RC_EACCES: 6779 return EPERM; 6780 case IXL_AQ_RC_EAGAIN: 6781 return EAGAIN; 6782 default: 6783 return EIO; 6784 } 6785 6786 return 0; 6787 } 6788 6789 static void 6790 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6791 { 6792 int rv, retry, retry_limit; 6793 6794 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6795 retry_limit = 5; 6796 } else { 6797 retry_limit = 0; 6798 } 6799 6800 for (retry = 0; retry < retry_limit; retry++) { 6801 rv = ixl_rx_ctl_write(sc, reg, value); 6802 if (rv == 0) 6803 return; 6804 else if (rv == EAGAIN) 6805 delaymsec(1); 6806 else 6807 break; 6808 } 6809 6810 ixl_wr(sc, reg, value); 6811 } 6812 6813 static int 6814 ixl_nvm_lock(struct ixl_softc *sc, char rw) 6815 { 6816 struct ixl_aq_desc iaq; 6817 struct ixl_aq_req_resource_param *param; 6818 int rv; 6819 6820 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6821 return 0; 6822 6823 memset(&iaq, 0, sizeof(iaq)); 6824 iaq.iaq_opcode = htole16(IXL_AQ_OP_REQUEST_RESOURCE); 6825 6826 param = (struct ixl_aq_req_resource_param *)&iaq.iaq_param; 6827 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6828 if (rw == 'R') { 6829 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_READ); 6830 } else { 6831 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_WRITE); 6832 } 6833 6834 rv = ixl_atq_poll(sc, &iaq, 250); 6835 6836 if (rv != 0) 6837 return ETIMEDOUT; 6838 6839 switch (le16toh(iaq.iaq_retval)) { 6840 case IXL_AQ_RC_OK: 6841 break; 6842 case IXL_AQ_RC_EACCES: 6843 return EACCES; 6844 case IXL_AQ_RC_EBUSY: 6845 return EBUSY; 6846 case IXL_AQ_RC_EPERM: 6847 return EPERM; 6848 } 6849 6850 return 0; 6851 } 6852 6853 static int 6854 ixl_nvm_unlock(struct ixl_softc *sc) 6855 { 6856 struct ixl_aq_desc iaq; 6857 struct ixl_aq_rel_resource_param *param; 6858 int rv; 6859 6860 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6861 return 0; 6862 6863 memset(&iaq, 0, sizeof(iaq)); 6864 iaq.iaq_opcode = htole16(IXL_AQ_OP_RELEASE_RESOURCE); 6865 6866 param = (struct ixl_aq_rel_resource_param *)&iaq.iaq_param; 6867 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6868 6869 rv = ixl_atq_poll(sc, &iaq, 250); 6870 6871 if (rv != 0) 6872 return ETIMEDOUT; 6873 6874 switch (le16toh(iaq.iaq_retval)) { 6875 case IXL_AQ_RC_OK: 6876 break; 6877 default: 6878 return EIO; 6879 } 6880 return 0; 6881 } 6882 6883 static int 6884 ixl_srdone_poll(struct ixl_softc *sc) 6885 { 6886 int wait_count; 6887 uint32_t reg; 6888 6889 for (wait_count = 0; wait_count < IXL_SRRD_SRCTL_ATTEMPTS; 6890 wait_count++) { 6891 reg = ixl_rd(sc, I40E_GLNVM_SRCTL); 6892 if (ISSET(reg, I40E_GLNVM_SRCTL_DONE_MASK)) 6893 break; 6894 6895 delaymsec(5); 6896 } 6897 6898 if (wait_count == IXL_SRRD_SRCTL_ATTEMPTS) 6899 return -1; 6900 6901 return 0; 6902 } 6903 6904 static int 6905 ixl_nvm_read_srctl(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 6906 { 6907 uint32_t reg; 6908 6909 if (ixl_srdone_poll(sc) != 0) 6910 return ETIMEDOUT; 6911 6912 reg = ((uint32_t)offset << I40E_GLNVM_SRCTL_ADDR_SHIFT) | 6913 __BIT(I40E_GLNVM_SRCTL_START_SHIFT); 6914 ixl_wr(sc, I40E_GLNVM_SRCTL, reg); 6915 6916 if (ixl_srdone_poll(sc) != 0) { 6917 aprint_debug("NVM read error: couldn't access " 6918 "Shadow RAM address: 0x%x\n", offset); 6919 return ETIMEDOUT; 6920 } 6921 6922 reg = ixl_rd(sc, I40E_GLNVM_SRDATA); 6923 *data = (uint16_t)__SHIFTOUT(reg, I40E_GLNVM_SRDATA_RDDATA_MASK); 6924 6925 return 0; 6926 } 6927 6928 static int 6929 ixl_nvm_read_aq(struct ixl_softc *sc, uint16_t offset_word, 6930 void *data, size_t len) 6931 { 6932 struct ixl_dmamem *idm; 6933 struct ixl_aq_desc iaq; 6934 struct ixl_aq_nvm_param *param; 6935 uint32_t offset_bytes; 6936 int rv; 6937 6938 idm = &sc->sc_aqbuf; 6939 if (len > IXL_DMA_LEN(idm)) 6940 return ENOMEM; 6941 6942 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 6943 memset(&iaq, 0, sizeof(iaq)); 6944 iaq.iaq_opcode = htole16(IXL_AQ_OP_NVM_READ); 6945 iaq.iaq_flags = htole16(IXL_AQ_BUF | 6946 ((len > I40E_AQ_LARGE_BUF) ? IXL_AQ_LB : 0)); 6947 iaq.iaq_datalen = htole16(len); 6948 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 6949 6950 param = (struct ixl_aq_nvm_param *)iaq.iaq_param; 6951 param->command_flags = IXL_AQ_NVM_LAST_CMD; 6952 param->module_pointer = 0; 6953 param->length = htole16(len); 6954 offset_bytes = (uint32_t)offset_word * 2; 6955 offset_bytes &= 0x00FFFFFF; 6956 param->offset = htole32(offset_bytes); 6957 6958 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 6959 BUS_DMASYNC_PREREAD); 6960 6961 rv = ixl_atq_poll(sc, &iaq, 250); 6962 6963 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 6964 BUS_DMASYNC_POSTREAD); 6965 6966 if (rv != 0) { 6967 return ETIMEDOUT; 6968 } 6969 6970 switch (le16toh(iaq.iaq_retval)) { 6971 case IXL_AQ_RC_OK: 6972 break; 6973 case IXL_AQ_RC_EPERM: 6974 return EPERM; 6975 case IXL_AQ_RC_EINVAL: 6976 return EINVAL; 6977 case IXL_AQ_RC_EBUSY: 6978 return EBUSY; 6979 case IXL_AQ_RC_EIO: 6980 default: 6981 return EIO; 6982 } 6983 6984 memcpy(data, IXL_DMA_KVA(idm), len); 6985 6986 return 0; 6987 } 6988 6989 static int 6990 ixl_rd16_nvm(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 6991 { 6992 int error; 6993 uint16_t buf; 6994 6995 error = ixl_nvm_lock(sc, 'R'); 6996 if (error) 6997 return error; 6998 6999 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMREAD)) { 7000 error = ixl_nvm_read_aq(sc, offset, 7001 &buf, sizeof(buf)); 7002 if (error == 0) 7003 *data = le16toh(buf); 7004 } else { 7005 error = ixl_nvm_read_srctl(sc, offset, &buf); 7006 if (error == 0) 7007 *data = buf; 7008 } 7009 7010 ixl_nvm_unlock(sc); 7011 7012 return error; 7013 } 7014 7015 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 7016 7017 #ifdef _MODULE 7018 #include "ioconf.c" 7019 #endif 7020 7021 #ifdef _MODULE 7022 static void 7023 ixl_parse_modprop(prop_dictionary_t dict) 7024 { 7025 prop_object_t obj; 7026 int64_t val; 7027 uint64_t uval; 7028 7029 if (dict == NULL) 7030 return; 7031 7032 obj = prop_dictionary_get(dict, "nomsix"); 7033 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_BOOL) { 7034 ixl_param_nomsix = prop_bool_true((prop_bool_t)obj); 7035 } 7036 7037 obj = prop_dictionary_get(dict, "stats_interval"); 7038 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7039 val = prop_number_integer_value((prop_number_t)obj); 7040 7041 /* the range has no reason */ 7042 if (100 < val && val < 180000) { 7043 ixl_param_stats_interval = val; 7044 } 7045 } 7046 7047 obj = prop_dictionary_get(dict, "nqps_limit"); 7048 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7049 val = prop_number_integer_value((prop_number_t)obj); 7050 7051 if (val <= INT32_MAX) 7052 ixl_param_nqps_limit = val; 7053 } 7054 7055 obj = prop_dictionary_get(dict, "rx_ndescs"); 7056 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7057 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7058 7059 if (uval > 8) 7060 ixl_param_rx_ndescs = uval; 7061 } 7062 7063 obj = prop_dictionary_get(dict, "tx_ndescs"); 7064 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7065 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7066 7067 if (uval > IXL_TX_PKT_DESCS) 7068 ixl_param_tx_ndescs = uval; 7069 } 7070 7071 } 7072 #endif 7073 7074 static int 7075 if_ixl_modcmd(modcmd_t cmd, void *opaque) 7076 { 7077 int error = 0; 7078 7079 #ifdef _MODULE 7080 switch (cmd) { 7081 case MODULE_CMD_INIT: 7082 ixl_parse_modprop((prop_dictionary_t)opaque); 7083 error = config_init_component(cfdriver_ioconf_if_ixl, 7084 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7085 break; 7086 case MODULE_CMD_FINI: 7087 error = config_fini_component(cfdriver_ioconf_if_ixl, 7088 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7089 break; 7090 default: 7091 error = ENOTTY; 7092 break; 7093 } 7094 #endif 7095 7096 return error; 7097 } 7098
Indexes created Sat Sep 20 22:09:52 GMT 2025