if_ixl.c revision 1.59
1 /* $NetBSD: if_ixl.c,v 1.59 2020/03/03 04:34:45 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 __KERNEL_RCSID(0, "$NetBSD: if_ixl.c,v 1.59 2020/03/03 04:34:45 yamaguchi Exp $"); 78 79 #ifdef _KERNEL_OPT 80 #include "opt_net_mpsafe.h" 81 #include "opt_if_ixl.h" 82 #endif 83 84 #include <sys/param.h> 85 #include <sys/types.h> 86 87 #include <sys/cpu.h> 88 #include <sys/device.h> 89 #include <sys/evcnt.h> 90 #include <sys/interrupt.h> 91 #include <sys/kmem.h> 92 #include <sys/module.h> 93 #include <sys/mutex.h> 94 #include <sys/pcq.h> 95 #include <sys/syslog.h> 96 #include <sys/workqueue.h> 97 98 #include <sys/bus.h> 99 100 #include <net/bpf.h> 101 #include <net/if.h> 102 #include <net/if_dl.h> 103 #include <net/if_media.h> 104 #include <net/if_ether.h> 105 #include <net/rss_config.h> 106 107 #include <netinet/tcp.h> /* for struct tcphdr */ 108 #include <netinet/udp.h> /* for struct udphdr */ 109 110 #include <dev/pci/pcivar.h> 111 #include <dev/pci/pcidevs.h> 112 113 #include <dev/pci/if_ixlreg.h> 114 #include <dev/pci/if_ixlvar.h> 115 116 #include <prop/proplib.h> 117 118 struct ixl_softc; /* defined */ 119 120 #define I40E_PF_RESET_WAIT_COUNT 200 121 #define I40E_AQ_LARGE_BUF 512 122 123 /* bitfields for Tx queue mapping in QTX_CTL */ 124 #define I40E_QTX_CTL_VF_QUEUE 0x0 125 #define I40E_QTX_CTL_VM_QUEUE 0x1 126 #define I40E_QTX_CTL_PF_QUEUE 0x2 127 128 #define I40E_QUEUE_TYPE_EOL 0x7ff 129 #define I40E_INTR_NOTX_QUEUE 0 130 131 #define I40E_QUEUE_TYPE_RX 0x0 132 #define I40E_QUEUE_TYPE_TX 0x1 133 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 134 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 135 136 #define I40E_ITR_INDEX_RX 0x0 137 #define I40E_ITR_INDEX_TX 0x1 138 #define I40E_ITR_INDEX_OTHER 0x2 139 #define I40E_ITR_INDEX_NONE 0x3 140 141 #define I40E_INTR_NOTX_QUEUE 0 142 #define I40E_INTR_NOTX_INTR 0 143 #define I40E_INTR_NOTX_RX_QUEUE 0 144 #define I40E_INTR_NOTX_TX_QUEUE 1 145 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 146 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 147 148 #define BIT_ULL(a) (1ULL << (a)) 149 #define IXL_RSS_HENA_DEFAULT_BASE \ 150 (BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 151 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 152 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 153 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 154 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 155 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 156 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 157 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 158 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 159 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 160 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 161 #define IXL_RSS_HENA_DEFAULT_XL710 IXL_RSS_HENA_DEFAULT_BASE 162 #define IXL_RSS_HENA_DEFAULT_X722 (IXL_RSS_HENA_DEFAULT_XL710 | \ 163 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 164 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 165 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 166 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 167 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 168 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 169 #define I40E_HASH_LUT_SIZE_128 0 170 #define IXL_RSS_KEY_SIZE_REG 13 171 172 #define IXL_ICR0_CRIT_ERR_MASK \ 173 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 174 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 175 I40E_PFINT_ICR0_PE_CRITERR_MASK) 176 177 #define IXL_QUEUE_MAX_XL710 64 178 #define IXL_QUEUE_MAX_X722 128 179 180 #define IXL_TX_PKT_DESCS 8 181 #define IXL_TX_PKT_MAXSIZE (MCLBYTES * IXL_TX_PKT_DESCS) 182 #define IXL_TX_QUEUE_ALIGN 128 183 #define IXL_RX_QUEUE_ALIGN 128 184 185 #define IXL_MCLBYTES (MCLBYTES - ETHER_ALIGN) 186 #define IXL_MTU_ETHERLEN ETHER_HDR_LEN \ 187 + ETHER_CRC_LEN 188 #if 0 189 #define IXL_MAX_MTU (9728 - IXL_MTU_ETHERLEN) 190 #else 191 /* (dbuff * 5) - ETHER_HDR_LEN - ETHER_CRC_LEN */ 192 #define IXL_MAX_MTU (9600 - IXL_MTU_ETHERLEN) 193 #endif 194 #define IXL_MIN_MTU (ETHER_MIN_LEN - ETHER_CRC_LEN) 195 196 #define IXL_PCIREG PCI_MAPREG_START 197 198 #define IXL_ITR0 0x0 199 #define IXL_ITR1 0x1 200 #define IXL_ITR2 0x2 201 #define IXL_NOITR 0x3 202 203 #define IXL_AQ_NUM 256 204 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 205 #define IXL_AQ_ALIGN 64 /* lol */ 206 #define IXL_AQ_BUFLEN 4096 207 208 #define IXL_HMC_ROUNDUP 512 209 #define IXL_HMC_PGSIZE 4096 210 #define IXL_HMC_DVASZ sizeof(uint64_t) 211 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 212 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 213 #define IXL_HMC_PDVALID 1ULL 214 215 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 216 217 #define IXL_SRRD_SRCTL_ATTEMPTS 100000 218 219 struct ixl_aq_regs { 220 bus_size_t atq_tail; 221 bus_size_t atq_head; 222 bus_size_t atq_len; 223 bus_size_t atq_bal; 224 bus_size_t atq_bah; 225 226 bus_size_t arq_tail; 227 bus_size_t arq_head; 228 bus_size_t arq_len; 229 bus_size_t arq_bal; 230 bus_size_t arq_bah; 231 232 uint32_t atq_len_enable; 233 uint32_t atq_tail_mask; 234 uint32_t atq_head_mask; 235 236 uint32_t arq_len_enable; 237 uint32_t arq_tail_mask; 238 uint32_t arq_head_mask; 239 }; 240 241 struct ixl_phy_type { 242 uint64_t phy_type; 243 uint64_t ifm_type; 244 }; 245 246 struct ixl_speed_type { 247 uint8_t dev_speed; 248 uint64_t net_speed; 249 }; 250 251 struct ixl_aq_buf { 252 SIMPLEQ_ENTRY(ixl_aq_buf) 253 aqb_entry; 254 void *aqb_data; 255 bus_dmamap_t aqb_map; 256 bus_dma_segment_t aqb_seg; 257 size_t aqb_size; 258 int aqb_nsegs; 259 }; 260 SIMPLEQ_HEAD(ixl_aq_bufs, ixl_aq_buf); 261 262 struct ixl_dmamem { 263 bus_dmamap_t ixm_map; 264 bus_dma_segment_t ixm_seg; 265 int ixm_nsegs; 266 size_t ixm_size; 267 void *ixm_kva; 268 }; 269 270 #define IXL_DMA_MAP(_ixm) ((_ixm)->ixm_map) 271 #define IXL_DMA_DVA(_ixm) ((_ixm)->ixm_map->dm_segs[0].ds_addr) 272 #define IXL_DMA_KVA(_ixm) ((void *)(_ixm)->ixm_kva) 273 #define IXL_DMA_LEN(_ixm) ((_ixm)->ixm_size) 274 275 struct ixl_hmc_entry { 276 uint64_t hmc_base; 277 uint32_t hmc_count; 278 uint64_t hmc_size; 279 }; 280 281 enum ixl_hmc_types { 282 IXL_HMC_LAN_TX = 0, 283 IXL_HMC_LAN_RX, 284 IXL_HMC_FCOE_CTX, 285 IXL_HMC_FCOE_FILTER, 286 IXL_HMC_COUNT 287 }; 288 289 struct ixl_hmc_pack { 290 uint16_t offset; 291 uint16_t width; 292 uint16_t lsb; 293 }; 294 295 /* 296 * these hmc objects have weird sizes and alignments, so these are abstract 297 * representations of them that are nice for c to populate. 298 * 299 * the packing code relies on little-endian values being stored in the fields, 300 * no high bits in the fields being set, and the fields must be packed in the 301 * same order as they are in the ctx structure. 302 */ 303 304 struct ixl_hmc_rxq { 305 uint16_t head; 306 uint8_t cpuid; 307 uint64_t base; 308 #define IXL_HMC_RXQ_BASE_UNIT 128 309 uint16_t qlen; 310 uint16_t dbuff; 311 #define IXL_HMC_RXQ_DBUFF_UNIT 128 312 uint8_t hbuff; 313 #define IXL_HMC_RXQ_HBUFF_UNIT 64 314 uint8_t dtype; 315 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 316 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 317 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 318 uint8_t dsize; 319 #define IXL_HMC_RXQ_DSIZE_16 0 320 #define IXL_HMC_RXQ_DSIZE_32 1 321 uint8_t crcstrip; 322 uint8_t fc_ena; 323 uint8_t l2sel; 324 uint8_t hsplit_0; 325 uint8_t hsplit_1; 326 uint8_t showiv; 327 uint16_t rxmax; 328 uint8_t tphrdesc_ena; 329 uint8_t tphwdesc_ena; 330 uint8_t tphdata_ena; 331 uint8_t tphhead_ena; 332 uint8_t lrxqthresh; 333 uint8_t prefena; 334 }; 335 336 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 337 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 338 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 339 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 340 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 341 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 342 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 343 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 344 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 345 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 346 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 347 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 348 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 349 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 350 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 351 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 352 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 353 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 354 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 355 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 356 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 357 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 358 }; 359 360 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 361 362 struct ixl_hmc_txq { 363 uint16_t head; 364 uint8_t new_context; 365 uint64_t base; 366 #define IXL_HMC_TXQ_BASE_UNIT 128 367 uint8_t fc_ena; 368 uint8_t timesync_ena; 369 uint8_t fd_ena; 370 uint8_t alt_vlan_ena; 371 uint8_t cpuid; 372 uint16_t thead_wb; 373 uint8_t head_wb_ena; 374 #define IXL_HMC_TXQ_DESC_WB 0 375 #define IXL_HMC_TXQ_HEAD_WB 1 376 uint16_t qlen; 377 uint8_t tphrdesc_ena; 378 uint8_t tphrpacket_ena; 379 uint8_t tphwdesc_ena; 380 uint64_t head_wb_addr; 381 uint32_t crc; 382 uint16_t rdylist; 383 uint8_t rdylist_act; 384 }; 385 386 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 387 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 388 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 389 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 390 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 391 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 392 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 393 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 394 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 395 /* line 1 */ 396 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 397 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 398 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 399 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 400 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 401 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 402 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 403 /* line 7 */ 404 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 405 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 406 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 407 }; 408 409 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 410 411 struct ixl_work { 412 struct work ixw_cookie; 413 void (*ixw_func)(void *); 414 void *ixw_arg; 415 unsigned int ixw_added; 416 }; 417 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 418 419 struct ixl_tx_map { 420 struct mbuf *txm_m; 421 bus_dmamap_t txm_map; 422 unsigned int txm_eop; 423 }; 424 425 struct ixl_tx_ring { 426 kmutex_t txr_lock; 427 struct ixl_softc *txr_sc; 428 429 unsigned int txr_prod; 430 unsigned int txr_cons; 431 432 struct ixl_tx_map *txr_maps; 433 struct ixl_dmamem txr_mem; 434 435 bus_size_t txr_tail; 436 unsigned int txr_qid; 437 pcq_t *txr_intrq; 438 void *txr_si; 439 440 struct evcnt txr_defragged; 441 struct evcnt txr_defrag_failed; 442 struct evcnt txr_pcqdrop; 443 struct evcnt txr_transmitdef; 444 struct evcnt txr_intr; 445 struct evcnt txr_defer; 446 }; 447 448 struct ixl_rx_map { 449 struct mbuf *rxm_m; 450 bus_dmamap_t rxm_map; 451 }; 452 453 struct ixl_rx_ring { 454 kmutex_t rxr_lock; 455 456 unsigned int rxr_prod; 457 unsigned int rxr_cons; 458 459 struct ixl_rx_map *rxr_maps; 460 struct ixl_dmamem rxr_mem; 461 462 struct mbuf *rxr_m_head; 463 struct mbuf **rxr_m_tail; 464 465 bus_size_t rxr_tail; 466 unsigned int rxr_qid; 467 468 struct evcnt rxr_mgethdr_failed; 469 struct evcnt rxr_mgetcl_failed; 470 struct evcnt rxr_mbuf_load_failed; 471 struct evcnt rxr_intr; 472 struct evcnt rxr_defer; 473 }; 474 475 struct ixl_queue_pair { 476 struct ixl_softc *qp_sc; 477 struct ixl_tx_ring *qp_txr; 478 struct ixl_rx_ring *qp_rxr; 479 480 char qp_name[16]; 481 482 void *qp_si; 483 struct work qp_work; 484 bool qp_workqueue; 485 }; 486 487 struct ixl_atq { 488 struct ixl_aq_desc iatq_desc; 489 void (*iatq_fn)(struct ixl_softc *, 490 const struct ixl_aq_desc *); 491 }; 492 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 493 494 struct ixl_product { 495 unsigned int vendor_id; 496 unsigned int product_id; 497 }; 498 499 struct ixl_stats_counters { 500 bool isc_has_offset; 501 struct evcnt isc_crc_errors; 502 uint64_t isc_crc_errors_offset; 503 struct evcnt isc_illegal_bytes; 504 uint64_t isc_illegal_bytes_offset; 505 struct evcnt isc_rx_bytes; 506 uint64_t isc_rx_bytes_offset; 507 struct evcnt isc_rx_discards; 508 uint64_t isc_rx_discards_offset; 509 struct evcnt isc_rx_unicast; 510 uint64_t isc_rx_unicast_offset; 511 struct evcnt isc_rx_multicast; 512 uint64_t isc_rx_multicast_offset; 513 struct evcnt isc_rx_broadcast; 514 uint64_t isc_rx_broadcast_offset; 515 struct evcnt isc_rx_size_64; 516 uint64_t isc_rx_size_64_offset; 517 struct evcnt isc_rx_size_127; 518 uint64_t isc_rx_size_127_offset; 519 struct evcnt isc_rx_size_255; 520 uint64_t isc_rx_size_255_offset; 521 struct evcnt isc_rx_size_511; 522 uint64_t isc_rx_size_511_offset; 523 struct evcnt isc_rx_size_1023; 524 uint64_t isc_rx_size_1023_offset; 525 struct evcnt isc_rx_size_1522; 526 uint64_t isc_rx_size_1522_offset; 527 struct evcnt isc_rx_size_big; 528 uint64_t isc_rx_size_big_offset; 529 struct evcnt isc_rx_undersize; 530 uint64_t isc_rx_undersize_offset; 531 struct evcnt isc_rx_oversize; 532 uint64_t isc_rx_oversize_offset; 533 struct evcnt isc_rx_fragments; 534 uint64_t isc_rx_fragments_offset; 535 struct evcnt isc_rx_jabber; 536 uint64_t isc_rx_jabber_offset; 537 struct evcnt isc_tx_bytes; 538 uint64_t isc_tx_bytes_offset; 539 struct evcnt isc_tx_dropped_link_down; 540 uint64_t isc_tx_dropped_link_down_offset; 541 struct evcnt isc_tx_unicast; 542 uint64_t isc_tx_unicast_offset; 543 struct evcnt isc_tx_multicast; 544 uint64_t isc_tx_multicast_offset; 545 struct evcnt isc_tx_broadcast; 546 uint64_t isc_tx_broadcast_offset; 547 struct evcnt isc_tx_size_64; 548 uint64_t isc_tx_size_64_offset; 549 struct evcnt isc_tx_size_127; 550 uint64_t isc_tx_size_127_offset; 551 struct evcnt isc_tx_size_255; 552 uint64_t isc_tx_size_255_offset; 553 struct evcnt isc_tx_size_511; 554 uint64_t isc_tx_size_511_offset; 555 struct evcnt isc_tx_size_1023; 556 uint64_t isc_tx_size_1023_offset; 557 struct evcnt isc_tx_size_1522; 558 uint64_t isc_tx_size_1522_offset; 559 struct evcnt isc_tx_size_big; 560 uint64_t isc_tx_size_big_offset; 561 struct evcnt isc_mac_local_faults; 562 uint64_t isc_mac_local_faults_offset; 563 struct evcnt isc_mac_remote_faults; 564 uint64_t isc_mac_remote_faults_offset; 565 struct evcnt isc_link_xon_rx; 566 uint64_t isc_link_xon_rx_offset; 567 struct evcnt isc_link_xon_tx; 568 uint64_t isc_link_xon_tx_offset; 569 struct evcnt isc_link_xoff_rx; 570 uint64_t isc_link_xoff_rx_offset; 571 struct evcnt isc_link_xoff_tx; 572 uint64_t isc_link_xoff_tx_offset; 573 struct evcnt isc_vsi_rx_discards; 574 uint64_t isc_vsi_rx_discards_offset; 575 struct evcnt isc_vsi_rx_bytes; 576 uint64_t isc_vsi_rx_bytes_offset; 577 struct evcnt isc_vsi_rx_unicast; 578 uint64_t isc_vsi_rx_unicast_offset; 579 struct evcnt isc_vsi_rx_multicast; 580 uint64_t isc_vsi_rx_multicast_offset; 581 struct evcnt isc_vsi_rx_broadcast; 582 uint64_t isc_vsi_rx_broadcast_offset; 583 struct evcnt isc_vsi_tx_errors; 584 uint64_t isc_vsi_tx_errors_offset; 585 struct evcnt isc_vsi_tx_bytes; 586 uint64_t isc_vsi_tx_bytes_offset; 587 struct evcnt isc_vsi_tx_unicast; 588 uint64_t isc_vsi_tx_unicast_offset; 589 struct evcnt isc_vsi_tx_multicast; 590 uint64_t isc_vsi_tx_multicast_offset; 591 struct evcnt isc_vsi_tx_broadcast; 592 uint64_t isc_vsi_tx_broadcast_offset; 593 }; 594 595 /* 596 * Locking notes: 597 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 598 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 599 * - more than one lock of them cannot be held at once. 600 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 601 * (a spin mutex). 602 * - the lock cannot held with txr_lock or rxr_lock. 603 * + a field named sc_arq_* is not protected by any lock. 604 * - operations for sc_arq_* is done in one context related to 605 * sc_arq_task. 606 * + other fields in ixl_softc is protected by sc_cfg_lock 607 * (an adaptive mutex) 608 * - It must be held before another lock is held, and It can be 609 * released after the other lock is released. 610 * */ 611 612 struct ixl_softc { 613 device_t sc_dev; 614 struct ethercom sc_ec; 615 bool sc_attached; 616 bool sc_dead; 617 uint32_t sc_port; 618 struct sysctllog *sc_sysctllog; 619 struct workqueue *sc_workq; 620 struct workqueue *sc_workq_txrx; 621 int sc_stats_intval; 622 callout_t sc_stats_callout; 623 struct ixl_work sc_stats_task; 624 struct ixl_stats_counters 625 sc_stats_counters; 626 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 627 struct ifmedia sc_media; 628 uint64_t sc_media_status; 629 uint64_t sc_media_active; 630 uint64_t sc_phy_types; 631 uint8_t sc_phy_abilities; 632 uint8_t sc_phy_linkspeed; 633 uint8_t sc_phy_fec_cfg; 634 uint16_t sc_eee_cap; 635 uint32_t sc_eeer_val; 636 uint8_t sc_d3_lpan; 637 kmutex_t sc_cfg_lock; 638 enum i40e_mac_type sc_mac_type; 639 uint32_t sc_rss_table_size; 640 uint32_t sc_rss_table_entry_width; 641 bool sc_txrx_workqueue; 642 u_int sc_tx_process_limit; 643 u_int sc_rx_process_limit; 644 u_int sc_tx_intr_process_limit; 645 u_int sc_rx_intr_process_limit; 646 647 int sc_cur_ec_capenable; 648 649 struct pci_attach_args sc_pa; 650 pci_intr_handle_t *sc_ihp; 651 void **sc_ihs; 652 unsigned int sc_nintrs; 653 654 bus_dma_tag_t sc_dmat; 655 bus_space_tag_t sc_memt; 656 bus_space_handle_t sc_memh; 657 bus_size_t sc_mems; 658 659 uint8_t sc_pf_id; 660 uint16_t sc_uplink_seid; /* le */ 661 uint16_t sc_downlink_seid; /* le */ 662 uint16_t sc_vsi_number; 663 uint16_t sc_vsi_stat_counter_idx; 664 uint16_t sc_seid; 665 unsigned int sc_base_queue; 666 667 pci_intr_type_t sc_intrtype; 668 unsigned int sc_msix_vector_queue; 669 670 struct ixl_dmamem sc_scratch; 671 struct ixl_dmamem sc_aqbuf; 672 673 const struct ixl_aq_regs * 674 sc_aq_regs; 675 uint32_t sc_aq_flags; 676 #define IXL_SC_AQ_FLAG_RXCTL __BIT(0) 677 #define IXL_SC_AQ_FLAG_NVMLOCK __BIT(1) 678 #define IXL_SC_AQ_FLAG_NVMREAD __BIT(2) 679 #define IXL_SC_AQ_FLAG_RSS __BIT(3) 680 681 kmutex_t sc_atq_lock; 682 kcondvar_t sc_atq_cv; 683 struct ixl_dmamem sc_atq; 684 unsigned int sc_atq_prod; 685 unsigned int sc_atq_cons; 686 687 struct ixl_dmamem sc_arq; 688 struct ixl_work sc_arq_task; 689 struct ixl_aq_bufs sc_arq_idle; 690 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 691 unsigned int sc_arq_prod; 692 unsigned int sc_arq_cons; 693 694 struct ixl_work sc_link_state_task; 695 struct ixl_atq sc_link_state_atq; 696 697 struct ixl_dmamem sc_hmc_sd; 698 struct ixl_dmamem sc_hmc_pd; 699 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 700 701 unsigned int sc_tx_ring_ndescs; 702 unsigned int sc_rx_ring_ndescs; 703 unsigned int sc_nqueue_pairs; 704 unsigned int sc_nqueue_pairs_max; 705 unsigned int sc_nqueue_pairs_device; 706 struct ixl_queue_pair *sc_qps; 707 708 struct evcnt sc_event_atq; 709 struct evcnt sc_event_link; 710 struct evcnt sc_event_ecc_err; 711 struct evcnt sc_event_pci_exception; 712 struct evcnt sc_event_crit_err; 713 }; 714 715 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 716 #define IXL_TX_PROCESS_LIMIT 256 717 #define IXL_RX_PROCESS_LIMIT 256 718 #define IXL_TX_INTR_PROCESS_LIMIT 256 719 #define IXL_RX_INTR_PROCESS_LIMIT 0U 720 721 #define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx | \ 722 IFCAP_CSUM_TCPv4_Rx | \ 723 IFCAP_CSUM_UDPv4_Rx | \ 724 IFCAP_CSUM_TCPv6_Rx | \ 725 IFCAP_CSUM_UDPv6_Rx) 726 #define IXL_IFCAP_TXCSUM (IFCAP_CSUM_IPv4_Tx | \ 727 IFCAP_CSUM_TCPv4_Tx | \ 728 IFCAP_CSUM_UDPv4_Tx | \ 729 IFCAP_CSUM_TCPv6_Tx | \ 730 IFCAP_CSUM_UDPv6_Tx) 731 #define IXL_CSUM_ALL_OFFLOAD (M_CSUM_IPv4 | \ 732 M_CSUM_TCPv4 | M_CSUM_TCPv6 | \ 733 M_CSUM_UDPv4 | M_CSUM_UDPv6) 734 735 #define delaymsec(_x) DELAY(1000 * (_x)) 736 #ifdef IXL_DEBUG 737 #define DDPRINTF(sc, fmt, args...) \ 738 do { \ 739 if ((sc) != NULL) { \ 740 device_printf( \ 741 ((struct ixl_softc *)(sc))->sc_dev, \ 742 ""); \ 743 } \ 744 printf("%s:\t" fmt, __func__, ##args); \ 745 } while (0) 746 #else 747 #define DDPRINTF(sc, fmt, args...) __nothing 748 #endif 749 #ifndef IXL_STATS_INTERVAL_MSEC 750 #define IXL_STATS_INTERVAL_MSEC 10000 751 #endif 752 #ifndef IXL_QUEUE_NUM 753 #define IXL_QUEUE_NUM 0 754 #endif 755 756 static bool ixl_param_nomsix = false; 757 static int ixl_param_stats_interval = IXL_STATS_INTERVAL_MSEC; 758 static int ixl_param_nqps_limit = IXL_QUEUE_NUM; 759 static unsigned int ixl_param_tx_ndescs = 1024; 760 static unsigned int ixl_param_rx_ndescs = 1024; 761 762 static enum i40e_mac_type 763 ixl_mactype(pci_product_id_t); 764 static void ixl_clear_hw(struct ixl_softc *); 765 static int ixl_pf_reset(struct ixl_softc *); 766 767 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 768 bus_size_t, bus_size_t); 769 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 770 771 static int ixl_arq_fill(struct ixl_softc *); 772 static void ixl_arq_unfill(struct ixl_softc *); 773 774 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 775 unsigned int); 776 static void ixl_atq_set(struct ixl_atq *, 777 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 778 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 779 static void ixl_atq_done(struct ixl_softc *); 780 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 781 static int ixl_atq_exec_locked(struct ixl_softc *, struct ixl_atq *); 782 static int ixl_get_version(struct ixl_softc *); 783 static int ixl_get_nvm_version(struct ixl_softc *); 784 static int ixl_get_hw_capabilities(struct ixl_softc *); 785 static int ixl_pxe_clear(struct ixl_softc *); 786 static int ixl_lldp_shut(struct ixl_softc *); 787 static int ixl_get_mac(struct ixl_softc *); 788 static int ixl_get_switch_config(struct ixl_softc *); 789 static int ixl_phy_mask_ints(struct ixl_softc *); 790 static int ixl_get_phy_info(struct ixl_softc *); 791 static int ixl_set_phy_config(struct ixl_softc *, uint8_t, uint8_t, bool); 792 static int ixl_set_phy_autoselect(struct ixl_softc *); 793 static int ixl_restart_an(struct ixl_softc *); 794 static int ixl_hmc(struct ixl_softc *); 795 static void ixl_hmc_free(struct ixl_softc *); 796 static int ixl_get_vsi(struct ixl_softc *); 797 static int ixl_set_vsi(struct ixl_softc *); 798 static void ixl_set_filter_control(struct ixl_softc *); 799 static void ixl_get_link_status(void *); 800 static int ixl_get_link_status_poll(struct ixl_softc *, int *); 801 static void ixl_get_link_status_done(struct ixl_softc *, 802 const struct ixl_aq_desc *); 803 static int ixl_set_link_status(struct ixl_softc *, 804 const struct ixl_aq_desc *); 805 static uint64_t ixl_search_link_speed(uint8_t); 806 static uint8_t ixl_search_baudrate(uint64_t); 807 static void ixl_config_rss(struct ixl_softc *); 808 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 809 uint16_t, uint16_t); 810 static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *, 811 uint16_t, uint16_t); 812 static void ixl_arq(void *); 813 static void ixl_hmc_pack(void *, const void *, 814 const struct ixl_hmc_pack *, unsigned int); 815 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 816 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 817 static int ixl_rd16_nvm(struct ixl_softc *, uint16_t, uint16_t *); 818 819 static int ixl_match(device_t, cfdata_t, void *); 820 static void ixl_attach(device_t, device_t, void *); 821 static int ixl_detach(device_t, int); 822 823 static void ixl_media_add(struct ixl_softc *); 824 static int ixl_media_change(struct ifnet *); 825 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 826 static void ixl_watchdog(struct ifnet *); 827 static int ixl_ioctl(struct ifnet *, u_long, void *); 828 static void ixl_start(struct ifnet *); 829 static int ixl_transmit(struct ifnet *, struct mbuf *); 830 static void ixl_deferred_transmit(void *); 831 static int ixl_intr(void *); 832 static int ixl_queue_intr(void *); 833 static int ixl_other_intr(void *); 834 static void ixl_handle_queue(void *); 835 static void ixl_handle_queue_wk(struct work *, void *); 836 static void ixl_sched_handle_queue(struct ixl_softc *, 837 struct ixl_queue_pair *); 838 static int ixl_init(struct ifnet *); 839 static int ixl_init_locked(struct ixl_softc *); 840 static void ixl_stop(struct ifnet *, int); 841 static void ixl_stop_locked(struct ixl_softc *); 842 static int ixl_iff(struct ixl_softc *); 843 static int ixl_ifflags_cb(struct ethercom *); 844 static int ixl_setup_interrupts(struct ixl_softc *); 845 static int ixl_establish_intx(struct ixl_softc *); 846 static int ixl_establish_msix(struct ixl_softc *); 847 static void ixl_enable_queue_intr(struct ixl_softc *, 848 struct ixl_queue_pair *); 849 static void ixl_disable_queue_intr(struct ixl_softc *, 850 struct ixl_queue_pair *); 851 static void ixl_enable_other_intr(struct ixl_softc *); 852 static void ixl_disable_other_intr(struct ixl_softc *); 853 static void ixl_config_queue_intr(struct ixl_softc *); 854 static void ixl_config_other_intr(struct ixl_softc *); 855 856 static struct ixl_tx_ring * 857 ixl_txr_alloc(struct ixl_softc *, unsigned int); 858 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 859 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 860 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 861 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 862 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 863 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 864 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 865 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 866 867 static struct ixl_rx_ring * 868 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 869 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 870 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 871 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 872 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 873 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 874 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 875 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 876 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 877 878 static struct workqueue * 879 ixl_workq_create(const char *, pri_t, int, int); 880 static void ixl_workq_destroy(struct workqueue *); 881 static int ixl_workqs_teardown(device_t); 882 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 883 static void ixl_work_add(struct workqueue *, struct ixl_work *); 884 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 885 static void ixl_workq_work(struct work *, void *); 886 static const struct ixl_product * 887 ixl_lookup(const struct pci_attach_args *pa); 888 static void ixl_link_state_update(struct ixl_softc *, 889 const struct ixl_aq_desc *); 890 static int ixl_vlan_cb(struct ethercom *, uint16_t, bool); 891 static int ixl_setup_vlan_hwfilter(struct ixl_softc *); 892 static void ixl_teardown_vlan_hwfilter(struct ixl_softc *); 893 static int ixl_update_macvlan(struct ixl_softc *); 894 static int ixl_setup_interrupts(struct ixl_softc *);; 895 static void ixl_teardown_interrupts(struct ixl_softc *); 896 static int ixl_setup_stats(struct ixl_softc *); 897 static void ixl_teardown_stats(struct ixl_softc *); 898 static void ixl_stats_callout(void *); 899 static void ixl_stats_update(void *); 900 static int ixl_setup_sysctls(struct ixl_softc *); 901 static void ixl_teardown_sysctls(struct ixl_softc *); 902 static int ixl_queue_pairs_alloc(struct ixl_softc *); 903 static void ixl_queue_pairs_free(struct ixl_softc *); 904 905 static const struct ixl_phy_type ixl_phy_type_map[] = { 906 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 907 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 908 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 909 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 910 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 911 { 1ULL << IXL_PHY_TYPE_XAUI | 912 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 913 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 914 { 1ULL << IXL_PHY_TYPE_XLAUI | 915 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 916 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 917 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 918 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 919 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 920 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 921 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 922 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 923 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 924 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 925 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 926 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 927 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 928 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 929 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 930 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 931 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 932 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 933 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 934 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 935 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 936 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 937 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 938 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 939 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_ACC }, 940 }; 941 942 static const struct ixl_speed_type ixl_speed_type_map[] = { 943 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 944 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 945 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 946 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 947 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 948 }; 949 950 static const struct ixl_aq_regs ixl_pf_aq_regs = { 951 .atq_tail = I40E_PF_ATQT, 952 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 953 .atq_head = I40E_PF_ATQH, 954 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 955 .atq_len = I40E_PF_ATQLEN, 956 .atq_bal = I40E_PF_ATQBAL, 957 .atq_bah = I40E_PF_ATQBAH, 958 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 959 960 .arq_tail = I40E_PF_ARQT, 961 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 962 .arq_head = I40E_PF_ARQH, 963 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 964 .arq_len = I40E_PF_ARQLEN, 965 .arq_bal = I40E_PF_ARQBAL, 966 .arq_bah = I40E_PF_ARQBAH, 967 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 968 }; 969 970 #define ixl_rd(_s, _r) \ 971 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 972 #define ixl_wr(_s, _r, _v) \ 973 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 974 #define ixl_barrier(_s, _r, _l, _o) \ 975 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 976 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 977 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 978 979 static inline uint32_t 980 ixl_dmamem_hi(struct ixl_dmamem *ixm) 981 { 982 uint32_t retval; 983 uint64_t val; 984 985 if (sizeof(IXL_DMA_DVA(ixm)) > 4) { 986 val = (intptr_t)IXL_DMA_DVA(ixm); 987 retval = (uint32_t)(val >> 32); 988 } else { 989 retval = 0; 990 } 991 992 return retval; 993 } 994 995 static inline uint32_t 996 ixl_dmamem_lo(struct ixl_dmamem *ixm) 997 { 998 999 return (uint32_t)IXL_DMA_DVA(ixm); 1000 } 1001 1002 static inline void 1003 ixl_aq_dva(struct ixl_aq_desc *iaq, bus_addr_t addr) 1004 { 1005 uint64_t val; 1006 1007 if (sizeof(addr) > 4) { 1008 val = (intptr_t)addr; 1009 iaq->iaq_param[2] = htole32(val >> 32); 1010 } else { 1011 iaq->iaq_param[2] = htole32(0); 1012 } 1013 1014 iaq->iaq_param[3] = htole32(addr); 1015 } 1016 1017 static inline unsigned int 1018 ixl_rxr_unrefreshed(unsigned int prod, unsigned int cons, unsigned int ndescs) 1019 { 1020 unsigned int num; 1021 1022 if (prod < cons) 1023 num = cons - prod; 1024 else 1025 num = (ndescs - prod) + cons; 1026 1027 if (__predict_true(num > 0)) { 1028 /* device cannot receive packets if all descripter is filled */ 1029 num -= 1; 1030 } 1031 1032 return num; 1033 } 1034 1035 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 1036 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 1037 DVF_DETACH_SHUTDOWN); 1038 1039 static const struct ixl_product ixl_products[] = { 1040 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 1041 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 1042 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 1043 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 1044 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 1045 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 1046 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T }, 1047 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 1048 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 1049 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 1050 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 1051 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 1052 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 1053 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 1054 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 1055 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 1056 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 1057 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 1058 /* required last entry */ 1059 {0, 0} 1060 }; 1061 1062 static const struct ixl_product * 1063 ixl_lookup(const struct pci_attach_args *pa) 1064 { 1065 const struct ixl_product *ixlp; 1066 1067 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 1068 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 1069 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 1070 return ixlp; 1071 } 1072 1073 return NULL; 1074 } 1075 1076 static int 1077 ixl_match(device_t parent, cfdata_t match, void *aux) 1078 { 1079 const struct pci_attach_args *pa = aux; 1080 1081 return (ixl_lookup(pa) != NULL) ? 1 : 0; 1082 } 1083 1084 static void 1085 ixl_attach(device_t parent, device_t self, void *aux) 1086 { 1087 struct ixl_softc *sc; 1088 struct pci_attach_args *pa = aux; 1089 struct ifnet *ifp; 1090 pcireg_t memtype; 1091 uint32_t firstq, port, ari, func; 1092 char xnamebuf[32]; 1093 int tries, rv, link; 1094 1095 sc = device_private(self); 1096 sc->sc_dev = self; 1097 ifp = &sc->sc_ec.ec_if; 1098 1099 sc->sc_pa = *pa; 1100 sc->sc_dmat = (pci_dma64_available(pa)) ? 1101 pa->pa_dmat64 : pa->pa_dmat; 1102 sc->sc_aq_regs = &ixl_pf_aq_regs; 1103 1104 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(pa->pa_id)); 1105 1106 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 1107 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 1108 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 1109 aprint_error(": unable to map registers\n"); 1110 return; 1111 } 1112 1113 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 1114 1115 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 1116 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 1117 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 1118 sc->sc_base_queue = firstq; 1119 1120 ixl_clear_hw(sc); 1121 if (ixl_pf_reset(sc) == -1) { 1122 /* error printed by ixl pf_reset */ 1123 goto unmap; 1124 } 1125 1126 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 1127 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 1128 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 1129 sc->sc_port = port; 1130 aprint_normal(": port %u", sc->sc_port); 1131 1132 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 1133 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 1134 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 1135 1136 func = ixl_rd(sc, I40E_PF_FUNC_RID); 1137 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 1138 1139 /* initialise the adminq */ 1140 1141 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 1142 1143 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 1144 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1145 aprint_error("\n" "%s: unable to allocate atq\n", 1146 device_xname(self)); 1147 goto unmap; 1148 } 1149 1150 SIMPLEQ_INIT(&sc->sc_arq_idle); 1151 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 1152 sc->sc_arq_cons = 0; 1153 sc->sc_arq_prod = 0; 1154 1155 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 1156 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1157 aprint_error("\n" "%s: unable to allocate arq\n", 1158 device_xname(self)); 1159 goto free_atq; 1160 } 1161 1162 if (!ixl_arq_fill(sc)) { 1163 aprint_error("\n" "%s: unable to fill arq descriptors\n", 1164 device_xname(self)); 1165 goto free_arq; 1166 } 1167 1168 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1169 0, IXL_DMA_LEN(&sc->sc_atq), 1170 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1171 1172 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1173 0, IXL_DMA_LEN(&sc->sc_arq), 1174 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1175 1176 for (tries = 0; tries < 10; tries++) { 1177 sc->sc_atq_cons = 0; 1178 sc->sc_atq_prod = 0; 1179 1180 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1181 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1182 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1183 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1184 1185 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1186 1187 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1188 ixl_dmamem_lo(&sc->sc_atq)); 1189 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1190 ixl_dmamem_hi(&sc->sc_atq)); 1191 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1192 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1193 1194 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1195 ixl_dmamem_lo(&sc->sc_arq)); 1196 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1197 ixl_dmamem_hi(&sc->sc_arq)); 1198 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1199 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1200 1201 rv = ixl_get_version(sc); 1202 if (rv == 0) 1203 break; 1204 if (rv != ETIMEDOUT) { 1205 aprint_error(", unable to get firmware version\n"); 1206 goto shutdown; 1207 } 1208 1209 delaymsec(100); 1210 } 1211 1212 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1213 1214 if (ixl_dmamem_alloc(sc, &sc->sc_aqbuf, IXL_AQ_BUFLEN, 0) != 0) { 1215 aprint_error_dev(self, ", unable to allocate nvm buffer\n"); 1216 goto shutdown; 1217 } 1218 1219 ixl_get_nvm_version(sc); 1220 1221 if (sc->sc_mac_type == I40E_MAC_X722) 1222 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_X722; 1223 else 1224 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_XL710; 1225 1226 rv = ixl_get_hw_capabilities(sc); 1227 if (rv != 0) { 1228 aprint_error(", GET HW CAPABILITIES %s\n", 1229 rv == ETIMEDOUT ? "timeout" : "error"); 1230 goto free_aqbuf; 1231 } 1232 1233 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1234 if (ixl_param_nqps_limit > 0) { 1235 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_max, 1236 ixl_param_nqps_limit); 1237 } 1238 1239 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1240 sc->sc_tx_ring_ndescs = ixl_param_tx_ndescs; 1241 sc->sc_rx_ring_ndescs = ixl_param_rx_ndescs; 1242 1243 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1244 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1245 1246 if (ixl_get_mac(sc) != 0) { 1247 /* error printed by ixl_get_mac */ 1248 goto free_aqbuf; 1249 } 1250 1251 aprint_normal("\n"); 1252 aprint_naive("\n"); 1253 1254 aprint_normal_dev(self, "Ethernet address %s\n", 1255 ether_sprintf(sc->sc_enaddr)); 1256 1257 rv = ixl_pxe_clear(sc); 1258 if (rv != 0) { 1259 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1260 rv == ETIMEDOUT ? "timeout" : "error"); 1261 } 1262 1263 ixl_set_filter_control(sc); 1264 1265 if (ixl_hmc(sc) != 0) { 1266 /* error printed by ixl_hmc */ 1267 goto free_aqbuf; 1268 } 1269 1270 if (ixl_lldp_shut(sc) != 0) { 1271 /* error printed by ixl_lldp_shut */ 1272 goto free_hmc; 1273 } 1274 1275 if (ixl_phy_mask_ints(sc) != 0) { 1276 /* error printed by ixl_phy_mask_ints */ 1277 goto free_hmc; 1278 } 1279 1280 if (ixl_restart_an(sc) != 0) { 1281 /* error printed by ixl_restart_an */ 1282 goto free_hmc; 1283 } 1284 1285 if (ixl_get_switch_config(sc) != 0) { 1286 /* error printed by ixl_get_switch_config */ 1287 goto free_hmc; 1288 } 1289 1290 rv = ixl_get_link_status_poll(sc, NULL); 1291 if (rv != 0) { 1292 aprint_error_dev(self, "GET LINK STATUS %s\n", 1293 rv == ETIMEDOUT ? "timeout" : "error"); 1294 goto free_hmc; 1295 } 1296 1297 /* 1298 * The FW often returns EIO in "Get PHY Abilities" command 1299 * if there is no delay 1300 */ 1301 DELAY(500); 1302 if (ixl_get_phy_info(sc) != 0) { 1303 /* error printed by ixl_get_phy_info */ 1304 goto free_hmc; 1305 } 1306 1307 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1308 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1309 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1310 goto free_hmc; 1311 } 1312 1313 rv = ixl_get_vsi(sc); 1314 if (rv != 0) { 1315 aprint_error_dev(self, "GET VSI %s %d\n", 1316 rv == ETIMEDOUT ? "timeout" : "error", rv); 1317 goto free_scratch; 1318 } 1319 1320 rv = ixl_set_vsi(sc); 1321 if (rv != 0) { 1322 aprint_error_dev(self, "UPDATE VSI error %s %d\n", 1323 rv == ETIMEDOUT ? "timeout" : "error", rv); 1324 goto free_scratch; 1325 } 1326 1327 if (ixl_queue_pairs_alloc(sc) != 0) { 1328 /* error printed by ixl_queue_pairs_alloc */ 1329 goto free_scratch; 1330 } 1331 1332 if (ixl_setup_interrupts(sc) != 0) { 1333 /* error printed by ixl_setup_interrupts */ 1334 goto free_queue_pairs; 1335 } 1336 1337 if (ixl_setup_stats(sc) != 0) { 1338 aprint_error_dev(self, "failed to setup event counters\n"); 1339 goto teardown_intrs; 1340 } 1341 1342 if (ixl_setup_sysctls(sc) != 0) { 1343 /* error printed by ixl_setup_sysctls */ 1344 goto teardown_stats; 1345 } 1346 1347 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1348 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1349 IPL_NET, WQ_MPSAFE); 1350 if (sc->sc_workq == NULL) 1351 goto teardown_sysctls; 1352 1353 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1354 rv = workqueue_create(&sc->sc_workq_txrx, xnamebuf, ixl_handle_queue_wk, 1355 sc, IXL_WORKQUEUE_PRI, IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1356 if (rv != 0) { 1357 sc->sc_workq_txrx = NULL; 1358 goto teardown_wqs; 1359 } 1360 1361 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1362 cv_init(&sc->sc_atq_cv, xnamebuf); 1363 1364 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1365 1366 ifp->if_softc = sc; 1367 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1368 ifp->if_extflags = IFEF_MPSAFE; 1369 ifp->if_ioctl = ixl_ioctl; 1370 ifp->if_start = ixl_start; 1371 ifp->if_transmit = ixl_transmit; 1372 ifp->if_watchdog = ixl_watchdog; 1373 ifp->if_init = ixl_init; 1374 ifp->if_stop = ixl_stop; 1375 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1376 IFQ_SET_READY(&ifp->if_snd); 1377 ifp->if_capabilities |= IXL_IFCAP_RXCSUM; 1378 ifp->if_capabilities |= IXL_IFCAP_TXCSUM; 1379 #if 0 1380 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1381 #endif 1382 ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb); 1383 sc->sc_ec.ec_capabilities |= ETHERCAP_JUMBO_MTU; 1384 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1385 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1386 1387 sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities; 1388 /* Disable VLAN_HWFILTER by default */ 1389 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1390 1391 sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable; 1392 1393 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1394 ifmedia_init(&sc->sc_media, IFM_IMASK, ixl_media_change, 1395 ixl_media_status); 1396 1397 ixl_media_add(sc); 1398 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1399 if (ISSET(sc->sc_phy_abilities, 1400 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1401 ifmedia_add(&sc->sc_media, 1402 IFM_ETHER | IFM_AUTO | IFM_FLOW, 0, NULL); 1403 } 1404 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_NONE, 0, NULL); 1405 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1406 1407 if_attach(ifp); 1408 if_deferred_start_init(ifp, NULL); 1409 ether_ifattach(ifp, sc->sc_enaddr); 1410 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1411 1412 rv = ixl_get_link_status_poll(sc, &link); 1413 if (rv != 0) 1414 link = LINK_STATE_UNKNOWN; 1415 if_link_state_change(ifp, link); 1416 1417 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 1418 ixl_work_set(&sc->sc_link_state_task, ixl_get_link_status, sc); 1419 1420 ixl_config_other_intr(sc); 1421 ixl_enable_other_intr(sc); 1422 1423 ixl_set_phy_autoselect(sc); 1424 1425 /* remove default mac filter and replace it so we can see vlans */ 1426 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 1427 if (rv != ENOENT) { 1428 aprint_debug_dev(self, 1429 "unable to remove macvlan %u\n", rv); 1430 } 1431 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1432 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1433 if (rv != ENOENT) { 1434 aprint_debug_dev(self, 1435 "unable to remove macvlan, ignore vlan %u\n", rv); 1436 } 1437 1438 if (ixl_update_macvlan(sc) != 0) { 1439 aprint_debug_dev(self, 1440 "couldn't enable vlan hardware filter\n"); 1441 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1442 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1443 } 1444 1445 sc->sc_txrx_workqueue = true; 1446 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1447 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1448 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1449 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1450 1451 ixl_stats_update(sc); 1452 sc->sc_stats_counters.isc_has_offset = true; 1453 1454 if (pmf_device_register(self, NULL, NULL) != true) 1455 aprint_debug_dev(self, "couldn't establish power handler\n"); 1456 sc->sc_attached = true; 1457 return; 1458 1459 teardown_wqs: 1460 config_finalize_register(self, ixl_workqs_teardown); 1461 teardown_sysctls: 1462 ixl_teardown_sysctls(sc); 1463 teardown_stats: 1464 ixl_teardown_stats(sc); 1465 teardown_intrs: 1466 ixl_teardown_interrupts(sc); 1467 free_queue_pairs: 1468 ixl_queue_pairs_free(sc); 1469 free_scratch: 1470 ixl_dmamem_free(sc, &sc->sc_scratch); 1471 free_hmc: 1472 ixl_hmc_free(sc); 1473 free_aqbuf: 1474 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1475 shutdown: 1476 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1477 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1478 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1479 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1480 1481 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1482 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1483 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1484 1485 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1486 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1487 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1488 1489 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1490 0, IXL_DMA_LEN(&sc->sc_arq), 1491 BUS_DMASYNC_POSTREAD); 1492 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1493 0, IXL_DMA_LEN(&sc->sc_atq), 1494 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1495 1496 ixl_arq_unfill(sc); 1497 free_arq: 1498 ixl_dmamem_free(sc, &sc->sc_arq); 1499 free_atq: 1500 ixl_dmamem_free(sc, &sc->sc_atq); 1501 unmap: 1502 mutex_destroy(&sc->sc_atq_lock); 1503 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1504 mutex_destroy(&sc->sc_cfg_lock); 1505 sc->sc_mems = 0; 1506 1507 sc->sc_attached = false; 1508 } 1509 1510 static int 1511 ixl_detach(device_t self, int flags) 1512 { 1513 struct ixl_softc *sc = device_private(self); 1514 struct ifnet *ifp = &sc->sc_ec.ec_if; 1515 1516 if (!sc->sc_attached) 1517 return 0; 1518 1519 ixl_stop(ifp, 1); 1520 1521 ixl_disable_other_intr(sc); 1522 1523 callout_halt(&sc->sc_stats_callout, NULL); 1524 ixl_work_wait(sc->sc_workq, &sc->sc_stats_task); 1525 1526 /* wait for ATQ handler */ 1527 mutex_enter(&sc->sc_atq_lock); 1528 mutex_exit(&sc->sc_atq_lock); 1529 1530 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1531 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1532 1533 if (sc->sc_workq != NULL) { 1534 ixl_workq_destroy(sc->sc_workq); 1535 sc->sc_workq = NULL; 1536 } 1537 1538 if (sc->sc_workq_txrx != NULL) { 1539 workqueue_destroy(sc->sc_workq_txrx); 1540 sc->sc_workq_txrx = NULL; 1541 } 1542 1543 ether_ifdetach(ifp); 1544 if_detach(ifp); 1545 ifmedia_fini(&sc->sc_media); 1546 1547 ixl_teardown_interrupts(sc); 1548 ixl_teardown_stats(sc); 1549 ixl_teardown_sysctls(sc); 1550 1551 ixl_queue_pairs_free(sc); 1552 1553 ixl_dmamem_free(sc, &sc->sc_scratch); 1554 ixl_hmc_free(sc); 1555 1556 /* shutdown */ 1557 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1558 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1559 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1560 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1561 1562 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1563 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1564 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1565 1566 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1567 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1568 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1569 1570 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1571 0, IXL_DMA_LEN(&sc->sc_arq), 1572 BUS_DMASYNC_POSTREAD); 1573 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1574 0, IXL_DMA_LEN(&sc->sc_atq), 1575 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1576 1577 ixl_arq_unfill(sc); 1578 1579 ixl_dmamem_free(sc, &sc->sc_arq); 1580 ixl_dmamem_free(sc, &sc->sc_atq); 1581 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1582 1583 cv_destroy(&sc->sc_atq_cv); 1584 mutex_destroy(&sc->sc_atq_lock); 1585 1586 if (sc->sc_mems != 0) { 1587 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1588 sc->sc_mems = 0; 1589 } 1590 1591 mutex_destroy(&sc->sc_cfg_lock); 1592 1593 return 0; 1594 } 1595 1596 static int 1597 ixl_workqs_teardown(device_t self) 1598 { 1599 struct ixl_softc *sc = device_private(self); 1600 1601 if (sc->sc_workq != NULL) { 1602 ixl_workq_destroy(sc->sc_workq); 1603 sc->sc_workq = NULL; 1604 } 1605 1606 if (sc->sc_workq_txrx != NULL) { 1607 workqueue_destroy(sc->sc_workq_txrx); 1608 sc->sc_workq_txrx = NULL; 1609 } 1610 1611 return 0; 1612 } 1613 1614 static int 1615 ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 1616 { 1617 struct ifnet *ifp = &ec->ec_if; 1618 struct ixl_softc *sc = ifp->if_softc; 1619 int rv; 1620 1621 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1622 return 0; 1623 } 1624 1625 if (set) { 1626 rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid, 1627 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1628 if (rv == 0) { 1629 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 1630 vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1631 } 1632 } else { 1633 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid, 1634 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1635 (void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid, 1636 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1637 } 1638 1639 return rv; 1640 } 1641 1642 static void 1643 ixl_media_add(struct ixl_softc *sc) 1644 { 1645 struct ifmedia *ifm = &sc->sc_media; 1646 const struct ixl_phy_type *itype; 1647 unsigned int i; 1648 bool flow; 1649 1650 if (ISSET(sc->sc_phy_abilities, 1651 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1652 flow = true; 1653 } else { 1654 flow = false; 1655 } 1656 1657 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1658 itype = &ixl_phy_type_map[i]; 1659 1660 if (ISSET(sc->sc_phy_types, itype->phy_type)) { 1661 ifmedia_add(ifm, 1662 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1663 1664 if (flow) { 1665 ifmedia_add(ifm, 1666 IFM_ETHER | IFM_FDX | IFM_FLOW | 1667 itype->ifm_type, 0, NULL); 1668 } 1669 1670 if (itype->ifm_type != IFM_100_TX) 1671 continue; 1672 1673 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1674 0, NULL); 1675 if (flow) { 1676 ifmedia_add(ifm, 1677 IFM_ETHER | IFM_FLOW | itype->ifm_type, 1678 0, NULL); 1679 } 1680 } 1681 } 1682 } 1683 1684 static void 1685 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1686 { 1687 struct ixl_softc *sc = ifp->if_softc; 1688 1689 ifmr->ifm_status = sc->sc_media_status; 1690 ifmr->ifm_active = sc->sc_media_active; 1691 1692 mutex_enter(&sc->sc_cfg_lock); 1693 if (ifp->if_link_state == LINK_STATE_UP) 1694 SET(ifmr->ifm_status, IFM_ACTIVE); 1695 mutex_exit(&sc->sc_cfg_lock); 1696 } 1697 1698 static int 1699 ixl_media_change(struct ifnet *ifp) 1700 { 1701 struct ixl_softc *sc = ifp->if_softc; 1702 struct ifmedia *ifm = &sc->sc_media; 1703 uint64_t ifm_active = sc->sc_media_active; 1704 uint8_t link_speed, abilities; 1705 1706 switch (IFM_SUBTYPE(ifm_active)) { 1707 case IFM_1000_SGMII: 1708 case IFM_1000_KX: 1709 case IFM_10G_KX4: 1710 case IFM_10G_KR: 1711 case IFM_40G_KR4: 1712 case IFM_20G_KR2: 1713 case IFM_25G_KR: 1714 /* backplanes */ 1715 return EINVAL; 1716 } 1717 1718 abilities = IXL_PHY_ABILITY_AUTONEGO | IXL_PHY_ABILITY_LINKUP; 1719 1720 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1721 case IFM_AUTO: 1722 link_speed = sc->sc_phy_linkspeed; 1723 break; 1724 case IFM_NONE: 1725 link_speed = 0; 1726 CLR(abilities, IXL_PHY_ABILITY_LINKUP); 1727 break; 1728 default: 1729 link_speed = ixl_search_baudrate( 1730 ifmedia_baudrate(ifm->ifm_media)); 1731 } 1732 1733 if (ISSET(abilities, IXL_PHY_ABILITY_LINKUP)) { 1734 if (ISSET(link_speed, sc->sc_phy_linkspeed) == 0) 1735 return EINVAL; 1736 } 1737 1738 if (ifm->ifm_media & IFM_FLOW) { 1739 abilities |= sc->sc_phy_abilities & 1740 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX); 1741 } 1742 1743 return ixl_set_phy_config(sc, link_speed, abilities, false); 1744 } 1745 1746 static void 1747 ixl_watchdog(struct ifnet *ifp) 1748 { 1749 1750 } 1751 1752 static void 1753 ixl_del_all_multiaddr(struct ixl_softc *sc) 1754 { 1755 struct ethercom *ec = &sc->sc_ec; 1756 struct ether_multi *enm; 1757 struct ether_multistep step; 1758 1759 ETHER_LOCK(ec); 1760 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1761 ETHER_NEXT_MULTI(step, enm)) { 1762 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1763 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1764 } 1765 ETHER_UNLOCK(ec); 1766 } 1767 1768 static int 1769 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1770 { 1771 struct ifnet *ifp = &sc->sc_ec.ec_if; 1772 int rv; 1773 1774 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1775 return 0; 1776 1777 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1778 ixl_del_all_multiaddr(sc); 1779 SET(ifp->if_flags, IFF_ALLMULTI); 1780 return ENETRESET; 1781 } 1782 1783 /* multicast address can not use VLAN HWFILTER */ 1784 rv = ixl_add_macvlan(sc, addrlo, 0, 1785 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1786 1787 if (rv == ENOSPC) { 1788 ixl_del_all_multiaddr(sc); 1789 SET(ifp->if_flags, IFF_ALLMULTI); 1790 return ENETRESET; 1791 } 1792 1793 return rv; 1794 } 1795 1796 static int 1797 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1798 { 1799 struct ifnet *ifp = &sc->sc_ec.ec_if; 1800 struct ethercom *ec = &sc->sc_ec; 1801 struct ether_multi *enm, *enm_last; 1802 struct ether_multistep step; 1803 int error, rv = 0; 1804 1805 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1806 ixl_remove_macvlan(sc, addrlo, 0, 1807 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1808 return 0; 1809 } 1810 1811 ETHER_LOCK(ec); 1812 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1813 ETHER_NEXT_MULTI(step, enm)) { 1814 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1815 ETHER_ADDR_LEN) != 0) { 1816 goto out; 1817 } 1818 } 1819 1820 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1821 ETHER_NEXT_MULTI(step, enm)) { 1822 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1823 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1824 if (error != 0) 1825 break; 1826 } 1827 1828 if (enm != NULL) { 1829 enm_last = enm; 1830 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1831 ETHER_NEXT_MULTI(step, enm)) { 1832 if (enm == enm_last) 1833 break; 1834 1835 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1836 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1837 } 1838 } else { 1839 CLR(ifp->if_flags, IFF_ALLMULTI); 1840 rv = ENETRESET; 1841 } 1842 1843 out: 1844 ETHER_UNLOCK(ec); 1845 return rv; 1846 } 1847 1848 static int 1849 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1850 { 1851 struct ifreq *ifr = (struct ifreq *)data; 1852 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1853 const struct sockaddr *sa; 1854 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1855 int s, error = 0; 1856 unsigned int nmtu; 1857 1858 switch (cmd) { 1859 case SIOCSIFMTU: 1860 nmtu = ifr->ifr_mtu; 1861 1862 if (nmtu < IXL_MIN_MTU || nmtu > IXL_MAX_MTU) { 1863 error = EINVAL; 1864 break; 1865 } 1866 if (ifp->if_mtu != nmtu) { 1867 s = splnet(); 1868 error = ether_ioctl(ifp, cmd, data); 1869 splx(s); 1870 if (error == ENETRESET) 1871 error = ixl_init(ifp); 1872 } 1873 break; 1874 case SIOCADDMULTI: 1875 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1876 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1877 error = ether_multiaddr(sa, addrlo, addrhi); 1878 if (error != 0) 1879 return error; 1880 1881 error = ixl_add_multi(sc, addrlo, addrhi); 1882 if (error != 0 && error != ENETRESET) { 1883 ether_delmulti(sa, &sc->sc_ec); 1884 error = EIO; 1885 } 1886 } 1887 break; 1888 1889 case SIOCDELMULTI: 1890 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1891 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1892 error = ether_multiaddr(sa, addrlo, addrhi); 1893 if (error != 0) 1894 return error; 1895 1896 error = ixl_del_multi(sc, addrlo, addrhi); 1897 } 1898 break; 1899 1900 default: 1901 s = splnet(); 1902 error = ether_ioctl(ifp, cmd, data); 1903 splx(s); 1904 } 1905 1906 if (error == ENETRESET) 1907 error = ixl_iff(sc); 1908 1909 return error; 1910 } 1911 1912 static enum i40e_mac_type 1913 ixl_mactype(pci_product_id_t id) 1914 { 1915 1916 switch (id) { 1917 case PCI_PRODUCT_INTEL_XL710_SFP: 1918 case PCI_PRODUCT_INTEL_XL710_KX_B: 1919 case PCI_PRODUCT_INTEL_XL710_KX_C: 1920 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1921 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1922 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1923 case PCI_PRODUCT_INTEL_X710_10G_T: 1924 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1925 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1926 case PCI_PRODUCT_INTEL_X710_T4_10G: 1927 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1928 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1929 return I40E_MAC_XL710; 1930 1931 case PCI_PRODUCT_INTEL_X722_KX: 1932 case PCI_PRODUCT_INTEL_X722_QSFP: 1933 case PCI_PRODUCT_INTEL_X722_SFP: 1934 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1935 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1936 case PCI_PRODUCT_INTEL_X722_I_SFP: 1937 return I40E_MAC_X722; 1938 } 1939 1940 return I40E_MAC_GENERIC; 1941 } 1942 1943 static inline void * 1944 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1945 { 1946 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1947 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1948 1949 if (i >= e->hmc_count) 1950 return NULL; 1951 1952 kva += e->hmc_base; 1953 kva += i * e->hmc_size; 1954 1955 return kva; 1956 } 1957 1958 static inline size_t 1959 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1960 { 1961 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1962 1963 return e->hmc_size; 1964 } 1965 1966 static void 1967 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1968 { 1969 struct ixl_rx_ring *rxr = qp->qp_rxr; 1970 1971 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1972 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1973 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1974 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1975 ixl_flush(sc); 1976 } 1977 1978 static void 1979 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1980 { 1981 struct ixl_rx_ring *rxr = qp->qp_rxr; 1982 1983 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1984 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1985 ixl_flush(sc); 1986 } 1987 1988 static void 1989 ixl_enable_other_intr(struct ixl_softc *sc) 1990 { 1991 1992 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1993 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1994 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1995 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1996 ixl_flush(sc); 1997 } 1998 1999 static void 2000 ixl_disable_other_intr(struct ixl_softc *sc) 2001 { 2002 2003 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 2004 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 2005 ixl_flush(sc); 2006 } 2007 2008 static int 2009 ixl_reinit(struct ixl_softc *sc) 2010 { 2011 struct ixl_rx_ring *rxr; 2012 struct ixl_tx_ring *txr; 2013 unsigned int i; 2014 uint32_t reg; 2015 2016 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2017 2018 if (ixl_get_vsi(sc) != 0) 2019 return EIO; 2020 2021 if (ixl_set_vsi(sc) != 0) 2022 return EIO; 2023 2024 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2025 txr = sc->sc_qps[i].qp_txr; 2026 rxr = sc->sc_qps[i].qp_rxr; 2027 2028 ixl_txr_config(sc, txr); 2029 ixl_rxr_config(sc, rxr); 2030 } 2031 2032 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2033 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 2034 2035 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2036 txr = sc->sc_qps[i].qp_txr; 2037 rxr = sc->sc_qps[i].qp_rxr; 2038 2039 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 2040 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 2041 ixl_flush(sc); 2042 2043 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 2044 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 2045 2046 /* ixl_rxfill() needs lock held */ 2047 mutex_enter(&rxr->rxr_lock); 2048 ixl_rxfill(sc, rxr); 2049 mutex_exit(&rxr->rxr_lock); 2050 2051 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2052 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2053 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2054 if (ixl_rxr_enabled(sc, rxr) != 0) 2055 goto stop; 2056 2057 ixl_txr_qdis(sc, txr, 1); 2058 2059 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2060 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2061 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2062 2063 if (ixl_txr_enabled(sc, txr) != 0) 2064 goto stop; 2065 } 2066 2067 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2068 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2069 2070 return 0; 2071 2072 stop: 2073 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2074 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2075 2076 return ETIMEDOUT; 2077 } 2078 2079 static int 2080 ixl_init_locked(struct ixl_softc *sc) 2081 { 2082 struct ifnet *ifp = &sc->sc_ec.ec_if; 2083 unsigned int i; 2084 int error, eccap_change; 2085 2086 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2087 2088 if (ISSET(ifp->if_flags, IFF_RUNNING)) 2089 ixl_stop_locked(sc); 2090 2091 if (sc->sc_dead) { 2092 return ENXIO; 2093 } 2094 2095 eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable; 2096 if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING)) 2097 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 2098 2099 if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) { 2100 if (ixl_update_macvlan(sc) == 0) { 2101 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 2102 } else { 2103 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 2104 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 2105 } 2106 } 2107 2108 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 2109 sc->sc_nqueue_pairs = 1; 2110 else 2111 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 2112 2113 error = ixl_reinit(sc); 2114 if (error) { 2115 ixl_stop_locked(sc); 2116 return error; 2117 } 2118 2119 SET(ifp->if_flags, IFF_RUNNING); 2120 CLR(ifp->if_flags, IFF_OACTIVE); 2121 2122 (void)ixl_get_link_status(sc); 2123 2124 ixl_config_rss(sc); 2125 ixl_config_queue_intr(sc); 2126 2127 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2128 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 2129 } 2130 2131 error = ixl_iff(sc); 2132 if (error) { 2133 ixl_stop_locked(sc); 2134 return error; 2135 } 2136 2137 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 2138 2139 return 0; 2140 } 2141 2142 static int 2143 ixl_init(struct ifnet *ifp) 2144 { 2145 struct ixl_softc *sc = ifp->if_softc; 2146 int error; 2147 2148 mutex_enter(&sc->sc_cfg_lock); 2149 error = ixl_init_locked(sc); 2150 mutex_exit(&sc->sc_cfg_lock); 2151 2152 return error; 2153 } 2154 2155 static int 2156 ixl_iff(struct ixl_softc *sc) 2157 { 2158 struct ifnet *ifp = &sc->sc_ec.ec_if; 2159 struct ixl_atq iatq; 2160 struct ixl_aq_desc *iaq; 2161 struct ixl_aq_vsi_promisc_param *param; 2162 uint16_t flag_add, flag_del; 2163 int error; 2164 2165 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2166 return 0; 2167 2168 memset(&iatq, 0, sizeof(iatq)); 2169 2170 iaq = &iatq.iatq_desc; 2171 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 2172 2173 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 2174 param->flags = htole16(0); 2175 2176 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER) 2177 || ISSET(ifp->if_flags, IFF_PROMISC)) { 2178 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2179 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2180 } 2181 2182 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 2183 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2184 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2185 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 2186 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2187 } 2188 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2189 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2190 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2191 param->seid = sc->sc_seid; 2192 2193 error = ixl_atq_exec(sc, &iatq); 2194 if (error) 2195 return error; 2196 2197 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 2198 return EIO; 2199 2200 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 2201 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 2202 flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH; 2203 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH; 2204 } else { 2205 flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN; 2206 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN; 2207 } 2208 2209 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del); 2210 2211 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2212 ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add); 2213 } 2214 return 0; 2215 } 2216 2217 static void 2218 ixl_stop_rendezvous(struct ixl_softc *sc) 2219 { 2220 struct ixl_tx_ring *txr; 2221 struct ixl_rx_ring *rxr; 2222 unsigned int i; 2223 2224 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2225 txr = sc->sc_qps[i].qp_txr; 2226 rxr = sc->sc_qps[i].qp_rxr; 2227 2228 mutex_enter(&txr->txr_lock); 2229 mutex_exit(&txr->txr_lock); 2230 2231 mutex_enter(&rxr->rxr_lock); 2232 mutex_exit(&rxr->rxr_lock); 2233 2234 sc->sc_qps[i].qp_workqueue = false; 2235 workqueue_wait(sc->sc_workq_txrx, 2236 &sc->sc_qps[i].qp_work); 2237 } 2238 } 2239 2240 static void 2241 ixl_stop_locked(struct ixl_softc *sc) 2242 { 2243 struct ifnet *ifp = &sc->sc_ec.ec_if; 2244 struct ixl_rx_ring *rxr; 2245 struct ixl_tx_ring *txr; 2246 unsigned int i; 2247 uint32_t reg; 2248 2249 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2250 2251 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 2252 callout_stop(&sc->sc_stats_callout); 2253 2254 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2255 txr = sc->sc_qps[i].qp_txr; 2256 rxr = sc->sc_qps[i].qp_rxr; 2257 2258 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 2259 2260 mutex_enter(&txr->txr_lock); 2261 ixl_txr_qdis(sc, txr, 0); 2262 mutex_exit(&txr->txr_lock); 2263 } 2264 2265 /* XXX wait at least 400 usec for all tx queues in one go */ 2266 ixl_flush(sc); 2267 DELAY(500); 2268 2269 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2270 txr = sc->sc_qps[i].qp_txr; 2271 rxr = sc->sc_qps[i].qp_rxr; 2272 2273 mutex_enter(&txr->txr_lock); 2274 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2275 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2276 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2277 mutex_exit(&txr->txr_lock); 2278 2279 mutex_enter(&rxr->rxr_lock); 2280 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2281 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2282 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2283 mutex_exit(&rxr->rxr_lock); 2284 } 2285 2286 /* XXX short wait for all queue disables to settle */ 2287 ixl_flush(sc); 2288 DELAY(50); 2289 2290 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2291 txr = sc->sc_qps[i].qp_txr; 2292 rxr = sc->sc_qps[i].qp_rxr; 2293 2294 mutex_enter(&txr->txr_lock); 2295 if (ixl_txr_disabled(sc, txr) != 0) { 2296 mutex_exit(&txr->txr_lock); 2297 goto die; 2298 } 2299 mutex_exit(&txr->txr_lock); 2300 2301 mutex_enter(&rxr->rxr_lock); 2302 if (ixl_rxr_disabled(sc, rxr) != 0) { 2303 mutex_exit(&rxr->rxr_lock); 2304 goto die; 2305 } 2306 mutex_exit(&rxr->rxr_lock); 2307 } 2308 2309 ixl_stop_rendezvous(sc); 2310 2311 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2312 txr = sc->sc_qps[i].qp_txr; 2313 rxr = sc->sc_qps[i].qp_rxr; 2314 2315 mutex_enter(&txr->txr_lock); 2316 ixl_txr_unconfig(sc, txr); 2317 mutex_exit(&txr->txr_lock); 2318 2319 mutex_enter(&rxr->rxr_lock); 2320 ixl_rxr_unconfig(sc, rxr); 2321 mutex_exit(&rxr->rxr_lock); 2322 2323 ixl_txr_clean(sc, txr); 2324 ixl_rxr_clean(sc, rxr); 2325 } 2326 2327 return; 2328 die: 2329 sc->sc_dead = true; 2330 log(LOG_CRIT, "%s: failed to shut down rings", 2331 device_xname(sc->sc_dev)); 2332 return; 2333 } 2334 2335 static void 2336 ixl_stop(struct ifnet *ifp, int disable) 2337 { 2338 struct ixl_softc *sc = ifp->if_softc; 2339 2340 mutex_enter(&sc->sc_cfg_lock); 2341 ixl_stop_locked(sc); 2342 mutex_exit(&sc->sc_cfg_lock); 2343 } 2344 2345 static int 2346 ixl_queue_pairs_alloc(struct ixl_softc *sc) 2347 { 2348 struct ixl_queue_pair *qp; 2349 unsigned int i; 2350 size_t sz; 2351 2352 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2353 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 2354 2355 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2356 qp = &sc->sc_qps[i]; 2357 2358 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2359 ixl_handle_queue, qp); 2360 if (qp->qp_si == NULL) 2361 goto free; 2362 2363 qp->qp_txr = ixl_txr_alloc(sc, i); 2364 if (qp->qp_txr == NULL) 2365 goto free; 2366 2367 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2368 if (qp->qp_rxr == NULL) 2369 goto free; 2370 2371 qp->qp_sc = sc; 2372 snprintf(qp->qp_name, sizeof(qp->qp_name), 2373 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2374 } 2375 2376 return 0; 2377 free: 2378 if (sc->sc_qps != NULL) { 2379 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2380 qp = &sc->sc_qps[i]; 2381 2382 if (qp->qp_txr != NULL) 2383 ixl_txr_free(sc, qp->qp_txr); 2384 if (qp->qp_rxr != NULL) 2385 ixl_rxr_free(sc, qp->qp_rxr); 2386 if (qp->qp_si != NULL) 2387 softint_disestablish(qp->qp_si); 2388 } 2389 2390 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2391 kmem_free(sc->sc_qps, sz); 2392 sc->sc_qps = NULL; 2393 } 2394 2395 return -1; 2396 } 2397 2398 static void 2399 ixl_queue_pairs_free(struct ixl_softc *sc) 2400 { 2401 struct ixl_queue_pair *qp; 2402 unsigned int i; 2403 size_t sz; 2404 2405 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2406 qp = &sc->sc_qps[i]; 2407 ixl_txr_free(sc, qp->qp_txr); 2408 ixl_rxr_free(sc, qp->qp_rxr); 2409 softint_disestablish(qp->qp_si); 2410 } 2411 2412 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2413 kmem_free(sc->sc_qps, sz); 2414 sc->sc_qps = NULL; 2415 } 2416 2417 static struct ixl_tx_ring * 2418 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2419 { 2420 struct ixl_tx_ring *txr = NULL; 2421 struct ixl_tx_map *maps = NULL, *txm; 2422 unsigned int i; 2423 2424 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2425 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2426 KM_SLEEP); 2427 2428 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2429 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2430 IXL_TX_QUEUE_ALIGN) != 0) 2431 goto free; 2432 2433 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2434 txm = &maps[i]; 2435 2436 if (bus_dmamap_create(sc->sc_dmat, IXL_TX_PKT_MAXSIZE, 2437 IXL_TX_PKT_DESCS, IXL_TX_PKT_MAXSIZE, 0, 2438 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2439 goto uncreate; 2440 2441 txm->txm_eop = -1; 2442 txm->txm_m = NULL; 2443 } 2444 2445 txr->txr_cons = txr->txr_prod = 0; 2446 txr->txr_maps = maps; 2447 2448 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2449 if (txr->txr_intrq == NULL) 2450 goto uncreate; 2451 2452 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2453 ixl_deferred_transmit, txr); 2454 if (txr->txr_si == NULL) 2455 goto destroy_pcq; 2456 2457 txr->txr_tail = I40E_QTX_TAIL(qid); 2458 txr->txr_qid = qid; 2459 txr->txr_sc = sc; 2460 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2461 2462 return txr; 2463 2464 destroy_pcq: 2465 pcq_destroy(txr->txr_intrq); 2466 uncreate: 2467 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2468 txm = &maps[i]; 2469 2470 if (txm->txm_map == NULL) 2471 continue; 2472 2473 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2474 } 2475 2476 ixl_dmamem_free(sc, &txr->txr_mem); 2477 free: 2478 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2479 kmem_free(txr, sizeof(*txr)); 2480 2481 return NULL; 2482 } 2483 2484 static void 2485 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2486 { 2487 unsigned int qid; 2488 bus_size_t reg; 2489 uint32_t r; 2490 2491 qid = txr->txr_qid + sc->sc_base_queue; 2492 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2493 qid %= 128; 2494 2495 r = ixl_rd(sc, reg); 2496 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2497 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2498 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2499 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2500 ixl_wr(sc, reg, r); 2501 } 2502 2503 static void 2504 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2505 { 2506 struct ixl_hmc_txq txq; 2507 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2508 void *hmc; 2509 2510 memset(&txq, 0, sizeof(txq)); 2511 txq.head = htole16(txr->txr_cons); 2512 txq.new_context = 1; 2513 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2514 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2515 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2516 txq.tphrdesc_ena = 0; 2517 txq.tphrpacket_ena = 0; 2518 txq.tphwdesc_ena = 0; 2519 txq.rdylist = data->qs_handle[0]; 2520 2521 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2522 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2523 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2524 __arraycount(ixl_hmc_pack_txq)); 2525 } 2526 2527 static void 2528 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2529 { 2530 void *hmc; 2531 2532 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2533 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2534 txr->txr_cons = txr->txr_prod = 0; 2535 } 2536 2537 static void 2538 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2539 { 2540 struct ixl_tx_map *maps, *txm; 2541 bus_dmamap_t map; 2542 unsigned int i; 2543 2544 maps = txr->txr_maps; 2545 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2546 txm = &maps[i]; 2547 2548 if (txm->txm_m == NULL) 2549 continue; 2550 2551 map = txm->txm_map; 2552 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2553 BUS_DMASYNC_POSTWRITE); 2554 bus_dmamap_unload(sc->sc_dmat, map); 2555 2556 m_freem(txm->txm_m); 2557 txm->txm_m = NULL; 2558 } 2559 } 2560 2561 static int 2562 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2563 { 2564 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2565 uint32_t reg; 2566 int i; 2567 2568 for (i = 0; i < 10; i++) { 2569 reg = ixl_rd(sc, ena); 2570 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2571 return 0; 2572 2573 delaymsec(10); 2574 } 2575 2576 return ETIMEDOUT; 2577 } 2578 2579 static int 2580 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2581 { 2582 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2583 uint32_t reg; 2584 int i; 2585 2586 KASSERT(mutex_owned(&txr->txr_lock)); 2587 2588 for (i = 0; i < 10; i++) { 2589 reg = ixl_rd(sc, ena); 2590 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2591 return 0; 2592 2593 delaymsec(10); 2594 } 2595 2596 return ETIMEDOUT; 2597 } 2598 2599 static void 2600 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2601 { 2602 struct ixl_tx_map *maps, *txm; 2603 struct mbuf *m; 2604 unsigned int i; 2605 2606 softint_disestablish(txr->txr_si); 2607 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2608 m_freem(m); 2609 pcq_destroy(txr->txr_intrq); 2610 2611 maps = txr->txr_maps; 2612 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2613 txm = &maps[i]; 2614 2615 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2616 } 2617 2618 ixl_dmamem_free(sc, &txr->txr_mem); 2619 mutex_destroy(&txr->txr_lock); 2620 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2621 kmem_free(txr, sizeof(*txr)); 2622 } 2623 2624 static inline int 2625 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2626 struct ixl_tx_ring *txr) 2627 { 2628 struct mbuf *m; 2629 int error; 2630 2631 KASSERT(mutex_owned(&txr->txr_lock)); 2632 2633 m = *m0; 2634 2635 error = bus_dmamap_load_mbuf(dmat, map, m, 2636 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2637 if (error != EFBIG) 2638 return error; 2639 2640 m = m_defrag(m, M_DONTWAIT); 2641 if (m != NULL) { 2642 *m0 = m; 2643 txr->txr_defragged.ev_count++; 2644 2645 error = bus_dmamap_load_mbuf(dmat, map, m, 2646 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2647 } else { 2648 txr->txr_defrag_failed.ev_count++; 2649 error = ENOBUFS; 2650 } 2651 2652 return error; 2653 } 2654 2655 static inline int 2656 ixl_tx_setup_offloads(struct mbuf *m, uint64_t *cmd_txd) 2657 { 2658 struct ether_header *eh; 2659 size_t len; 2660 uint64_t cmd; 2661 2662 cmd = 0; 2663 2664 eh = mtod(m, struct ether_header *); 2665 switch (htons(eh->ether_type)) { 2666 case ETHERTYPE_IP: 2667 case ETHERTYPE_IPV6: 2668 len = ETHER_HDR_LEN; 2669 break; 2670 case ETHERTYPE_VLAN: 2671 len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2672 break; 2673 default: 2674 len = 0; 2675 } 2676 cmd |= ((len >> 1) << IXL_TX_DESC_MACLEN_SHIFT); 2677 2678 if (m->m_pkthdr.csum_flags & 2679 (M_CSUM_TSOv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) { 2680 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4; 2681 } 2682 if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) { 2683 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4_CSUM; 2684 } 2685 2686 if (m->m_pkthdr.csum_flags & 2687 (M_CSUM_TSOv6 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) { 2688 cmd |= IXL_TX_DESC_CMD_IIPT_IPV6; 2689 } 2690 2691 switch (cmd & IXL_TX_DESC_CMD_IIPT_MASK) { 2692 case IXL_TX_DESC_CMD_IIPT_IPV4: 2693 case IXL_TX_DESC_CMD_IIPT_IPV4_CSUM: 2694 len = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); 2695 break; 2696 case IXL_TX_DESC_CMD_IIPT_IPV6: 2697 len = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); 2698 break; 2699 default: 2700 len = 0; 2701 } 2702 cmd |= ((len >> 2) << IXL_TX_DESC_IPLEN_SHIFT); 2703 2704 if (m->m_pkthdr.csum_flags & 2705 (M_CSUM_TSOv4 | M_CSUM_TSOv6 | M_CSUM_TCPv4 | M_CSUM_TCPv6)) { 2706 len = sizeof(struct tcphdr); 2707 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_TCP; 2708 } else if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) { 2709 len = sizeof(struct udphdr); 2710 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_UDP; 2711 } else { 2712 len = 0; 2713 } 2714 cmd |= ((len >> 2) << IXL_TX_DESC_L4LEN_SHIFT); 2715 2716 *cmd_txd |= cmd; 2717 return 0; 2718 } 2719 2720 static void 2721 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2722 bool is_transmit) 2723 { 2724 struct ixl_softc *sc = ifp->if_softc; 2725 struct ixl_tx_desc *ring, *txd; 2726 struct ixl_tx_map *txm; 2727 bus_dmamap_t map; 2728 struct mbuf *m; 2729 uint64_t cmd, cmd_txd; 2730 unsigned int prod, free, last, i; 2731 unsigned int mask; 2732 int post = 0; 2733 2734 KASSERT(mutex_owned(&txr->txr_lock)); 2735 2736 if (ifp->if_link_state != LINK_STATE_UP 2737 || !ISSET(ifp->if_flags, IFF_RUNNING) 2738 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2739 if (!is_transmit) 2740 IFQ_PURGE(&ifp->if_snd); 2741 return; 2742 } 2743 2744 prod = txr->txr_prod; 2745 free = txr->txr_cons; 2746 if (free <= prod) 2747 free += sc->sc_tx_ring_ndescs; 2748 free -= prod; 2749 2750 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2751 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2752 2753 ring = IXL_DMA_KVA(&txr->txr_mem); 2754 mask = sc->sc_tx_ring_ndescs - 1; 2755 last = prod; 2756 cmd = 0; 2757 txd = NULL; 2758 2759 for (;;) { 2760 if (free <= IXL_TX_PKT_DESCS) { 2761 if (!is_transmit) 2762 SET(ifp->if_flags, IFF_OACTIVE); 2763 break; 2764 } 2765 2766 if (is_transmit) 2767 m = pcq_get(txr->txr_intrq); 2768 else 2769 IFQ_DEQUEUE(&ifp->if_snd, m); 2770 2771 if (m == NULL) 2772 break; 2773 2774 txm = &txr->txr_maps[prod]; 2775 map = txm->txm_map; 2776 2777 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2778 if_statinc(ifp, if_oerrors); 2779 m_freem(m); 2780 continue; 2781 } 2782 2783 cmd_txd = 0; 2784 if (m->m_pkthdr.csum_flags & IXL_CSUM_ALL_OFFLOAD) { 2785 ixl_tx_setup_offloads(m, &cmd_txd); 2786 } 2787 2788 if (vlan_has_tag(m)) { 2789 cmd_txd |= (uint64_t)vlan_get_tag(m) << 2790 IXL_TX_DESC_L2TAG1_SHIFT; 2791 cmd_txd |= IXL_TX_DESC_CMD_IL2TAG1; 2792 } 2793 2794 bus_dmamap_sync(sc->sc_dmat, map, 0, 2795 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2796 2797 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2798 txd = &ring[prod]; 2799 2800 cmd = (uint64_t)map->dm_segs[i].ds_len << 2801 IXL_TX_DESC_BSIZE_SHIFT; 2802 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2803 cmd |= cmd_txd; 2804 2805 txd->addr = htole64(map->dm_segs[i].ds_addr); 2806 txd->cmd = htole64(cmd); 2807 2808 last = prod; 2809 2810 prod++; 2811 prod &= mask; 2812 } 2813 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2814 txd->cmd = htole64(cmd); 2815 2816 txm->txm_m = m; 2817 txm->txm_eop = last; 2818 2819 bpf_mtap(ifp, m, BPF_D_OUT); 2820 2821 free -= i; 2822 post = 1; 2823 } 2824 2825 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2826 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2827 2828 if (post) { 2829 txr->txr_prod = prod; 2830 ixl_wr(sc, txr->txr_tail, prod); 2831 } 2832 } 2833 2834 static int 2835 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2836 { 2837 struct ifnet *ifp = &sc->sc_ec.ec_if; 2838 struct ixl_tx_desc *ring, *txd; 2839 struct ixl_tx_map *txm; 2840 struct mbuf *m; 2841 bus_dmamap_t map; 2842 unsigned int cons, prod, last; 2843 unsigned int mask; 2844 uint64_t dtype; 2845 int done = 0, more = 0; 2846 2847 KASSERT(mutex_owned(&txr->txr_lock)); 2848 2849 prod = txr->txr_prod; 2850 cons = txr->txr_cons; 2851 2852 if (cons == prod) 2853 return 0; 2854 2855 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2856 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2857 2858 ring = IXL_DMA_KVA(&txr->txr_mem); 2859 mask = sc->sc_tx_ring_ndescs - 1; 2860 2861 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2862 2863 do { 2864 if (txlimit-- <= 0) { 2865 more = 1; 2866 break; 2867 } 2868 2869 txm = &txr->txr_maps[cons]; 2870 last = txm->txm_eop; 2871 txd = &ring[last]; 2872 2873 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2874 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2875 break; 2876 2877 map = txm->txm_map; 2878 2879 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2880 BUS_DMASYNC_POSTWRITE); 2881 bus_dmamap_unload(sc->sc_dmat, map); 2882 2883 m = txm->txm_m; 2884 if (m != NULL) { 2885 if_statinc_ref(nsr, if_opackets); 2886 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 2887 if (ISSET(m->m_flags, M_MCAST)) 2888 if_statinc_ref(nsr, if_omcasts); 2889 m_freem(m); 2890 } 2891 2892 txm->txm_m = NULL; 2893 txm->txm_eop = -1; 2894 2895 cons = last + 1; 2896 cons &= mask; 2897 done = 1; 2898 } while (cons != prod); 2899 2900 IF_STAT_PUTREF(ifp); 2901 2902 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2903 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2904 2905 txr->txr_cons = cons; 2906 2907 if (done) { 2908 softint_schedule(txr->txr_si); 2909 if (txr->txr_qid == 0) { 2910 CLR(ifp->if_flags, IFF_OACTIVE); 2911 if_schedule_deferred_start(ifp); 2912 } 2913 } 2914 2915 return more; 2916 } 2917 2918 static void 2919 ixl_start(struct ifnet *ifp) 2920 { 2921 struct ixl_softc *sc; 2922 struct ixl_tx_ring *txr; 2923 2924 sc = ifp->if_softc; 2925 txr = sc->sc_qps[0].qp_txr; 2926 2927 mutex_enter(&txr->txr_lock); 2928 ixl_tx_common_locked(ifp, txr, false); 2929 mutex_exit(&txr->txr_lock); 2930 } 2931 2932 static inline unsigned int 2933 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2934 { 2935 u_int cpuid; 2936 2937 cpuid = cpu_index(curcpu()); 2938 2939 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2940 } 2941 2942 static int 2943 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2944 { 2945 struct ixl_softc *sc; 2946 struct ixl_tx_ring *txr; 2947 unsigned int qid; 2948 2949 sc = ifp->if_softc; 2950 qid = ixl_select_txqueue(sc, m); 2951 2952 txr = sc->sc_qps[qid].qp_txr; 2953 2954 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2955 mutex_enter(&txr->txr_lock); 2956 txr->txr_pcqdrop.ev_count++; 2957 mutex_exit(&txr->txr_lock); 2958 2959 m_freem(m); 2960 return ENOBUFS; 2961 } 2962 2963 if (mutex_tryenter(&txr->txr_lock)) { 2964 ixl_tx_common_locked(ifp, txr, true); 2965 mutex_exit(&txr->txr_lock); 2966 } else { 2967 kpreempt_disable(); 2968 softint_schedule(txr->txr_si); 2969 kpreempt_enable(); 2970 } 2971 2972 return 0; 2973 } 2974 2975 static void 2976 ixl_deferred_transmit(void *xtxr) 2977 { 2978 struct ixl_tx_ring *txr = xtxr; 2979 struct ixl_softc *sc = txr->txr_sc; 2980 struct ifnet *ifp = &sc->sc_ec.ec_if; 2981 2982 mutex_enter(&txr->txr_lock); 2983 txr->txr_transmitdef.ev_count++; 2984 if (pcq_peek(txr->txr_intrq) != NULL) 2985 ixl_tx_common_locked(ifp, txr, true); 2986 mutex_exit(&txr->txr_lock); 2987 } 2988 2989 static struct ixl_rx_ring * 2990 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2991 { 2992 struct ixl_rx_ring *rxr = NULL; 2993 struct ixl_rx_map *maps = NULL, *rxm; 2994 unsigned int i; 2995 2996 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2997 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2998 KM_SLEEP); 2999 3000 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 3001 sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs, 3002 IXL_RX_QUEUE_ALIGN) != 0) 3003 goto free; 3004 3005 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3006 rxm = &maps[i]; 3007 3008 if (bus_dmamap_create(sc->sc_dmat, 3009 IXL_MCLBYTES, 1, IXL_MCLBYTES, 0, 3010 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 3011 goto uncreate; 3012 3013 rxm->rxm_m = NULL; 3014 } 3015 3016 rxr->rxr_cons = rxr->rxr_prod = 0; 3017 rxr->rxr_m_head = NULL; 3018 rxr->rxr_m_tail = &rxr->rxr_m_head; 3019 rxr->rxr_maps = maps; 3020 3021 rxr->rxr_tail = I40E_QRX_TAIL(qid); 3022 rxr->rxr_qid = qid; 3023 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 3024 3025 return rxr; 3026 3027 uncreate: 3028 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3029 rxm = &maps[i]; 3030 3031 if (rxm->rxm_map == NULL) 3032 continue; 3033 3034 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 3035 } 3036 3037 ixl_dmamem_free(sc, &rxr->rxr_mem); 3038 free: 3039 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3040 kmem_free(rxr, sizeof(*rxr)); 3041 3042 return NULL; 3043 } 3044 3045 static void 3046 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3047 { 3048 struct ixl_rx_map *maps, *rxm; 3049 bus_dmamap_t map; 3050 unsigned int i; 3051 3052 maps = rxr->rxr_maps; 3053 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3054 rxm = &maps[i]; 3055 3056 if (rxm->rxm_m == NULL) 3057 continue; 3058 3059 map = rxm->rxm_map; 3060 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3061 BUS_DMASYNC_POSTWRITE); 3062 bus_dmamap_unload(sc->sc_dmat, map); 3063 3064 m_freem(rxm->rxm_m); 3065 rxm->rxm_m = NULL; 3066 } 3067 3068 m_freem(rxr->rxr_m_head); 3069 rxr->rxr_m_head = NULL; 3070 rxr->rxr_m_tail = &rxr->rxr_m_head; 3071 3072 rxr->rxr_prod = rxr->rxr_cons = 0; 3073 } 3074 3075 static int 3076 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3077 { 3078 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3079 uint32_t reg; 3080 int i; 3081 3082 for (i = 0; i < 10; i++) { 3083 reg = ixl_rd(sc, ena); 3084 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 3085 return 0; 3086 3087 delaymsec(10); 3088 } 3089 3090 return ETIMEDOUT; 3091 } 3092 3093 static int 3094 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3095 { 3096 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3097 uint32_t reg; 3098 int i; 3099 3100 KASSERT(mutex_owned(&rxr->rxr_lock)); 3101 3102 for (i = 0; i < 10; i++) { 3103 reg = ixl_rd(sc, ena); 3104 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 3105 return 0; 3106 3107 delaymsec(10); 3108 } 3109 3110 return ETIMEDOUT; 3111 } 3112 3113 static void 3114 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3115 { 3116 struct ixl_hmc_rxq rxq; 3117 struct ifnet *ifp = &sc->sc_ec.ec_if; 3118 uint16_t rxmax; 3119 void *hmc; 3120 3121 memset(&rxq, 0, sizeof(rxq)); 3122 rxmax = ifp->if_mtu + IXL_MTU_ETHERLEN; 3123 3124 rxq.head = htole16(rxr->rxr_cons); 3125 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 3126 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 3127 rxq.dbuff = htole16(IXL_MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 3128 rxq.hbuff = 0; 3129 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 3130 rxq.dsize = IXL_HMC_RXQ_DSIZE_32; 3131 rxq.crcstrip = 1; 3132 rxq.l2sel = 1; 3133 rxq.showiv = 1; 3134 rxq.rxmax = htole16(rxmax); 3135 rxq.tphrdesc_ena = 0; 3136 rxq.tphwdesc_ena = 0; 3137 rxq.tphdata_ena = 0; 3138 rxq.tphhead_ena = 0; 3139 rxq.lrxqthresh = 0; 3140 rxq.prefena = 1; 3141 3142 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3143 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3144 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 3145 __arraycount(ixl_hmc_pack_rxq)); 3146 } 3147 3148 static void 3149 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3150 { 3151 void *hmc; 3152 3153 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3154 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3155 rxr->rxr_cons = rxr->rxr_prod = 0; 3156 } 3157 3158 static void 3159 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3160 { 3161 struct ixl_rx_map *maps, *rxm; 3162 unsigned int i; 3163 3164 maps = rxr->rxr_maps; 3165 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3166 rxm = &maps[i]; 3167 3168 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 3169 } 3170 3171 ixl_dmamem_free(sc, &rxr->rxr_mem); 3172 mutex_destroy(&rxr->rxr_lock); 3173 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3174 kmem_free(rxr, sizeof(*rxr)); 3175 } 3176 3177 static inline void 3178 ixl_rx_csum(struct mbuf *m, uint64_t qword) 3179 { 3180 int flags_mask; 3181 3182 if (!ISSET(qword, IXL_RX_DESC_L3L4P)) { 3183 /* No L3 or L4 checksum was calculated */ 3184 return; 3185 } 3186 3187 switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) { 3188 case IXL_RX_DESC_PTYPE_IPV4FRAG: 3189 case IXL_RX_DESC_PTYPE_IPV4: 3190 case IXL_RX_DESC_PTYPE_SCTPV4: 3191 case IXL_RX_DESC_PTYPE_ICMPV4: 3192 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3193 break; 3194 case IXL_RX_DESC_PTYPE_TCPV4: 3195 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3196 flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD; 3197 break; 3198 case IXL_RX_DESC_PTYPE_UDPV4: 3199 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3200 flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD; 3201 break; 3202 case IXL_RX_DESC_PTYPE_TCPV6: 3203 flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD; 3204 break; 3205 case IXL_RX_DESC_PTYPE_UDPV6: 3206 flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD; 3207 break; 3208 default: 3209 flags_mask = 0; 3210 } 3211 3212 m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 | 3213 M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6)); 3214 3215 if (ISSET(qword, IXL_RX_DESC_IPE)) { 3216 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD); 3217 } 3218 3219 if (ISSET(qword, IXL_RX_DESC_L4E)) { 3220 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD); 3221 } 3222 } 3223 3224 static int 3225 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 3226 { 3227 struct ifnet *ifp = &sc->sc_ec.ec_if; 3228 struct ixl_rx_wb_desc_32 *ring, *rxd; 3229 struct ixl_rx_map *rxm; 3230 bus_dmamap_t map; 3231 unsigned int cons, prod; 3232 struct mbuf *m; 3233 uint64_t word, word0; 3234 unsigned int len; 3235 unsigned int mask; 3236 int done = 0, more = 0; 3237 3238 KASSERT(mutex_owned(&rxr->rxr_lock)); 3239 3240 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3241 return 0; 3242 3243 prod = rxr->rxr_prod; 3244 cons = rxr->rxr_cons; 3245 3246 if (cons == prod) 3247 return 0; 3248 3249 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3250 0, IXL_DMA_LEN(&rxr->rxr_mem), 3251 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3252 3253 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3254 mask = sc->sc_rx_ring_ndescs - 1; 3255 3256 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 3257 3258 do { 3259 if (rxlimit-- <= 0) { 3260 more = 1; 3261 break; 3262 } 3263 3264 rxd = &ring[cons]; 3265 3266 word = le64toh(rxd->qword1); 3267 3268 if (!ISSET(word, IXL_RX_DESC_DD)) 3269 break; 3270 3271 rxm = &rxr->rxr_maps[cons]; 3272 3273 map = rxm->rxm_map; 3274 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3275 BUS_DMASYNC_POSTREAD); 3276 bus_dmamap_unload(sc->sc_dmat, map); 3277 3278 m = rxm->rxm_m; 3279 rxm->rxm_m = NULL; 3280 3281 KASSERT(m != NULL); 3282 3283 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 3284 m->m_len = len; 3285 m->m_pkthdr.len = 0; 3286 3287 m->m_next = NULL; 3288 *rxr->rxr_m_tail = m; 3289 rxr->rxr_m_tail = &m->m_next; 3290 3291 m = rxr->rxr_m_head; 3292 m->m_pkthdr.len += len; 3293 3294 if (ISSET(word, IXL_RX_DESC_EOP)) { 3295 word0 = le64toh(rxd->qword0); 3296 3297 if (ISSET(word, IXL_RX_DESC_L2TAG1P)) { 3298 vlan_set_tag(m, 3299 __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK)); 3300 } 3301 3302 if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0) 3303 ixl_rx_csum(m, word); 3304 3305 if (!ISSET(word, 3306 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 3307 m_set_rcvif(m, ifp); 3308 if_statinc_ref(nsr, if_ipackets); 3309 if_statadd_ref(nsr, if_ibytes, 3310 m->m_pkthdr.len); 3311 if_percpuq_enqueue(ifp->if_percpuq, m); 3312 } else { 3313 if_statinc_ref(nsr, if_ierrors); 3314 m_freem(m); 3315 } 3316 3317 rxr->rxr_m_head = NULL; 3318 rxr->rxr_m_tail = &rxr->rxr_m_head; 3319 } 3320 3321 cons++; 3322 cons &= mask; 3323 3324 done = 1; 3325 } while (cons != prod); 3326 3327 if (done) { 3328 rxr->rxr_cons = cons; 3329 if (ixl_rxfill(sc, rxr) == -1) 3330 if_statinc_ref(nsr, if_iqdrops); 3331 } 3332 3333 IF_STAT_PUTREF(ifp); 3334 3335 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3336 0, IXL_DMA_LEN(&rxr->rxr_mem), 3337 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3338 3339 return more; 3340 } 3341 3342 static int 3343 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3344 { 3345 struct ixl_rx_rd_desc_32 *ring, *rxd; 3346 struct ixl_rx_map *rxm; 3347 bus_dmamap_t map; 3348 struct mbuf *m; 3349 unsigned int prod; 3350 unsigned int slots; 3351 unsigned int mask; 3352 int post = 0, error = 0; 3353 3354 KASSERT(mutex_owned(&rxr->rxr_lock)); 3355 3356 prod = rxr->rxr_prod; 3357 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 3358 sc->sc_rx_ring_ndescs); 3359 3360 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3361 mask = sc->sc_rx_ring_ndescs - 1; 3362 3363 if (__predict_false(slots <= 0)) 3364 return -1; 3365 3366 do { 3367 rxm = &rxr->rxr_maps[prod]; 3368 3369 MGETHDR(m, M_DONTWAIT, MT_DATA); 3370 if (m == NULL) { 3371 rxr->rxr_mgethdr_failed.ev_count++; 3372 error = -1; 3373 break; 3374 } 3375 3376 MCLGET(m, M_DONTWAIT); 3377 if (!ISSET(m->m_flags, M_EXT)) { 3378 rxr->rxr_mgetcl_failed.ev_count++; 3379 error = -1; 3380 m_freem(m); 3381 break; 3382 } 3383 3384 m->m_len = m->m_pkthdr.len = MCLBYTES; 3385 m_adj(m, ETHER_ALIGN); 3386 3387 map = rxm->rxm_map; 3388 3389 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 3390 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 3391 rxr->rxr_mbuf_load_failed.ev_count++; 3392 error = -1; 3393 m_freem(m); 3394 break; 3395 } 3396 3397 rxm->rxm_m = m; 3398 3399 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3400 BUS_DMASYNC_PREREAD); 3401 3402 rxd = &ring[prod]; 3403 3404 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 3405 rxd->haddr = htole64(0); 3406 3407 prod++; 3408 prod &= mask; 3409 3410 post = 1; 3411 3412 } while (--slots); 3413 3414 if (post) { 3415 rxr->rxr_prod = prod; 3416 ixl_wr(sc, rxr->rxr_tail, prod); 3417 } 3418 3419 return error; 3420 } 3421 3422 static inline int 3423 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 3424 u_int txlimit, struct evcnt *txevcnt, 3425 u_int rxlimit, struct evcnt *rxevcnt) 3426 { 3427 struct ixl_tx_ring *txr = qp->qp_txr; 3428 struct ixl_rx_ring *rxr = qp->qp_rxr; 3429 int txmore, rxmore; 3430 int rv; 3431 3432 mutex_enter(&txr->txr_lock); 3433 txevcnt->ev_count++; 3434 txmore = ixl_txeof(sc, txr, txlimit); 3435 mutex_exit(&txr->txr_lock); 3436 3437 mutex_enter(&rxr->rxr_lock); 3438 rxevcnt->ev_count++; 3439 rxmore = ixl_rxeof(sc, rxr, rxlimit); 3440 mutex_exit(&rxr->rxr_lock); 3441 3442 rv = txmore | (rxmore << 1); 3443 3444 return rv; 3445 } 3446 3447 static void 3448 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 3449 { 3450 3451 if (qp->qp_workqueue) 3452 workqueue_enqueue(sc->sc_workq_txrx, &qp->qp_work, NULL); 3453 else 3454 softint_schedule(qp->qp_si); 3455 } 3456 3457 static int 3458 ixl_intr(void *xsc) 3459 { 3460 struct ixl_softc *sc = xsc; 3461 struct ixl_tx_ring *txr; 3462 struct ixl_rx_ring *rxr; 3463 uint32_t icr, rxintr, txintr; 3464 int rv = 0; 3465 unsigned int i; 3466 3467 KASSERT(sc != NULL); 3468 3469 ixl_enable_other_intr(sc); 3470 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3471 3472 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3473 atomic_inc_64(&sc->sc_event_atq.ev_count); 3474 ixl_atq_done(sc); 3475 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3476 rv = 1; 3477 } 3478 3479 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3480 atomic_inc_64(&sc->sc_event_link.ev_count); 3481 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3482 rv = 1; 3483 } 3484 3485 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 3486 txintr = icr & I40E_INTR_NOTX_TX_MASK; 3487 3488 if (txintr || rxintr) { 3489 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 3490 txr = sc->sc_qps[i].qp_txr; 3491 rxr = sc->sc_qps[i].qp_rxr; 3492 3493 ixl_handle_queue_common(sc, &sc->sc_qps[i], 3494 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 3495 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 3496 } 3497 rv = 1; 3498 } 3499 3500 return rv; 3501 } 3502 3503 static int 3504 ixl_queue_intr(void *xqp) 3505 { 3506 struct ixl_queue_pair *qp = xqp; 3507 struct ixl_tx_ring *txr = qp->qp_txr; 3508 struct ixl_rx_ring *rxr = qp->qp_rxr; 3509 struct ixl_softc *sc = qp->qp_sc; 3510 u_int txlimit, rxlimit; 3511 int more; 3512 3513 txlimit = sc->sc_tx_intr_process_limit; 3514 rxlimit = sc->sc_rx_intr_process_limit; 3515 qp->qp_workqueue = sc->sc_txrx_workqueue; 3516 3517 more = ixl_handle_queue_common(sc, qp, 3518 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3519 3520 if (more != 0) { 3521 ixl_sched_handle_queue(sc, qp); 3522 } else { 3523 /* for ALTQ */ 3524 if (txr->txr_qid == 0) 3525 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3526 softint_schedule(txr->txr_si); 3527 3528 ixl_enable_queue_intr(sc, qp); 3529 } 3530 3531 return 1; 3532 } 3533 3534 static void 3535 ixl_handle_queue_wk(struct work *wk, void *xsc) 3536 { 3537 struct ixl_queue_pair *qp; 3538 3539 qp = container_of(wk, struct ixl_queue_pair, qp_work); 3540 ixl_handle_queue(qp); 3541 } 3542 3543 static void 3544 ixl_handle_queue(void *xqp) 3545 { 3546 struct ixl_queue_pair *qp = xqp; 3547 struct ixl_softc *sc = qp->qp_sc; 3548 struct ixl_tx_ring *txr = qp->qp_txr; 3549 struct ixl_rx_ring *rxr = qp->qp_rxr; 3550 u_int txlimit, rxlimit; 3551 int more; 3552 3553 txlimit = sc->sc_tx_process_limit; 3554 rxlimit = sc->sc_rx_process_limit; 3555 3556 more = ixl_handle_queue_common(sc, qp, 3557 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3558 3559 if (more != 0) 3560 ixl_sched_handle_queue(sc, qp); 3561 else 3562 ixl_enable_queue_intr(sc, qp); 3563 } 3564 3565 static inline void 3566 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3567 { 3568 uint32_t hmc_idx, hmc_isvf; 3569 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3570 3571 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3572 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3573 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3574 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3575 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3576 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3577 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3578 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3579 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3580 3581 device_printf(sc->sc_dev, 3582 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3583 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3584 } 3585 3586 static int 3587 ixl_other_intr(void *xsc) 3588 { 3589 struct ixl_softc *sc = xsc; 3590 uint32_t icr, mask, reg; 3591 int rv; 3592 3593 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3594 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3595 3596 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3597 atomic_inc_64(&sc->sc_event_atq.ev_count); 3598 ixl_atq_done(sc); 3599 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3600 rv = 1; 3601 } 3602 3603 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3604 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3605 device_printf(sc->sc_dev, "link stat changed\n"); 3606 3607 atomic_inc_64(&sc->sc_event_link.ev_count); 3608 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3609 rv = 1; 3610 } 3611 3612 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3613 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3614 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3615 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3616 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3617 3618 device_printf(sc->sc_dev, "GRST: %s\n", 3619 reg == I40E_RESET_CORER ? "CORER" : 3620 reg == I40E_RESET_GLOBR ? "GLOBR" : 3621 reg == I40E_RESET_EMPR ? "EMPR" : 3622 "POR"); 3623 } 3624 3625 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3626 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3627 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3628 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3629 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3630 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3631 3632 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3633 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3634 device_printf(sc->sc_dev, "critical error\n"); 3635 } 3636 3637 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3638 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3639 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3640 ixl_print_hmc_error(sc, reg); 3641 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3642 } 3643 3644 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3645 ixl_flush(sc); 3646 ixl_enable_other_intr(sc); 3647 return rv; 3648 } 3649 3650 static void 3651 ixl_get_link_status_done(struct ixl_softc *sc, 3652 const struct ixl_aq_desc *iaq) 3653 { 3654 3655 ixl_link_state_update(sc, iaq); 3656 } 3657 3658 static void 3659 ixl_get_link_status(void *xsc) 3660 { 3661 struct ixl_softc *sc = xsc; 3662 struct ixl_aq_desc *iaq; 3663 struct ixl_aq_link_param *param; 3664 int error; 3665 3666 mutex_enter(&sc->sc_atq_lock); 3667 3668 iaq = &sc->sc_link_state_atq.iatq_desc; 3669 memset(iaq, 0, sizeof(*iaq)); 3670 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3671 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3672 param->notify = IXL_AQ_LINK_NOTIFY; 3673 3674 error = ixl_atq_exec_locked(sc, &sc->sc_link_state_atq); 3675 if (error == 0) { 3676 ixl_get_link_status_done(sc, iaq); 3677 } 3678 3679 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 3680 3681 mutex_exit(&sc->sc_atq_lock); 3682 } 3683 3684 static void 3685 ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3686 { 3687 struct ifnet *ifp = &sc->sc_ec.ec_if; 3688 int link_state; 3689 3690 KASSERT(kpreempt_disabled()); 3691 3692 link_state = ixl_set_link_status(sc, iaq); 3693 3694 if (ifp->if_link_state != link_state) 3695 if_link_state_change(ifp, link_state); 3696 3697 if (link_state != LINK_STATE_DOWN) { 3698 if_schedule_deferred_start(ifp); 3699 } 3700 } 3701 3702 static void 3703 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3704 const char *msg) 3705 { 3706 char buf[512]; 3707 size_t len; 3708 3709 len = sizeof(buf); 3710 buf[--len] = '\0'; 3711 3712 device_printf(sc->sc_dev, "%s\n", msg); 3713 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3714 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3715 buf, le16toh(iaq->iaq_opcode)); 3716 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3717 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3718 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3719 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3720 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3721 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3722 } 3723 3724 static void 3725 ixl_arq(void *xsc) 3726 { 3727 struct ixl_softc *sc = xsc; 3728 struct ixl_aq_desc *arq, *iaq; 3729 struct ixl_aq_buf *aqb; 3730 unsigned int cons = sc->sc_arq_cons; 3731 unsigned int prod; 3732 int done = 0; 3733 3734 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3735 sc->sc_aq_regs->arq_head_mask; 3736 3737 if (cons == prod) 3738 goto done; 3739 3740 arq = IXL_DMA_KVA(&sc->sc_arq); 3741 3742 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3743 0, IXL_DMA_LEN(&sc->sc_arq), 3744 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3745 3746 do { 3747 iaq = &arq[cons]; 3748 aqb = sc->sc_arq_live[cons]; 3749 3750 KASSERT(aqb != NULL); 3751 3752 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3753 BUS_DMASYNC_POSTREAD); 3754 3755 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3756 ixl_aq_dump(sc, iaq, "arq event"); 3757 3758 switch (iaq->iaq_opcode) { 3759 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3760 kpreempt_disable(); 3761 ixl_link_state_update(sc, iaq); 3762 kpreempt_enable(); 3763 break; 3764 } 3765 3766 memset(iaq, 0, sizeof(*iaq)); 3767 sc->sc_arq_live[cons] = NULL; 3768 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3769 3770 cons++; 3771 cons &= IXL_AQ_MASK; 3772 3773 done = 1; 3774 } while (cons != prod); 3775 3776 if (done) { 3777 sc->sc_arq_cons = cons; 3778 ixl_arq_fill(sc); 3779 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3780 0, IXL_DMA_LEN(&sc->sc_arq), 3781 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3782 } 3783 3784 done: 3785 ixl_enable_other_intr(sc); 3786 } 3787 3788 static void 3789 ixl_atq_set(struct ixl_atq *iatq, 3790 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3791 { 3792 3793 iatq->iatq_fn = fn; 3794 } 3795 3796 static int 3797 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3798 { 3799 struct ixl_aq_desc *atq, *slot; 3800 unsigned int prod, cons, prod_next; 3801 3802 /* assert locked */ 3803 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3804 3805 atq = IXL_DMA_KVA(&sc->sc_atq); 3806 prod = sc->sc_atq_prod; 3807 cons = sc->sc_atq_cons; 3808 prod_next = (prod +1) & IXL_AQ_MASK; 3809 3810 if (cons == prod_next) 3811 return ENOMEM; 3812 3813 slot = &atq[prod]; 3814 3815 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3816 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3817 3818 KASSERT(iatq->iatq_fn != NULL); 3819 *slot = iatq->iatq_desc; 3820 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3821 3822 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3823 ixl_aq_dump(sc, slot, "atq command"); 3824 3825 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3826 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3827 3828 sc->sc_atq_prod = prod_next; 3829 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3830 3831 return 0; 3832 } 3833 3834 static void 3835 ixl_atq_done_locked(struct ixl_softc *sc) 3836 { 3837 struct ixl_aq_desc *atq, *slot; 3838 struct ixl_atq *iatq; 3839 unsigned int cons; 3840 unsigned int prod; 3841 3842 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3843 3844 prod = sc->sc_atq_prod; 3845 cons = sc->sc_atq_cons; 3846 3847 if (prod == cons) 3848 return; 3849 3850 atq = IXL_DMA_KVA(&sc->sc_atq); 3851 3852 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3853 0, IXL_DMA_LEN(&sc->sc_atq), 3854 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3855 3856 do { 3857 slot = &atq[cons]; 3858 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3859 break; 3860 3861 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3862 iatq->iatq_desc = *slot; 3863 3864 memset(slot, 0, sizeof(*slot)); 3865 3866 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3867 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3868 3869 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3870 3871 cons++; 3872 cons &= IXL_AQ_MASK; 3873 } while (cons != prod); 3874 3875 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3876 0, IXL_DMA_LEN(&sc->sc_atq), 3877 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3878 3879 sc->sc_atq_cons = cons; 3880 } 3881 3882 static void 3883 ixl_atq_done(struct ixl_softc *sc) 3884 { 3885 3886 mutex_enter(&sc->sc_atq_lock); 3887 ixl_atq_done_locked(sc); 3888 mutex_exit(&sc->sc_atq_lock); 3889 } 3890 3891 static void 3892 ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3893 { 3894 3895 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3896 3897 cv_signal(&sc->sc_atq_cv); 3898 } 3899 3900 static int 3901 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3902 { 3903 int error; 3904 3905 mutex_enter(&sc->sc_atq_lock); 3906 error = ixl_atq_exec_locked(sc, iatq); 3907 mutex_exit(&sc->sc_atq_lock); 3908 3909 return error; 3910 } 3911 3912 static int 3913 ixl_atq_exec_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3914 { 3915 int error; 3916 3917 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3918 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3919 3920 ixl_atq_set(iatq, ixl_wakeup); 3921 3922 error = ixl_atq_post_locked(sc, iatq); 3923 if (error) 3924 return error; 3925 3926 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3927 IXL_ATQ_EXEC_TIMEOUT); 3928 3929 return error; 3930 } 3931 3932 static int 3933 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3934 { 3935 struct ixl_aq_desc *atq, *slot; 3936 unsigned int prod; 3937 unsigned int t = 0; 3938 3939 mutex_enter(&sc->sc_atq_lock); 3940 3941 atq = IXL_DMA_KVA(&sc->sc_atq); 3942 prod = sc->sc_atq_prod; 3943 slot = atq + prod; 3944 3945 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3946 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3947 3948 *slot = *iaq; 3949 slot->iaq_flags |= htole16(IXL_AQ_SI); 3950 3951 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3952 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3953 3954 prod++; 3955 prod &= IXL_AQ_MASK; 3956 sc->sc_atq_prod = prod; 3957 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3958 3959 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3960 delaymsec(1); 3961 3962 if (t++ > tm) { 3963 mutex_exit(&sc->sc_atq_lock); 3964 return ETIMEDOUT; 3965 } 3966 } 3967 3968 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3969 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3970 *iaq = *slot; 3971 memset(slot, 0, sizeof(*slot)); 3972 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3973 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3974 3975 sc->sc_atq_cons = prod; 3976 3977 mutex_exit(&sc->sc_atq_lock); 3978 3979 return 0; 3980 } 3981 3982 static int 3983 ixl_get_version(struct ixl_softc *sc) 3984 { 3985 struct ixl_aq_desc iaq; 3986 uint32_t fwbuild, fwver, apiver; 3987 uint16_t api_maj_ver, api_min_ver; 3988 3989 memset(&iaq, 0, sizeof(iaq)); 3990 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3991 3992 iaq.iaq_retval = le16toh(23); 3993 3994 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3995 return ETIMEDOUT; 3996 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3997 return EIO; 3998 3999 fwbuild = le32toh(iaq.iaq_param[1]); 4000 fwver = le32toh(iaq.iaq_param[2]); 4001 apiver = le32toh(iaq.iaq_param[3]); 4002 4003 api_maj_ver = (uint16_t)apiver; 4004 api_min_ver = (uint16_t)(apiver >> 16); 4005 4006 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 4007 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 4008 4009 if (sc->sc_mac_type == I40E_MAC_X722) { 4010 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK | 4011 IXL_SC_AQ_FLAG_NVMREAD); 4012 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 4013 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS); 4014 } 4015 4016 #define IXL_API_VER(maj, min) (((uint32_t)(maj) << 16) | (min)) 4017 if (IXL_API_VER(api_maj_ver, api_min_ver) >= IXL_API_VER(1, 5)) { 4018 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 4019 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK); 4020 } 4021 #undef IXL_API_VER 4022 4023 return 0; 4024 } 4025 4026 static int 4027 ixl_get_nvm_version(struct ixl_softc *sc) 4028 { 4029 uint16_t nvmver, cfg_ptr, eetrack_hi, eetrack_lo, oem_hi, oem_lo; 4030 uint32_t eetrack, oem; 4031 uint16_t nvm_maj_ver, nvm_min_ver, oem_build; 4032 uint8_t oem_ver, oem_patch; 4033 4034 nvmver = cfg_ptr = eetrack_hi = eetrack_lo = oem_hi = oem_lo = 0; 4035 ixl_rd16_nvm(sc, I40E_SR_NVM_DEV_STARTER_VERSION, &nvmver); 4036 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_HI, &eetrack_hi); 4037 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_LO, &eetrack_lo); 4038 ixl_rd16_nvm(sc, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr); 4039 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF, &oem_hi); 4040 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF + 1, &oem_lo); 4041 4042 nvm_maj_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_HI_MASK); 4043 nvm_min_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_LO_MASK); 4044 eetrack = ((uint32_t)eetrack_hi << 16) | eetrack_lo; 4045 oem = ((uint32_t)oem_hi << 16) | oem_lo; 4046 oem_ver = __SHIFTOUT(oem, IXL_NVM_OEMVERSION_MASK); 4047 oem_build = __SHIFTOUT(oem, IXL_NVM_OEMBUILD_MASK); 4048 oem_patch = __SHIFTOUT(oem, IXL_NVM_OEMPATCH_MASK); 4049 4050 aprint_normal(" nvm %x.%02x etid %08x oem %d.%d.%d", 4051 nvm_maj_ver, nvm_min_ver, eetrack, 4052 oem_ver, oem_build, oem_patch); 4053 4054 return 0; 4055 } 4056 4057 static int 4058 ixl_pxe_clear(struct ixl_softc *sc) 4059 { 4060 struct ixl_aq_desc iaq; 4061 int rv; 4062 4063 memset(&iaq, 0, sizeof(iaq)); 4064 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 4065 iaq.iaq_param[0] = htole32(0x2); 4066 4067 rv = ixl_atq_poll(sc, &iaq, 250); 4068 4069 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 4070 4071 if (rv != 0) 4072 return ETIMEDOUT; 4073 4074 switch (iaq.iaq_retval) { 4075 case htole16(IXL_AQ_RC_OK): 4076 case htole16(IXL_AQ_RC_EEXIST): 4077 break; 4078 default: 4079 return EIO; 4080 } 4081 4082 return 0; 4083 } 4084 4085 static int 4086 ixl_lldp_shut(struct ixl_softc *sc) 4087 { 4088 struct ixl_aq_desc iaq; 4089 4090 memset(&iaq, 0, sizeof(iaq)); 4091 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 4092 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 4093 4094 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4095 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 4096 return -1; 4097 } 4098 4099 switch (iaq.iaq_retval) { 4100 case htole16(IXL_AQ_RC_EMODE): 4101 case htole16(IXL_AQ_RC_EPERM): 4102 /* ignore silently */ 4103 default: 4104 break; 4105 } 4106 4107 return 0; 4108 } 4109 4110 static void 4111 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 4112 { 4113 uint16_t id; 4114 uint32_t number, logical_id; 4115 4116 id = le16toh(cap->cap_id); 4117 number = le32toh(cap->number); 4118 logical_id = le32toh(cap->logical_id); 4119 4120 switch (id) { 4121 case IXL_AQ_CAP_RSS: 4122 sc->sc_rss_table_size = number; 4123 sc->sc_rss_table_entry_width = logical_id; 4124 break; 4125 case IXL_AQ_CAP_RXQ: 4126 case IXL_AQ_CAP_TXQ: 4127 sc->sc_nqueue_pairs_device = MIN(number, 4128 sc->sc_nqueue_pairs_device); 4129 break; 4130 } 4131 } 4132 4133 static int 4134 ixl_get_hw_capabilities(struct ixl_softc *sc) 4135 { 4136 struct ixl_dmamem idm; 4137 struct ixl_aq_desc iaq; 4138 struct ixl_aq_capability *caps; 4139 size_t i, ncaps; 4140 bus_size_t caps_size; 4141 uint16_t status; 4142 int rv; 4143 4144 caps_size = sizeof(caps[0]) * 40; 4145 memset(&iaq, 0, sizeof(iaq)); 4146 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 4147 4148 do { 4149 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 4150 return -1; 4151 } 4152 4153 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4154 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4155 iaq.iaq_datalen = htole16(caps_size); 4156 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4157 4158 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4159 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 4160 4161 rv = ixl_atq_poll(sc, &iaq, 250); 4162 4163 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4164 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 4165 4166 if (rv != 0) { 4167 aprint_error(", HW capabilities timeout\n"); 4168 goto done; 4169 } 4170 4171 status = le16toh(iaq.iaq_retval); 4172 4173 if (status == IXL_AQ_RC_ENOMEM) { 4174 caps_size = le16toh(iaq.iaq_datalen); 4175 ixl_dmamem_free(sc, &idm); 4176 } 4177 } while (status == IXL_AQ_RC_ENOMEM); 4178 4179 if (status != IXL_AQ_RC_OK) { 4180 aprint_error(", HW capabilities error\n"); 4181 goto done; 4182 } 4183 4184 caps = IXL_DMA_KVA(&idm); 4185 ncaps = le16toh(iaq.iaq_param[1]); 4186 4187 for (i = 0; i < ncaps; i++) { 4188 ixl_parse_hw_capability(sc, &caps[i]); 4189 } 4190 4191 done: 4192 ixl_dmamem_free(sc, &idm); 4193 return rv; 4194 } 4195 4196 static int 4197 ixl_get_mac(struct ixl_softc *sc) 4198 { 4199 struct ixl_dmamem idm; 4200 struct ixl_aq_desc iaq; 4201 struct ixl_aq_mac_addresses *addrs; 4202 int rv; 4203 4204 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 4205 aprint_error(", unable to allocate mac addresses\n"); 4206 return -1; 4207 } 4208 4209 memset(&iaq, 0, sizeof(iaq)); 4210 iaq.iaq_flags = htole16(IXL_AQ_BUF); 4211 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 4212 iaq.iaq_datalen = htole16(sizeof(*addrs)); 4213 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4214 4215 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4216 BUS_DMASYNC_PREREAD); 4217 4218 rv = ixl_atq_poll(sc, &iaq, 250); 4219 4220 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4221 BUS_DMASYNC_POSTREAD); 4222 4223 if (rv != 0) { 4224 aprint_error(", MAC ADDRESS READ timeout\n"); 4225 rv = -1; 4226 goto done; 4227 } 4228 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4229 aprint_error(", MAC ADDRESS READ error\n"); 4230 rv = -1; 4231 goto done; 4232 } 4233 4234 addrs = IXL_DMA_KVA(&idm); 4235 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 4236 printf(", port address is not valid\n"); 4237 goto done; 4238 } 4239 4240 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 4241 rv = 0; 4242 4243 done: 4244 ixl_dmamem_free(sc, &idm); 4245 return rv; 4246 } 4247 4248 static int 4249 ixl_get_switch_config(struct ixl_softc *sc) 4250 { 4251 struct ixl_dmamem idm; 4252 struct ixl_aq_desc iaq; 4253 struct ixl_aq_switch_config *hdr; 4254 struct ixl_aq_switch_config_element *elms, *elm; 4255 unsigned int nelm, i; 4256 int rv; 4257 4258 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4259 aprint_error_dev(sc->sc_dev, 4260 "unable to allocate switch config buffer\n"); 4261 return -1; 4262 } 4263 4264 memset(&iaq, 0, sizeof(iaq)); 4265 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4266 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4267 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 4268 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 4269 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4270 4271 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4272 BUS_DMASYNC_PREREAD); 4273 4274 rv = ixl_atq_poll(sc, &iaq, 250); 4275 4276 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4277 BUS_DMASYNC_POSTREAD); 4278 4279 if (rv != 0) { 4280 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 4281 rv = -1; 4282 goto done; 4283 } 4284 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4285 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 4286 rv = -1; 4287 goto done; 4288 } 4289 4290 hdr = IXL_DMA_KVA(&idm); 4291 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 4292 4293 nelm = le16toh(hdr->num_reported); 4294 if (nelm < 1) { 4295 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 4296 rv = -1; 4297 goto done; 4298 } 4299 4300 for (i = 0; i < nelm; i++) { 4301 elm = &elms[i]; 4302 4303 aprint_debug_dev(sc->sc_dev, 4304 "type %x revision %u seid %04x\n", 4305 elm->type, elm->revision, le16toh(elm->seid)); 4306 aprint_debug_dev(sc->sc_dev, 4307 "uplink %04x downlink %04x\n", 4308 le16toh(elm->uplink_seid), 4309 le16toh(elm->downlink_seid)); 4310 aprint_debug_dev(sc->sc_dev, 4311 "conntype %x scheduler %04x extra %04x\n", 4312 elm->connection_type, 4313 le16toh(elm->scheduler_id), 4314 le16toh(elm->element_info)); 4315 } 4316 4317 elm = &elms[0]; 4318 4319 sc->sc_uplink_seid = elm->uplink_seid; 4320 sc->sc_downlink_seid = elm->downlink_seid; 4321 sc->sc_seid = elm->seid; 4322 4323 if ((sc->sc_uplink_seid == htole16(0)) != 4324 (sc->sc_downlink_seid == htole16(0))) { 4325 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 4326 rv = -1; 4327 goto done; 4328 } 4329 4330 done: 4331 ixl_dmamem_free(sc, &idm); 4332 return rv; 4333 } 4334 4335 static int 4336 ixl_phy_mask_ints(struct ixl_softc *sc) 4337 { 4338 struct ixl_aq_desc iaq; 4339 4340 memset(&iaq, 0, sizeof(iaq)); 4341 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 4342 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 4343 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 4344 IXL_AQ_PHY_EV_MEDIA_NA)); 4345 4346 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4347 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 4348 return -1; 4349 } 4350 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4351 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 4352 return -1; 4353 } 4354 4355 return 0; 4356 } 4357 4358 static int 4359 ixl_get_phy_abilities(struct ixl_softc *sc,struct ixl_dmamem *idm) 4360 { 4361 struct ixl_aq_desc iaq; 4362 int rv; 4363 4364 memset(&iaq, 0, sizeof(iaq)); 4365 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4366 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4367 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 4368 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 4369 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 4370 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 4371 4372 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4373 BUS_DMASYNC_PREREAD); 4374 4375 rv = ixl_atq_poll(sc, &iaq, 250); 4376 4377 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4378 BUS_DMASYNC_POSTREAD); 4379 4380 if (rv != 0) 4381 return -1; 4382 4383 return le16toh(iaq.iaq_retval); 4384 } 4385 4386 static int 4387 ixl_get_phy_info(struct ixl_softc *sc) 4388 { 4389 struct ixl_dmamem idm; 4390 struct ixl_aq_phy_abilities *phy; 4391 int rv; 4392 4393 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4394 aprint_error_dev(sc->sc_dev, 4395 "unable to allocate phy abilities buffer\n"); 4396 return -1; 4397 } 4398 4399 rv = ixl_get_phy_abilities(sc, &idm); 4400 switch (rv) { 4401 case -1: 4402 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 4403 goto done; 4404 case IXL_AQ_RC_OK: 4405 break; 4406 case IXL_AQ_RC_EIO: 4407 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 4408 goto done; 4409 default: 4410 aprint_error_dev(sc->sc_dev, 4411 "GET PHY ABILITIIES error %u\n", rv); 4412 goto done; 4413 } 4414 4415 phy = IXL_DMA_KVA(&idm); 4416 4417 sc->sc_phy_types = le32toh(phy->phy_type); 4418 sc->sc_phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 4419 4420 sc->sc_phy_abilities = phy->abilities; 4421 sc->sc_phy_linkspeed = phy->link_speed; 4422 sc->sc_phy_fec_cfg = phy->fec_cfg_curr_mod_ext_info & 4423 (IXL_AQ_ENABLE_FEC_KR | IXL_AQ_ENABLE_FEC_RS | 4424 IXL_AQ_REQUEST_FEC_KR | IXL_AQ_REQUEST_FEC_RS); 4425 sc->sc_eee_cap = phy->eee_capability; 4426 sc->sc_eeer_val = phy->eeer_val; 4427 sc->sc_d3_lpan = phy->d3_lpan; 4428 4429 rv = 0; 4430 4431 done: 4432 ixl_dmamem_free(sc, &idm); 4433 return rv; 4434 } 4435 4436 static int 4437 ixl_set_phy_config(struct ixl_softc *sc, 4438 uint8_t link_speed, uint8_t abilities, bool polling) 4439 { 4440 struct ixl_aq_phy_param *param; 4441 struct ixl_atq iatq; 4442 struct ixl_aq_desc *iaq; 4443 int error; 4444 4445 memset(&iatq, 0, sizeof(iatq)); 4446 4447 iaq = &iatq.iatq_desc; 4448 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_CONFIG); 4449 param = (struct ixl_aq_phy_param *)&iaq->iaq_param; 4450 param->phy_types = htole32((uint32_t)sc->sc_phy_types); 4451 param->phy_type_ext = (uint8_t)(sc->sc_phy_types >> 32); 4452 param->link_speed = link_speed; 4453 param->abilities = abilities | IXL_AQ_PHY_ABILITY_AUTO_LINK; 4454 param->fec_cfg = sc->sc_phy_fec_cfg; 4455 param->eee_capability = sc->sc_eee_cap; 4456 param->eeer_val = sc->sc_eeer_val; 4457 param->d3_lpan = sc->sc_d3_lpan; 4458 4459 if (polling) 4460 error = ixl_atq_poll(sc, iaq, 250); 4461 else 4462 error = ixl_atq_exec(sc, &iatq); 4463 4464 if (error != 0) 4465 return error; 4466 4467 switch (le16toh(iaq->iaq_retval)) { 4468 case IXL_AQ_RC_OK: 4469 break; 4470 case IXL_AQ_RC_EPERM: 4471 return EPERM; 4472 default: 4473 return EIO; 4474 } 4475 4476 return 0; 4477 } 4478 4479 static int 4480 ixl_set_phy_autoselect(struct ixl_softc *sc) 4481 { 4482 uint8_t link_speed, abilities; 4483 4484 link_speed = sc->sc_phy_linkspeed; 4485 abilities = IXL_PHY_ABILITY_LINKUP | IXL_PHY_ABILITY_AUTONEGO; 4486 4487 return ixl_set_phy_config(sc, link_speed, abilities, true); 4488 } 4489 4490 static int 4491 ixl_get_link_status_poll(struct ixl_softc *sc, int *l) 4492 { 4493 struct ixl_aq_desc iaq; 4494 struct ixl_aq_link_param *param; 4495 int link; 4496 4497 memset(&iaq, 0, sizeof(iaq)); 4498 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 4499 param = (struct ixl_aq_link_param *)iaq.iaq_param; 4500 param->notify = IXL_AQ_LINK_NOTIFY; 4501 4502 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4503 return ETIMEDOUT; 4504 } 4505 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4506 return EIO; 4507 } 4508 4509 link = ixl_set_link_status(sc, &iaq); 4510 4511 if (l != NULL) 4512 *l = link; 4513 4514 return 0; 4515 } 4516 4517 static int 4518 ixl_get_vsi(struct ixl_softc *sc) 4519 { 4520 struct ixl_dmamem *vsi = &sc->sc_scratch; 4521 struct ixl_aq_desc iaq; 4522 struct ixl_aq_vsi_param *param; 4523 struct ixl_aq_vsi_reply *reply; 4524 struct ixl_aq_vsi_data *data; 4525 int rv; 4526 4527 /* grumble, vsi info isn't "known" at compile time */ 4528 4529 memset(&iaq, 0, sizeof(iaq)); 4530 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4531 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4532 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 4533 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4534 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4535 4536 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4537 param->uplink_seid = sc->sc_seid; 4538 4539 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4540 BUS_DMASYNC_PREREAD); 4541 4542 rv = ixl_atq_poll(sc, &iaq, 250); 4543 4544 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4545 BUS_DMASYNC_POSTREAD); 4546 4547 if (rv != 0) { 4548 return ETIMEDOUT; 4549 } 4550 4551 switch (le16toh(iaq.iaq_retval)) { 4552 case IXL_AQ_RC_OK: 4553 break; 4554 case IXL_AQ_RC_ENOENT: 4555 return ENOENT; 4556 case IXL_AQ_RC_EACCES: 4557 return EACCES; 4558 default: 4559 return EIO; 4560 } 4561 4562 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 4563 sc->sc_vsi_number = le16toh(reply->vsi_number); 4564 data = IXL_DMA_KVA(vsi); 4565 sc->sc_vsi_stat_counter_idx = le16toh(data->stat_counter_idx); 4566 4567 return 0; 4568 } 4569 4570 static int 4571 ixl_set_vsi(struct ixl_softc *sc) 4572 { 4573 struct ixl_dmamem *vsi = &sc->sc_scratch; 4574 struct ixl_aq_desc iaq; 4575 struct ixl_aq_vsi_param *param; 4576 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 4577 unsigned int qnum; 4578 uint16_t val; 4579 int rv; 4580 4581 qnum = sc->sc_nqueue_pairs - 1; 4582 4583 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 4584 IXL_AQ_VSI_VALID_VLAN); 4585 4586 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 4587 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 4588 data->queue_mapping[0] = htole16(0); 4589 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 4590 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 4591 4592 val = le16toh(data->port_vlan_flags); 4593 CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK); 4594 SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL); 4595 4596 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { 4597 SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH); 4598 } else { 4599 SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING); 4600 } 4601 4602 data->port_vlan_flags = htole16(val); 4603 4604 /* grumble, vsi info isn't "known" at compile time */ 4605 4606 memset(&iaq, 0, sizeof(iaq)); 4607 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4608 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4609 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 4610 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4611 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4612 4613 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4614 param->uplink_seid = sc->sc_seid; 4615 4616 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4617 BUS_DMASYNC_PREWRITE); 4618 4619 rv = ixl_atq_poll(sc, &iaq, 250); 4620 4621 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4622 BUS_DMASYNC_POSTWRITE); 4623 4624 if (rv != 0) { 4625 return ETIMEDOUT; 4626 } 4627 4628 switch (le16toh(iaq.iaq_retval)) { 4629 case IXL_AQ_RC_OK: 4630 break; 4631 case IXL_AQ_RC_ENOENT: 4632 return ENOENT; 4633 case IXL_AQ_RC_EACCES: 4634 return EACCES; 4635 default: 4636 return EIO; 4637 } 4638 4639 return 0; 4640 } 4641 4642 static void 4643 ixl_set_filter_control(struct ixl_softc *sc) 4644 { 4645 uint32_t reg; 4646 4647 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4648 4649 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4650 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4651 4652 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4653 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4654 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4655 4656 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4657 } 4658 4659 static inline void 4660 ixl_get_default_rss_key(uint32_t *buf, size_t len) 4661 { 4662 size_t cplen; 4663 uint8_t rss_seed[RSS_KEYSIZE]; 4664 4665 rss_getkey(rss_seed); 4666 memset(buf, 0, len); 4667 4668 cplen = MIN(len, sizeof(rss_seed)); 4669 memcpy(buf, rss_seed, cplen); 4670 } 4671 4672 static int 4673 ixl_set_rss_key(struct ixl_softc *sc, uint8_t *key, size_t keylen) 4674 { 4675 struct ixl_dmamem *idm; 4676 struct ixl_atq iatq; 4677 struct ixl_aq_desc *iaq; 4678 struct ixl_aq_rss_key_param *param; 4679 struct ixl_aq_rss_key_data *data; 4680 size_t len, datalen, stdlen, extlen; 4681 uint16_t vsi_id; 4682 int rv; 4683 4684 memset(&iatq, 0, sizeof(iatq)); 4685 iaq = &iatq.iatq_desc; 4686 idm = &sc->sc_aqbuf; 4687 4688 datalen = sizeof(*data); 4689 4690 /*XXX The buf size has to be less than the size of the register */ 4691 datalen = MIN(IXL_RSS_KEY_SIZE_REG * sizeof(uint32_t), datalen); 4692 4693 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4694 (datalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4695 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_KEY); 4696 iaq->iaq_datalen = htole16(datalen); 4697 4698 param = (struct ixl_aq_rss_key_param *)iaq->iaq_param; 4699 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSKEY_VSI_ID_SHIFT) | 4700 IXL_AQ_RSSKEY_VSI_VALID; 4701 param->vsi_id = htole16(vsi_id); 4702 4703 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4704 data = IXL_DMA_KVA(idm); 4705 4706 len = MIN(keylen, datalen); 4707 stdlen = MIN(sizeof(data->standard_rss_key), len); 4708 memcpy(data->standard_rss_key, key, stdlen); 4709 len = (len > stdlen) ? (len - stdlen) : 0; 4710 4711 extlen = MIN(sizeof(data->extended_hash_key), len); 4712 extlen = (stdlen < keylen) ? 0 : keylen - stdlen; 4713 memcpy(data->extended_hash_key, key + stdlen, extlen); 4714 4715 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4716 4717 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4718 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4719 4720 rv = ixl_atq_exec(sc, &iatq); 4721 4722 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4723 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4724 4725 if (rv != 0) { 4726 return ETIMEDOUT; 4727 } 4728 4729 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4730 return EIO; 4731 } 4732 4733 return 0; 4734 } 4735 4736 static int 4737 ixl_set_rss_lut(struct ixl_softc *sc, uint8_t *lut, size_t lutlen) 4738 { 4739 struct ixl_dmamem *idm; 4740 struct ixl_atq iatq; 4741 struct ixl_aq_desc *iaq; 4742 struct ixl_aq_rss_lut_param *param; 4743 uint16_t vsi_id; 4744 uint8_t *data; 4745 size_t dmalen; 4746 int rv; 4747 4748 memset(&iatq, 0, sizeof(iatq)); 4749 iaq = &iatq.iatq_desc; 4750 idm = &sc->sc_aqbuf; 4751 4752 dmalen = MIN(lutlen, IXL_DMA_LEN(idm)); 4753 4754 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4755 (dmalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4756 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_LUT); 4757 iaq->iaq_datalen = htole16(dmalen); 4758 4759 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4760 data = IXL_DMA_KVA(idm); 4761 memcpy(data, lut, dmalen); 4762 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4763 4764 param = (struct ixl_aq_rss_lut_param *)iaq->iaq_param; 4765 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSLUT_VSI_ID_SHIFT) | 4766 IXL_AQ_RSSLUT_VSI_VALID; 4767 param->vsi_id = htole16(vsi_id); 4768 param->flags = htole16(IXL_AQ_RSSLUT_TABLE_TYPE_PF << 4769 IXL_AQ_RSSLUT_TABLE_TYPE_SHIFT); 4770 4771 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4772 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4773 4774 rv = ixl_atq_exec(sc, &iatq); 4775 4776 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4777 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4778 4779 if (rv != 0) { 4780 return ETIMEDOUT; 4781 } 4782 4783 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4784 return EIO; 4785 } 4786 4787 return 0; 4788 } 4789 4790 static int 4791 ixl_register_rss_key(struct ixl_softc *sc) 4792 { 4793 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4794 int rv; 4795 size_t i; 4796 4797 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4798 4799 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)){ 4800 rv = ixl_set_rss_key(sc, (uint8_t*)rss_seed, 4801 sizeof(rss_seed)); 4802 } else { 4803 rv = 0; 4804 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4805 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4806 } 4807 } 4808 4809 return rv; 4810 } 4811 4812 static void 4813 ixl_register_rss_pctype(struct ixl_softc *sc) 4814 { 4815 uint64_t set_hena = 0; 4816 uint32_t hena0, hena1; 4817 4818 /* 4819 * We use TCP/UDP with IPv4/IPv6 by default. 4820 * Note: the device can not use just IP header in each 4821 * TCP/UDP packets for the RSS hash calculation. 4822 */ 4823 if (sc->sc_mac_type == I40E_MAC_X722) 4824 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4825 else 4826 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4827 4828 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4829 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4830 4831 SET(hena0, set_hena); 4832 SET(hena1, set_hena >> 32); 4833 4834 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4835 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4836 } 4837 4838 static int 4839 ixl_register_rss_hlut(struct ixl_softc *sc) 4840 { 4841 unsigned int qid; 4842 uint8_t hlut_buf[512], lut_mask; 4843 uint32_t *hluts; 4844 size_t i, hluts_num; 4845 int rv; 4846 4847 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4848 4849 for (i = 0; i < sc->sc_rss_table_size; i++) { 4850 qid = i % sc->sc_nqueue_pairs; 4851 hlut_buf[i] = qid & lut_mask; 4852 } 4853 4854 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4855 rv = ixl_set_rss_lut(sc, hlut_buf, sizeof(hlut_buf)); 4856 } else { 4857 rv = 0; 4858 hluts = (uint32_t *)hlut_buf; 4859 hluts_num = sc->sc_rss_table_size >> 2; 4860 for (i = 0; i < hluts_num; i++) { 4861 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4862 } 4863 ixl_flush(sc); 4864 } 4865 4866 return rv; 4867 } 4868 4869 static void 4870 ixl_config_rss(struct ixl_softc *sc) 4871 { 4872 4873 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4874 4875 ixl_register_rss_key(sc); 4876 ixl_register_rss_pctype(sc); 4877 ixl_register_rss_hlut(sc); 4878 } 4879 4880 static const struct ixl_phy_type * 4881 ixl_search_phy_type(uint8_t phy_type) 4882 { 4883 const struct ixl_phy_type *itype; 4884 uint64_t mask; 4885 unsigned int i; 4886 4887 if (phy_type >= 64) 4888 return NULL; 4889 4890 mask = 1ULL << phy_type; 4891 4892 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4893 itype = &ixl_phy_type_map[i]; 4894 4895 if (ISSET(itype->phy_type, mask)) 4896 return itype; 4897 } 4898 4899 return NULL; 4900 } 4901 4902 static uint64_t 4903 ixl_search_link_speed(uint8_t link_speed) 4904 { 4905 const struct ixl_speed_type *type; 4906 unsigned int i; 4907 4908 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4909 type = &ixl_speed_type_map[i]; 4910 4911 if (ISSET(type->dev_speed, link_speed)) 4912 return type->net_speed; 4913 } 4914 4915 return 0; 4916 } 4917 4918 static uint8_t 4919 ixl_search_baudrate(uint64_t baudrate) 4920 { 4921 const struct ixl_speed_type *type; 4922 unsigned int i; 4923 4924 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4925 type = &ixl_speed_type_map[i]; 4926 4927 if (type->net_speed == baudrate) { 4928 return type->dev_speed; 4929 } 4930 } 4931 4932 return 0; 4933 } 4934 4935 static int 4936 ixl_restart_an(struct ixl_softc *sc) 4937 { 4938 struct ixl_aq_desc iaq; 4939 4940 memset(&iaq, 0, sizeof(iaq)); 4941 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4942 iaq.iaq_param[0] = 4943 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4944 4945 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4946 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4947 return -1; 4948 } 4949 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4950 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4951 return -1; 4952 } 4953 4954 return 0; 4955 } 4956 4957 static int 4958 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4959 uint16_t vlan, uint16_t flags) 4960 { 4961 struct ixl_aq_desc iaq; 4962 struct ixl_aq_add_macvlan *param; 4963 struct ixl_aq_add_macvlan_elem *elem; 4964 4965 memset(&iaq, 0, sizeof(iaq)); 4966 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4967 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4968 iaq.iaq_datalen = htole16(sizeof(*elem)); 4969 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4970 4971 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4972 param->num_addrs = htole16(1); 4973 param->seid0 = htole16(0x8000) | sc->sc_seid; 4974 param->seid1 = 0; 4975 param->seid2 = 0; 4976 4977 elem = IXL_DMA_KVA(&sc->sc_scratch); 4978 memset(elem, 0, sizeof(*elem)); 4979 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4980 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4981 elem->vlan = htole16(vlan); 4982 4983 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4984 return IXL_AQ_RC_EINVAL; 4985 } 4986 4987 switch (le16toh(iaq.iaq_retval)) { 4988 case IXL_AQ_RC_OK: 4989 break; 4990 case IXL_AQ_RC_ENOSPC: 4991 return ENOSPC; 4992 case IXL_AQ_RC_ENOENT: 4993 return ENOENT; 4994 case IXL_AQ_RC_EACCES: 4995 return EACCES; 4996 case IXL_AQ_RC_EEXIST: 4997 return EEXIST; 4998 case IXL_AQ_RC_EINVAL: 4999 return EINVAL; 5000 default: 5001 return EIO; 5002 } 5003 5004 return 0; 5005 } 5006 5007 static int 5008 ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 5009 uint16_t vlan, uint16_t flags) 5010 { 5011 struct ixl_aq_desc iaq; 5012 struct ixl_aq_remove_macvlan *param; 5013 struct ixl_aq_remove_macvlan_elem *elem; 5014 5015 memset(&iaq, 0, sizeof(iaq)); 5016 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 5017 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 5018 iaq.iaq_datalen = htole16(sizeof(*elem)); 5019 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 5020 5021 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 5022 param->num_addrs = htole16(1); 5023 param->seid0 = htole16(0x8000) | sc->sc_seid; 5024 param->seid1 = 0; 5025 param->seid2 = 0; 5026 5027 elem = IXL_DMA_KVA(&sc->sc_scratch); 5028 memset(elem, 0, sizeof(*elem)); 5029 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 5030 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 5031 elem->vlan = htole16(vlan); 5032 5033 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 5034 return EINVAL; 5035 } 5036 5037 switch (le16toh(iaq.iaq_retval)) { 5038 case IXL_AQ_RC_OK: 5039 break; 5040 case IXL_AQ_RC_ENOENT: 5041 return ENOENT; 5042 case IXL_AQ_RC_EACCES: 5043 return EACCES; 5044 case IXL_AQ_RC_EINVAL: 5045 return EINVAL; 5046 default: 5047 return EIO; 5048 } 5049 5050 return 0; 5051 } 5052 5053 static int 5054 ixl_hmc(struct ixl_softc *sc) 5055 { 5056 struct { 5057 uint32_t count; 5058 uint32_t minsize; 5059 bus_size_t objsiz; 5060 bus_size_t setoff; 5061 bus_size_t setcnt; 5062 } regs[] = { 5063 { 5064 0, 5065 IXL_HMC_TXQ_MINSIZE, 5066 I40E_GLHMC_LANTXOBJSZ, 5067 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 5068 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 5069 }, 5070 { 5071 0, 5072 IXL_HMC_RXQ_MINSIZE, 5073 I40E_GLHMC_LANRXOBJSZ, 5074 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 5075 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 5076 }, 5077 { 5078 0, 5079 0, 5080 I40E_GLHMC_FCOEDDPOBJSZ, 5081 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 5082 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 5083 }, 5084 { 5085 0, 5086 0, 5087 I40E_GLHMC_FCOEFOBJSZ, 5088 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 5089 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 5090 }, 5091 }; 5092 struct ixl_hmc_entry *e; 5093 uint64_t size, dva; 5094 uint8_t *kva; 5095 uint64_t *sdpage; 5096 unsigned int i; 5097 int npages, tables; 5098 uint32_t reg; 5099 5100 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 5101 5102 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 5103 ixl_rd(sc, I40E_GLHMC_LANQMAX); 5104 5105 size = 0; 5106 for (i = 0; i < __arraycount(regs); i++) { 5107 e = &sc->sc_hmc_entries[i]; 5108 5109 e->hmc_count = regs[i].count; 5110 reg = ixl_rd(sc, regs[i].objsiz); 5111 e->hmc_size = BIT_ULL(0x3F & reg); 5112 e->hmc_base = size; 5113 5114 if ((e->hmc_size * 8) < regs[i].minsize) { 5115 aprint_error_dev(sc->sc_dev, 5116 "kernel hmc entry is too big\n"); 5117 return -1; 5118 } 5119 5120 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 5121 } 5122 size = roundup(size, IXL_HMC_PGSIZE); 5123 npages = size / IXL_HMC_PGSIZE; 5124 5125 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 5126 5127 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 5128 aprint_error_dev(sc->sc_dev, 5129 "unable to allocate hmc pd memory\n"); 5130 return -1; 5131 } 5132 5133 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 5134 IXL_HMC_PGSIZE) != 0) { 5135 aprint_error_dev(sc->sc_dev, 5136 "unable to allocate hmc sd memory\n"); 5137 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5138 return -1; 5139 } 5140 5141 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 5142 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 5143 5144 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 5145 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 5146 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5147 5148 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 5149 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 5150 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 5151 5152 for (i = 0; (int)i < npages; i++) { 5153 *sdpage = htole64(dva | IXL_HMC_PDVALID); 5154 sdpage++; 5155 5156 dva += IXL_HMC_PGSIZE; 5157 } 5158 5159 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 5160 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 5161 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5162 5163 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 5164 for (i = 0; (int)i < tables; i++) { 5165 uint32_t count; 5166 5167 KASSERT(npages >= 0); 5168 5169 count = ((unsigned int)npages > IXL_HMC_PGS) ? 5170 IXL_HMC_PGS : (unsigned int)npages; 5171 5172 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 5173 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 5174 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 5175 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 5176 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 5177 ixl_wr(sc, I40E_PFHMC_SDCMD, 5178 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 5179 5180 npages -= IXL_HMC_PGS; 5181 dva += IXL_HMC_PGSIZE; 5182 } 5183 5184 for (i = 0; i < __arraycount(regs); i++) { 5185 e = &sc->sc_hmc_entries[i]; 5186 5187 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 5188 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 5189 } 5190 5191 return 0; 5192 } 5193 5194 static void 5195 ixl_hmc_free(struct ixl_softc *sc) 5196 { 5197 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 5198 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5199 } 5200 5201 static void 5202 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 5203 unsigned int npacking) 5204 { 5205 uint8_t *dst = d; 5206 const uint8_t *src = s; 5207 unsigned int i; 5208 5209 for (i = 0; i < npacking; i++) { 5210 const struct ixl_hmc_pack *pack = &packing[i]; 5211 unsigned int offset = pack->lsb / 8; 5212 unsigned int align = pack->lsb % 8; 5213 const uint8_t *in = src + pack->offset; 5214 uint8_t *out = dst + offset; 5215 int width = pack->width; 5216 unsigned int inbits = 0; 5217 5218 if (align) { 5219 inbits = (*in++) << align; 5220 *out++ |= (inbits & 0xff); 5221 inbits >>= 8; 5222 5223 width -= 8 - align; 5224 } 5225 5226 while (width >= 8) { 5227 inbits |= (*in++) << align; 5228 *out++ = (inbits & 0xff); 5229 inbits >>= 8; 5230 5231 width -= 8; 5232 } 5233 5234 if (width > 0) { 5235 inbits |= (*in) << align; 5236 *out |= (inbits & ((1 << width) - 1)); 5237 } 5238 } 5239 } 5240 5241 static struct ixl_aq_buf * 5242 ixl_aqb_alloc(struct ixl_softc *sc) 5243 { 5244 struct ixl_aq_buf *aqb; 5245 5246 aqb = kmem_alloc(sizeof(*aqb), KM_SLEEP); 5247 5248 aqb->aqb_size = IXL_AQ_BUFLEN; 5249 5250 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 5251 aqb->aqb_size, 0, 5252 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 5253 goto free; 5254 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 5255 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 5256 BUS_DMA_WAITOK) != 0) 5257 goto destroy; 5258 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 5259 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 5260 goto dma_free; 5261 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 5262 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 5263 goto unmap; 5264 5265 return aqb; 5266 unmap: 5267 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5268 dma_free: 5269 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5270 destroy: 5271 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5272 free: 5273 kmem_free(aqb, sizeof(*aqb)); 5274 5275 return NULL; 5276 } 5277 5278 static void 5279 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 5280 { 5281 5282 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 5283 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5284 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5285 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5286 kmem_free(aqb, sizeof(*aqb)); 5287 } 5288 5289 static int 5290 ixl_arq_fill(struct ixl_softc *sc) 5291 { 5292 struct ixl_aq_buf *aqb; 5293 struct ixl_aq_desc *arq, *iaq; 5294 unsigned int prod = sc->sc_arq_prod; 5295 unsigned int n; 5296 int post = 0; 5297 5298 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 5299 IXL_AQ_NUM); 5300 arq = IXL_DMA_KVA(&sc->sc_arq); 5301 5302 if (__predict_false(n <= 0)) 5303 return 0; 5304 5305 do { 5306 aqb = sc->sc_arq_live[prod]; 5307 iaq = &arq[prod]; 5308 5309 if (aqb == NULL) { 5310 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 5311 if (aqb != NULL) { 5312 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5313 ixl_aq_buf, aqb_entry); 5314 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 5315 break; 5316 } 5317 5318 sc->sc_arq_live[prod] = aqb; 5319 memset(aqb->aqb_data, 0, aqb->aqb_size); 5320 5321 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 5322 aqb->aqb_size, BUS_DMASYNC_PREREAD); 5323 5324 iaq->iaq_flags = htole16(IXL_AQ_BUF | 5325 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 5326 IXL_AQ_LB : 0)); 5327 iaq->iaq_opcode = 0; 5328 iaq->iaq_datalen = htole16(aqb->aqb_size); 5329 iaq->iaq_retval = 0; 5330 iaq->iaq_cookie = 0; 5331 iaq->iaq_param[0] = 0; 5332 iaq->iaq_param[1] = 0; 5333 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 5334 } 5335 5336 prod++; 5337 prod &= IXL_AQ_MASK; 5338 5339 post = 1; 5340 5341 } while (--n); 5342 5343 if (post) { 5344 sc->sc_arq_prod = prod; 5345 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 5346 } 5347 5348 return post; 5349 } 5350 5351 static void 5352 ixl_arq_unfill(struct ixl_softc *sc) 5353 { 5354 struct ixl_aq_buf *aqb; 5355 unsigned int i; 5356 5357 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 5358 aqb = sc->sc_arq_live[i]; 5359 if (aqb == NULL) 5360 continue; 5361 5362 sc->sc_arq_live[i] = NULL; 5363 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 5364 BUS_DMASYNC_POSTREAD); 5365 ixl_aqb_free(sc, aqb); 5366 } 5367 5368 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 5369 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5370 ixl_aq_buf, aqb_entry); 5371 ixl_aqb_free(sc, aqb); 5372 } 5373 } 5374 5375 static void 5376 ixl_clear_hw(struct ixl_softc *sc) 5377 { 5378 uint32_t num_queues, base_queue; 5379 uint32_t num_pf_int; 5380 uint32_t num_vf_int; 5381 uint32_t num_vfs; 5382 uint32_t i, j; 5383 uint32_t val; 5384 uint32_t eol = 0x7ff; 5385 5386 /* get number of interrupts, queues, and vfs */ 5387 val = ixl_rd(sc, I40E_GLPCI_CNF2); 5388 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 5389 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 5390 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 5391 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 5392 5393 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 5394 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 5395 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 5396 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 5397 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 5398 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 5399 num_queues = (j - base_queue) + 1; 5400 else 5401 num_queues = 0; 5402 5403 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 5404 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 5405 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 5406 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 5407 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 5408 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 5409 num_vfs = (j - i) + 1; 5410 else 5411 num_vfs = 0; 5412 5413 /* stop all the interrupts */ 5414 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5415 ixl_flush(sc); 5416 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 5417 for (i = 0; i < num_pf_int - 2; i++) 5418 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 5419 ixl_flush(sc); 5420 5421 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 5422 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5423 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 5424 for (i = 0; i < num_pf_int - 2; i++) 5425 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 5426 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5427 for (i = 0; i < num_vfs; i++) 5428 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 5429 for (i = 0; i < num_vf_int - 2; i++) 5430 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 5431 5432 /* warn the HW of the coming Tx disables */ 5433 for (i = 0; i < num_queues; i++) { 5434 uint32_t abs_queue_idx = base_queue + i; 5435 uint32_t reg_block = 0; 5436 5437 if (abs_queue_idx >= 128) { 5438 reg_block = abs_queue_idx / 128; 5439 abs_queue_idx %= 128; 5440 } 5441 5442 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 5443 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 5444 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 5445 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 5446 5447 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 5448 } 5449 delaymsec(400); 5450 5451 /* stop all the queues */ 5452 for (i = 0; i < num_queues; i++) { 5453 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 5454 ixl_wr(sc, I40E_QTX_ENA(i), 0); 5455 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 5456 ixl_wr(sc, I40E_QRX_ENA(i), 0); 5457 } 5458 5459 /* short wait for all queue disables to settle */ 5460 delaymsec(50); 5461 } 5462 5463 static int 5464 ixl_pf_reset(struct ixl_softc *sc) 5465 { 5466 uint32_t cnt = 0; 5467 uint32_t cnt1 = 0; 5468 uint32_t reg = 0, reg0 = 0; 5469 uint32_t grst_del; 5470 5471 /* 5472 * Poll for Global Reset steady state in case of recent GRST. 5473 * The grst delay value is in 100ms units, and we'll wait a 5474 * couple counts longer to be sure we don't just miss the end. 5475 */ 5476 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 5477 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 5478 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 5479 5480 grst_del = grst_del * 20; 5481 5482 for (cnt = 0; cnt < grst_del; cnt++) { 5483 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 5484 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 5485 break; 5486 delaymsec(100); 5487 } 5488 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5489 aprint_error(", Global reset polling failed to complete\n"); 5490 return -1; 5491 } 5492 5493 /* Now Wait for the FW to be ready */ 5494 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 5495 reg = ixl_rd(sc, I40E_GLNVM_ULD); 5496 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5497 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 5498 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5499 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 5500 break; 5501 5502 delaymsec(10); 5503 } 5504 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5505 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 5506 aprint_error(", wait for FW Reset complete timed out " 5507 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 5508 return -1; 5509 } 5510 5511 /* 5512 * If there was a Global Reset in progress when we got here, 5513 * we don't need to do the PF Reset 5514 */ 5515 if (cnt == 0) { 5516 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5517 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 5518 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 5519 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5520 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 5521 break; 5522 delaymsec(1); 5523 5524 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 5525 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5526 aprint_error(", Core reset upcoming." 5527 " Skipping PF reset reset request\n"); 5528 return -1; 5529 } 5530 } 5531 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 5532 aprint_error(", PF reset polling failed to complete" 5533 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 5534 return -1; 5535 } 5536 } 5537 5538 return 0; 5539 } 5540 5541 static int 5542 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 5543 bus_size_t size, bus_size_t align) 5544 { 5545 ixm->ixm_size = size; 5546 5547 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 5548 ixm->ixm_size, 0, 5549 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 5550 &ixm->ixm_map) != 0) 5551 return 1; 5552 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 5553 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 5554 BUS_DMA_WAITOK) != 0) 5555 goto destroy; 5556 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 5557 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 5558 goto free; 5559 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 5560 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 5561 goto unmap; 5562 5563 memset(ixm->ixm_kva, 0, ixm->ixm_size); 5564 5565 return 0; 5566 unmap: 5567 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5568 free: 5569 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5570 destroy: 5571 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5572 return 1; 5573 } 5574 5575 static void 5576 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 5577 { 5578 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 5579 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5580 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5581 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5582 } 5583 5584 static int 5585 ixl_setup_vlan_hwfilter(struct ixl_softc *sc) 5586 { 5587 struct ethercom *ec = &sc->sc_ec; 5588 struct vlanid_list *vlanidp; 5589 int rv; 5590 5591 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5592 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5593 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5594 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5595 5596 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5597 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5598 if (rv != 0) 5599 return rv; 5600 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5601 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5602 if (rv != 0) 5603 return rv; 5604 5605 ETHER_LOCK(ec); 5606 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5607 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 5608 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5609 if (rv != 0) 5610 break; 5611 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 5612 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5613 if (rv != 0) 5614 break; 5615 } 5616 ETHER_UNLOCK(ec); 5617 5618 return rv; 5619 } 5620 5621 static void 5622 ixl_teardown_vlan_hwfilter(struct ixl_softc *sc) 5623 { 5624 struct vlanid_list *vlanidp; 5625 struct ethercom *ec = &sc->sc_ec; 5626 5627 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5628 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5629 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5630 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5631 5632 ETHER_LOCK(ec); 5633 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5634 ixl_remove_macvlan(sc, sc->sc_enaddr, 5635 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5636 ixl_remove_macvlan(sc, etherbroadcastaddr, 5637 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5638 } 5639 ETHER_UNLOCK(ec); 5640 5641 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5642 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5643 ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5644 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5645 } 5646 5647 static int 5648 ixl_update_macvlan(struct ixl_softc *sc) 5649 { 5650 int rv = 0; 5651 int next_ec_capenable = sc->sc_ec.ec_capenable; 5652 5653 if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 5654 rv = ixl_setup_vlan_hwfilter(sc); 5655 if (rv != 0) 5656 ixl_teardown_vlan_hwfilter(sc); 5657 } else { 5658 ixl_teardown_vlan_hwfilter(sc); 5659 } 5660 5661 return rv; 5662 } 5663 5664 static int 5665 ixl_ifflags_cb(struct ethercom *ec) 5666 { 5667 struct ifnet *ifp = &ec->ec_if; 5668 struct ixl_softc *sc = ifp->if_softc; 5669 int rv, change; 5670 5671 mutex_enter(&sc->sc_cfg_lock); 5672 5673 change = ec->ec_capenable ^ sc->sc_cur_ec_capenable; 5674 5675 if (ISSET(change, ETHERCAP_VLAN_HWTAGGING)) { 5676 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 5677 rv = ENETRESET; 5678 goto out; 5679 } 5680 5681 if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) { 5682 rv = ixl_update_macvlan(sc); 5683 if (rv == 0) { 5684 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 5685 } else { 5686 CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER); 5687 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 5688 } 5689 } 5690 5691 rv = ixl_iff(sc); 5692 out: 5693 mutex_exit(&sc->sc_cfg_lock); 5694 5695 return rv; 5696 } 5697 5698 static int 5699 ixl_set_link_status(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 5700 { 5701 const struct ixl_aq_link_status *status; 5702 const struct ixl_phy_type *itype; 5703 5704 uint64_t ifm_active = IFM_ETHER; 5705 uint64_t ifm_status = IFM_AVALID; 5706 int link_state = LINK_STATE_DOWN; 5707 uint64_t baudrate = 0; 5708 5709 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 5710 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) { 5711 ifm_active |= IFM_NONE; 5712 goto done; 5713 } 5714 5715 ifm_active |= IFM_FDX; 5716 ifm_status |= IFM_ACTIVE; 5717 link_state = LINK_STATE_UP; 5718 5719 itype = ixl_search_phy_type(status->phy_type); 5720 if (itype != NULL) 5721 ifm_active |= itype->ifm_type; 5722 5723 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 5724 ifm_active |= IFM_ETH_TXPAUSE; 5725 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 5726 ifm_active |= IFM_ETH_RXPAUSE; 5727 5728 baudrate = ixl_search_link_speed(status->link_speed); 5729 5730 done: 5731 sc->sc_media_active = ifm_active; 5732 sc->sc_media_status = ifm_status; 5733 5734 sc->sc_ec.ec_if.if_baudrate = baudrate; 5735 5736 return link_state; 5737 } 5738 5739 static int 5740 ixl_establish_intx(struct ixl_softc *sc) 5741 { 5742 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5743 pci_intr_handle_t *intr; 5744 char xnamebuf[32]; 5745 char intrbuf[PCI_INTRSTR_LEN]; 5746 char const *intrstr; 5747 5748 KASSERT(sc->sc_nintrs == 1); 5749 5750 intr = &sc->sc_ihp[0]; 5751 5752 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 5753 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 5754 device_xname(sc->sc_dev)); 5755 5756 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 5757 sc, xnamebuf); 5758 5759 if (sc->sc_ihs[0] == NULL) { 5760 aprint_error_dev(sc->sc_dev, 5761 "unable to establish interrupt at %s\n", intrstr); 5762 return -1; 5763 } 5764 5765 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 5766 return 0; 5767 } 5768 5769 static int 5770 ixl_establish_msix(struct ixl_softc *sc) 5771 { 5772 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5773 kcpuset_t *affinity; 5774 unsigned int vector = 0; 5775 unsigned int i; 5776 int affinity_to, r; 5777 char xnamebuf[32]; 5778 char intrbuf[PCI_INTRSTR_LEN]; 5779 char const *intrstr; 5780 5781 kcpuset_create(&affinity, false); 5782 5783 /* the "other" intr is mapped to vector 0 */ 5784 vector = 0; 5785 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5786 intrbuf, sizeof(intrbuf)); 5787 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 5788 device_xname(sc->sc_dev)); 5789 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5790 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 5791 sc, xnamebuf); 5792 if (sc->sc_ihs[vector] == NULL) { 5793 aprint_error_dev(sc->sc_dev, 5794 "unable to establish interrupt at %s\n", intrstr); 5795 goto fail; 5796 } 5797 5798 aprint_normal_dev(sc->sc_dev, "other interrupt at %s", intrstr); 5799 5800 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5801 affinity_to = (affinity_to + sc->sc_nqueue_pairs_max) % ncpu; 5802 5803 kcpuset_zero(affinity); 5804 kcpuset_set(affinity, affinity_to); 5805 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5806 if (r == 0) { 5807 aprint_normal(", affinity to %u", affinity_to); 5808 } 5809 aprint_normal("\n"); 5810 vector++; 5811 5812 sc->sc_msix_vector_queue = vector; 5813 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5814 5815 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5816 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5817 intrbuf, sizeof(intrbuf)); 5818 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 5819 device_xname(sc->sc_dev), i); 5820 5821 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5822 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 5823 (void *)&sc->sc_qps[i], xnamebuf); 5824 5825 if (sc->sc_ihs[vector] == NULL) { 5826 aprint_error_dev(sc->sc_dev, 5827 "unable to establish interrupt at %s\n", intrstr); 5828 goto fail; 5829 } 5830 5831 aprint_normal_dev(sc->sc_dev, 5832 "for TXRX%d interrupt at %s",i , intrstr); 5833 5834 kcpuset_zero(affinity); 5835 kcpuset_set(affinity, affinity_to); 5836 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5837 if (r == 0) { 5838 aprint_normal(", affinity to %u", affinity_to); 5839 affinity_to = (affinity_to + 1) % ncpu; 5840 } 5841 aprint_normal("\n"); 5842 vector++; 5843 } 5844 5845 kcpuset_destroy(affinity); 5846 5847 return 0; 5848 fail: 5849 for (i = 0; i < vector; i++) { 5850 pci_intr_disestablish(pc, sc->sc_ihs[i]); 5851 } 5852 5853 sc->sc_msix_vector_queue = 0; 5854 sc->sc_msix_vector_queue = 0; 5855 kcpuset_destroy(affinity); 5856 5857 return -1; 5858 } 5859 5860 static void 5861 ixl_config_queue_intr(struct ixl_softc *sc) 5862 { 5863 unsigned int i, vector; 5864 5865 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5866 vector = sc->sc_msix_vector_queue; 5867 } else { 5868 vector = I40E_INTR_NOTX_INTR; 5869 5870 ixl_wr(sc, I40E_PFINT_LNKLST0, 5871 (I40E_INTR_NOTX_QUEUE << 5872 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5873 (I40E_QUEUE_TYPE_RX << 5874 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5875 } 5876 5877 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5878 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5879 ixl_flush(sc); 5880 5881 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5882 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5883 (I40E_QUEUE_TYPE_RX << 5884 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5885 5886 ixl_wr(sc, I40E_QINT_RQCTL(i), 5887 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5888 (I40E_ITR_INDEX_RX << 5889 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5890 (I40E_INTR_NOTX_RX_QUEUE << 5891 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5892 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5893 (I40E_QUEUE_TYPE_TX << 5894 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5895 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5896 5897 ixl_wr(sc, I40E_QINT_TQCTL(i), 5898 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5899 (I40E_ITR_INDEX_TX << 5900 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5901 (I40E_INTR_NOTX_TX_QUEUE << 5902 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5903 (I40E_QUEUE_TYPE_EOL << 5904 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5905 (I40E_QUEUE_TYPE_RX << 5906 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5907 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5908 5909 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) 5910 vector++; 5911 } 5912 ixl_flush(sc); 5913 5914 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), 0x7a); 5915 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), 0x7a); 5916 ixl_flush(sc); 5917 } 5918 5919 static void 5920 ixl_config_other_intr(struct ixl_softc *sc) 5921 { 5922 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5923 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5924 5925 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5926 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5927 I40E_PFINT_ICR0_ENA_GRST_MASK | 5928 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5929 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5930 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5931 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5932 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5933 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5934 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5935 5936 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5937 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5938 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5939 (I40E_ITR_INDEX_OTHER << 5940 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5941 ixl_flush(sc); 5942 } 5943 5944 static int 5945 ixl_setup_interrupts(struct ixl_softc *sc) 5946 { 5947 struct pci_attach_args *pa = &sc->sc_pa; 5948 pci_intr_type_t max_type, intr_type; 5949 int counts[PCI_INTR_TYPE_SIZE]; 5950 int error; 5951 unsigned int i; 5952 bool retry; 5953 5954 memset(counts, 0, sizeof(counts)); 5955 max_type = PCI_INTR_TYPE_MSIX; 5956 /* QPs + other interrupt */ 5957 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5958 counts[PCI_INTR_TYPE_INTX] = 1; 5959 5960 if (ixl_param_nomsix) 5961 counts[PCI_INTR_TYPE_MSIX] = 0; 5962 5963 do { 5964 retry = false; 5965 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5966 if (error != 0) { 5967 aprint_error_dev(sc->sc_dev, 5968 "couldn't map interrupt\n"); 5969 break; 5970 } 5971 5972 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5973 sc->sc_nintrs = counts[intr_type]; 5974 KASSERT(sc->sc_nintrs > 0); 5975 5976 for (i = 0; i < sc->sc_nintrs; i++) { 5977 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5978 PCI_INTR_MPSAFE, true); 5979 } 5980 5981 sc->sc_ihs = kmem_alloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5982 KM_SLEEP); 5983 5984 if (intr_type == PCI_INTR_TYPE_MSIX) { 5985 error = ixl_establish_msix(sc); 5986 if (error) { 5987 counts[PCI_INTR_TYPE_MSIX] = 0; 5988 retry = true; 5989 } 5990 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5991 error = ixl_establish_intx(sc); 5992 } else { 5993 error = -1; 5994 } 5995 5996 if (error) { 5997 kmem_free(sc->sc_ihs, 5998 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5999 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 6000 } else { 6001 sc->sc_intrtype = intr_type; 6002 } 6003 } while (retry); 6004 6005 return error; 6006 } 6007 6008 static void 6009 ixl_teardown_interrupts(struct ixl_softc *sc) 6010 { 6011 struct pci_attach_args *pa = &sc->sc_pa; 6012 unsigned int i; 6013 6014 for (i = 0; i < sc->sc_nintrs; i++) { 6015 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 6016 } 6017 6018 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 6019 6020 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 6021 sc->sc_ihs = NULL; 6022 sc->sc_nintrs = 0; 6023 } 6024 6025 static int 6026 ixl_setup_stats(struct ixl_softc *sc) 6027 { 6028 struct ixl_queue_pair *qp; 6029 struct ixl_tx_ring *txr; 6030 struct ixl_rx_ring *rxr; 6031 struct ixl_stats_counters *isc; 6032 unsigned int i; 6033 6034 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6035 qp = &sc->sc_qps[i]; 6036 txr = qp->qp_txr; 6037 rxr = qp->qp_rxr; 6038 6039 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 6040 NULL, qp->qp_name, "m_defrag successed"); 6041 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 6042 NULL, qp->qp_name, "m_defrag_failed"); 6043 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 6044 NULL, qp->qp_name, "Dropped in pcq"); 6045 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 6046 NULL, qp->qp_name, "Deferred transmit"); 6047 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 6048 NULL, qp->qp_name, "Interrupt on queue"); 6049 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 6050 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6051 6052 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 6053 NULL, qp->qp_name, "MGETHDR failed"); 6054 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 6055 NULL, qp->qp_name, "MCLGET failed"); 6056 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 6057 EVCNT_TYPE_MISC, NULL, qp->qp_name, 6058 "bus_dmamap_load_mbuf failed"); 6059 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 6060 NULL, qp->qp_name, "Interrupt on queue"); 6061 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 6062 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6063 } 6064 6065 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 6066 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 6067 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 6068 NULL, device_xname(sc->sc_dev), "Link status event"); 6069 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 6070 NULL, device_xname(sc->sc_dev), "ECC error"); 6071 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 6072 NULL, device_xname(sc->sc_dev), "PCI exception"); 6073 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 6074 NULL, device_xname(sc->sc_dev), "Critical error"); 6075 6076 isc = &sc->sc_stats_counters; 6077 evcnt_attach_dynamic(&isc->isc_crc_errors, EVCNT_TYPE_MISC, 6078 NULL, device_xname(sc->sc_dev), "CRC errors"); 6079 evcnt_attach_dynamic(&isc->isc_illegal_bytes, EVCNT_TYPE_MISC, 6080 NULL, device_xname(sc->sc_dev), "Illegal bytes"); 6081 evcnt_attach_dynamic(&isc->isc_mac_local_faults, EVCNT_TYPE_MISC, 6082 NULL, device_xname(sc->sc_dev), "Mac local faults"); 6083 evcnt_attach_dynamic(&isc->isc_mac_remote_faults, EVCNT_TYPE_MISC, 6084 NULL, device_xname(sc->sc_dev), "Mac remote faults"); 6085 evcnt_attach_dynamic(&isc->isc_link_xon_rx, EVCNT_TYPE_MISC, 6086 NULL, device_xname(sc->sc_dev), "Rx xon"); 6087 evcnt_attach_dynamic(&isc->isc_link_xon_tx, EVCNT_TYPE_MISC, 6088 NULL, device_xname(sc->sc_dev), "Tx xon"); 6089 evcnt_attach_dynamic(&isc->isc_link_xoff_rx, EVCNT_TYPE_MISC, 6090 NULL, device_xname(sc->sc_dev), "Rx xoff"); 6091 evcnt_attach_dynamic(&isc->isc_link_xoff_tx, EVCNT_TYPE_MISC, 6092 NULL, device_xname(sc->sc_dev), "Tx xoff"); 6093 evcnt_attach_dynamic(&isc->isc_rx_fragments, EVCNT_TYPE_MISC, 6094 NULL, device_xname(sc->sc_dev), "Rx fragments"); 6095 evcnt_attach_dynamic(&isc->isc_rx_jabber, EVCNT_TYPE_MISC, 6096 NULL, device_xname(sc->sc_dev), "Rx jabber"); 6097 6098 evcnt_attach_dynamic(&isc->isc_rx_size_64, EVCNT_TYPE_MISC, 6099 NULL, device_xname(sc->sc_dev), "Rx size 64"); 6100 evcnt_attach_dynamic(&isc->isc_rx_size_127, EVCNT_TYPE_MISC, 6101 NULL, device_xname(sc->sc_dev), "Rx size 127"); 6102 evcnt_attach_dynamic(&isc->isc_rx_size_255, EVCNT_TYPE_MISC, 6103 NULL, device_xname(sc->sc_dev), "Rx size 255"); 6104 evcnt_attach_dynamic(&isc->isc_rx_size_511, EVCNT_TYPE_MISC, 6105 NULL, device_xname(sc->sc_dev), "Rx size 511"); 6106 evcnt_attach_dynamic(&isc->isc_rx_size_1023, EVCNT_TYPE_MISC, 6107 NULL, device_xname(sc->sc_dev), "Rx size 1023"); 6108 evcnt_attach_dynamic(&isc->isc_rx_size_1522, EVCNT_TYPE_MISC, 6109 NULL, device_xname(sc->sc_dev), "Rx size 1522"); 6110 evcnt_attach_dynamic(&isc->isc_rx_size_big, EVCNT_TYPE_MISC, 6111 NULL, device_xname(sc->sc_dev), "Rx jumbo packets"); 6112 evcnt_attach_dynamic(&isc->isc_rx_undersize, EVCNT_TYPE_MISC, 6113 NULL, device_xname(sc->sc_dev), "Rx under size"); 6114 evcnt_attach_dynamic(&isc->isc_rx_oversize, EVCNT_TYPE_MISC, 6115 NULL, device_xname(sc->sc_dev), "Rx over size"); 6116 6117 evcnt_attach_dynamic(&isc->isc_rx_bytes, EVCNT_TYPE_MISC, 6118 NULL, device_xname(sc->sc_dev), "Rx bytes / port"); 6119 evcnt_attach_dynamic(&isc->isc_rx_discards, EVCNT_TYPE_MISC, 6120 NULL, device_xname(sc->sc_dev), "Rx discards / port"); 6121 evcnt_attach_dynamic(&isc->isc_rx_unicast, EVCNT_TYPE_MISC, 6122 NULL, device_xname(sc->sc_dev), "Rx unicast / port"); 6123 evcnt_attach_dynamic(&isc->isc_rx_multicast, EVCNT_TYPE_MISC, 6124 NULL, device_xname(sc->sc_dev), "Rx multicast / port"); 6125 evcnt_attach_dynamic(&isc->isc_rx_broadcast, EVCNT_TYPE_MISC, 6126 NULL, device_xname(sc->sc_dev), "Rx broadcast / port"); 6127 6128 evcnt_attach_dynamic(&isc->isc_vsi_rx_bytes, EVCNT_TYPE_MISC, 6129 NULL, device_xname(sc->sc_dev), "Rx bytes / vsi"); 6130 evcnt_attach_dynamic(&isc->isc_vsi_rx_discards, EVCNT_TYPE_MISC, 6131 NULL, device_xname(sc->sc_dev), "Rx discard / vsi"); 6132 evcnt_attach_dynamic(&isc->isc_vsi_rx_unicast, EVCNT_TYPE_MISC, 6133 NULL, device_xname(sc->sc_dev), "Rx unicast / vsi"); 6134 evcnt_attach_dynamic(&isc->isc_vsi_rx_multicast, EVCNT_TYPE_MISC, 6135 NULL, device_xname(sc->sc_dev), "Rx multicast / vsi"); 6136 evcnt_attach_dynamic(&isc->isc_vsi_rx_broadcast, EVCNT_TYPE_MISC, 6137 NULL, device_xname(sc->sc_dev), "Rx broadcast / vsi"); 6138 6139 evcnt_attach_dynamic(&isc->isc_tx_size_64, EVCNT_TYPE_MISC, 6140 NULL, device_xname(sc->sc_dev), "Tx size 64"); 6141 evcnt_attach_dynamic(&isc->isc_tx_size_127, EVCNT_TYPE_MISC, 6142 NULL, device_xname(sc->sc_dev), "Tx size 127"); 6143 evcnt_attach_dynamic(&isc->isc_tx_size_255, EVCNT_TYPE_MISC, 6144 NULL, device_xname(sc->sc_dev), "Tx size 255"); 6145 evcnt_attach_dynamic(&isc->isc_tx_size_511, EVCNT_TYPE_MISC, 6146 NULL, device_xname(sc->sc_dev), "Tx size 511"); 6147 evcnt_attach_dynamic(&isc->isc_tx_size_1023, EVCNT_TYPE_MISC, 6148 NULL, device_xname(sc->sc_dev), "Tx size 1023"); 6149 evcnt_attach_dynamic(&isc->isc_tx_size_1522, EVCNT_TYPE_MISC, 6150 NULL, device_xname(sc->sc_dev), "Tx size 1522"); 6151 evcnt_attach_dynamic(&isc->isc_tx_size_big, EVCNT_TYPE_MISC, 6152 NULL, device_xname(sc->sc_dev), "Tx jumbo packets"); 6153 6154 evcnt_attach_dynamic(&isc->isc_tx_bytes, EVCNT_TYPE_MISC, 6155 NULL, device_xname(sc->sc_dev), "Tx bytes / port"); 6156 evcnt_attach_dynamic(&isc->isc_tx_dropped_link_down, EVCNT_TYPE_MISC, 6157 NULL, device_xname(sc->sc_dev), 6158 "Tx dropped due to link down / port"); 6159 evcnt_attach_dynamic(&isc->isc_tx_unicast, EVCNT_TYPE_MISC, 6160 NULL, device_xname(sc->sc_dev), "Tx unicast / port"); 6161 evcnt_attach_dynamic(&isc->isc_tx_multicast, EVCNT_TYPE_MISC, 6162 NULL, device_xname(sc->sc_dev), "Tx multicast / port"); 6163 evcnt_attach_dynamic(&isc->isc_tx_broadcast, EVCNT_TYPE_MISC, 6164 NULL, device_xname(sc->sc_dev), "Tx broadcast / port"); 6165 6166 evcnt_attach_dynamic(&isc->isc_vsi_tx_bytes, EVCNT_TYPE_MISC, 6167 NULL, device_xname(sc->sc_dev), "Tx bytes / vsi"); 6168 evcnt_attach_dynamic(&isc->isc_vsi_tx_errors, EVCNT_TYPE_MISC, 6169 NULL, device_xname(sc->sc_dev), "Tx errors / vsi"); 6170 evcnt_attach_dynamic(&isc->isc_vsi_tx_unicast, EVCNT_TYPE_MISC, 6171 NULL, device_xname(sc->sc_dev), "Tx unicast / vsi"); 6172 evcnt_attach_dynamic(&isc->isc_vsi_tx_multicast, EVCNT_TYPE_MISC, 6173 NULL, device_xname(sc->sc_dev), "Tx multicast / vsi"); 6174 evcnt_attach_dynamic(&isc->isc_vsi_tx_broadcast, EVCNT_TYPE_MISC, 6175 NULL, device_xname(sc->sc_dev), "Tx broadcast / vsi"); 6176 6177 sc->sc_stats_intval = ixl_param_stats_interval; 6178 callout_init(&sc->sc_stats_callout, CALLOUT_MPSAFE); 6179 callout_setfunc(&sc->sc_stats_callout, ixl_stats_callout, sc); 6180 ixl_work_set(&sc->sc_stats_task, ixl_stats_update, sc); 6181 6182 return 0; 6183 } 6184 6185 static void 6186 ixl_teardown_stats(struct ixl_softc *sc) 6187 { 6188 struct ixl_tx_ring *txr; 6189 struct ixl_rx_ring *rxr; 6190 struct ixl_stats_counters *isc; 6191 unsigned int i; 6192 6193 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6194 txr = sc->sc_qps[i].qp_txr; 6195 rxr = sc->sc_qps[i].qp_rxr; 6196 6197 evcnt_detach(&txr->txr_defragged); 6198 evcnt_detach(&txr->txr_defrag_failed); 6199 evcnt_detach(&txr->txr_pcqdrop); 6200 evcnt_detach(&txr->txr_transmitdef); 6201 evcnt_detach(&txr->txr_intr); 6202 evcnt_detach(&txr->txr_defer); 6203 6204 evcnt_detach(&rxr->rxr_mgethdr_failed); 6205 evcnt_detach(&rxr->rxr_mgetcl_failed); 6206 evcnt_detach(&rxr->rxr_mbuf_load_failed); 6207 evcnt_detach(&rxr->rxr_intr); 6208 evcnt_detach(&rxr->rxr_defer); 6209 } 6210 6211 isc = &sc->sc_stats_counters; 6212 evcnt_detach(&isc->isc_crc_errors); 6213 evcnt_detach(&isc->isc_illegal_bytes); 6214 evcnt_detach(&isc->isc_mac_local_faults); 6215 evcnt_detach(&isc->isc_mac_remote_faults); 6216 evcnt_detach(&isc->isc_link_xon_rx); 6217 evcnt_detach(&isc->isc_link_xon_tx); 6218 evcnt_detach(&isc->isc_link_xoff_rx); 6219 evcnt_detach(&isc->isc_link_xoff_tx); 6220 evcnt_detach(&isc->isc_rx_fragments); 6221 evcnt_detach(&isc->isc_rx_jabber); 6222 evcnt_detach(&isc->isc_rx_bytes); 6223 evcnt_detach(&isc->isc_rx_discards); 6224 evcnt_detach(&isc->isc_rx_unicast); 6225 evcnt_detach(&isc->isc_rx_multicast); 6226 evcnt_detach(&isc->isc_rx_broadcast); 6227 evcnt_detach(&isc->isc_rx_size_64); 6228 evcnt_detach(&isc->isc_rx_size_127); 6229 evcnt_detach(&isc->isc_rx_size_255); 6230 evcnt_detach(&isc->isc_rx_size_511); 6231 evcnt_detach(&isc->isc_rx_size_1023); 6232 evcnt_detach(&isc->isc_rx_size_1522); 6233 evcnt_detach(&isc->isc_rx_size_big); 6234 evcnt_detach(&isc->isc_rx_undersize); 6235 evcnt_detach(&isc->isc_rx_oversize); 6236 evcnt_detach(&isc->isc_tx_bytes); 6237 evcnt_detach(&isc->isc_tx_dropped_link_down); 6238 evcnt_detach(&isc->isc_tx_unicast); 6239 evcnt_detach(&isc->isc_tx_multicast); 6240 evcnt_detach(&isc->isc_tx_broadcast); 6241 evcnt_detach(&isc->isc_tx_size_64); 6242 evcnt_detach(&isc->isc_tx_size_127); 6243 evcnt_detach(&isc->isc_tx_size_255); 6244 evcnt_detach(&isc->isc_tx_size_511); 6245 evcnt_detach(&isc->isc_tx_size_1023); 6246 evcnt_detach(&isc->isc_tx_size_1522); 6247 evcnt_detach(&isc->isc_tx_size_big); 6248 evcnt_detach(&isc->isc_vsi_rx_discards); 6249 evcnt_detach(&isc->isc_vsi_rx_bytes); 6250 evcnt_detach(&isc->isc_vsi_rx_unicast); 6251 evcnt_detach(&isc->isc_vsi_rx_multicast); 6252 evcnt_detach(&isc->isc_vsi_rx_broadcast); 6253 evcnt_detach(&isc->isc_vsi_tx_errors); 6254 evcnt_detach(&isc->isc_vsi_tx_bytes); 6255 evcnt_detach(&isc->isc_vsi_tx_unicast); 6256 evcnt_detach(&isc->isc_vsi_tx_multicast); 6257 evcnt_detach(&isc->isc_vsi_tx_broadcast); 6258 6259 evcnt_detach(&sc->sc_event_atq); 6260 evcnt_detach(&sc->sc_event_link); 6261 evcnt_detach(&sc->sc_event_ecc_err); 6262 evcnt_detach(&sc->sc_event_pci_exception); 6263 evcnt_detach(&sc->sc_event_crit_err); 6264 6265 callout_destroy(&sc->sc_stats_callout); 6266 } 6267 6268 static void 6269 ixl_stats_callout(void *xsc) 6270 { 6271 struct ixl_softc *sc = xsc; 6272 6273 ixl_work_add(sc->sc_workq, &sc->sc_stats_task); 6274 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 6275 } 6276 6277 static uint64_t 6278 ixl_stat_delta(struct ixl_softc *sc, uint32_t reg_hi, uint32_t reg_lo, 6279 uint64_t *offset, bool has_offset) 6280 { 6281 uint64_t value, delta; 6282 int bitwidth; 6283 6284 bitwidth = reg_hi == 0 ? 32 : 48; 6285 6286 value = ixl_rd(sc, reg_lo); 6287 6288 if (bitwidth > 32) { 6289 value |= ((uint64_t)ixl_rd(sc, reg_hi) << 32); 6290 } 6291 6292 if (__predict_true(has_offset)) { 6293 delta = value; 6294 if (value < *offset) 6295 delta += ((uint64_t)1 << bitwidth); 6296 delta -= *offset; 6297 } else { 6298 delta = 0; 6299 } 6300 atomic_swap_64(offset, value); 6301 6302 return delta; 6303 } 6304 6305 static void 6306 ixl_stats_update(void *xsc) 6307 { 6308 struct ixl_softc *sc = xsc; 6309 struct ixl_stats_counters *isc; 6310 uint64_t delta; 6311 6312 isc = &sc->sc_stats_counters; 6313 6314 /* errors */ 6315 delta = ixl_stat_delta(sc, 6316 0, I40E_GLPRT_CRCERRS(sc->sc_port), 6317 &isc->isc_crc_errors_offset, isc->isc_has_offset); 6318 atomic_add_64(&isc->isc_crc_errors.ev_count, delta); 6319 6320 delta = ixl_stat_delta(sc, 6321 0, I40E_GLPRT_ILLERRC(sc->sc_port), 6322 &isc->isc_illegal_bytes_offset, isc->isc_has_offset); 6323 atomic_add_64(&isc->isc_illegal_bytes.ev_count, delta); 6324 6325 /* rx */ 6326 delta = ixl_stat_delta(sc, 6327 I40E_GLPRT_GORCH(sc->sc_port), I40E_GLPRT_GORCL(sc->sc_port), 6328 &isc->isc_rx_bytes_offset, isc->isc_has_offset); 6329 atomic_add_64(&isc->isc_rx_bytes.ev_count, delta); 6330 6331 delta = ixl_stat_delta(sc, 6332 0, I40E_GLPRT_RDPC(sc->sc_port), 6333 &isc->isc_rx_discards_offset, isc->isc_has_offset); 6334 atomic_add_64(&isc->isc_rx_discards.ev_count, delta); 6335 6336 delta = ixl_stat_delta(sc, 6337 I40E_GLPRT_UPRCH(sc->sc_port), I40E_GLPRT_UPRCL(sc->sc_port), 6338 &isc->isc_rx_unicast_offset, isc->isc_has_offset); 6339 atomic_add_64(&isc->isc_rx_unicast.ev_count, delta); 6340 6341 delta = ixl_stat_delta(sc, 6342 I40E_GLPRT_MPRCH(sc->sc_port), I40E_GLPRT_MPRCL(sc->sc_port), 6343 &isc->isc_rx_multicast_offset, isc->isc_has_offset); 6344 atomic_add_64(&isc->isc_rx_multicast.ev_count, delta); 6345 6346 delta = ixl_stat_delta(sc, 6347 I40E_GLPRT_BPRCH(sc->sc_port), I40E_GLPRT_BPRCL(sc->sc_port), 6348 &isc->isc_rx_broadcast_offset, isc->isc_has_offset); 6349 atomic_add_64(&isc->isc_rx_broadcast.ev_count, delta); 6350 6351 /* Packet size stats rx */ 6352 delta = ixl_stat_delta(sc, 6353 I40E_GLPRT_PRC64H(sc->sc_port), I40E_GLPRT_PRC64L(sc->sc_port), 6354 &isc->isc_rx_size_64_offset, isc->isc_has_offset); 6355 atomic_add_64(&isc->isc_rx_size_64.ev_count, delta); 6356 6357 delta = ixl_stat_delta(sc, 6358 I40E_GLPRT_PRC127H(sc->sc_port), I40E_GLPRT_PRC127L(sc->sc_port), 6359 &isc->isc_rx_size_127_offset, isc->isc_has_offset); 6360 atomic_add_64(&isc->isc_rx_size_127.ev_count, delta); 6361 6362 delta = ixl_stat_delta(sc, 6363 I40E_GLPRT_PRC255H(sc->sc_port), I40E_GLPRT_PRC255L(sc->sc_port), 6364 &isc->isc_rx_size_255_offset, isc->isc_has_offset); 6365 atomic_add_64(&isc->isc_rx_size_255.ev_count, delta); 6366 6367 delta = ixl_stat_delta(sc, 6368 I40E_GLPRT_PRC511H(sc->sc_port), I40E_GLPRT_PRC511L(sc->sc_port), 6369 &isc->isc_rx_size_511_offset, isc->isc_has_offset); 6370 atomic_add_64(&isc->isc_rx_size_511.ev_count, delta); 6371 6372 delta = ixl_stat_delta(sc, 6373 I40E_GLPRT_PRC1023H(sc->sc_port), I40E_GLPRT_PRC1023L(sc->sc_port), 6374 &isc->isc_rx_size_1023_offset, isc->isc_has_offset); 6375 atomic_add_64(&isc->isc_rx_size_1023.ev_count, delta); 6376 6377 delta = ixl_stat_delta(sc, 6378 I40E_GLPRT_PRC1522H(sc->sc_port), I40E_GLPRT_PRC1522L(sc->sc_port), 6379 &isc->isc_rx_size_1522_offset, isc->isc_has_offset); 6380 atomic_add_64(&isc->isc_rx_size_1522.ev_count, delta); 6381 6382 delta = ixl_stat_delta(sc, 6383 I40E_GLPRT_PRC9522H(sc->sc_port), I40E_GLPRT_PRC9522L(sc->sc_port), 6384 &isc->isc_rx_size_big_offset, isc->isc_has_offset); 6385 atomic_add_64(&isc->isc_rx_size_big.ev_count, delta); 6386 6387 delta = ixl_stat_delta(sc, 6388 0, I40E_GLPRT_RUC(sc->sc_port), 6389 &isc->isc_rx_undersize_offset, isc->isc_has_offset); 6390 atomic_add_64(&isc->isc_rx_undersize.ev_count, delta); 6391 6392 delta = ixl_stat_delta(sc, 6393 0, I40E_GLPRT_ROC(sc->sc_port), 6394 &isc->isc_rx_oversize_offset, isc->isc_has_offset); 6395 atomic_add_64(&isc->isc_rx_oversize.ev_count, delta); 6396 6397 /* tx */ 6398 delta = ixl_stat_delta(sc, 6399 I40E_GLPRT_GOTCH(sc->sc_port), I40E_GLPRT_GOTCL(sc->sc_port), 6400 &isc->isc_tx_bytes_offset, isc->isc_has_offset); 6401 atomic_add_64(&isc->isc_tx_bytes.ev_count, delta); 6402 6403 delta = ixl_stat_delta(sc, 6404 0, I40E_GLPRT_TDOLD(sc->sc_port), 6405 &isc->isc_tx_dropped_link_down_offset, isc->isc_has_offset); 6406 atomic_add_64(&isc->isc_tx_dropped_link_down.ev_count, delta); 6407 6408 delta = ixl_stat_delta(sc, 6409 I40E_GLPRT_UPTCH(sc->sc_port), I40E_GLPRT_UPTCL(sc->sc_port), 6410 &isc->isc_tx_unicast_offset, isc->isc_has_offset); 6411 atomic_add_64(&isc->isc_tx_unicast.ev_count, delta); 6412 6413 delta = ixl_stat_delta(sc, 6414 I40E_GLPRT_MPTCH(sc->sc_port), I40E_GLPRT_MPTCL(sc->sc_port), 6415 &isc->isc_tx_multicast_offset, isc->isc_has_offset); 6416 atomic_add_64(&isc->isc_tx_multicast.ev_count, delta); 6417 6418 delta = ixl_stat_delta(sc, 6419 I40E_GLPRT_BPTCH(sc->sc_port), I40E_GLPRT_BPTCL(sc->sc_port), 6420 &isc->isc_tx_broadcast_offset, isc->isc_has_offset); 6421 atomic_add_64(&isc->isc_tx_broadcast.ev_count, delta); 6422 6423 /* Packet size stats tx */ 6424 delta = ixl_stat_delta(sc, 6425 I40E_GLPRT_PTC64L(sc->sc_port), I40E_GLPRT_PTC64L(sc->sc_port), 6426 &isc->isc_tx_size_64_offset, isc->isc_has_offset); 6427 atomic_add_64(&isc->isc_tx_size_64.ev_count, delta); 6428 6429 delta = ixl_stat_delta(sc, 6430 I40E_GLPRT_PTC127H(sc->sc_port), I40E_GLPRT_PTC127L(sc->sc_port), 6431 &isc->isc_tx_size_127_offset, isc->isc_has_offset); 6432 atomic_add_64(&isc->isc_tx_size_127.ev_count, delta); 6433 6434 delta = ixl_stat_delta(sc, 6435 I40E_GLPRT_PTC255H(sc->sc_port), I40E_GLPRT_PTC255L(sc->sc_port), 6436 &isc->isc_tx_size_255_offset, isc->isc_has_offset); 6437 atomic_add_64(&isc->isc_tx_size_255.ev_count, delta); 6438 6439 delta = ixl_stat_delta(sc, 6440 I40E_GLPRT_PTC511H(sc->sc_port), I40E_GLPRT_PTC511L(sc->sc_port), 6441 &isc->isc_tx_size_511_offset, isc->isc_has_offset); 6442 atomic_add_64(&isc->isc_tx_size_511.ev_count, delta); 6443 6444 delta = ixl_stat_delta(sc, 6445 I40E_GLPRT_PTC1023H(sc->sc_port), I40E_GLPRT_PTC1023L(sc->sc_port), 6446 &isc->isc_tx_size_1023_offset, isc->isc_has_offset); 6447 atomic_add_64(&isc->isc_tx_size_1023.ev_count, delta); 6448 6449 delta = ixl_stat_delta(sc, 6450 I40E_GLPRT_PTC1522H(sc->sc_port), I40E_GLPRT_PTC1522L(sc->sc_port), 6451 &isc->isc_tx_size_1522_offset, isc->isc_has_offset); 6452 atomic_add_64(&isc->isc_tx_size_1522.ev_count, delta); 6453 6454 delta = ixl_stat_delta(sc, 6455 I40E_GLPRT_PTC9522H(sc->sc_port), I40E_GLPRT_PTC9522L(sc->sc_port), 6456 &isc->isc_tx_size_big_offset, isc->isc_has_offset); 6457 atomic_add_64(&isc->isc_tx_size_big.ev_count, delta); 6458 6459 /* mac faults */ 6460 delta = ixl_stat_delta(sc, 6461 0, I40E_GLPRT_MLFC(sc->sc_port), 6462 &isc->isc_mac_local_faults_offset, isc->isc_has_offset); 6463 atomic_add_64(&isc->isc_mac_local_faults.ev_count, delta); 6464 6465 delta = ixl_stat_delta(sc, 6466 0, I40E_GLPRT_MRFC(sc->sc_port), 6467 &isc->isc_mac_remote_faults_offset, isc->isc_has_offset); 6468 atomic_add_64(&isc->isc_mac_remote_faults.ev_count, delta); 6469 6470 /* Flow control (LFC) stats */ 6471 delta = ixl_stat_delta(sc, 6472 0, I40E_GLPRT_LXONRXC(sc->sc_port), 6473 &isc->isc_link_xon_rx_offset, isc->isc_has_offset); 6474 atomic_add_64(&isc->isc_link_xon_rx.ev_count, delta); 6475 6476 delta = ixl_stat_delta(sc, 6477 0, I40E_GLPRT_LXONTXC(sc->sc_port), 6478 &isc->isc_link_xon_tx_offset, isc->isc_has_offset); 6479 atomic_add_64(&isc->isc_link_xon_tx.ev_count, delta); 6480 6481 delta = ixl_stat_delta(sc, 6482 0, I40E_GLPRT_LXOFFRXC(sc->sc_port), 6483 &isc->isc_link_xoff_rx_offset, isc->isc_has_offset); 6484 atomic_add_64(&isc->isc_link_xoff_rx.ev_count, delta); 6485 6486 delta = ixl_stat_delta(sc, 6487 0, I40E_GLPRT_LXOFFTXC(sc->sc_port), 6488 &isc->isc_link_xoff_tx_offset, isc->isc_has_offset); 6489 atomic_add_64(&isc->isc_link_xoff_tx.ev_count, delta); 6490 6491 /* fragments */ 6492 delta = ixl_stat_delta(sc, 6493 0, I40E_GLPRT_RFC(sc->sc_port), 6494 &isc->isc_rx_fragments_offset, isc->isc_has_offset); 6495 atomic_add_64(&isc->isc_rx_fragments.ev_count, delta); 6496 6497 delta = ixl_stat_delta(sc, 6498 0, I40E_GLPRT_RJC(sc->sc_port), 6499 &isc->isc_rx_jabber_offset, isc->isc_has_offset); 6500 atomic_add_64(&isc->isc_rx_jabber.ev_count, delta); 6501 6502 /* VSI rx counters */ 6503 delta = ixl_stat_delta(sc, 6504 0, I40E_GLV_RDPC(sc->sc_vsi_stat_counter_idx), 6505 &isc->isc_vsi_rx_discards_offset, isc->isc_has_offset); 6506 atomic_add_64(&isc->isc_vsi_rx_discards.ev_count, delta); 6507 6508 delta = ixl_stat_delta(sc, 6509 I40E_GLV_GORCH(sc->sc_vsi_stat_counter_idx), 6510 I40E_GLV_GORCL(sc->sc_vsi_stat_counter_idx), 6511 &isc->isc_vsi_rx_bytes_offset, isc->isc_has_offset); 6512 atomic_add_64(&isc->isc_vsi_rx_bytes.ev_count, delta); 6513 6514 delta = ixl_stat_delta(sc, 6515 I40E_GLV_UPRCH(sc->sc_vsi_stat_counter_idx), 6516 I40E_GLV_UPRCL(sc->sc_vsi_stat_counter_idx), 6517 &isc->isc_vsi_rx_unicast_offset, isc->isc_has_offset); 6518 atomic_add_64(&isc->isc_vsi_rx_unicast.ev_count, delta); 6519 6520 delta = ixl_stat_delta(sc, 6521 I40E_GLV_MPRCH(sc->sc_vsi_stat_counter_idx), 6522 I40E_GLV_MPRCL(sc->sc_vsi_stat_counter_idx), 6523 &isc->isc_vsi_rx_multicast_offset, isc->isc_has_offset); 6524 atomic_add_64(&isc->isc_vsi_rx_multicast.ev_count, delta); 6525 6526 delta = ixl_stat_delta(sc, 6527 I40E_GLV_BPRCH(sc->sc_vsi_stat_counter_idx), 6528 I40E_GLV_BPRCL(sc->sc_vsi_stat_counter_idx), 6529 &isc->isc_vsi_rx_broadcast_offset, isc->isc_has_offset); 6530 atomic_add_64(&isc->isc_vsi_rx_broadcast.ev_count, delta); 6531 6532 /* VSI tx counters */ 6533 delta = ixl_stat_delta(sc, 6534 0, I40E_GLV_TEPC(sc->sc_vsi_stat_counter_idx), 6535 &isc->isc_vsi_tx_errors_offset, isc->isc_has_offset); 6536 atomic_add_64(&isc->isc_vsi_tx_errors.ev_count, delta); 6537 6538 delta = ixl_stat_delta(sc, 6539 I40E_GLV_GOTCH(sc->sc_vsi_stat_counter_idx), 6540 I40E_GLV_GOTCL(sc->sc_vsi_stat_counter_idx), 6541 &isc->isc_vsi_tx_bytes_offset, isc->isc_has_offset); 6542 atomic_add_64(&isc->isc_vsi_tx_bytes.ev_count, delta); 6543 6544 delta = ixl_stat_delta(sc, 6545 I40E_GLV_UPTCH(sc->sc_vsi_stat_counter_idx), 6546 I40E_GLV_UPTCL(sc->sc_vsi_stat_counter_idx), 6547 &isc->isc_vsi_tx_unicast_offset, isc->isc_has_offset); 6548 atomic_add_64(&isc->isc_vsi_tx_unicast.ev_count, delta); 6549 6550 delta = ixl_stat_delta(sc, 6551 I40E_GLV_MPTCH(sc->sc_vsi_stat_counter_idx), 6552 I40E_GLV_MPTCL(sc->sc_vsi_stat_counter_idx), 6553 &isc->isc_vsi_tx_multicast_offset, isc->isc_has_offset); 6554 atomic_add_64(&isc->isc_vsi_tx_multicast.ev_count, delta); 6555 6556 delta = ixl_stat_delta(sc, 6557 I40E_GLV_BPTCH(sc->sc_vsi_stat_counter_idx), 6558 I40E_GLV_BPTCL(sc->sc_vsi_stat_counter_idx), 6559 &isc->isc_vsi_tx_broadcast_offset, isc->isc_has_offset); 6560 atomic_add_64(&isc->isc_vsi_tx_broadcast.ev_count, delta); 6561 } 6562 6563 static int 6564 ixl_setup_sysctls(struct ixl_softc *sc) 6565 { 6566 const char *devname; 6567 struct sysctllog **log; 6568 const struct sysctlnode *rnode, *rxnode, *txnode; 6569 int error; 6570 6571 log = &sc->sc_sysctllog; 6572 devname = device_xname(sc->sc_dev); 6573 6574 error = sysctl_createv(log, 0, NULL, &rnode, 6575 0, CTLTYPE_NODE, devname, 6576 SYSCTL_DESCR("ixl information and settings"), 6577 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 6578 if (error) 6579 goto out; 6580 6581 error = sysctl_createv(log, 0, &rnode, NULL, 6582 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 6583 SYSCTL_DESCR("Use workqueue for packet processing"), 6584 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 6585 if (error) 6586 goto out; 6587 6588 error = sysctl_createv(log, 0, &rnode, NULL, 6589 CTLFLAG_READONLY, CTLTYPE_INT, "stats_interval", 6590 SYSCTL_DESCR("Statistics collection interval in milliseconds"), 6591 NULL, 0, &sc->sc_stats_intval, 0, CTL_CREATE, CTL_EOL); 6592 6593 error = sysctl_createv(log, 0, &rnode, &rxnode, 6594 0, CTLTYPE_NODE, "rx", 6595 SYSCTL_DESCR("ixl information and settings for Rx"), 6596 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6597 if (error) 6598 goto out; 6599 6600 error = sysctl_createv(log, 0, &rxnode, NULL, 6601 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6602 SYSCTL_DESCR("max number of Rx packets" 6603 " to process for interrupt processing"), 6604 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6605 if (error) 6606 goto out; 6607 6608 error = sysctl_createv(log, 0, &rxnode, NULL, 6609 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6610 SYSCTL_DESCR("max number of Rx packets" 6611 " to process for deferred processing"), 6612 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 6613 if (error) 6614 goto out; 6615 6616 error = sysctl_createv(log, 0, &rnode, &txnode, 6617 0, CTLTYPE_NODE, "tx", 6618 SYSCTL_DESCR("ixl information and settings for Tx"), 6619 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6620 if (error) 6621 goto out; 6622 6623 error = sysctl_createv(log, 0, &txnode, NULL, 6624 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6625 SYSCTL_DESCR("max number of Tx packets" 6626 " to process for interrupt processing"), 6627 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6628 if (error) 6629 goto out; 6630 6631 error = sysctl_createv(log, 0, &txnode, NULL, 6632 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6633 SYSCTL_DESCR("max number of Tx packets" 6634 " to process for deferred processing"), 6635 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 6636 if (error) 6637 goto out; 6638 6639 out: 6640 if (error) { 6641 aprint_error_dev(sc->sc_dev, 6642 "unable to create sysctl node\n"); 6643 sysctl_teardown(log); 6644 } 6645 6646 return error; 6647 } 6648 6649 static void 6650 ixl_teardown_sysctls(struct ixl_softc *sc) 6651 { 6652 6653 sysctl_teardown(&sc->sc_sysctllog); 6654 } 6655 6656 static struct workqueue * 6657 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 6658 { 6659 struct workqueue *wq; 6660 int error; 6661 6662 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 6663 prio, ipl, flags); 6664 6665 if (error) 6666 return NULL; 6667 6668 return wq; 6669 } 6670 6671 static void 6672 ixl_workq_destroy(struct workqueue *wq) 6673 { 6674 6675 workqueue_destroy(wq); 6676 } 6677 6678 static void 6679 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 6680 { 6681 6682 memset(work, 0, sizeof(*work)); 6683 work->ixw_func = func; 6684 work->ixw_arg = arg; 6685 } 6686 6687 static void 6688 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 6689 { 6690 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 6691 return; 6692 6693 kpreempt_disable(); 6694 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 6695 kpreempt_enable(); 6696 } 6697 6698 static void 6699 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 6700 { 6701 6702 workqueue_wait(wq, &work->ixw_cookie); 6703 } 6704 6705 static void 6706 ixl_workq_work(struct work *wk, void *context) 6707 { 6708 struct ixl_work *work; 6709 6710 work = container_of(wk, struct ixl_work, ixw_cookie); 6711 6712 atomic_swap_uint(&work->ixw_added, 0); 6713 work->ixw_func(work->ixw_arg); 6714 } 6715 6716 static int 6717 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 6718 { 6719 struct ixl_aq_desc iaq; 6720 6721 memset(&iaq, 0, sizeof(iaq)); 6722 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 6723 iaq.iaq_param[1] = htole32(reg); 6724 6725 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6726 return ETIMEDOUT; 6727 6728 switch (htole16(iaq.iaq_retval)) { 6729 case IXL_AQ_RC_OK: 6730 /* success */ 6731 break; 6732 case IXL_AQ_RC_EACCES: 6733 return EPERM; 6734 case IXL_AQ_RC_EAGAIN: 6735 return EAGAIN; 6736 default: 6737 return EIO; 6738 } 6739 6740 *rv = htole32(iaq.iaq_param[3]); 6741 return 0; 6742 } 6743 6744 static uint32_t 6745 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 6746 { 6747 uint32_t val; 6748 int rv, retry, retry_limit; 6749 6750 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6751 retry_limit = 5; 6752 } else { 6753 retry_limit = 0; 6754 } 6755 6756 for (retry = 0; retry < retry_limit; retry++) { 6757 rv = ixl_rx_ctl_read(sc, reg, &val); 6758 if (rv == 0) 6759 return val; 6760 else if (rv == EAGAIN) 6761 delaymsec(1); 6762 else 6763 break; 6764 } 6765 6766 val = ixl_rd(sc, reg); 6767 6768 return val; 6769 } 6770 6771 static int 6772 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6773 { 6774 struct ixl_aq_desc iaq; 6775 6776 memset(&iaq, 0, sizeof(iaq)); 6777 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 6778 iaq.iaq_param[1] = htole32(reg); 6779 iaq.iaq_param[3] = htole32(value); 6780 6781 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6782 return ETIMEDOUT; 6783 6784 switch (htole16(iaq.iaq_retval)) { 6785 case IXL_AQ_RC_OK: 6786 /* success */ 6787 break; 6788 case IXL_AQ_RC_EACCES: 6789 return EPERM; 6790 case IXL_AQ_RC_EAGAIN: 6791 return EAGAIN; 6792 default: 6793 return EIO; 6794 } 6795 6796 return 0; 6797 } 6798 6799 static void 6800 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6801 { 6802 int rv, retry, retry_limit; 6803 6804 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6805 retry_limit = 5; 6806 } else { 6807 retry_limit = 0; 6808 } 6809 6810 for (retry = 0; retry < retry_limit; retry++) { 6811 rv = ixl_rx_ctl_write(sc, reg, value); 6812 if (rv == 0) 6813 return; 6814 else if (rv == EAGAIN) 6815 delaymsec(1); 6816 else 6817 break; 6818 } 6819 6820 ixl_wr(sc, reg, value); 6821 } 6822 6823 static int 6824 ixl_nvm_lock(struct ixl_softc *sc, char rw) 6825 { 6826 struct ixl_aq_desc iaq; 6827 struct ixl_aq_req_resource_param *param; 6828 int rv; 6829 6830 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6831 return 0; 6832 6833 memset(&iaq, 0, sizeof(iaq)); 6834 iaq.iaq_opcode = htole16(IXL_AQ_OP_REQUEST_RESOURCE); 6835 6836 param = (struct ixl_aq_req_resource_param *)&iaq.iaq_param; 6837 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6838 if (rw == 'R') { 6839 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_READ); 6840 } else { 6841 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_WRITE); 6842 } 6843 6844 rv = ixl_atq_poll(sc, &iaq, 250); 6845 6846 if (rv != 0) 6847 return ETIMEDOUT; 6848 6849 switch (le16toh(iaq.iaq_retval)) { 6850 case IXL_AQ_RC_OK: 6851 break; 6852 case IXL_AQ_RC_EACCES: 6853 return EACCES; 6854 case IXL_AQ_RC_EBUSY: 6855 return EBUSY; 6856 case IXL_AQ_RC_EPERM: 6857 return EPERM; 6858 } 6859 6860 return 0; 6861 } 6862 6863 static int 6864 ixl_nvm_unlock(struct ixl_softc *sc) 6865 { 6866 struct ixl_aq_desc iaq; 6867 struct ixl_aq_rel_resource_param *param; 6868 int rv; 6869 6870 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6871 return 0; 6872 6873 memset(&iaq, 0, sizeof(iaq)); 6874 iaq.iaq_opcode = htole16(IXL_AQ_OP_RELEASE_RESOURCE); 6875 6876 param = (struct ixl_aq_rel_resource_param *)&iaq.iaq_param; 6877 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6878 6879 rv = ixl_atq_poll(sc, &iaq, 250); 6880 6881 if (rv != 0) 6882 return ETIMEDOUT; 6883 6884 switch (le16toh(iaq.iaq_retval)) { 6885 case IXL_AQ_RC_OK: 6886 break; 6887 default: 6888 return EIO; 6889 } 6890 return 0; 6891 } 6892 6893 static int 6894 ixl_srdone_poll(struct ixl_softc *sc) 6895 { 6896 int wait_count; 6897 uint32_t reg; 6898 6899 for (wait_count = 0; wait_count < IXL_SRRD_SRCTL_ATTEMPTS; 6900 wait_count++) { 6901 reg = ixl_rd(sc, I40E_GLNVM_SRCTL); 6902 if (ISSET(reg, I40E_GLNVM_SRCTL_DONE_MASK)) 6903 break; 6904 6905 delaymsec(5); 6906 } 6907 6908 if (wait_count == IXL_SRRD_SRCTL_ATTEMPTS) 6909 return -1; 6910 6911 return 0; 6912 } 6913 6914 static int 6915 ixl_nvm_read_srctl(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 6916 { 6917 uint32_t reg; 6918 6919 if (ixl_srdone_poll(sc) != 0) 6920 return ETIMEDOUT; 6921 6922 reg = ((uint32_t)offset << I40E_GLNVM_SRCTL_ADDR_SHIFT) | 6923 __BIT(I40E_GLNVM_SRCTL_START_SHIFT); 6924 ixl_wr(sc, I40E_GLNVM_SRCTL, reg); 6925 6926 if (ixl_srdone_poll(sc) != 0) { 6927 aprint_debug("NVM read error: couldn't access " 6928 "Shadow RAM address: 0x%x\n", offset); 6929 return ETIMEDOUT; 6930 } 6931 6932 reg = ixl_rd(sc, I40E_GLNVM_SRDATA); 6933 *data = (uint16_t)__SHIFTOUT(reg, I40E_GLNVM_SRDATA_RDDATA_MASK); 6934 6935 return 0; 6936 } 6937 6938 static int 6939 ixl_nvm_read_aq(struct ixl_softc *sc, uint16_t offset_word, 6940 void *data, size_t len) 6941 { 6942 struct ixl_dmamem *idm; 6943 struct ixl_aq_desc iaq; 6944 struct ixl_aq_nvm_param *param; 6945 uint32_t offset_bytes; 6946 int rv; 6947 6948 idm = &sc->sc_aqbuf; 6949 if (len > IXL_DMA_LEN(idm)) 6950 return ENOMEM; 6951 6952 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 6953 memset(&iaq, 0, sizeof(iaq)); 6954 iaq.iaq_opcode = htole16(IXL_AQ_OP_NVM_READ); 6955 iaq.iaq_flags = htole16(IXL_AQ_BUF | 6956 ((len > I40E_AQ_LARGE_BUF) ? IXL_AQ_LB : 0)); 6957 iaq.iaq_datalen = htole16(len); 6958 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 6959 6960 param = (struct ixl_aq_nvm_param *)iaq.iaq_param; 6961 param->command_flags = IXL_AQ_NVM_LAST_CMD; 6962 param->module_pointer = 0; 6963 param->length = htole16(len); 6964 offset_bytes = (uint32_t)offset_word * 2; 6965 offset_bytes &= 0x00FFFFFF; 6966 param->offset = htole32(offset_bytes); 6967 6968 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 6969 BUS_DMASYNC_PREREAD); 6970 6971 rv = ixl_atq_poll(sc, &iaq, 250); 6972 6973 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 6974 BUS_DMASYNC_POSTREAD); 6975 6976 if (rv != 0) { 6977 return ETIMEDOUT; 6978 } 6979 6980 switch (le16toh(iaq.iaq_retval)) { 6981 case IXL_AQ_RC_OK: 6982 break; 6983 case IXL_AQ_RC_EPERM: 6984 return EPERM; 6985 case IXL_AQ_RC_EINVAL: 6986 return EINVAL; 6987 case IXL_AQ_RC_EBUSY: 6988 return EBUSY; 6989 case IXL_AQ_RC_EIO: 6990 default: 6991 return EIO; 6992 } 6993 6994 memcpy(data, IXL_DMA_KVA(idm), len); 6995 6996 return 0; 6997 } 6998 6999 static int 7000 ixl_rd16_nvm(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 7001 { 7002 int error; 7003 uint16_t buf; 7004 7005 error = ixl_nvm_lock(sc, 'R'); 7006 if (error) 7007 return error; 7008 7009 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMREAD)) { 7010 error = ixl_nvm_read_aq(sc, offset, 7011 &buf, sizeof(buf)); 7012 if (error == 0) 7013 *data = le16toh(buf); 7014 } else { 7015 error = ixl_nvm_read_srctl(sc, offset, &buf); 7016 if (error == 0) 7017 *data = buf; 7018 } 7019 7020 ixl_nvm_unlock(sc); 7021 7022 return error; 7023 } 7024 7025 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 7026 7027 #ifdef _MODULE 7028 #include "ioconf.c" 7029 #endif 7030 7031 #ifdef _MODULE 7032 static void 7033 ixl_parse_modprop(prop_dictionary_t dict) 7034 { 7035 prop_object_t obj; 7036 int64_t val; 7037 uint64_t uval; 7038 7039 if (dict == NULL) 7040 return; 7041 7042 obj = prop_dictionary_get(dict, "nomsix"); 7043 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_BOOL) { 7044 ixl_param_nomsix = prop_bool_true((prop_bool_t)obj); 7045 } 7046 7047 obj = prop_dictionary_get(dict, "stats_interval"); 7048 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7049 val = prop_number_integer_value((prop_number_t)obj); 7050 7051 /* the range has no reason */ 7052 if (100 < val && val < 180000) { 7053 ixl_param_stats_interval = val; 7054 } 7055 } 7056 7057 obj = prop_dictionary_get(dict, "nqps_limit"); 7058 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7059 val = prop_number_integer_value((prop_number_t)obj); 7060 7061 if (val <= INT32_MAX) 7062 ixl_param_nqps_limit = val; 7063 } 7064 7065 obj = prop_dictionary_get(dict, "rx_ndescs"); 7066 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7067 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7068 7069 if (uval > 8) 7070 ixl_param_rx_ndescs = uval; 7071 } 7072 7073 obj = prop_dictionary_get(dict, "tx_ndescs"); 7074 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7075 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7076 7077 if (uval > IXL_TX_PKT_DESCS) 7078 ixl_param_tx_ndescs = uval; 7079 } 7080 7081 } 7082 #endif 7083 7084 static int 7085 if_ixl_modcmd(modcmd_t cmd, void *opaque) 7086 { 7087 int error = 0; 7088 7089 #ifdef _MODULE 7090 switch (cmd) { 7091 case MODULE_CMD_INIT: 7092 ixl_parse_modprop((prop_dictionary_t)opaque); 7093 error = config_init_component(cfdriver_ioconf_if_ixl, 7094 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7095 break; 7096 case MODULE_CMD_FINI: 7097 error = config_fini_component(cfdriver_ioconf_if_ixl, 7098 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7099 break; 7100 default: 7101 error = ENOTTY; 7102 break; 7103 } 7104 #endif 7105 7106 return error; 7107 } 7108
Indexes created Thu Sep 25 16:09:42 GMT 2025