if_ixl.c revision 1.19
1 /* $NetBSD: if_ixl.c,v 1.19 2020/01/09 08:54:05 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 78 #ifdef _KERNEL_OPT 79 #include "opt_net_mpsafe.h" 80 #endif 81 82 #include <sys/param.h> 83 #include <sys/types.h> 84 85 #include <sys/cpu.h> 86 #include <sys/device.h> 87 #include <sys/evcnt.h> 88 #include <sys/interrupt.h> 89 #include <sys/kmem.h> 90 #include <sys/malloc.h> 91 #include <sys/module.h> 92 #include <sys/mutex.h> 93 #include <sys/pcq.h> 94 #include <sys/syslog.h> 95 #include <sys/workqueue.h> 96 97 #include <sys/bus.h> 98 99 #include <net/bpf.h> 100 #include <net/if.h> 101 #include <net/if_dl.h> 102 #include <net/if_media.h> 103 #include <net/if_ether.h> 104 #include <net/rss_config.h> 105 106 #include <dev/pci/pcivar.h> 107 #include <dev/pci/pcidevs.h> 108 109 #include <dev/pci/if_ixlreg.h> 110 #include <dev/pci/if_ixlvar.h> 111 112 #include <prop/proplib.h> 113 114 struct ixl_softc; /* defined */ 115 116 #define I40E_PF_RESET_WAIT_COUNT 200 117 #define I40E_AQ_LARGE_BUF 512 118 119 /* bitfields for Tx queue mapping in QTX_CTL */ 120 #define I40E_QTX_CTL_VF_QUEUE 0x0 121 #define I40E_QTX_CTL_VM_QUEUE 0x1 122 #define I40E_QTX_CTL_PF_QUEUE 0x2 123 124 #define I40E_QUEUE_TYPE_EOL 0x7ff 125 #define I40E_INTR_NOTX_QUEUE 0 126 127 #define I40E_QUEUE_TYPE_RX 0x0 128 #define I40E_QUEUE_TYPE_TX 0x1 129 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 130 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 131 132 #define I40E_ITR_INDEX_RX 0x0 133 #define I40E_ITR_INDEX_TX 0x1 134 #define I40E_ITR_INDEX_OTHER 0x2 135 #define I40E_ITR_INDEX_NONE 0x3 136 137 #define I40E_INTR_NOTX_QUEUE 0 138 #define I40E_INTR_NOTX_INTR 0 139 #define I40E_INTR_NOTX_RX_QUEUE 0 140 #define I40E_INTR_NOTX_TX_QUEUE 1 141 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 142 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 143 144 #define BIT_ULL(a) (1ULL << (a)) 145 #define IXL_RSS_HENA_DEFAULT_BASE \ 146 (BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 147 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 148 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 149 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 150 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 151 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 152 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 153 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 154 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 155 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 156 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 157 #define IXL_RSS_HENA_DEFAULT_XL710 IXL_RSS_HENA_DEFAULT_BASE 158 #define IXL_RSS_HENA_DEFAULT_X722 (IXL_RSS_HENA_DEFAULT_XL710 | \ 159 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 160 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 161 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 162 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 163 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 164 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 165 #define I40E_HASH_LUT_SIZE_128 0 166 #define IXL_RSS_KEY_SIZE_REG 13 167 168 #define IXL_ICR0_CRIT_ERR_MASK \ 169 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 170 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 171 I40E_PFINT_ICR0_PE_CRITERR_MASK) 172 173 #define IXL_TX_PKT_DESCS 8 174 #define IXL_TX_QUEUE_ALIGN 128 175 #define IXL_RX_QUEUE_ALIGN 128 176 177 #define IXL_HARDMTU 9712 /* 9726 - ETHER_HDR_LEN */ 178 179 #define IXL_PCIREG PCI_MAPREG_START 180 181 #define IXL_ITR0 0x0 182 #define IXL_ITR1 0x1 183 #define IXL_ITR2 0x2 184 #define IXL_NOITR 0x3 185 186 #define IXL_AQ_NUM 256 187 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 188 #define IXL_AQ_ALIGN 64 /* lol */ 189 #define IXL_AQ_BUFLEN 4096 190 191 #define IXL_HMC_ROUNDUP 512 192 #define IXL_HMC_PGSIZE 4096 193 #define IXL_HMC_DVASZ sizeof(uint64_t) 194 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 195 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 196 #define IXL_HMC_PDVALID 1ULL 197 198 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 199 200 struct ixl_aq_regs { 201 bus_size_t atq_tail; 202 bus_size_t atq_head; 203 bus_size_t atq_len; 204 bus_size_t atq_bal; 205 bus_size_t atq_bah; 206 207 bus_size_t arq_tail; 208 bus_size_t arq_head; 209 bus_size_t arq_len; 210 bus_size_t arq_bal; 211 bus_size_t arq_bah; 212 213 uint32_t atq_len_enable; 214 uint32_t atq_tail_mask; 215 uint32_t atq_head_mask; 216 217 uint32_t arq_len_enable; 218 uint32_t arq_tail_mask; 219 uint32_t arq_head_mask; 220 }; 221 222 struct ixl_phy_type { 223 uint64_t phy_type; 224 uint64_t ifm_type; 225 }; 226 227 struct ixl_speed_type { 228 uint8_t dev_speed; 229 uint64_t net_speed; 230 }; 231 232 struct ixl_aq_buf { 233 SIMPLEQ_ENTRY(ixl_aq_buf) 234 aqb_entry; 235 void *aqb_data; 236 bus_dmamap_t aqb_map; 237 bus_dma_segment_t aqb_seg; 238 size_t aqb_size; 239 int aqb_nsegs; 240 }; 241 SIMPLEQ_HEAD(ixl_aq_bufs, ixl_aq_buf); 242 243 struct ixl_dmamem { 244 bus_dmamap_t ixm_map; 245 bus_dma_segment_t ixm_seg; 246 int ixm_nsegs; 247 size_t ixm_size; 248 void *ixm_kva; 249 }; 250 251 #define IXL_DMA_MAP(_ixm) ((_ixm)->ixm_map) 252 #define IXL_DMA_DVA(_ixm) ((_ixm)->ixm_map->dm_segs[0].ds_addr) 253 #define IXL_DMA_KVA(_ixm) ((void *)(_ixm)->ixm_kva) 254 #define IXL_DMA_LEN(_ixm) ((_ixm)->ixm_size) 255 256 struct ixl_hmc_entry { 257 uint64_t hmc_base; 258 uint32_t hmc_count; 259 uint64_t hmc_size; 260 }; 261 262 enum ixl_hmc_types { 263 IXL_HMC_LAN_TX = 0, 264 IXL_HMC_LAN_RX, 265 IXL_HMC_FCOE_CTX, 266 IXL_HMC_FCOE_FILTER, 267 IXL_HMC_COUNT 268 }; 269 270 struct ixl_hmc_pack { 271 uint16_t offset; 272 uint16_t width; 273 uint16_t lsb; 274 }; 275 276 /* 277 * these hmc objects have weird sizes and alignments, so these are abstract 278 * representations of them that are nice for c to populate. 279 * 280 * the packing code relies on little-endian values being stored in the fields, 281 * no high bits in the fields being set, and the fields must be packed in the 282 * same order as they are in the ctx structure. 283 */ 284 285 struct ixl_hmc_rxq { 286 uint16_t head; 287 uint8_t cpuid; 288 uint64_t base; 289 #define IXL_HMC_RXQ_BASE_UNIT 128 290 uint16_t qlen; 291 uint16_t dbuff; 292 #define IXL_HMC_RXQ_DBUFF_UNIT 128 293 uint8_t hbuff; 294 #define IXL_HMC_RXQ_HBUFF_UNIT 64 295 uint8_t dtype; 296 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 297 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 298 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 299 uint8_t dsize; 300 #define IXL_HMC_RXQ_DSIZE_16 0 301 #define IXL_HMC_RXQ_DSIZE_32 1 302 uint8_t crcstrip; 303 uint8_t fc_ena; 304 uint8_t l2sel; 305 uint8_t hsplit_0; 306 uint8_t hsplit_1; 307 uint8_t showiv; 308 uint16_t rxmax; 309 uint8_t tphrdesc_ena; 310 uint8_t tphwdesc_ena; 311 uint8_t tphdata_ena; 312 uint8_t tphhead_ena; 313 uint8_t lrxqthresh; 314 uint8_t prefena; 315 }; 316 317 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 318 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 319 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 320 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 321 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 322 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 323 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 324 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 325 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 326 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 327 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 328 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 329 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 330 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 331 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 332 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 333 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 334 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 335 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 336 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 337 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 338 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 339 }; 340 341 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 342 343 struct ixl_hmc_txq { 344 uint16_t head; 345 uint8_t new_context; 346 uint64_t base; 347 #define IXL_HMC_TXQ_BASE_UNIT 128 348 uint8_t fc_ena; 349 uint8_t timesync_ena; 350 uint8_t fd_ena; 351 uint8_t alt_vlan_ena; 352 uint16_t thead_wb; 353 uint8_t cpuid; 354 uint8_t head_wb_ena; 355 #define IXL_HMC_TXQ_DESC_WB 0 356 #define IXL_HMC_TXQ_HEAD_WB 1 357 uint16_t qlen; 358 uint8_t tphrdesc_ena; 359 uint8_t tphrpacket_ena; 360 uint8_t tphwdesc_ena; 361 uint64_t head_wb_addr; 362 uint32_t crc; 363 uint16_t rdylist; 364 uint8_t rdylist_act; 365 }; 366 367 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 368 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 369 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 370 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 371 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 372 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 373 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 374 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 375 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 376 /* line 1 */ 377 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 378 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 379 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 380 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 381 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 382 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 383 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 384 /* line 7 */ 385 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 386 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 387 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 388 }; 389 390 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 391 392 struct ixl_work { 393 struct work ixw_cookie; 394 void (*ixw_func)(void *); 395 void *ixw_arg; 396 unsigned int ixw_added; 397 }; 398 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 399 400 struct ixl_tx_map { 401 struct mbuf *txm_m; 402 bus_dmamap_t txm_map; 403 unsigned int txm_eop; 404 }; 405 406 struct ixl_tx_ring { 407 kmutex_t txr_lock; 408 struct ixl_softc *txr_sc; 409 410 unsigned int txr_prod; 411 unsigned int txr_cons; 412 413 struct ixl_tx_map *txr_maps; 414 struct ixl_dmamem txr_mem; 415 416 bus_size_t txr_tail; 417 unsigned int txr_qid; 418 pcq_t *txr_intrq; 419 void *txr_si; 420 421 uint64_t txr_oerrors; /* if_oerrors */ 422 uint64_t txr_opackets; /* if_opackets */ 423 uint64_t txr_obytes; /* if_obytes */ 424 uint64_t txr_omcasts; /* if_omcasts */ 425 426 struct evcnt txr_defragged; 427 struct evcnt txr_defrag_failed; 428 struct evcnt txr_pcqdrop; 429 struct evcnt txr_transmitdef; 430 struct evcnt txr_intr; 431 struct evcnt txr_defer; 432 }; 433 434 struct ixl_rx_map { 435 struct mbuf *rxm_m; 436 bus_dmamap_t rxm_map; 437 }; 438 439 struct ixl_rx_ring { 440 kmutex_t rxr_lock; 441 442 unsigned int rxr_prod; 443 unsigned int rxr_cons; 444 445 struct ixl_rx_map *rxr_maps; 446 struct ixl_dmamem rxr_mem; 447 448 struct mbuf *rxr_m_head; 449 struct mbuf **rxr_m_tail; 450 451 bus_size_t rxr_tail; 452 unsigned int rxr_qid; 453 454 uint64_t rxr_ipackets; /* if_ipackets */ 455 uint64_t rxr_ibytes; /* if_ibytes */ 456 uint64_t rxr_iqdrops; /* iqdrops */ 457 uint64_t rxr_ierrors; /* if_ierrors */ 458 459 struct evcnt rxr_mgethdr_failed; 460 struct evcnt rxr_mgetcl_failed; 461 struct evcnt rxr_mbuf_load_failed; 462 struct evcnt rxr_intr; 463 struct evcnt rxr_defer; 464 }; 465 466 struct ixl_queue_pair { 467 struct ixl_softc *qp_sc; 468 struct ixl_tx_ring *qp_txr; 469 struct ixl_rx_ring *qp_rxr; 470 471 char qp_name[16]; 472 473 void *qp_si; 474 struct ixl_work qp_task; 475 bool qp_workqueue; 476 }; 477 478 struct ixl_atq { 479 struct ixl_aq_desc iatq_desc; 480 void (*iatq_fn)(struct ixl_softc *, 481 const struct ixl_aq_desc *); 482 }; 483 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 484 485 struct ixl_product { 486 unsigned int vendor_id; 487 unsigned int product_id; 488 }; 489 490 struct ixl_stats_counters { 491 bool isc_has_offset; 492 struct evcnt isc_crc_errors; 493 uint64_t isc_crc_errors_offset; 494 struct evcnt isc_illegal_bytes; 495 uint64_t isc_illegal_bytes_offset; 496 struct evcnt isc_rx_bytes; 497 uint64_t isc_rx_bytes_offset; 498 struct evcnt isc_rx_discards; 499 uint64_t isc_rx_discards_offset; 500 struct evcnt isc_rx_unicast; 501 uint64_t isc_rx_unicast_offset; 502 struct evcnt isc_rx_multicast; 503 uint64_t isc_rx_multicast_offset; 504 struct evcnt isc_rx_broadcast; 505 uint64_t isc_rx_broadcast_offset; 506 struct evcnt isc_rx_size_64; 507 uint64_t isc_rx_size_64_offset; 508 struct evcnt isc_rx_size_127; 509 uint64_t isc_rx_size_127_offset; 510 struct evcnt isc_rx_size_255; 511 uint64_t isc_rx_size_255_offset; 512 struct evcnt isc_rx_size_511; 513 uint64_t isc_rx_size_511_offset; 514 struct evcnt isc_rx_size_1023; 515 uint64_t isc_rx_size_1023_offset; 516 struct evcnt isc_rx_size_1522; 517 uint64_t isc_rx_size_1522_offset; 518 struct evcnt isc_rx_size_big; 519 uint64_t isc_rx_size_big_offset; 520 struct evcnt isc_rx_undersize; 521 uint64_t isc_rx_undersize_offset; 522 struct evcnt isc_rx_oversize; 523 uint64_t isc_rx_oversize_offset; 524 struct evcnt isc_rx_fragments; 525 uint64_t isc_rx_fragments_offset; 526 struct evcnt isc_rx_jabber; 527 uint64_t isc_rx_jabber_offset; 528 struct evcnt isc_tx_bytes; 529 uint64_t isc_tx_bytes_offset; 530 struct evcnt isc_tx_dropped_link_down; 531 uint64_t isc_tx_dropped_link_down_offset; 532 struct evcnt isc_tx_unicast; 533 uint64_t isc_tx_unicast_offset; 534 struct evcnt isc_tx_multicast; 535 uint64_t isc_tx_multicast_offset; 536 struct evcnt isc_tx_broadcast; 537 uint64_t isc_tx_broadcast_offset; 538 struct evcnt isc_tx_size_64; 539 uint64_t isc_tx_size_64_offset; 540 struct evcnt isc_tx_size_127; 541 uint64_t isc_tx_size_127_offset; 542 struct evcnt isc_tx_size_255; 543 uint64_t isc_tx_size_255_offset; 544 struct evcnt isc_tx_size_511; 545 uint64_t isc_tx_size_511_offset; 546 struct evcnt isc_tx_size_1023; 547 uint64_t isc_tx_size_1023_offset; 548 struct evcnt isc_tx_size_1522; 549 uint64_t isc_tx_size_1522_offset; 550 struct evcnt isc_tx_size_big; 551 uint64_t isc_tx_size_big_offset; 552 struct evcnt isc_mac_local_faults; 553 uint64_t isc_mac_local_faults_offset; 554 struct evcnt isc_mac_remote_faults; 555 uint64_t isc_mac_remote_faults_offset; 556 struct evcnt isc_link_xon_rx; 557 uint64_t isc_link_xon_rx_offset; 558 struct evcnt isc_link_xon_tx; 559 uint64_t isc_link_xon_tx_offset; 560 struct evcnt isc_link_xoff_rx; 561 uint64_t isc_link_xoff_rx_offset; 562 struct evcnt isc_link_xoff_tx; 563 uint64_t isc_link_xoff_tx_offset; 564 struct evcnt isc_vsi_rx_discards; 565 uint64_t isc_vsi_rx_discards_offset; 566 struct evcnt isc_vsi_rx_bytes; 567 uint64_t isc_vsi_rx_bytes_offset; 568 struct evcnt isc_vsi_rx_unicast; 569 uint64_t isc_vsi_rx_unicast_offset; 570 struct evcnt isc_vsi_rx_multicast; 571 uint64_t isc_vsi_rx_multicast_offset; 572 struct evcnt isc_vsi_rx_broadcast; 573 uint64_t isc_vsi_rx_broadcast_offset; 574 struct evcnt isc_vsi_tx_errors; 575 uint64_t isc_vsi_tx_errors_offset; 576 struct evcnt isc_vsi_tx_bytes; 577 uint64_t isc_vsi_tx_bytes_offset; 578 struct evcnt isc_vsi_tx_unicast; 579 uint64_t isc_vsi_tx_unicast_offset; 580 struct evcnt isc_vsi_tx_multicast; 581 uint64_t isc_vsi_tx_multicast_offset; 582 struct evcnt isc_vsi_tx_broadcast; 583 uint64_t isc_vsi_tx_broadcast_offset; 584 }; 585 586 /* 587 * Locking notes: 588 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 589 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 590 * - more than one lock of them cannot be held at once. 591 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 592 * (a spin mutex). 593 * - the lock cannot held with txr_lock or rxr_lock. 594 * + a field named sc_arq_* is not protected by any lock. 595 * - operations for sc_arq_* is done in one context related to 596 * sc_arq_task. 597 * + other fields in ixl_softc is protected by sc_cfg_lock 598 * (an adaptive mutex) 599 * - It must be held before another lock is held, and It can be 600 * released after the other lock is released. 601 * */ 602 603 struct ixl_softc { 604 device_t sc_dev; 605 struct ethercom sc_ec; 606 bool sc_attached; 607 bool sc_dead; 608 bool sc_rxctl_atq; 609 uint32_t sc_port; 610 struct sysctllog *sc_sysctllog; 611 struct workqueue *sc_workq; 612 struct workqueue *sc_workq_txrx; 613 int sc_stats_intval; 614 callout_t sc_stats_callout; 615 struct ixl_work sc_stats_task; 616 struct ixl_stats_counters 617 sc_stats_counters; 618 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 619 struct ifmedia sc_media; 620 uint64_t sc_media_status; 621 uint64_t sc_media_active; 622 kmutex_t sc_cfg_lock; 623 enum i40e_mac_type sc_mac_type; 624 uint32_t sc_rss_table_size; 625 uint32_t sc_rss_table_entry_width; 626 bool sc_txrx_workqueue; 627 u_int sc_tx_process_limit; 628 u_int sc_rx_process_limit; 629 u_int sc_tx_intr_process_limit; 630 u_int sc_rx_intr_process_limit; 631 632 int sc_cur_ec_capenable; 633 634 struct pci_attach_args sc_pa; 635 pci_intr_handle_t *sc_ihp; 636 void **sc_ihs; 637 unsigned int sc_nintrs; 638 639 bus_dma_tag_t sc_dmat; 640 bus_space_tag_t sc_memt; 641 bus_space_handle_t sc_memh; 642 bus_size_t sc_mems; 643 644 uint8_t sc_pf_id; 645 uint16_t sc_uplink_seid; /* le */ 646 uint16_t sc_downlink_seid; /* le */ 647 uint16_t sc_vsi_number; /* le */ 648 uint16_t sc_vsi_stat_counter_idx; 649 uint16_t sc_seid; 650 unsigned int sc_base_queue; 651 652 pci_intr_type_t sc_intrtype; 653 unsigned int sc_msix_vector_queue; 654 655 struct ixl_dmamem sc_scratch; 656 657 const struct ixl_aq_regs * 658 sc_aq_regs; 659 660 kmutex_t sc_atq_lock; 661 kcondvar_t sc_atq_cv; 662 struct ixl_dmamem sc_atq; 663 unsigned int sc_atq_prod; 664 unsigned int sc_atq_cons; 665 666 struct ixl_dmamem sc_arq; 667 struct ixl_work sc_arq_task; 668 struct ixl_aq_bufs sc_arq_idle; 669 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 670 unsigned int sc_arq_prod; 671 unsigned int sc_arq_cons; 672 673 struct ixl_work sc_link_state_task; 674 struct ixl_atq sc_link_state_atq; 675 676 struct ixl_dmamem sc_hmc_sd; 677 struct ixl_dmamem sc_hmc_pd; 678 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 679 680 unsigned int sc_tx_ring_ndescs; 681 unsigned int sc_rx_ring_ndescs; 682 unsigned int sc_nqueue_pairs; 683 unsigned int sc_nqueue_pairs_max; 684 unsigned int sc_nqueue_pairs_device; 685 struct ixl_queue_pair *sc_qps; 686 687 struct evcnt sc_event_atq; 688 struct evcnt sc_event_link; 689 struct evcnt sc_event_ecc_err; 690 struct evcnt sc_event_pci_exception; 691 struct evcnt sc_event_crit_err; 692 }; 693 694 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 695 #define IXL_TX_PROCESS_LIMIT 256 696 #define IXL_RX_PROCESS_LIMIT 256 697 #define IXL_TX_INTR_PROCESS_LIMIT 256 698 #define IXL_RX_INTR_PROCESS_LIMIT 0U 699 700 #define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx| \ 701 IFCAP_CSUM_TCPv4_Rx| \ 702 IFCAP_CSUM_UDPv4_Rx| \ 703 IFCAP_CSUM_TCPv6_Rx| \ 704 IFCAP_CSUM_UDPv6_Rx) 705 706 #define delaymsec(_x) DELAY(1000 * (_x)) 707 #ifdef IXL_DEBUG 708 #define DDPRINTF(sc, fmt, args...) \ 709 do { \ 710 if ((sc) != NULL) { \ 711 device_printf( \ 712 ((struct ixl_softc *)(sc))->sc_dev, \ 713 ""); \ 714 } \ 715 printf("%s:\t" fmt, __func__, ##args); \ 716 } while (0) 717 #else 718 #define DDPRINTF(sc, fmt, args...) __nothing 719 #endif 720 #ifndef IXL_STATS_INTERVAL_MSEC 721 #define IXL_STATS_INTERVAL_MSEC 10000 722 #endif 723 #ifndef IXL_QUEUE_NUM 724 #define IXL_QUEUE_NUM 0 725 #endif 726 727 static bool ixl_param_nomsix = false; 728 static int ixl_param_stats_interval = IXL_STATS_INTERVAL_MSEC; 729 static int ixl_param_nqps_limit = IXL_QUEUE_NUM; 730 static unsigned int ixl_param_tx_ndescs = 1024; 731 static unsigned int ixl_param_rx_ndescs = 1024; 732 733 static enum i40e_mac_type 734 ixl_mactype(pci_product_id_t); 735 static void ixl_clear_hw(struct ixl_softc *); 736 static int ixl_pf_reset(struct ixl_softc *); 737 738 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 739 bus_size_t, bus_size_t); 740 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 741 742 static int ixl_arq_fill(struct ixl_softc *); 743 static void ixl_arq_unfill(struct ixl_softc *); 744 745 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 746 unsigned int); 747 static void ixl_atq_set(struct ixl_atq *, 748 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 749 static int ixl_atq_post(struct ixl_softc *, struct ixl_atq *); 750 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 751 static void ixl_atq_done(struct ixl_softc *); 752 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 753 static int ixl_get_version(struct ixl_softc *); 754 static int ixl_get_hw_capabilities(struct ixl_softc *); 755 static int ixl_pxe_clear(struct ixl_softc *); 756 static int ixl_lldp_shut(struct ixl_softc *); 757 static int ixl_get_mac(struct ixl_softc *); 758 static int ixl_get_switch_config(struct ixl_softc *); 759 static int ixl_phy_mask_ints(struct ixl_softc *); 760 static int ixl_get_phy_types(struct ixl_softc *, uint64_t *); 761 static int ixl_restart_an(struct ixl_softc *); 762 static int ixl_hmc(struct ixl_softc *); 763 static void ixl_hmc_free(struct ixl_softc *); 764 static int ixl_get_vsi(struct ixl_softc *); 765 static int ixl_set_vsi(struct ixl_softc *); 766 static void ixl_set_filter_control(struct ixl_softc *); 767 static void ixl_get_link_status(void *); 768 static int ixl_get_link_status_poll(struct ixl_softc *); 769 static int ixl_set_link_status(struct ixl_softc *, 770 const struct ixl_aq_desc *); 771 static void ixl_config_rss(struct ixl_softc *); 772 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 773 uint16_t, uint16_t); 774 static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *, 775 uint16_t, uint16_t); 776 static void ixl_arq(void *); 777 static void ixl_hmc_pack(void *, const void *, 778 const struct ixl_hmc_pack *, unsigned int); 779 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 780 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 781 782 static int ixl_match(device_t, cfdata_t, void *); 783 static void ixl_attach(device_t, device_t, void *); 784 static int ixl_detach(device_t, int); 785 786 static void ixl_media_add(struct ixl_softc *, uint64_t); 787 static int ixl_media_change(struct ifnet *); 788 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 789 static void ixl_watchdog(struct ifnet *); 790 static int ixl_ioctl(struct ifnet *, u_long, void *); 791 static void ixl_start(struct ifnet *); 792 static int ixl_transmit(struct ifnet *, struct mbuf *); 793 static void ixl_deferred_transmit(void *); 794 static int ixl_intr(void *); 795 static int ixl_queue_intr(void *); 796 static int ixl_other_intr(void *); 797 static void ixl_handle_queue(void *); 798 static void ixl_sched_handle_queue(struct ixl_softc *, 799 struct ixl_queue_pair *); 800 static int ixl_init(struct ifnet *); 801 static int ixl_init_locked(struct ixl_softc *); 802 static void ixl_stop(struct ifnet *, int); 803 static void ixl_stop_locked(struct ixl_softc *); 804 static int ixl_iff(struct ixl_softc *); 805 static int ixl_ifflags_cb(struct ethercom *); 806 static int ixl_setup_interrupts(struct ixl_softc *); 807 static int ixl_establish_intx(struct ixl_softc *); 808 static int ixl_establish_msix(struct ixl_softc *); 809 static void ixl_set_affinity_msix(struct ixl_softc *); 810 static void ixl_enable_queue_intr(struct ixl_softc *, 811 struct ixl_queue_pair *); 812 static void ixl_disable_queue_intr(struct ixl_softc *, 813 struct ixl_queue_pair *); 814 static void ixl_enable_other_intr(struct ixl_softc *); 815 static void ixl_disable_other_intr(struct ixl_softc *); 816 static void ixl_config_queue_intr(struct ixl_softc *); 817 static void ixl_config_other_intr(struct ixl_softc *); 818 819 static struct ixl_tx_ring * 820 ixl_txr_alloc(struct ixl_softc *, unsigned int); 821 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 822 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 823 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 824 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 825 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 826 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 827 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 828 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 829 830 static struct ixl_rx_ring * 831 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 832 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 833 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 834 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 835 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 836 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 837 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 838 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 839 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 840 841 static struct workqueue * 842 ixl_workq_create(const char *, pri_t, int, int); 843 static void ixl_workq_destroy(struct workqueue *); 844 static int ixl_workqs_teardown(device_t); 845 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 846 static void ixl_work_add(struct workqueue *, struct ixl_work *); 847 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 848 static void ixl_workq_work(struct work *, void *); 849 static const struct ixl_product * 850 ixl_lookup(const struct pci_attach_args *pa); 851 static void ixl_link_state_update(struct ixl_softc *, 852 const struct ixl_aq_desc *); 853 static int ixl_vlan_cb(struct ethercom *, uint16_t, bool); 854 static int ixl_setup_vlan_hwfilter(struct ixl_softc *); 855 static void ixl_teardown_vlan_hwfilter(struct ixl_softc *); 856 static int ixl_update_macvlan(struct ixl_softc *); 857 static int ixl_setup_interrupts(struct ixl_softc *);; 858 static void ixl_teardown_interrupts(struct ixl_softc *); 859 static int ixl_setup_stats(struct ixl_softc *); 860 static void ixl_teardown_stats(struct ixl_softc *); 861 static void ixl_stats_callout(void *); 862 static void ixl_stats_update(void *); 863 static int ixl_setup_sysctls(struct ixl_softc *); 864 static void ixl_teardown_sysctls(struct ixl_softc *); 865 static int ixl_queue_pairs_alloc(struct ixl_softc *); 866 static void ixl_queue_pairs_free(struct ixl_softc *); 867 868 static const struct ixl_phy_type ixl_phy_type_map[] = { 869 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 870 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 871 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 872 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 873 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 874 { 1ULL << IXL_PHY_TYPE_XAUI | 875 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 876 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 877 { 1ULL << IXL_PHY_TYPE_XLAUI | 878 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 879 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 880 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 881 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 882 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 883 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 884 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 885 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 886 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 887 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 888 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 889 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 890 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 891 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 892 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 893 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 894 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 895 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 896 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 897 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 898 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 899 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 900 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 901 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 902 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_CR }, 903 }; 904 905 static const struct ixl_speed_type ixl_speed_type_map[] = { 906 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 907 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 908 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 909 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 910 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 911 }; 912 913 static const struct ixl_aq_regs ixl_pf_aq_regs = { 914 .atq_tail = I40E_PF_ATQT, 915 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 916 .atq_head = I40E_PF_ATQH, 917 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 918 .atq_len = I40E_PF_ATQLEN, 919 .atq_bal = I40E_PF_ATQBAL, 920 .atq_bah = I40E_PF_ATQBAH, 921 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 922 923 .arq_tail = I40E_PF_ARQT, 924 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 925 .arq_head = I40E_PF_ARQH, 926 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 927 .arq_len = I40E_PF_ARQLEN, 928 .arq_bal = I40E_PF_ARQBAL, 929 .arq_bah = I40E_PF_ARQBAH, 930 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 931 }; 932 933 #define ixl_rd(_s, _r) \ 934 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 935 #define ixl_wr(_s, _r, _v) \ 936 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 937 #define ixl_barrier(_s, _r, _l, _o) \ 938 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 939 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 940 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 941 942 static inline uint32_t 943 ixl_dmamem_hi(struct ixl_dmamem *ixm) 944 { 945 uint32_t retval; 946 uint64_t val; 947 948 if (sizeof(IXL_DMA_DVA(ixm)) > 4) { 949 val = (intptr_t)IXL_DMA_DVA(ixm); 950 retval = (uint32_t)(val >> 32); 951 } else { 952 retval = 0; 953 } 954 955 return retval; 956 } 957 958 static inline uint32_t 959 ixl_dmamem_lo(struct ixl_dmamem *ixm) 960 { 961 962 return (uint32_t)IXL_DMA_DVA(ixm); 963 } 964 965 static inline void 966 ixl_aq_dva(struct ixl_aq_desc *iaq, bus_addr_t addr) 967 { 968 uint64_t val; 969 970 if (sizeof(addr) > 4) { 971 val = (intptr_t)addr; 972 iaq->iaq_param[2] = htole32(val >> 32); 973 } else { 974 iaq->iaq_param[2] = htole32(0); 975 } 976 977 iaq->iaq_param[3] = htole32(addr); 978 } 979 980 static inline unsigned int 981 ixl_rxr_unrefreshed(unsigned int prod, unsigned int cons, unsigned int ndescs) 982 { 983 unsigned int num; 984 985 if (prod < cons) 986 num = cons - prod; 987 else 988 num = (ndescs - prod) + cons; 989 990 if (__predict_true(num > 0)) { 991 /* device cannot receive packets if all descripter is filled */ 992 num -= 1; 993 } 994 995 return num; 996 } 997 998 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 999 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 1000 DVF_DETACH_SHUTDOWN); 1001 1002 static const struct ixl_product ixl_products[] = { 1003 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 1004 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 1005 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 1006 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 1007 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 1008 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 1009 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T }, 1010 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 1011 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 1012 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 1013 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 1014 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 1015 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 1016 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 1017 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 1018 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 1019 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 1020 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 1021 /* required last entry */ 1022 {0, 0} 1023 }; 1024 1025 static const struct ixl_product * 1026 ixl_lookup(const struct pci_attach_args *pa) 1027 { 1028 const struct ixl_product *ixlp; 1029 1030 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 1031 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 1032 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 1033 return ixlp; 1034 } 1035 1036 return NULL; 1037 } 1038 1039 static int 1040 ixl_match(device_t parent, cfdata_t match, void *aux) 1041 { 1042 const struct pci_attach_args *pa = aux; 1043 1044 return (ixl_lookup(pa) != NULL) ? 1 : 0; 1045 } 1046 1047 static void 1048 ixl_attach(device_t parent, device_t self, void *aux) 1049 { 1050 struct ixl_softc *sc; 1051 struct pci_attach_args *pa = aux; 1052 struct ifnet *ifp; 1053 pcireg_t memtype, reg; 1054 uint32_t firstq, port, ari, func; 1055 uint64_t phy_types = 0; 1056 char xnamebuf[32]; 1057 int tries, rv; 1058 1059 sc = device_private(self); 1060 sc->sc_dev = self; 1061 ifp = &sc->sc_ec.ec_if; 1062 1063 sc->sc_pa = *pa; 1064 sc->sc_dmat = (pci_dma64_available(pa)) ? 1065 pa->pa_dmat64 : pa->pa_dmat; 1066 sc->sc_aq_regs = &ixl_pf_aq_regs; 1067 1068 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 1069 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(reg)); 1070 1071 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 1072 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 1073 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 1074 aprint_error(": unable to map registers\n"); 1075 return; 1076 } 1077 1078 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 1079 1080 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 1081 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 1082 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 1083 sc->sc_base_queue = firstq; 1084 1085 ixl_clear_hw(sc); 1086 if (ixl_pf_reset(sc) == -1) { 1087 /* error printed by ixl pf_reset */ 1088 goto unmap; 1089 } 1090 1091 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 1092 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 1093 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 1094 sc->sc_port = port; 1095 aprint_normal(": port %u", sc->sc_port); 1096 1097 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 1098 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 1099 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 1100 1101 func = ixl_rd(sc, I40E_PF_FUNC_RID); 1102 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 1103 1104 /* initialise the adminq */ 1105 1106 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 1107 1108 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 1109 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1110 aprint_error("\n" "%s: unable to allocate atq\n", 1111 device_xname(self)); 1112 goto unmap; 1113 } 1114 1115 SIMPLEQ_INIT(&sc->sc_arq_idle); 1116 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 1117 sc->sc_arq_cons = 0; 1118 sc->sc_arq_prod = 0; 1119 1120 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 1121 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1122 aprint_error("\n" "%s: unable to allocate arq\n", 1123 device_xname(self)); 1124 goto free_atq; 1125 } 1126 1127 if (!ixl_arq_fill(sc)) { 1128 aprint_error("\n" "%s: unable to fill arq descriptors\n", 1129 device_xname(self)); 1130 goto free_arq; 1131 } 1132 1133 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1134 0, IXL_DMA_LEN(&sc->sc_atq), 1135 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1136 1137 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1138 0, IXL_DMA_LEN(&sc->sc_arq), 1139 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1140 1141 for (tries = 0; tries < 10; tries++) { 1142 sc->sc_atq_cons = 0; 1143 sc->sc_atq_prod = 0; 1144 1145 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1146 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1147 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1148 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1149 1150 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1151 1152 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1153 ixl_dmamem_lo(&sc->sc_atq)); 1154 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1155 ixl_dmamem_hi(&sc->sc_atq)); 1156 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1157 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1158 1159 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1160 ixl_dmamem_lo(&sc->sc_arq)); 1161 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1162 ixl_dmamem_hi(&sc->sc_arq)); 1163 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1164 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1165 1166 rv = ixl_get_version(sc); 1167 if (rv == 0) 1168 break; 1169 if (rv != ETIMEDOUT) { 1170 aprint_error(", unable to get firmware version\n"); 1171 goto shutdown; 1172 } 1173 1174 delaymsec(100); 1175 } 1176 1177 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1178 1179 if (sc->sc_mac_type == I40E_MAC_X722) 1180 sc->sc_nqueue_pairs_device = 128; 1181 else 1182 sc->sc_nqueue_pairs_device = 64; 1183 1184 rv = ixl_get_hw_capabilities(sc); 1185 if (rv != 0) { 1186 aprint_error(", GET HW CAPABILITIES %s\n", 1187 rv == ETIMEDOUT ? "timeout" : "error"); 1188 goto shutdown; 1189 } 1190 1191 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1192 if (ixl_param_nqps_limit > 0) { 1193 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_max, 1194 ixl_param_nqps_limit); 1195 } 1196 1197 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1198 sc->sc_tx_ring_ndescs = ixl_param_tx_ndescs; 1199 sc->sc_rx_ring_ndescs = ixl_param_rx_ndescs; 1200 1201 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1202 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1203 1204 if (ixl_get_mac(sc) != 0) { 1205 /* error printed by ixl_get_mac */ 1206 goto shutdown; 1207 } 1208 1209 aprint_normal("\n"); 1210 aprint_naive("\n"); 1211 1212 aprint_normal_dev(self, "Ethernet address %s\n", 1213 ether_sprintf(sc->sc_enaddr)); 1214 1215 rv = ixl_pxe_clear(sc); 1216 if (rv != 0) { 1217 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1218 rv == ETIMEDOUT ? "timeout" : "error"); 1219 } 1220 1221 ixl_set_filter_control(sc); 1222 1223 if (ixl_hmc(sc) != 0) { 1224 /* error printed by ixl_hmc */ 1225 goto shutdown; 1226 } 1227 1228 if (ixl_lldp_shut(sc) != 0) { 1229 /* error printed by ixl_lldp_shut */ 1230 goto free_hmc; 1231 } 1232 1233 if (ixl_phy_mask_ints(sc) != 0) { 1234 /* error printed by ixl_phy_mask_ints */ 1235 goto free_hmc; 1236 } 1237 1238 if (ixl_restart_an(sc) != 0) { 1239 /* error printed by ixl_restart_an */ 1240 goto free_hmc; 1241 } 1242 1243 if (ixl_get_switch_config(sc) != 0) { 1244 /* error printed by ixl_get_switch_config */ 1245 goto free_hmc; 1246 } 1247 1248 if (ixl_get_phy_types(sc, &phy_types) != 0) { 1249 /* error printed by ixl_get_phy_abilities */ 1250 goto free_hmc; 1251 } 1252 1253 rv = ixl_get_link_status_poll(sc); 1254 if (rv != 0) { 1255 aprint_error_dev(self, "GET LINK STATUS %s\n", 1256 rv == ETIMEDOUT ? "timeout" : "error"); 1257 goto free_hmc; 1258 } 1259 1260 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1261 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1262 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1263 goto free_hmc; 1264 } 1265 1266 rv = ixl_get_vsi(sc); 1267 if (rv != 0) { 1268 aprint_error_dev(self, "GET VSI %s %d\n", 1269 rv == ETIMEDOUT ? "timeout" : "error", rv); 1270 goto free_scratch; 1271 } 1272 1273 rv = ixl_set_vsi(sc); 1274 if (rv != 0) { 1275 aprint_error_dev(self, "UPDATE VSI error %s %d\n", 1276 rv == ETIMEDOUT ? "timeout" : "error", rv); 1277 goto free_scratch; 1278 } 1279 1280 if (ixl_queue_pairs_alloc(sc) != 0) { 1281 /* error printed by ixl_queue_pairs_alloc */ 1282 goto free_scratch; 1283 } 1284 1285 if (ixl_setup_interrupts(sc) != 0) { 1286 /* error printed by ixl_setup_interrupts */ 1287 goto free_queue_pairs; 1288 } 1289 1290 if (ixl_setup_stats(sc) != 0) { 1291 aprint_error_dev(self, "failed to setup event counters\n"); 1292 goto teardown_intrs; 1293 } 1294 1295 if (ixl_setup_sysctls(sc) != 0) { 1296 /* error printed by ixl_setup_sysctls */ 1297 goto teardown_stats; 1298 } 1299 1300 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1301 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1302 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1303 if (sc->sc_workq == NULL) 1304 goto teardown_sysctls; 1305 1306 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1307 sc->sc_workq_txrx = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1308 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1309 if (sc->sc_workq_txrx == NULL) 1310 goto teardown_wqs; 1311 1312 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1313 cv_init(&sc->sc_atq_cv, xnamebuf); 1314 1315 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1316 1317 ifp->if_softc = sc; 1318 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1319 ifp->if_extflags = IFEF_MPSAFE; 1320 ifp->if_ioctl = ixl_ioctl; 1321 ifp->if_start = ixl_start; 1322 ifp->if_transmit = ixl_transmit; 1323 ifp->if_watchdog = ixl_watchdog; 1324 ifp->if_init = ixl_init; 1325 ifp->if_stop = ixl_stop; 1326 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1327 IFQ_SET_READY(&ifp->if_snd); 1328 ifp->if_capabilities |= IXL_IFCAP_RXCSUM; 1329 #if 0 1330 ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_UDPv4_Tx; 1331 #endif 1332 ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb); 1333 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_MTU; 1334 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1335 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1336 1337 sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities; 1338 /* Disable VLAN_HWFILTER by default */ 1339 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1340 1341 sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable; 1342 1343 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1344 ifmedia_init(&sc->sc_media, IFM_IMASK, ixl_media_change, 1345 ixl_media_status); 1346 1347 ixl_media_add(sc, phy_types); 1348 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1349 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1350 1351 if_attach(ifp); 1352 if_deferred_start_init(ifp, NULL); 1353 ether_ifattach(ifp, sc->sc_enaddr); 1354 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1355 1356 (void)ixl_get_link_status_poll(sc); 1357 ixl_work_set(&sc->sc_link_state_task, ixl_get_link_status, sc); 1358 1359 ixl_config_other_intr(sc); 1360 ixl_enable_other_intr(sc); 1361 1362 /* remove default mac filter and replace it so we can see vlans */ 1363 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 1364 if (rv != ENOENT) { 1365 aprint_debug_dev(self, 1366 "unable to remove macvlan %u\n", rv); 1367 } 1368 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1369 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1370 if (rv != ENOENT) { 1371 aprint_debug_dev(self, 1372 "unable to remove macvlan, ignore vlan %u\n", rv); 1373 } 1374 1375 if (ixl_update_macvlan(sc) != 0) { 1376 aprint_debug_dev(self, 1377 "couldn't enable vlan hardware filter\n"); 1378 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1379 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1380 } 1381 1382 sc->sc_txrx_workqueue = true; 1383 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1384 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1385 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1386 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1387 1388 ixl_stats_update(sc); 1389 sc->sc_stats_counters.isc_has_offset = true; 1390 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 1391 1392 if (pmf_device_register(self, NULL, NULL) != true) 1393 aprint_debug_dev(self, "couldn't establish power handler\n"); 1394 sc->sc_attached = true; 1395 return; 1396 1397 teardown_wqs: 1398 config_finalize_register(self, ixl_workqs_teardown); 1399 teardown_sysctls: 1400 ixl_teardown_sysctls(sc); 1401 teardown_stats: 1402 ixl_teardown_stats(sc); 1403 teardown_intrs: 1404 ixl_teardown_interrupts(sc); 1405 free_queue_pairs: 1406 ixl_queue_pairs_free(sc); 1407 free_scratch: 1408 ixl_dmamem_free(sc, &sc->sc_scratch); 1409 free_hmc: 1410 ixl_hmc_free(sc); 1411 shutdown: 1412 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1413 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1414 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1415 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1416 1417 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1418 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1419 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1420 1421 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1422 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1423 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1424 1425 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1426 0, IXL_DMA_LEN(&sc->sc_arq), 1427 BUS_DMASYNC_POSTREAD); 1428 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1429 0, IXL_DMA_LEN(&sc->sc_atq), 1430 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1431 1432 ixl_arq_unfill(sc); 1433 free_arq: 1434 ixl_dmamem_free(sc, &sc->sc_arq); 1435 free_atq: 1436 ixl_dmamem_free(sc, &sc->sc_atq); 1437 unmap: 1438 mutex_destroy(&sc->sc_atq_lock); 1439 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1440 mutex_destroy(&sc->sc_cfg_lock); 1441 sc->sc_mems = 0; 1442 1443 sc->sc_attached = false; 1444 } 1445 1446 static int 1447 ixl_detach(device_t self, int flags) 1448 { 1449 struct ixl_softc *sc = device_private(self); 1450 struct ifnet *ifp = &sc->sc_ec.ec_if; 1451 1452 if (!sc->sc_attached) 1453 return 0; 1454 1455 ixl_stop(ifp, 1); 1456 1457 ixl_disable_other_intr(sc); 1458 1459 callout_stop(&sc->sc_stats_callout); 1460 ixl_work_wait(sc->sc_workq, &sc->sc_stats_task); 1461 1462 /* wait for ATQ handler */ 1463 mutex_enter(&sc->sc_atq_lock); 1464 mutex_exit(&sc->sc_atq_lock); 1465 1466 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1467 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1468 1469 if (sc->sc_workq != NULL) { 1470 ixl_workq_destroy(sc->sc_workq); 1471 sc->sc_workq = NULL; 1472 } 1473 1474 if (sc->sc_workq_txrx != NULL) { 1475 ixl_workq_destroy(sc->sc_workq_txrx); 1476 sc->sc_workq_txrx = NULL; 1477 } 1478 1479 ifmedia_delete_instance(&sc->sc_media, IFM_INST_ANY); 1480 ether_ifdetach(ifp); 1481 if_detach(ifp); 1482 1483 ixl_teardown_interrupts(sc); 1484 ixl_teardown_stats(sc); 1485 ixl_teardown_sysctls(sc); 1486 1487 ixl_queue_pairs_free(sc); 1488 1489 ixl_dmamem_free(sc, &sc->sc_scratch); 1490 ixl_hmc_free(sc); 1491 1492 /* shutdown */ 1493 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1494 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1495 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1496 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1497 1498 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1499 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1500 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1501 1502 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1503 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1504 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1505 1506 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1507 0, IXL_DMA_LEN(&sc->sc_arq), 1508 BUS_DMASYNC_POSTREAD); 1509 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1510 0, IXL_DMA_LEN(&sc->sc_atq), 1511 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1512 1513 ixl_arq_unfill(sc); 1514 1515 ixl_dmamem_free(sc, &sc->sc_arq); 1516 ixl_dmamem_free(sc, &sc->sc_atq); 1517 1518 cv_destroy(&sc->sc_atq_cv); 1519 mutex_destroy(&sc->sc_atq_lock); 1520 1521 if (sc->sc_mems != 0) { 1522 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1523 sc->sc_mems = 0; 1524 } 1525 1526 mutex_destroy(&sc->sc_cfg_lock); 1527 1528 return 0; 1529 } 1530 1531 static int 1532 ixl_workqs_teardown(device_t self) 1533 { 1534 struct ixl_softc *sc = device_private(self); 1535 1536 if (sc->sc_workq != NULL) { 1537 ixl_workq_destroy(sc->sc_workq); 1538 sc->sc_workq = NULL; 1539 } 1540 1541 if (sc->sc_workq_txrx != NULL) { 1542 ixl_workq_destroy(sc->sc_workq_txrx); 1543 sc->sc_workq_txrx = NULL; 1544 } 1545 1546 return 0; 1547 } 1548 1549 static int 1550 ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 1551 { 1552 struct ifnet *ifp = &ec->ec_if; 1553 struct ixl_softc *sc = ifp->if_softc; 1554 int rv; 1555 1556 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1557 return 0; 1558 } 1559 1560 if (set) { 1561 rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid, 1562 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1563 if (rv == 0) { 1564 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 1565 vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1566 } 1567 } else { 1568 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid, 1569 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1570 (void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid, 1571 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1572 } 1573 1574 return rv; 1575 } 1576 1577 static void 1578 ixl_media_add(struct ixl_softc *sc, uint64_t phy_types) 1579 { 1580 struct ifmedia *ifm = &sc->sc_media; 1581 const struct ixl_phy_type *itype; 1582 unsigned int i; 1583 1584 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1585 itype = &ixl_phy_type_map[i]; 1586 1587 if (ISSET(phy_types, itype->phy_type)) { 1588 ifmedia_add(ifm, 1589 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1590 1591 if (itype->ifm_type == IFM_100_TX) { 1592 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1593 0, NULL); 1594 } 1595 } 1596 } 1597 } 1598 1599 static void 1600 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1601 { 1602 struct ixl_softc *sc = ifp->if_softc; 1603 1604 ifmr->ifm_status = sc->sc_media_status; 1605 ifmr->ifm_active = sc->sc_media_active; 1606 1607 mutex_enter(&sc->sc_cfg_lock); 1608 if (ifp->if_link_state == LINK_STATE_UP) 1609 SET(ifmr->ifm_status, IFM_ACTIVE); 1610 mutex_exit(&sc->sc_cfg_lock); 1611 } 1612 1613 static int 1614 ixl_media_change(struct ifnet *ifp) 1615 { 1616 1617 return 0; 1618 } 1619 1620 static void 1621 ixl_watchdog(struct ifnet *ifp) 1622 { 1623 1624 } 1625 1626 static void 1627 ixl_del_all_multiaddr(struct ixl_softc *sc) 1628 { 1629 struct ethercom *ec = &sc->sc_ec; 1630 struct ether_multi *enm; 1631 struct ether_multistep step; 1632 1633 ETHER_LOCK(ec); 1634 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1635 ETHER_NEXT_MULTI(step, enm)) { 1636 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1637 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1638 } 1639 ETHER_UNLOCK(ec); 1640 } 1641 1642 static int 1643 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1644 { 1645 struct ifnet *ifp = &sc->sc_ec.ec_if; 1646 int rv; 1647 1648 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1649 return 0; 1650 1651 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1652 ixl_del_all_multiaddr(sc); 1653 SET(ifp->if_flags, IFF_ALLMULTI); 1654 return ENETRESET; 1655 } 1656 1657 /* multicast address can not use VLAN HWFILTER */ 1658 rv = ixl_add_macvlan(sc, addrlo, 0, 1659 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1660 1661 if (rv == ENOSPC) { 1662 ixl_del_all_multiaddr(sc); 1663 SET(ifp->if_flags, IFF_ALLMULTI); 1664 return ENETRESET; 1665 } 1666 1667 return rv; 1668 } 1669 1670 static int 1671 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1672 { 1673 struct ifnet *ifp = &sc->sc_ec.ec_if; 1674 struct ethercom *ec = &sc->sc_ec; 1675 struct ether_multi *enm, *enm_last; 1676 struct ether_multistep step; 1677 int error, rv = 0; 1678 1679 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1680 ixl_remove_macvlan(sc, addrlo, 0, 1681 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1682 return 0; 1683 } 1684 1685 ETHER_LOCK(ec); 1686 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1687 ETHER_NEXT_MULTI(step, enm)) { 1688 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1689 ETHER_ADDR_LEN) != 0) { 1690 goto out; 1691 } 1692 } 1693 1694 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1695 ETHER_NEXT_MULTI(step, enm)) { 1696 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1697 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1698 if (error != 0) 1699 break; 1700 } 1701 1702 if (enm != NULL) { 1703 enm_last = enm; 1704 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1705 ETHER_NEXT_MULTI(step, enm)) { 1706 if (enm == enm_last) 1707 break; 1708 1709 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1710 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1711 } 1712 } else { 1713 CLR(ifp->if_flags, IFF_ALLMULTI); 1714 rv = ENETRESET; 1715 } 1716 1717 out: 1718 ETHER_UNLOCK(ec); 1719 return rv; 1720 } 1721 1722 static int 1723 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1724 { 1725 struct ifreq *ifr = (struct ifreq *)data; 1726 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1727 struct ixl_tx_ring *txr; 1728 struct ixl_rx_ring *rxr; 1729 const struct sockaddr *sa; 1730 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1731 int s, error = 0; 1732 unsigned int i; 1733 1734 switch (cmd) { 1735 case SIOCADDMULTI: 1736 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1737 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1738 error = ether_multiaddr(sa, addrlo, addrhi); 1739 if (error != 0) 1740 return error; 1741 1742 error = ixl_add_multi(sc, addrlo, addrhi); 1743 if (error != 0 && error != ENETRESET) { 1744 ether_delmulti(sa, &sc->sc_ec); 1745 error = EIO; 1746 } 1747 } 1748 break; 1749 1750 case SIOCDELMULTI: 1751 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1752 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1753 error = ether_multiaddr(sa, addrlo, addrhi); 1754 if (error != 0) 1755 return error; 1756 1757 error = ixl_del_multi(sc, addrlo, addrhi); 1758 } 1759 break; 1760 1761 case SIOCGIFDATA: 1762 case SIOCZIFDATA: 1763 ifp->if_ipackets = 0; 1764 ifp->if_ibytes = 0; 1765 ifp->if_iqdrops = 0; 1766 ifp->if_ierrors = 0; 1767 ifp->if_opackets = 0; 1768 ifp->if_obytes = 0; 1769 ifp->if_omcasts = 0; 1770 1771 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 1772 txr = sc->sc_qps[i].qp_txr; 1773 rxr = sc->sc_qps[i].qp_rxr; 1774 1775 mutex_enter(&rxr->rxr_lock); 1776 ifp->if_ipackets += rxr->rxr_ipackets; 1777 ifp->if_ibytes += rxr->rxr_ibytes; 1778 ifp->if_iqdrops += rxr->rxr_iqdrops; 1779 ifp->if_ierrors += rxr->rxr_ierrors; 1780 if (cmd == SIOCZIFDATA) { 1781 rxr->rxr_ipackets = 0; 1782 rxr->rxr_ibytes = 0; 1783 rxr->rxr_iqdrops = 0; 1784 rxr->rxr_ierrors = 0; 1785 } 1786 mutex_exit(&rxr->rxr_lock); 1787 1788 mutex_enter(&txr->txr_lock); 1789 ifp->if_opackets += txr->txr_opackets; 1790 ifp->if_obytes += txr->txr_opackets; 1791 ifp->if_omcasts += txr->txr_omcasts; 1792 if (cmd == SIOCZIFDATA) { 1793 txr->txr_opackets = 0; 1794 txr->txr_opackets = 0; 1795 txr->txr_omcasts = 0; 1796 } 1797 mutex_exit(&txr->txr_lock); 1798 } 1799 /* FALLTHROUGH */ 1800 default: 1801 s = splnet(); 1802 error = ether_ioctl(ifp, cmd, data); 1803 splx(s); 1804 } 1805 1806 if (error == ENETRESET) 1807 error = ixl_iff(sc); 1808 1809 return error; 1810 } 1811 1812 static enum i40e_mac_type 1813 ixl_mactype(pci_product_id_t id) 1814 { 1815 1816 switch (id) { 1817 case PCI_PRODUCT_INTEL_XL710_SFP: 1818 case PCI_PRODUCT_INTEL_XL710_KX_B: 1819 case PCI_PRODUCT_INTEL_XL710_KX_C: 1820 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1821 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1822 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1823 case PCI_PRODUCT_INTEL_X710_10G_T: 1824 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1825 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1826 case PCI_PRODUCT_INTEL_X710_T4_10G: 1827 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1828 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1829 return I40E_MAC_XL710; 1830 1831 case PCI_PRODUCT_INTEL_X722_KX: 1832 case PCI_PRODUCT_INTEL_X722_QSFP: 1833 case PCI_PRODUCT_INTEL_X722_SFP: 1834 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1835 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1836 case PCI_PRODUCT_INTEL_X722_I_SFP: 1837 return I40E_MAC_X722; 1838 } 1839 1840 return I40E_MAC_GENERIC; 1841 } 1842 1843 static inline void * 1844 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1845 { 1846 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1847 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1848 1849 if (i >= e->hmc_count) 1850 return NULL; 1851 1852 kva += e->hmc_base; 1853 kva += i * e->hmc_size; 1854 1855 return kva; 1856 } 1857 1858 static inline size_t 1859 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1860 { 1861 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1862 1863 return e->hmc_size; 1864 } 1865 1866 static void 1867 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1868 { 1869 struct ixl_rx_ring *rxr = qp->qp_rxr; 1870 1871 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1872 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1873 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1874 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1875 ixl_flush(sc); 1876 } 1877 1878 static void 1879 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1880 { 1881 struct ixl_rx_ring *rxr = qp->qp_rxr; 1882 1883 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1884 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1885 ixl_flush(sc); 1886 } 1887 1888 static void 1889 ixl_enable_other_intr(struct ixl_softc *sc) 1890 { 1891 1892 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1893 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1894 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1895 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1896 ixl_flush(sc); 1897 } 1898 1899 static void 1900 ixl_disable_other_intr(struct ixl_softc *sc) 1901 { 1902 1903 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1904 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1905 ixl_flush(sc); 1906 } 1907 1908 static int 1909 ixl_reinit(struct ixl_softc *sc) 1910 { 1911 struct ixl_rx_ring *rxr; 1912 struct ixl_tx_ring *txr; 1913 unsigned int i; 1914 uint32_t reg; 1915 1916 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1917 1918 if (ixl_get_vsi(sc) != 0) 1919 return EIO; 1920 1921 if (ixl_set_vsi(sc) != 0) 1922 return EIO; 1923 1924 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1925 txr = sc->sc_qps[i].qp_txr; 1926 rxr = sc->sc_qps[i].qp_rxr; 1927 1928 txr->txr_cons = txr->txr_prod = 0; 1929 rxr->rxr_cons = rxr->rxr_prod = 0; 1930 1931 ixl_txr_config(sc, txr); 1932 ixl_rxr_config(sc, rxr); 1933 } 1934 1935 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1936 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 1937 1938 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1939 txr = sc->sc_qps[i].qp_txr; 1940 rxr = sc->sc_qps[i].qp_rxr; 1941 1942 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 1943 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 1944 ixl_flush(sc); 1945 1946 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 1947 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 1948 1949 /* ixl_rxfill() needs lock held */ 1950 mutex_enter(&rxr->rxr_lock); 1951 ixl_rxfill(sc, rxr); 1952 mutex_exit(&rxr->rxr_lock); 1953 1954 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 1955 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 1956 ixl_wr(sc, I40E_QRX_ENA(i), reg); 1957 if (ixl_rxr_enabled(sc, rxr) != 0) 1958 goto stop; 1959 1960 ixl_txr_qdis(sc, txr, 1); 1961 1962 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 1963 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 1964 ixl_wr(sc, I40E_QTX_ENA(i), reg); 1965 1966 if (ixl_txr_enabled(sc, txr) != 0) 1967 goto stop; 1968 } 1969 1970 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1971 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 1972 1973 return 0; 1974 1975 stop: 1976 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1977 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 1978 1979 return ETIMEDOUT; 1980 } 1981 1982 static int 1983 ixl_init_locked(struct ixl_softc *sc) 1984 { 1985 struct ifnet *ifp = &sc->sc_ec.ec_if; 1986 unsigned int i; 1987 int error, eccap_change; 1988 1989 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1990 1991 if (ISSET(ifp->if_flags, IFF_RUNNING)) 1992 ixl_stop_locked(sc); 1993 1994 if (sc->sc_dead) { 1995 return ENXIO; 1996 } 1997 1998 eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable; 1999 if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING)) 2000 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 2001 2002 if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) { 2003 if (ixl_update_macvlan(sc) == 0) { 2004 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 2005 } else { 2006 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 2007 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 2008 } 2009 } 2010 2011 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 2012 sc->sc_nqueue_pairs = 1; 2013 else 2014 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 2015 2016 error = ixl_reinit(sc); 2017 if (error) { 2018 ixl_stop_locked(sc); 2019 return error; 2020 } 2021 2022 SET(ifp->if_flags, IFF_RUNNING); 2023 CLR(ifp->if_flags, IFF_OACTIVE); 2024 2025 (void)ixl_get_link_status(sc); 2026 2027 ixl_config_rss(sc); 2028 ixl_config_queue_intr(sc); 2029 2030 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2031 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 2032 } 2033 2034 error = ixl_iff(sc); 2035 if (error) { 2036 ixl_stop_locked(sc); 2037 return error; 2038 } 2039 2040 return 0; 2041 } 2042 2043 static int 2044 ixl_init(struct ifnet *ifp) 2045 { 2046 struct ixl_softc *sc = ifp->if_softc; 2047 int error; 2048 2049 mutex_enter(&sc->sc_cfg_lock); 2050 error = ixl_init_locked(sc); 2051 mutex_exit(&sc->sc_cfg_lock); 2052 2053 return error; 2054 } 2055 2056 static int 2057 ixl_iff(struct ixl_softc *sc) 2058 { 2059 struct ifnet *ifp = &sc->sc_ec.ec_if; 2060 struct ixl_atq iatq; 2061 struct ixl_aq_desc *iaq; 2062 struct ixl_aq_vsi_promisc_param *param; 2063 uint16_t flag_add, flag_del; 2064 int error; 2065 2066 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2067 return 0; 2068 2069 memset(&iatq, 0, sizeof(iatq)); 2070 2071 iaq = &iatq.iatq_desc; 2072 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 2073 2074 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 2075 param->flags = htole16(0); 2076 2077 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER) 2078 || ISSET(ifp->if_flags, IFF_PROMISC)) { 2079 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2080 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2081 } 2082 2083 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 2084 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2085 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2086 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 2087 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2088 } 2089 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2090 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2091 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2092 param->seid = sc->sc_seid; 2093 2094 error = ixl_atq_exec(sc, &iatq); 2095 if (error) 2096 return error; 2097 2098 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 2099 return EIO; 2100 2101 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 2102 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 2103 flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH; 2104 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH; 2105 } else { 2106 flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN; 2107 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN; 2108 } 2109 2110 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del); 2111 2112 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2113 ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add); 2114 } 2115 return 0; 2116 } 2117 2118 static void 2119 ixl_stop_rendezvous(struct ixl_softc *sc) 2120 { 2121 struct ixl_tx_ring *txr; 2122 struct ixl_rx_ring *rxr; 2123 unsigned int i; 2124 2125 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2126 txr = sc->sc_qps[i].qp_txr; 2127 rxr = sc->sc_qps[i].qp_rxr; 2128 2129 mutex_enter(&txr->txr_lock); 2130 mutex_exit(&txr->txr_lock); 2131 2132 mutex_enter(&rxr->rxr_lock); 2133 mutex_exit(&rxr->rxr_lock); 2134 2135 ixl_work_wait(sc->sc_workq_txrx, 2136 &sc->sc_qps[i].qp_task); 2137 } 2138 } 2139 2140 static void 2141 ixl_stop_locked(struct ixl_softc *sc) 2142 { 2143 struct ifnet *ifp = &sc->sc_ec.ec_if; 2144 struct ixl_rx_ring *rxr; 2145 struct ixl_tx_ring *txr; 2146 unsigned int i; 2147 uint32_t reg; 2148 2149 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2150 2151 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 2152 2153 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2154 txr = sc->sc_qps[i].qp_txr; 2155 rxr = sc->sc_qps[i].qp_rxr; 2156 2157 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 2158 2159 mutex_enter(&txr->txr_lock); 2160 ixl_txr_qdis(sc, txr, 0); 2161 /* XXX wait at least 400 usec for all tx queues in one go */ 2162 ixl_flush(sc); 2163 DELAY(500); 2164 2165 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2166 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2167 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2168 /* XXX wait 50ms from completaion of the TX queue disable*/ 2169 ixl_flush(sc); 2170 DELAY(50); 2171 2172 if (ixl_txr_disabled(sc, txr) != 0) { 2173 mutex_exit(&txr->txr_lock); 2174 goto die; 2175 } 2176 mutex_exit(&txr->txr_lock); 2177 2178 mutex_enter(&rxr->rxr_lock); 2179 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2180 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2181 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2182 /* XXX wait 50ms from completion of the RX queue disable */ 2183 ixl_flush(sc); 2184 DELAY(50); 2185 2186 if (ixl_rxr_disabled(sc, rxr) != 0) { 2187 mutex_exit(&rxr->rxr_lock); 2188 goto die; 2189 } 2190 mutex_exit(&rxr->rxr_lock); 2191 } 2192 2193 ixl_stop_rendezvous(sc); 2194 2195 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2196 txr = sc->sc_qps[i].qp_txr; 2197 rxr = sc->sc_qps[i].qp_rxr; 2198 2199 ixl_txr_unconfig(sc, txr); 2200 ixl_rxr_unconfig(sc, rxr); 2201 2202 ixl_txr_clean(sc, txr); 2203 ixl_rxr_clean(sc, rxr); 2204 } 2205 2206 return; 2207 die: 2208 sc->sc_dead = true; 2209 log(LOG_CRIT, "%s: failed to shut down rings", 2210 device_xname(sc->sc_dev)); 2211 return; 2212 } 2213 2214 static void 2215 ixl_stop(struct ifnet *ifp, int disable) 2216 { 2217 struct ixl_softc *sc = ifp->if_softc; 2218 2219 mutex_enter(&sc->sc_cfg_lock); 2220 ixl_stop_locked(sc); 2221 mutex_exit(&sc->sc_cfg_lock); 2222 } 2223 2224 static int 2225 ixl_queue_pairs_alloc(struct ixl_softc *sc) 2226 { 2227 struct ixl_queue_pair *qp; 2228 unsigned int i; 2229 size_t sz; 2230 2231 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2232 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 2233 2234 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2235 qp = &sc->sc_qps[i]; 2236 2237 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2238 ixl_handle_queue, qp); 2239 if (qp->qp_si == NULL) 2240 goto free; 2241 2242 qp->qp_txr = ixl_txr_alloc(sc, i); 2243 if (qp->qp_txr == NULL) 2244 goto free; 2245 2246 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2247 if (qp->qp_rxr == NULL) 2248 goto free; 2249 2250 qp->qp_sc = sc; 2251 ixl_work_set(&qp->qp_task, ixl_handle_queue, qp); 2252 snprintf(qp->qp_name, sizeof(qp->qp_name), 2253 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2254 } 2255 2256 return 0; 2257 free: 2258 if (sc->sc_qps != NULL) { 2259 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2260 qp = &sc->sc_qps[i]; 2261 2262 if (qp->qp_txr != NULL) 2263 ixl_txr_free(sc, qp->qp_txr); 2264 if (qp->qp_rxr != NULL) 2265 ixl_rxr_free(sc, qp->qp_rxr); 2266 if (qp->qp_si != NULL) 2267 softint_disestablish(qp->qp_si); 2268 } 2269 2270 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2271 kmem_free(sc->sc_qps, sz); 2272 sc->sc_qps = NULL; 2273 } 2274 2275 return -1; 2276 } 2277 2278 static void 2279 ixl_queue_pairs_free(struct ixl_softc *sc) 2280 { 2281 struct ixl_queue_pair *qp; 2282 unsigned int i; 2283 size_t sz; 2284 2285 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2286 qp = &sc->sc_qps[i]; 2287 ixl_txr_free(sc, qp->qp_txr); 2288 ixl_rxr_free(sc, qp->qp_rxr); 2289 softint_disestablish(qp->qp_si); 2290 } 2291 2292 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2293 kmem_free(sc->sc_qps, sz); 2294 sc->sc_qps = NULL; 2295 } 2296 2297 static struct ixl_tx_ring * 2298 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2299 { 2300 struct ixl_tx_ring *txr = NULL; 2301 struct ixl_tx_map *maps = NULL, *txm; 2302 unsigned int i; 2303 2304 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2305 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2306 KM_SLEEP); 2307 2308 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2309 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2310 IXL_TX_QUEUE_ALIGN) != 0) 2311 goto free; 2312 2313 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2314 txm = &maps[i]; 2315 2316 if (bus_dmamap_create(sc->sc_dmat, 2317 IXL_HARDMTU, IXL_TX_PKT_DESCS, IXL_HARDMTU, 0, 2318 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2319 goto uncreate; 2320 2321 txm->txm_eop = -1; 2322 txm->txm_m = NULL; 2323 } 2324 2325 txr->txr_cons = txr->txr_prod = 0; 2326 txr->txr_maps = maps; 2327 2328 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2329 if (txr->txr_intrq == NULL) 2330 goto uncreate; 2331 2332 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2333 ixl_deferred_transmit, txr); 2334 if (txr->txr_si == NULL) 2335 goto destroy_pcq; 2336 2337 txr->txr_tail = I40E_QTX_TAIL(qid); 2338 txr->txr_qid = qid; 2339 txr->txr_sc = sc; 2340 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2341 2342 return txr; 2343 2344 destroy_pcq: 2345 pcq_destroy(txr->txr_intrq); 2346 uncreate: 2347 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2348 txm = &maps[i]; 2349 2350 if (txm->txm_map == NULL) 2351 continue; 2352 2353 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2354 } 2355 2356 ixl_dmamem_free(sc, &txr->txr_mem); 2357 free: 2358 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2359 kmem_free(txr, sizeof(*txr)); 2360 2361 return NULL; 2362 } 2363 2364 static void 2365 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2366 { 2367 unsigned int qid; 2368 bus_size_t reg; 2369 uint32_t r; 2370 2371 qid = txr->txr_qid + sc->sc_base_queue; 2372 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2373 qid %= 128; 2374 2375 r = ixl_rd(sc, reg); 2376 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2377 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2378 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2379 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2380 ixl_wr(sc, reg, r); 2381 } 2382 2383 static void 2384 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2385 { 2386 struct ixl_hmc_txq txq; 2387 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2388 void *hmc; 2389 2390 memset(&txq, 0, sizeof(txq)); 2391 txq.head = htole16(txr->txr_cons); 2392 txq.new_context = 1; 2393 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2394 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2395 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2396 txq.tphrdesc_ena = 0; 2397 txq.tphrpacket_ena = 0; 2398 txq.tphwdesc_ena = 0; 2399 txq.rdylist = data->qs_handle[0]; 2400 2401 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2402 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2403 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2404 __arraycount(ixl_hmc_pack_txq)); 2405 } 2406 2407 static void 2408 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2409 { 2410 void *hmc; 2411 2412 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2413 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2414 } 2415 2416 static void 2417 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2418 { 2419 struct ixl_tx_map *maps, *txm; 2420 bus_dmamap_t map; 2421 unsigned int i; 2422 2423 maps = txr->txr_maps; 2424 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2425 txm = &maps[i]; 2426 2427 if (txm->txm_m == NULL) 2428 continue; 2429 2430 map = txm->txm_map; 2431 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2432 BUS_DMASYNC_POSTWRITE); 2433 bus_dmamap_unload(sc->sc_dmat, map); 2434 2435 m_freem(txm->txm_m); 2436 txm->txm_m = NULL; 2437 } 2438 } 2439 2440 static int 2441 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2442 { 2443 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2444 uint32_t reg; 2445 int i; 2446 2447 for (i = 0; i < 10; i++) { 2448 reg = ixl_rd(sc, ena); 2449 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2450 return 0; 2451 2452 delaymsec(10); 2453 } 2454 2455 return ETIMEDOUT; 2456 } 2457 2458 static int 2459 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2460 { 2461 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2462 uint32_t reg; 2463 int i; 2464 2465 KASSERT(mutex_owned(&txr->txr_lock)); 2466 2467 for (i = 0; i < 20; i++) { 2468 reg = ixl_rd(sc, ena); 2469 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2470 return 0; 2471 2472 delaymsec(10); 2473 } 2474 2475 return ETIMEDOUT; 2476 } 2477 2478 static void 2479 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2480 { 2481 struct ixl_tx_map *maps, *txm; 2482 struct mbuf *m; 2483 unsigned int i; 2484 2485 softint_disestablish(txr->txr_si); 2486 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2487 m_freem(m); 2488 pcq_destroy(txr->txr_intrq); 2489 2490 maps = txr->txr_maps; 2491 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2492 txm = &maps[i]; 2493 2494 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2495 } 2496 2497 ixl_dmamem_free(sc, &txr->txr_mem); 2498 mutex_destroy(&txr->txr_lock); 2499 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2500 kmem_free(txr, sizeof(*txr)); 2501 } 2502 2503 static inline int 2504 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2505 struct ixl_tx_ring *txr) 2506 { 2507 struct mbuf *m; 2508 int error; 2509 2510 KASSERT(mutex_owned(&txr->txr_lock)); 2511 2512 m = *m0; 2513 2514 error = bus_dmamap_load_mbuf(dmat, map, m, 2515 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2516 if (error != EFBIG) 2517 return error; 2518 2519 m = m_defrag(m, M_DONTWAIT); 2520 if (m != NULL) { 2521 *m0 = m; 2522 txr->txr_defragged.ev_count++; 2523 2524 error = bus_dmamap_load_mbuf(dmat, map, m, 2525 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2526 } else { 2527 txr->txr_defrag_failed.ev_count++; 2528 error = ENOBUFS; 2529 } 2530 2531 return error; 2532 } 2533 2534 static void 2535 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2536 bool is_transmit) 2537 { 2538 struct ixl_softc *sc = ifp->if_softc; 2539 struct ixl_tx_desc *ring, *txd; 2540 struct ixl_tx_map *txm; 2541 bus_dmamap_t map; 2542 struct mbuf *m; 2543 uint64_t cmd, cmd_vlan; 2544 unsigned int prod, free, last, i; 2545 unsigned int mask; 2546 int post = 0; 2547 2548 KASSERT(mutex_owned(&txr->txr_lock)); 2549 2550 if (ifp->if_link_state != LINK_STATE_UP 2551 || !ISSET(ifp->if_flags, IFF_RUNNING) 2552 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2553 if (!is_transmit) 2554 IFQ_PURGE(&ifp->if_snd); 2555 return; 2556 } 2557 2558 prod = txr->txr_prod; 2559 free = txr->txr_cons; 2560 if (free <= prod) 2561 free += sc->sc_tx_ring_ndescs; 2562 free -= prod; 2563 2564 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2565 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2566 2567 ring = IXL_DMA_KVA(&txr->txr_mem); 2568 mask = sc->sc_tx_ring_ndescs - 1; 2569 last = prod; 2570 cmd = 0; 2571 txd = NULL; 2572 2573 for (;;) { 2574 if (free <= IXL_TX_PKT_DESCS) { 2575 if (!is_transmit) 2576 SET(ifp->if_flags, IFF_OACTIVE); 2577 break; 2578 } 2579 2580 if (is_transmit) 2581 m = pcq_get(txr->txr_intrq); 2582 else 2583 IFQ_DEQUEUE(&ifp->if_snd, m); 2584 2585 if (m == NULL) 2586 break; 2587 2588 txm = &txr->txr_maps[prod]; 2589 map = txm->txm_map; 2590 2591 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2592 txr->txr_oerrors++; 2593 m_freem(m); 2594 continue; 2595 } 2596 2597 if (vlan_has_tag(m)) { 2598 cmd_vlan = (uint64_t)vlan_get_tag(m) << 2599 IXL_TX_DESC_L2TAG1_SHIFT; 2600 cmd_vlan |= IXL_TX_DESC_CMD_IL2TAG1; 2601 } else { 2602 cmd_vlan = 0; 2603 } 2604 2605 bus_dmamap_sync(sc->sc_dmat, map, 0, 2606 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2607 2608 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2609 txd = &ring[prod]; 2610 2611 cmd = (uint64_t)map->dm_segs[i].ds_len << 2612 IXL_TX_DESC_BSIZE_SHIFT; 2613 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2614 cmd |= cmd_vlan; 2615 2616 txd->addr = htole64(map->dm_segs[i].ds_addr); 2617 txd->cmd = htole64(cmd); 2618 2619 last = prod; 2620 2621 prod++; 2622 prod &= mask; 2623 } 2624 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2625 txd->cmd = htole64(cmd); 2626 2627 txm->txm_m = m; 2628 txm->txm_eop = last; 2629 2630 bpf_mtap(ifp, m, BPF_D_OUT); 2631 2632 free -= i; 2633 post = 1; 2634 } 2635 2636 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2637 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2638 2639 if (post) { 2640 txr->txr_prod = prod; 2641 ixl_wr(sc, txr->txr_tail, prod); 2642 } 2643 } 2644 2645 static int 2646 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2647 { 2648 struct ifnet *ifp = &sc->sc_ec.ec_if; 2649 struct ixl_tx_desc *ring, *txd; 2650 struct ixl_tx_map *txm; 2651 struct mbuf *m; 2652 bus_dmamap_t map; 2653 unsigned int cons, prod, last; 2654 unsigned int mask; 2655 uint64_t dtype; 2656 int done = 0, more = 0; 2657 2658 KASSERT(mutex_owned(&txr->txr_lock)); 2659 2660 prod = txr->txr_prod; 2661 cons = txr->txr_cons; 2662 2663 if (cons == prod) 2664 return 0; 2665 2666 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2667 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2668 2669 ring = IXL_DMA_KVA(&txr->txr_mem); 2670 mask = sc->sc_tx_ring_ndescs - 1; 2671 2672 do { 2673 if (txlimit-- <= 0) { 2674 more = 1; 2675 break; 2676 } 2677 2678 txm = &txr->txr_maps[cons]; 2679 last = txm->txm_eop; 2680 txd = &ring[last]; 2681 2682 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2683 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2684 break; 2685 2686 map = txm->txm_map; 2687 2688 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2689 BUS_DMASYNC_POSTWRITE); 2690 bus_dmamap_unload(sc->sc_dmat, map); 2691 2692 m = txm->txm_m; 2693 if (m != NULL) { 2694 txr->txr_opackets++; 2695 txr->txr_obytes += m->m_pkthdr.len; 2696 if (ISSET(m->m_flags, M_MCAST)) 2697 txr->txr_omcasts++; 2698 m_freem(m); 2699 } 2700 2701 txm->txm_m = NULL; 2702 txm->txm_eop = -1; 2703 2704 cons = last + 1; 2705 cons &= mask; 2706 done = 1; 2707 } while (cons != prod); 2708 2709 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2710 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2711 2712 txr->txr_cons = cons; 2713 2714 if (done) { 2715 softint_schedule(txr->txr_si); 2716 if (txr->txr_qid == 0) { 2717 CLR(ifp->if_flags, IFF_OACTIVE); 2718 if_schedule_deferred_start(ifp); 2719 } 2720 } 2721 2722 return more; 2723 } 2724 2725 static void 2726 ixl_start(struct ifnet *ifp) 2727 { 2728 struct ixl_softc *sc; 2729 struct ixl_tx_ring *txr; 2730 2731 sc = ifp->if_softc; 2732 txr = sc->sc_qps[0].qp_txr; 2733 2734 mutex_enter(&txr->txr_lock); 2735 ixl_tx_common_locked(ifp, txr, false); 2736 mutex_exit(&txr->txr_lock); 2737 } 2738 2739 static inline unsigned int 2740 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2741 { 2742 u_int cpuid; 2743 2744 cpuid = cpu_index(curcpu()); 2745 2746 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2747 } 2748 2749 static int 2750 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2751 { 2752 struct ixl_softc *sc; 2753 struct ixl_tx_ring *txr; 2754 unsigned int qid; 2755 2756 sc = ifp->if_softc; 2757 qid = ixl_select_txqueue(sc, m); 2758 2759 txr = sc->sc_qps[qid].qp_txr; 2760 2761 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2762 mutex_enter(&txr->txr_lock); 2763 txr->txr_pcqdrop.ev_count++; 2764 mutex_exit(&txr->txr_lock); 2765 2766 m_freem(m); 2767 return ENOBUFS; 2768 } 2769 2770 if (mutex_tryenter(&txr->txr_lock)) { 2771 ixl_tx_common_locked(ifp, txr, true); 2772 mutex_exit(&txr->txr_lock); 2773 } else { 2774 softint_schedule(txr->txr_si); 2775 } 2776 2777 return 0; 2778 } 2779 2780 static void 2781 ixl_deferred_transmit(void *xtxr) 2782 { 2783 struct ixl_tx_ring *txr = xtxr; 2784 struct ixl_softc *sc = txr->txr_sc; 2785 struct ifnet *ifp = &sc->sc_ec.ec_if; 2786 2787 mutex_enter(&txr->txr_lock); 2788 txr->txr_transmitdef.ev_count++; 2789 if (pcq_peek(txr->txr_intrq) != NULL) 2790 ixl_tx_common_locked(ifp, txr, true); 2791 mutex_exit(&txr->txr_lock); 2792 } 2793 2794 static struct ixl_rx_ring * 2795 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2796 { 2797 struct ixl_rx_ring *rxr = NULL; 2798 struct ixl_rx_map *maps = NULL, *rxm; 2799 unsigned int i; 2800 2801 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2802 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2803 KM_SLEEP); 2804 2805 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2806 sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs, 2807 IXL_RX_QUEUE_ALIGN) != 0) 2808 goto free; 2809 2810 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2811 rxm = &maps[i]; 2812 2813 if (bus_dmamap_create(sc->sc_dmat, 2814 IXL_HARDMTU, 1, IXL_HARDMTU, 0, 2815 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2816 goto uncreate; 2817 2818 rxm->rxm_m = NULL; 2819 } 2820 2821 rxr->rxr_cons = rxr->rxr_prod = 0; 2822 rxr->rxr_m_head = NULL; 2823 rxr->rxr_m_tail = &rxr->rxr_m_head; 2824 rxr->rxr_maps = maps; 2825 2826 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2827 rxr->rxr_qid = qid; 2828 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2829 2830 return rxr; 2831 2832 uncreate: 2833 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2834 rxm = &maps[i]; 2835 2836 if (rxm->rxm_map == NULL) 2837 continue; 2838 2839 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2840 } 2841 2842 ixl_dmamem_free(sc, &rxr->rxr_mem); 2843 free: 2844 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2845 kmem_free(rxr, sizeof(*rxr)); 2846 2847 return NULL; 2848 } 2849 2850 static void 2851 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2852 { 2853 struct ixl_rx_map *maps, *rxm; 2854 bus_dmamap_t map; 2855 unsigned int i; 2856 2857 maps = rxr->rxr_maps; 2858 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2859 rxm = &maps[i]; 2860 2861 if (rxm->rxm_m == NULL) 2862 continue; 2863 2864 map = rxm->rxm_map; 2865 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2866 BUS_DMASYNC_POSTWRITE); 2867 bus_dmamap_unload(sc->sc_dmat, map); 2868 2869 m_freem(rxm->rxm_m); 2870 rxm->rxm_m = NULL; 2871 } 2872 2873 m_freem(rxr->rxr_m_head); 2874 rxr->rxr_m_head = NULL; 2875 rxr->rxr_m_tail = &rxr->rxr_m_head; 2876 2877 rxr->rxr_prod = rxr->rxr_cons = 0; 2878 } 2879 2880 static int 2881 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2882 { 2883 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 2884 uint32_t reg; 2885 int i; 2886 2887 for (i = 0; i < 10; i++) { 2888 reg = ixl_rd(sc, ena); 2889 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 2890 return 0; 2891 2892 delaymsec(10); 2893 } 2894 2895 return ETIMEDOUT; 2896 } 2897 2898 static int 2899 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2900 { 2901 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 2902 uint32_t reg; 2903 int i; 2904 2905 KASSERT(mutex_owned(&rxr->rxr_lock)); 2906 2907 for (i = 0; i < 20; i++) { 2908 reg = ixl_rd(sc, ena); 2909 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 2910 return 0; 2911 2912 delaymsec(10); 2913 } 2914 2915 return ETIMEDOUT; 2916 } 2917 2918 static void 2919 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2920 { 2921 struct ixl_hmc_rxq rxq; 2922 void *hmc; 2923 2924 memset(&rxq, 0, sizeof(rxq)); 2925 2926 rxq.head = htole16(rxr->rxr_cons); 2927 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 2928 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 2929 rxq.dbuff = htole16(MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 2930 rxq.hbuff = 0; 2931 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 2932 rxq.dsize = IXL_HMC_RXQ_DSIZE_32; 2933 rxq.crcstrip = 1; 2934 rxq.l2sel = 1; 2935 rxq.showiv = 1; 2936 rxq.rxmax = htole16(IXL_HARDMTU); 2937 rxq.tphrdesc_ena = 0; 2938 rxq.tphwdesc_ena = 0; 2939 rxq.tphdata_ena = 0; 2940 rxq.tphhead_ena = 0; 2941 rxq.lrxqthresh = 0; 2942 rxq.prefena = 1; 2943 2944 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 2945 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 2946 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 2947 __arraycount(ixl_hmc_pack_rxq)); 2948 } 2949 2950 static void 2951 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2952 { 2953 void *hmc; 2954 2955 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 2956 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 2957 } 2958 2959 static void 2960 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2961 { 2962 struct ixl_rx_map *maps, *rxm; 2963 unsigned int i; 2964 2965 maps = rxr->rxr_maps; 2966 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2967 rxm = &maps[i]; 2968 2969 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2970 } 2971 2972 ixl_dmamem_free(sc, &rxr->rxr_mem); 2973 mutex_destroy(&rxr->rxr_lock); 2974 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2975 kmem_free(rxr, sizeof(*rxr)); 2976 } 2977 2978 static inline void 2979 ixl_rx_csum(struct mbuf *m, uint64_t qword) 2980 { 2981 int flags_mask; 2982 2983 if (!ISSET(qword, IXL_RX_DESC_L3L4P)) { 2984 /* No L3 or L4 checksum was calculated */ 2985 return; 2986 } 2987 2988 switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) { 2989 case IXL_RX_DESC_PTYPE_IPV4FRAG: 2990 case IXL_RX_DESC_PTYPE_IPV4: 2991 case IXL_RX_DESC_PTYPE_SCTPV4: 2992 case IXL_RX_DESC_PTYPE_ICMPV4: 2993 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 2994 break; 2995 case IXL_RX_DESC_PTYPE_TCPV4: 2996 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 2997 flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD; 2998 break; 2999 case IXL_RX_DESC_PTYPE_UDPV4: 3000 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3001 flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD; 3002 break; 3003 case IXL_RX_DESC_PTYPE_TCPV6: 3004 flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD; 3005 break; 3006 case IXL_RX_DESC_PTYPE_UDPV6: 3007 flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD; 3008 break; 3009 default: 3010 flags_mask = 0; 3011 } 3012 3013 m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 | 3014 M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6)); 3015 3016 if (ISSET(qword, IXL_RX_DESC_IPE)) { 3017 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD); 3018 } 3019 3020 if (ISSET(qword, IXL_RX_DESC_L4E)) { 3021 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD); 3022 } 3023 } 3024 3025 static int 3026 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 3027 { 3028 struct ifnet *ifp = &sc->sc_ec.ec_if; 3029 struct ixl_rx_wb_desc_32 *ring, *rxd; 3030 struct ixl_rx_map *rxm; 3031 bus_dmamap_t map; 3032 unsigned int cons, prod; 3033 struct mbuf *m; 3034 uint64_t word, word0; 3035 unsigned int len; 3036 unsigned int mask; 3037 int done = 0, more = 0; 3038 3039 KASSERT(mutex_owned(&rxr->rxr_lock)); 3040 3041 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3042 return 0; 3043 3044 prod = rxr->rxr_prod; 3045 cons = rxr->rxr_cons; 3046 3047 if (cons == prod) 3048 return 0; 3049 3050 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3051 0, IXL_DMA_LEN(&rxr->rxr_mem), 3052 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3053 3054 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3055 mask = sc->sc_rx_ring_ndescs - 1; 3056 3057 do { 3058 if (rxlimit-- <= 0) { 3059 more = 1; 3060 break; 3061 } 3062 3063 rxd = &ring[cons]; 3064 3065 word = le64toh(rxd->qword1); 3066 3067 if (!ISSET(word, IXL_RX_DESC_DD)) 3068 break; 3069 3070 rxm = &rxr->rxr_maps[cons]; 3071 3072 map = rxm->rxm_map; 3073 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3074 BUS_DMASYNC_POSTREAD); 3075 bus_dmamap_unload(sc->sc_dmat, map); 3076 3077 m = rxm->rxm_m; 3078 rxm->rxm_m = NULL; 3079 3080 KASSERT(m != NULL); 3081 3082 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 3083 m->m_len = len; 3084 m->m_pkthdr.len = 0; 3085 3086 m->m_next = NULL; 3087 *rxr->rxr_m_tail = m; 3088 rxr->rxr_m_tail = &m->m_next; 3089 3090 m = rxr->rxr_m_head; 3091 m->m_pkthdr.len += len; 3092 3093 if (ISSET(word, IXL_RX_DESC_EOP)) { 3094 word0 = le64toh(rxd->qword0); 3095 3096 if (ISSET(word, IXL_RX_DESC_L2TAG1P)) { 3097 vlan_set_tag(m, 3098 __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK)); 3099 } 3100 3101 if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0) 3102 ixl_rx_csum(m, word); 3103 3104 if (!ISSET(word, 3105 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 3106 m_set_rcvif(m, ifp); 3107 rxr->rxr_ipackets++; 3108 rxr->rxr_ibytes += m->m_pkthdr.len; 3109 if_percpuq_enqueue(ifp->if_percpuq, m); 3110 } else { 3111 rxr->rxr_ierrors++; 3112 m_freem(m); 3113 } 3114 3115 rxr->rxr_m_head = NULL; 3116 rxr->rxr_m_tail = &rxr->rxr_m_head; 3117 } 3118 3119 cons++; 3120 cons &= mask; 3121 3122 done = 1; 3123 } while (cons != prod); 3124 3125 if (done) { 3126 rxr->rxr_cons = cons; 3127 if (ixl_rxfill(sc, rxr) == -1) 3128 rxr->rxr_iqdrops++; 3129 } 3130 3131 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3132 0, IXL_DMA_LEN(&rxr->rxr_mem), 3133 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3134 3135 return more; 3136 } 3137 3138 static int 3139 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3140 { 3141 struct ixl_rx_rd_desc_32 *ring, *rxd; 3142 struct ixl_rx_map *rxm; 3143 bus_dmamap_t map; 3144 struct mbuf *m; 3145 unsigned int prod; 3146 unsigned int slots; 3147 unsigned int mask; 3148 int post = 0, error = 0; 3149 3150 KASSERT(mutex_owned(&rxr->rxr_lock)); 3151 3152 prod = rxr->rxr_prod; 3153 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 3154 sc->sc_rx_ring_ndescs); 3155 3156 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3157 mask = sc->sc_rx_ring_ndescs - 1; 3158 3159 if (__predict_false(slots <= 0)) 3160 return -1; 3161 3162 do { 3163 rxm = &rxr->rxr_maps[prod]; 3164 3165 MGETHDR(m, M_DONTWAIT, MT_DATA); 3166 if (m == NULL) { 3167 rxr->rxr_mgethdr_failed.ev_count++; 3168 error = -1; 3169 break; 3170 } 3171 3172 MCLGET(m, M_DONTWAIT); 3173 if (!ISSET(m->m_flags, M_EXT)) { 3174 rxr->rxr_mgetcl_failed.ev_count++; 3175 error = -1; 3176 m_freem(m); 3177 break; 3178 } 3179 3180 m->m_len = m->m_pkthdr.len = MCLBYTES + ETHER_ALIGN; 3181 m_adj(m, ETHER_ALIGN); 3182 3183 map = rxm->rxm_map; 3184 3185 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 3186 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 3187 rxr->rxr_mbuf_load_failed.ev_count++; 3188 error = -1; 3189 m_freem(m); 3190 break; 3191 } 3192 3193 rxm->rxm_m = m; 3194 3195 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3196 BUS_DMASYNC_PREREAD); 3197 3198 rxd = &ring[prod]; 3199 3200 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 3201 rxd->haddr = htole64(0); 3202 3203 prod++; 3204 prod &= mask; 3205 3206 post = 1; 3207 3208 } while (--slots); 3209 3210 if (post) { 3211 rxr->rxr_prod = prod; 3212 ixl_wr(sc, rxr->rxr_tail, prod); 3213 } 3214 3215 return error; 3216 } 3217 3218 static inline int 3219 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 3220 u_int txlimit, struct evcnt *txevcnt, 3221 u_int rxlimit, struct evcnt *rxevcnt) 3222 { 3223 struct ixl_tx_ring *txr = qp->qp_txr; 3224 struct ixl_rx_ring *rxr = qp->qp_rxr; 3225 int txmore, rxmore; 3226 int rv; 3227 3228 KASSERT(!mutex_owned(&txr->txr_lock)); 3229 KASSERT(!mutex_owned(&rxr->rxr_lock)); 3230 3231 mutex_enter(&txr->txr_lock); 3232 txevcnt->ev_count++; 3233 txmore = ixl_txeof(sc, txr, txlimit); 3234 mutex_exit(&txr->txr_lock); 3235 3236 mutex_enter(&rxr->rxr_lock); 3237 rxevcnt->ev_count++; 3238 rxmore = ixl_rxeof(sc, rxr, rxlimit); 3239 mutex_exit(&rxr->rxr_lock); 3240 3241 rv = txmore | (rxmore << 1); 3242 3243 return rv; 3244 } 3245 3246 static void 3247 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 3248 { 3249 3250 if (qp->qp_workqueue) 3251 ixl_work_add(sc->sc_workq_txrx, &qp->qp_task); 3252 else 3253 softint_schedule(qp->qp_si); 3254 } 3255 3256 static int 3257 ixl_intr(void *xsc) 3258 { 3259 struct ixl_softc *sc = xsc; 3260 struct ixl_tx_ring *txr; 3261 struct ixl_rx_ring *rxr; 3262 uint32_t icr, rxintr, txintr; 3263 int rv = 0; 3264 unsigned int i; 3265 3266 KASSERT(sc != NULL); 3267 3268 ixl_enable_other_intr(sc); 3269 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3270 3271 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3272 atomic_inc_64(&sc->sc_event_atq.ev_count); 3273 ixl_atq_done(sc); 3274 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3275 rv = 1; 3276 } 3277 3278 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3279 atomic_inc_64(&sc->sc_event_link.ev_count); 3280 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3281 rv = 1; 3282 } 3283 3284 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 3285 txintr = icr & I40E_INTR_NOTX_TX_MASK; 3286 3287 if (txintr || rxintr) { 3288 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 3289 txr = sc->sc_qps[i].qp_txr; 3290 rxr = sc->sc_qps[i].qp_rxr; 3291 3292 ixl_handle_queue_common(sc, &sc->sc_qps[i], 3293 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 3294 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 3295 } 3296 rv = 1; 3297 } 3298 3299 return rv; 3300 } 3301 3302 static int 3303 ixl_queue_intr(void *xqp) 3304 { 3305 struct ixl_queue_pair *qp = xqp; 3306 struct ixl_tx_ring *txr = qp->qp_txr; 3307 struct ixl_rx_ring *rxr = qp->qp_rxr; 3308 struct ixl_softc *sc = qp->qp_sc; 3309 u_int txlimit, rxlimit; 3310 int more; 3311 3312 txlimit = sc->sc_tx_intr_process_limit; 3313 rxlimit = sc->sc_rx_intr_process_limit; 3314 qp->qp_workqueue = sc->sc_txrx_workqueue; 3315 3316 more = ixl_handle_queue_common(sc, qp, 3317 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3318 3319 if (more != 0) { 3320 ixl_sched_handle_queue(sc, qp); 3321 } else { 3322 /* for ALTQ */ 3323 if (txr->txr_qid == 0) 3324 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3325 softint_schedule(txr->txr_si); 3326 3327 ixl_enable_queue_intr(sc, qp); 3328 } 3329 3330 return 1; 3331 } 3332 3333 static void 3334 ixl_handle_queue(void *xqp) 3335 { 3336 struct ixl_queue_pair *qp = xqp; 3337 struct ixl_softc *sc = qp->qp_sc; 3338 struct ixl_tx_ring *txr = qp->qp_txr; 3339 struct ixl_rx_ring *rxr = qp->qp_rxr; 3340 u_int txlimit, rxlimit; 3341 int more; 3342 3343 txlimit = sc->sc_tx_process_limit; 3344 rxlimit = sc->sc_rx_process_limit; 3345 3346 more = ixl_handle_queue_common(sc, qp, 3347 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3348 3349 if (more != 0) 3350 ixl_sched_handle_queue(sc, qp); 3351 else 3352 ixl_enable_queue_intr(sc, qp); 3353 } 3354 3355 static inline void 3356 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3357 { 3358 uint32_t hmc_idx, hmc_isvf; 3359 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3360 3361 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3362 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3363 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3364 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3365 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3366 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3367 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3368 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3369 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3370 3371 device_printf(sc->sc_dev, 3372 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3373 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3374 } 3375 3376 static int 3377 ixl_other_intr(void *xsc) 3378 { 3379 struct ixl_softc *sc = xsc; 3380 uint32_t icr, mask, reg; 3381 int rv; 3382 3383 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3384 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3385 3386 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3387 atomic_inc_64(&sc->sc_event_atq.ev_count); 3388 ixl_atq_done(sc); 3389 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3390 rv = 1; 3391 } 3392 3393 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3394 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3395 device_printf(sc->sc_dev, "link stat changed\n"); 3396 3397 atomic_inc_64(&sc->sc_event_link.ev_count); 3398 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3399 rv = 1; 3400 } 3401 3402 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3403 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3404 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3405 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3406 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3407 3408 device_printf(sc->sc_dev, "GRST: %s\n", 3409 reg == I40E_RESET_CORER ? "CORER" : 3410 reg == I40E_RESET_GLOBR ? "GLOBR" : 3411 reg == I40E_RESET_EMPR ? "EMPR" : 3412 "POR"); 3413 } 3414 3415 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3416 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3417 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3418 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3419 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3420 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3421 3422 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3423 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3424 device_printf(sc->sc_dev, "critical error\n"); 3425 } 3426 3427 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3428 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3429 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3430 ixl_print_hmc_error(sc, reg); 3431 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3432 } 3433 3434 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3435 ixl_flush(sc); 3436 ixl_enable_other_intr(sc); 3437 return rv; 3438 } 3439 3440 static void 3441 ixl_get_link_status_done(struct ixl_softc *sc, 3442 const struct ixl_aq_desc *iaq) 3443 { 3444 3445 ixl_link_state_update(sc, iaq); 3446 } 3447 3448 static void 3449 ixl_get_link_status(void *xsc) 3450 { 3451 struct ixl_softc *sc = xsc; 3452 struct ixl_aq_desc *iaq; 3453 struct ixl_aq_link_param *param; 3454 3455 memset(&sc->sc_link_state_atq, 0, sizeof(sc->sc_link_state_atq)); 3456 iaq = &sc->sc_link_state_atq.iatq_desc; 3457 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3458 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3459 param->notify = IXL_AQ_LINK_NOTIFY; 3460 3461 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 3462 (void)ixl_atq_post(sc, &sc->sc_link_state_atq); 3463 } 3464 3465 static void 3466 ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3467 { 3468 struct ifnet *ifp = &sc->sc_ec.ec_if; 3469 int link_state; 3470 3471 KASSERT(kpreempt_disabled()); 3472 3473 link_state = ixl_set_link_status(sc, iaq); 3474 3475 if (ifp->if_link_state != link_state) 3476 if_link_state_change(ifp, link_state); 3477 3478 if (link_state != LINK_STATE_DOWN) { 3479 if_schedule_deferred_start(ifp); 3480 } 3481 } 3482 3483 static void 3484 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3485 const char *msg) 3486 { 3487 char buf[512]; 3488 size_t len; 3489 3490 len = sizeof(buf); 3491 buf[--len] = '\0'; 3492 3493 device_printf(sc->sc_dev, "%s\n", msg); 3494 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3495 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3496 buf, le16toh(iaq->iaq_opcode)); 3497 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3498 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3499 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3500 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3501 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3502 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3503 } 3504 3505 static void 3506 ixl_arq(void *xsc) 3507 { 3508 struct ixl_softc *sc = xsc; 3509 struct ixl_aq_desc *arq, *iaq; 3510 struct ixl_aq_buf *aqb; 3511 unsigned int cons = sc->sc_arq_cons; 3512 unsigned int prod; 3513 int done = 0; 3514 3515 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3516 sc->sc_aq_regs->arq_head_mask; 3517 3518 if (cons == prod) 3519 goto done; 3520 3521 arq = IXL_DMA_KVA(&sc->sc_arq); 3522 3523 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3524 0, IXL_DMA_LEN(&sc->sc_arq), 3525 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3526 3527 do { 3528 iaq = &arq[cons]; 3529 aqb = sc->sc_arq_live[cons]; 3530 3531 KASSERT(aqb != NULL); 3532 3533 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3534 BUS_DMASYNC_POSTREAD); 3535 3536 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3537 ixl_aq_dump(sc, iaq, "arq event"); 3538 3539 switch (iaq->iaq_opcode) { 3540 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3541 kpreempt_disable(); 3542 ixl_link_state_update(sc, iaq); 3543 kpreempt_enable(); 3544 break; 3545 } 3546 3547 memset(iaq, 0, sizeof(*iaq)); 3548 sc->sc_arq_live[cons] = NULL; 3549 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3550 3551 cons++; 3552 cons &= IXL_AQ_MASK; 3553 3554 done = 1; 3555 } while (cons != prod); 3556 3557 if (done) { 3558 sc->sc_arq_cons = cons; 3559 ixl_arq_fill(sc); 3560 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3561 0, IXL_DMA_LEN(&sc->sc_arq), 3562 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3563 } 3564 3565 done: 3566 ixl_enable_other_intr(sc); 3567 } 3568 3569 static void 3570 ixl_atq_set(struct ixl_atq *iatq, 3571 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3572 { 3573 3574 iatq->iatq_fn = fn; 3575 } 3576 3577 static int 3578 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3579 { 3580 struct ixl_aq_desc *atq, *slot; 3581 unsigned int prod, cons, prod_next; 3582 3583 /* assert locked */ 3584 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3585 3586 atq = IXL_DMA_KVA(&sc->sc_atq); 3587 prod = sc->sc_atq_prod; 3588 cons = sc->sc_atq_cons; 3589 prod_next = (prod +1) & IXL_AQ_MASK; 3590 3591 if (cons == prod_next) 3592 return ENOMEM; 3593 3594 slot = &atq[prod]; 3595 3596 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3597 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3598 3599 *slot = iatq->iatq_desc; 3600 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3601 3602 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3603 ixl_aq_dump(sc, slot, "atq command"); 3604 3605 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3606 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3607 3608 sc->sc_atq_prod = prod_next; 3609 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3610 3611 return 0; 3612 } 3613 3614 static int 3615 ixl_atq_post(struct ixl_softc *sc, struct ixl_atq *iatq) 3616 { 3617 int rv; 3618 3619 mutex_enter(&sc->sc_atq_lock); 3620 rv = ixl_atq_post_locked(sc, iatq); 3621 mutex_exit(&sc->sc_atq_lock); 3622 3623 return rv; 3624 } 3625 3626 static void 3627 ixl_atq_done_locked(struct ixl_softc *sc) 3628 { 3629 struct ixl_aq_desc *atq, *slot; 3630 struct ixl_atq *iatq; 3631 unsigned int cons; 3632 unsigned int prod; 3633 3634 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3635 3636 prod = sc->sc_atq_prod; 3637 cons = sc->sc_atq_cons; 3638 3639 if (prod == cons) 3640 return; 3641 3642 atq = IXL_DMA_KVA(&sc->sc_atq); 3643 3644 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3645 0, IXL_DMA_LEN(&sc->sc_atq), 3646 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3647 3648 do { 3649 slot = &atq[cons]; 3650 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3651 break; 3652 3653 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3654 iatq->iatq_desc = *slot; 3655 3656 memset(slot, 0, sizeof(*slot)); 3657 3658 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3659 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3660 3661 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3662 3663 cons++; 3664 cons &= IXL_AQ_MASK; 3665 } while (cons != prod); 3666 3667 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3668 0, IXL_DMA_LEN(&sc->sc_atq), 3669 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3670 3671 sc->sc_atq_cons = cons; 3672 } 3673 3674 static void 3675 ixl_atq_done(struct ixl_softc *sc) 3676 { 3677 3678 mutex_enter(&sc->sc_atq_lock); 3679 ixl_atq_done_locked(sc); 3680 mutex_exit(&sc->sc_atq_lock); 3681 } 3682 3683 static void 3684 ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3685 { 3686 3687 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3688 3689 cv_signal(&sc->sc_atq_cv); 3690 } 3691 3692 static int 3693 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3694 { 3695 int error; 3696 3697 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3698 3699 ixl_atq_set(iatq, ixl_wakeup); 3700 3701 mutex_enter(&sc->sc_atq_lock); 3702 error = ixl_atq_post_locked(sc, iatq); 3703 if (error) { 3704 mutex_exit(&sc->sc_atq_lock); 3705 return error; 3706 } 3707 3708 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3709 IXL_ATQ_EXEC_TIMEOUT); 3710 mutex_exit(&sc->sc_atq_lock); 3711 3712 return error; 3713 } 3714 3715 static int 3716 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3717 { 3718 struct ixl_aq_desc *atq, *slot; 3719 unsigned int prod; 3720 unsigned int t = 0; 3721 3722 mutex_enter(&sc->sc_atq_lock); 3723 3724 atq = IXL_DMA_KVA(&sc->sc_atq); 3725 prod = sc->sc_atq_prod; 3726 slot = atq + prod; 3727 3728 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3729 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3730 3731 *slot = *iaq; 3732 slot->iaq_flags |= htole16(IXL_AQ_SI); 3733 3734 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3735 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3736 3737 prod++; 3738 prod &= IXL_AQ_MASK; 3739 sc->sc_atq_prod = prod; 3740 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3741 3742 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3743 delaymsec(1); 3744 3745 if (t++ > tm) { 3746 mutex_exit(&sc->sc_atq_lock); 3747 return ETIMEDOUT; 3748 } 3749 } 3750 3751 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3752 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3753 *iaq = *slot; 3754 memset(slot, 0, sizeof(*slot)); 3755 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3756 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3757 3758 sc->sc_atq_cons = prod; 3759 3760 mutex_exit(&sc->sc_atq_lock); 3761 3762 return 0; 3763 } 3764 3765 static int 3766 ixl_get_version(struct ixl_softc *sc) 3767 { 3768 struct ixl_aq_desc iaq; 3769 uint32_t fwbuild, fwver, apiver; 3770 uint16_t api_maj_ver, api_min_ver; 3771 3772 memset(&iaq, 0, sizeof(iaq)); 3773 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3774 3775 iaq.iaq_retval = le16toh(23); 3776 3777 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3778 return ETIMEDOUT; 3779 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3780 return EIO; 3781 3782 fwbuild = le32toh(iaq.iaq_param[1]); 3783 fwver = le32toh(iaq.iaq_param[2]); 3784 apiver = le32toh(iaq.iaq_param[3]); 3785 3786 api_maj_ver = (uint16_t)apiver; 3787 api_min_ver = (uint16_t)(apiver >> 16); 3788 3789 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 3790 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 3791 3792 sc->sc_rxctl_atq = true; 3793 if (sc->sc_mac_type == I40E_MAC_X722) { 3794 if (api_maj_ver == 1 && api_min_ver < 5) { 3795 sc->sc_rxctl_atq = false; 3796 } 3797 } 3798 3799 return 0; 3800 } 3801 3802 static int 3803 ixl_pxe_clear(struct ixl_softc *sc) 3804 { 3805 struct ixl_aq_desc iaq; 3806 int rv; 3807 3808 memset(&iaq, 0, sizeof(iaq)); 3809 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 3810 iaq.iaq_param[0] = htole32(0x2); 3811 3812 rv = ixl_atq_poll(sc, &iaq, 250); 3813 3814 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 3815 3816 if (rv != 0) 3817 return ETIMEDOUT; 3818 3819 switch (iaq.iaq_retval) { 3820 case htole16(IXL_AQ_RC_OK): 3821 case htole16(IXL_AQ_RC_EEXIST): 3822 break; 3823 default: 3824 return EIO; 3825 } 3826 3827 return 0; 3828 } 3829 3830 static int 3831 ixl_lldp_shut(struct ixl_softc *sc) 3832 { 3833 struct ixl_aq_desc iaq; 3834 3835 memset(&iaq, 0, sizeof(iaq)); 3836 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 3837 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 3838 3839 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 3840 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 3841 return -1; 3842 } 3843 3844 switch (iaq.iaq_retval) { 3845 case htole16(IXL_AQ_RC_EMODE): 3846 case htole16(IXL_AQ_RC_EPERM): 3847 /* ignore silently */ 3848 default: 3849 break; 3850 } 3851 3852 return 0; 3853 } 3854 3855 static void 3856 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 3857 { 3858 uint16_t id; 3859 uint32_t number, logical_id; 3860 3861 id = le16toh(cap->cap_id); 3862 number = le32toh(cap->number); 3863 logical_id = le32toh(cap->logical_id); 3864 3865 switch (id) { 3866 case IXL_AQ_CAP_RSS: 3867 sc->sc_rss_table_size = number; 3868 sc->sc_rss_table_entry_width = logical_id; 3869 break; 3870 case IXL_AQ_CAP_RXQ: 3871 case IXL_AQ_CAP_TXQ: 3872 sc->sc_nqueue_pairs_device = MIN(number, 3873 sc->sc_nqueue_pairs_device); 3874 break; 3875 } 3876 } 3877 3878 static int 3879 ixl_get_hw_capabilities(struct ixl_softc *sc) 3880 { 3881 struct ixl_dmamem idm; 3882 struct ixl_aq_desc iaq; 3883 struct ixl_aq_capability *caps; 3884 size_t i, ncaps; 3885 bus_size_t caps_size; 3886 uint16_t status; 3887 int rv; 3888 3889 caps_size = sizeof(caps[0]) * 40; 3890 memset(&iaq, 0, sizeof(iaq)); 3891 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 3892 3893 do { 3894 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 3895 return -1; 3896 } 3897 3898 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3899 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3900 iaq.iaq_datalen = htole16(caps_size); 3901 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3902 3903 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 3904 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 3905 3906 rv = ixl_atq_poll(sc, &iaq, 250); 3907 3908 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 3909 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 3910 3911 if (rv != 0) { 3912 aprint_error(", HW capabilities timeout\n"); 3913 goto done; 3914 } 3915 3916 status = le16toh(iaq.iaq_retval); 3917 3918 if (status == IXL_AQ_RC_ENOMEM) { 3919 caps_size = le16toh(iaq.iaq_datalen); 3920 ixl_dmamem_free(sc, &idm); 3921 } 3922 } while (status == IXL_AQ_RC_ENOMEM); 3923 3924 if (status != IXL_AQ_RC_OK) { 3925 aprint_error(", HW capabilities error\n"); 3926 goto done; 3927 } 3928 3929 caps = IXL_DMA_KVA(&idm); 3930 ncaps = le16toh(iaq.iaq_param[1]); 3931 3932 for (i = 0; i < ncaps; i++) { 3933 ixl_parse_hw_capability(sc, &caps[i]); 3934 } 3935 3936 done: 3937 ixl_dmamem_free(sc, &idm); 3938 return rv; 3939 } 3940 3941 static int 3942 ixl_get_mac(struct ixl_softc *sc) 3943 { 3944 struct ixl_dmamem idm; 3945 struct ixl_aq_desc iaq; 3946 struct ixl_aq_mac_addresses *addrs; 3947 int rv; 3948 3949 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 3950 aprint_error(", unable to allocate mac addresses\n"); 3951 return -1; 3952 } 3953 3954 memset(&iaq, 0, sizeof(iaq)); 3955 iaq.iaq_flags = htole16(IXL_AQ_BUF); 3956 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 3957 iaq.iaq_datalen = htole16(sizeof(*addrs)); 3958 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3959 3960 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3961 BUS_DMASYNC_PREREAD); 3962 3963 rv = ixl_atq_poll(sc, &iaq, 250); 3964 3965 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3966 BUS_DMASYNC_POSTREAD); 3967 3968 if (rv != 0) { 3969 aprint_error(", MAC ADDRESS READ timeout\n"); 3970 rv = -1; 3971 goto done; 3972 } 3973 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3974 aprint_error(", MAC ADDRESS READ error\n"); 3975 rv = -1; 3976 goto done; 3977 } 3978 3979 addrs = IXL_DMA_KVA(&idm); 3980 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 3981 printf(", port address is not valid\n"); 3982 goto done; 3983 } 3984 3985 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 3986 rv = 0; 3987 3988 done: 3989 ixl_dmamem_free(sc, &idm); 3990 return rv; 3991 } 3992 3993 static int 3994 ixl_get_switch_config(struct ixl_softc *sc) 3995 { 3996 struct ixl_dmamem idm; 3997 struct ixl_aq_desc iaq; 3998 struct ixl_aq_switch_config *hdr; 3999 struct ixl_aq_switch_config_element *elms, *elm; 4000 unsigned int nelm, i; 4001 int rv; 4002 4003 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4004 aprint_error_dev(sc->sc_dev, 4005 "unable to allocate switch config buffer\n"); 4006 return -1; 4007 } 4008 4009 memset(&iaq, 0, sizeof(iaq)); 4010 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4011 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4012 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 4013 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 4014 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4015 4016 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4017 BUS_DMASYNC_PREREAD); 4018 4019 rv = ixl_atq_poll(sc, &iaq, 250); 4020 4021 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4022 BUS_DMASYNC_POSTREAD); 4023 4024 if (rv != 0) { 4025 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 4026 rv = -1; 4027 goto done; 4028 } 4029 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4030 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 4031 rv = -1; 4032 goto done; 4033 } 4034 4035 hdr = IXL_DMA_KVA(&idm); 4036 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 4037 4038 nelm = le16toh(hdr->num_reported); 4039 if (nelm < 1) { 4040 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 4041 rv = -1; 4042 goto done; 4043 } 4044 4045 for (i = 0; i < nelm; i++) { 4046 elm = &elms[i]; 4047 4048 aprint_debug_dev(sc->sc_dev, 4049 "type %x revision %u seid %04x\n", 4050 elm->type, elm->revision, le16toh(elm->seid)); 4051 aprint_debug_dev(sc->sc_dev, 4052 "uplink %04x downlink %04x\n", 4053 le16toh(elm->uplink_seid), 4054 le16toh(elm->downlink_seid)); 4055 aprint_debug_dev(sc->sc_dev, 4056 "conntype %x scheduler %04x extra %04x\n", 4057 elm->connection_type, 4058 le16toh(elm->scheduler_id), 4059 le16toh(elm->element_info)); 4060 } 4061 4062 elm = &elms[0]; 4063 4064 sc->sc_uplink_seid = elm->uplink_seid; 4065 sc->sc_downlink_seid = elm->downlink_seid; 4066 sc->sc_seid = elm->seid; 4067 4068 if ((sc->sc_uplink_seid == htole16(0)) != 4069 (sc->sc_downlink_seid == htole16(0))) { 4070 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 4071 rv = -1; 4072 goto done; 4073 } 4074 4075 done: 4076 ixl_dmamem_free(sc, &idm); 4077 return rv; 4078 } 4079 4080 static int 4081 ixl_phy_mask_ints(struct ixl_softc *sc) 4082 { 4083 struct ixl_aq_desc iaq; 4084 4085 memset(&iaq, 0, sizeof(iaq)); 4086 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 4087 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 4088 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 4089 IXL_AQ_PHY_EV_MEDIA_NA)); 4090 4091 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4092 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 4093 return -1; 4094 } 4095 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4096 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 4097 return -1; 4098 } 4099 4100 return 0; 4101 } 4102 4103 static int 4104 ixl_get_phy_abilities(struct ixl_softc *sc,struct ixl_dmamem *idm) 4105 { 4106 struct ixl_aq_desc iaq; 4107 int rv; 4108 4109 memset(&iaq, 0, sizeof(iaq)); 4110 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4111 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4112 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 4113 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 4114 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 4115 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 4116 4117 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4118 BUS_DMASYNC_PREREAD); 4119 4120 rv = ixl_atq_poll(sc, &iaq, 250); 4121 4122 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4123 BUS_DMASYNC_POSTREAD); 4124 4125 if (rv != 0) 4126 return -1; 4127 4128 return le16toh(iaq.iaq_retval); 4129 } 4130 4131 static int 4132 ixl_get_phy_types(struct ixl_softc *sc, uint64_t *phy_types_ptr) 4133 { 4134 struct ixl_dmamem idm; 4135 struct ixl_aq_phy_abilities *phy; 4136 uint64_t phy_types; 4137 int rv; 4138 4139 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4140 aprint_error_dev(sc->sc_dev, 4141 "unable to allocate switch config buffer\n"); 4142 return -1; 4143 } 4144 4145 rv = ixl_get_phy_abilities(sc, &idm); 4146 switch (rv) { 4147 case -1: 4148 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 4149 goto done; 4150 case IXL_AQ_RC_OK: 4151 break; 4152 case IXL_AQ_RC_EIO: 4153 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 4154 break; 4155 default: 4156 aprint_error_dev(sc->sc_dev, 4157 "GET PHY ABILITIIES error %u\n", rv); 4158 goto done; 4159 } 4160 4161 phy = IXL_DMA_KVA(&idm); 4162 4163 phy_types = le32toh(phy->phy_type); 4164 phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 4165 4166 *phy_types_ptr = phy_types; 4167 4168 rv = 0; 4169 4170 done: 4171 ixl_dmamem_free(sc, &idm); 4172 return rv; 4173 } 4174 4175 static int 4176 ixl_get_link_status_poll(struct ixl_softc *sc) 4177 { 4178 struct ixl_aq_desc iaq; 4179 struct ixl_aq_link_param *param; 4180 int link; 4181 4182 memset(&iaq, 0, sizeof(iaq)); 4183 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 4184 param = (struct ixl_aq_link_param *)iaq.iaq_param; 4185 param->notify = IXL_AQ_LINK_NOTIFY; 4186 4187 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4188 return ETIMEDOUT; 4189 } 4190 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4191 return EIO; 4192 } 4193 4194 link = ixl_set_link_status(sc, &iaq); 4195 sc->sc_ec.ec_if.if_link_state = link; 4196 4197 return 0; 4198 } 4199 4200 static int 4201 ixl_get_vsi(struct ixl_softc *sc) 4202 { 4203 struct ixl_dmamem *vsi = &sc->sc_scratch; 4204 struct ixl_aq_desc iaq; 4205 struct ixl_aq_vsi_param *param; 4206 struct ixl_aq_vsi_reply *reply; 4207 struct ixl_aq_vsi_data *data; 4208 int rv; 4209 4210 /* grumble, vsi info isn't "known" at compile time */ 4211 4212 memset(&iaq, 0, sizeof(iaq)); 4213 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4214 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4215 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 4216 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4217 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4218 4219 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4220 param->uplink_seid = sc->sc_seid; 4221 4222 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4223 BUS_DMASYNC_PREREAD); 4224 4225 rv = ixl_atq_poll(sc, &iaq, 250); 4226 4227 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4228 BUS_DMASYNC_POSTREAD); 4229 4230 if (rv != 0) { 4231 return ETIMEDOUT; 4232 } 4233 4234 switch (le16toh(iaq.iaq_retval)) { 4235 case IXL_AQ_RC_OK: 4236 break; 4237 case IXL_AQ_RC_ENOENT: 4238 return ENOENT; 4239 case IXL_AQ_RC_EACCES: 4240 return EACCES; 4241 default: 4242 return EIO; 4243 } 4244 4245 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 4246 sc->sc_vsi_number = reply->vsi_number; 4247 data = IXL_DMA_KVA(vsi); 4248 sc->sc_vsi_stat_counter_idx = le16toh(data->stat_counter_idx); 4249 4250 return 0; 4251 } 4252 4253 static int 4254 ixl_set_vsi(struct ixl_softc *sc) 4255 { 4256 struct ixl_dmamem *vsi = &sc->sc_scratch; 4257 struct ixl_aq_desc iaq; 4258 struct ixl_aq_vsi_param *param; 4259 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 4260 unsigned int qnum; 4261 uint16_t val; 4262 int rv; 4263 4264 qnum = sc->sc_nqueue_pairs - 1; 4265 4266 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 4267 IXL_AQ_VSI_VALID_VLAN); 4268 4269 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 4270 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 4271 data->queue_mapping[0] = htole16(0); 4272 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 4273 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 4274 4275 val = le16toh(data->port_vlan_flags); 4276 CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK); 4277 SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL); 4278 4279 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { 4280 SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH); 4281 } else { 4282 SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING); 4283 } 4284 4285 data->port_vlan_flags = htole16(val); 4286 4287 /* grumble, vsi info isn't "known" at compile time */ 4288 4289 memset(&iaq, 0, sizeof(iaq)); 4290 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4291 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4292 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 4293 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4294 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4295 4296 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4297 param->uplink_seid = sc->sc_seid; 4298 4299 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4300 BUS_DMASYNC_PREWRITE); 4301 4302 rv = ixl_atq_poll(sc, &iaq, 250); 4303 4304 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4305 BUS_DMASYNC_POSTWRITE); 4306 4307 if (rv != 0) { 4308 return ETIMEDOUT; 4309 } 4310 4311 switch (le16toh(iaq.iaq_retval)) { 4312 case IXL_AQ_RC_OK: 4313 break; 4314 case IXL_AQ_RC_ENOENT: 4315 return ENOENT; 4316 case IXL_AQ_RC_EACCES: 4317 return EACCES; 4318 default: 4319 return EIO; 4320 } 4321 4322 return 0; 4323 } 4324 4325 static void 4326 ixl_set_filter_control(struct ixl_softc *sc) 4327 { 4328 uint32_t reg; 4329 4330 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4331 4332 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4333 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4334 4335 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4336 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4337 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4338 4339 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4340 } 4341 4342 static inline void 4343 ixl_get_default_rss_key(uint32_t *buf, size_t len) 4344 { 4345 size_t cplen; 4346 uint8_t rss_seed[RSS_KEYSIZE]; 4347 4348 rss_getkey(rss_seed); 4349 memset(buf, 0, len); 4350 4351 cplen = MIN(len, sizeof(rss_seed)); 4352 memcpy(buf, rss_seed, cplen); 4353 } 4354 4355 static void 4356 ixl_set_rss_key(struct ixl_softc *sc) 4357 { 4358 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4359 size_t i; 4360 4361 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4362 4363 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4364 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4365 } 4366 } 4367 4368 static void 4369 ixl_set_rss_pctype(struct ixl_softc *sc) 4370 { 4371 uint64_t set_hena = 0; 4372 uint32_t hena0, hena1; 4373 4374 if (sc->sc_mac_type == I40E_MAC_X722) 4375 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4376 else 4377 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4378 4379 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4380 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4381 4382 SET(hena0, set_hena); 4383 SET(hena1, set_hena >> 32); 4384 4385 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4386 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4387 } 4388 4389 static void 4390 ixl_set_rss_hlut(struct ixl_softc *sc) 4391 { 4392 unsigned int qid; 4393 uint8_t hlut_buf[512], lut_mask; 4394 uint32_t *hluts; 4395 size_t i, hluts_num; 4396 4397 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4398 4399 for (i = 0; i < sc->sc_rss_table_size; i++) { 4400 qid = i % sc->sc_nqueue_pairs; 4401 hlut_buf[i] = qid & lut_mask; 4402 } 4403 4404 hluts = (uint32_t *)hlut_buf; 4405 hluts_num = sc->sc_rss_table_size >> 2; 4406 for (i = 0; i < hluts_num; i++) { 4407 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4408 } 4409 ixl_flush(sc); 4410 } 4411 4412 static void 4413 ixl_config_rss(struct ixl_softc *sc) 4414 { 4415 4416 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4417 4418 ixl_set_rss_key(sc); 4419 ixl_set_rss_pctype(sc); 4420 ixl_set_rss_hlut(sc); 4421 } 4422 4423 static const struct ixl_phy_type * 4424 ixl_search_phy_type(uint8_t phy_type) 4425 { 4426 const struct ixl_phy_type *itype; 4427 uint64_t mask; 4428 unsigned int i; 4429 4430 if (phy_type >= 64) 4431 return NULL; 4432 4433 mask = 1ULL << phy_type; 4434 4435 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4436 itype = &ixl_phy_type_map[i]; 4437 4438 if (ISSET(itype->phy_type, mask)) 4439 return itype; 4440 } 4441 4442 return NULL; 4443 } 4444 4445 static uint64_t 4446 ixl_search_link_speed(uint8_t link_speed) 4447 { 4448 const struct ixl_speed_type *type; 4449 unsigned int i; 4450 4451 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4452 type = &ixl_speed_type_map[i]; 4453 4454 if (ISSET(type->dev_speed, link_speed)) 4455 return type->net_speed; 4456 } 4457 4458 return 0; 4459 } 4460 4461 static int 4462 ixl_restart_an(struct ixl_softc *sc) 4463 { 4464 struct ixl_aq_desc iaq; 4465 4466 memset(&iaq, 0, sizeof(iaq)); 4467 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4468 iaq.iaq_param[0] = 4469 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4470 4471 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4472 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4473 return -1; 4474 } 4475 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4476 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4477 return -1; 4478 } 4479 4480 return 0; 4481 } 4482 4483 static int 4484 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4485 uint16_t vlan, uint16_t flags) 4486 { 4487 struct ixl_aq_desc iaq; 4488 struct ixl_aq_add_macvlan *param; 4489 struct ixl_aq_add_macvlan_elem *elem; 4490 4491 memset(&iaq, 0, sizeof(iaq)); 4492 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4493 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4494 iaq.iaq_datalen = htole16(sizeof(*elem)); 4495 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4496 4497 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4498 param->num_addrs = htole16(1); 4499 param->seid0 = htole16(0x8000) | sc->sc_seid; 4500 param->seid1 = 0; 4501 param->seid2 = 0; 4502 4503 elem = IXL_DMA_KVA(&sc->sc_scratch); 4504 memset(elem, 0, sizeof(*elem)); 4505 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4506 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4507 elem->vlan = htole16(vlan); 4508 4509 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4510 return IXL_AQ_RC_EINVAL; 4511 } 4512 4513 switch (le16toh(iaq.iaq_retval)) { 4514 case IXL_AQ_RC_OK: 4515 break; 4516 case IXL_AQ_RC_ENOSPC: 4517 return ENOSPC; 4518 case IXL_AQ_RC_ENOENT: 4519 return ENOENT; 4520 case IXL_AQ_RC_EACCES: 4521 return EACCES; 4522 case IXL_AQ_RC_EEXIST: 4523 return EEXIST; 4524 case IXL_AQ_RC_EINVAL: 4525 return EINVAL; 4526 default: 4527 return EIO; 4528 } 4529 4530 return 0; 4531 } 4532 4533 static int 4534 ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4535 uint16_t vlan, uint16_t flags) 4536 { 4537 struct ixl_aq_desc iaq; 4538 struct ixl_aq_remove_macvlan *param; 4539 struct ixl_aq_remove_macvlan_elem *elem; 4540 4541 memset(&iaq, 0, sizeof(iaq)); 4542 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4543 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 4544 iaq.iaq_datalen = htole16(sizeof(*elem)); 4545 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4546 4547 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 4548 param->num_addrs = htole16(1); 4549 param->seid0 = htole16(0x8000) | sc->sc_seid; 4550 param->seid1 = 0; 4551 param->seid2 = 0; 4552 4553 elem = IXL_DMA_KVA(&sc->sc_scratch); 4554 memset(elem, 0, sizeof(*elem)); 4555 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4556 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 4557 elem->vlan = htole16(vlan); 4558 4559 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4560 return EINVAL; 4561 } 4562 4563 switch (le16toh(iaq.iaq_retval)) { 4564 case IXL_AQ_RC_OK: 4565 break; 4566 case IXL_AQ_RC_ENOENT: 4567 return ENOENT; 4568 case IXL_AQ_RC_EACCES: 4569 return EACCES; 4570 case IXL_AQ_RC_EINVAL: 4571 return EINVAL; 4572 default: 4573 return EIO; 4574 } 4575 4576 return 0; 4577 } 4578 4579 static int 4580 ixl_hmc(struct ixl_softc *sc) 4581 { 4582 struct { 4583 uint32_t count; 4584 uint32_t minsize; 4585 bus_size_t objsiz; 4586 bus_size_t setoff; 4587 bus_size_t setcnt; 4588 } regs[] = { 4589 { 4590 0, 4591 IXL_HMC_TXQ_MINSIZE, 4592 I40E_GLHMC_LANTXOBJSZ, 4593 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 4594 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 4595 }, 4596 { 4597 0, 4598 IXL_HMC_RXQ_MINSIZE, 4599 I40E_GLHMC_LANRXOBJSZ, 4600 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 4601 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 4602 }, 4603 { 4604 0, 4605 0, 4606 I40E_GLHMC_FCOEDDPOBJSZ, 4607 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 4608 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 4609 }, 4610 { 4611 0, 4612 0, 4613 I40E_GLHMC_FCOEFOBJSZ, 4614 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 4615 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 4616 }, 4617 }; 4618 struct ixl_hmc_entry *e; 4619 uint64_t size, dva; 4620 uint8_t *kva; 4621 uint64_t *sdpage; 4622 unsigned int i; 4623 int npages, tables; 4624 uint32_t reg; 4625 4626 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 4627 4628 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 4629 ixl_rd(sc, I40E_GLHMC_LANQMAX); 4630 4631 size = 0; 4632 for (i = 0; i < __arraycount(regs); i++) { 4633 e = &sc->sc_hmc_entries[i]; 4634 4635 e->hmc_count = regs[i].count; 4636 reg = ixl_rd(sc, regs[i].objsiz); 4637 e->hmc_size = BIT_ULL(0x3F & reg); 4638 e->hmc_base = size; 4639 4640 if ((e->hmc_size * 8) < regs[i].minsize) { 4641 aprint_error_dev(sc->sc_dev, 4642 "kernel hmc entry is too big\n"); 4643 return -1; 4644 } 4645 4646 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 4647 } 4648 size = roundup(size, IXL_HMC_PGSIZE); 4649 npages = size / IXL_HMC_PGSIZE; 4650 4651 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 4652 4653 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 4654 aprint_error_dev(sc->sc_dev, 4655 "unable to allocate hmc pd memory\n"); 4656 return -1; 4657 } 4658 4659 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 4660 IXL_HMC_PGSIZE) != 0) { 4661 aprint_error_dev(sc->sc_dev, 4662 "unable to allocate hmc sd memory\n"); 4663 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 4664 return -1; 4665 } 4666 4667 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 4668 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 4669 4670 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 4671 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 4672 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4673 4674 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 4675 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 4676 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 4677 4678 for (i = 0; (int)i < npages; i++) { 4679 *sdpage = htole64(dva | IXL_HMC_PDVALID); 4680 sdpage++; 4681 4682 dva += IXL_HMC_PGSIZE; 4683 } 4684 4685 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 4686 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 4687 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4688 4689 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 4690 for (i = 0; (int)i < tables; i++) { 4691 uint32_t count; 4692 4693 KASSERT(npages >= 0); 4694 4695 count = ((unsigned int)npages > IXL_HMC_PGS) ? 4696 IXL_HMC_PGS : (unsigned int)npages; 4697 4698 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 4699 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 4700 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 4701 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 4702 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 4703 ixl_wr(sc, I40E_PFHMC_SDCMD, 4704 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 4705 4706 npages -= IXL_HMC_PGS; 4707 dva += IXL_HMC_PGSIZE; 4708 } 4709 4710 for (i = 0; i < __arraycount(regs); i++) { 4711 e = &sc->sc_hmc_entries[i]; 4712 4713 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 4714 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 4715 } 4716 4717 return 0; 4718 } 4719 4720 static void 4721 ixl_hmc_free(struct ixl_softc *sc) 4722 { 4723 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 4724 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 4725 } 4726 4727 static void 4728 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 4729 unsigned int npacking) 4730 { 4731 uint8_t *dst = d; 4732 const uint8_t *src = s; 4733 unsigned int i; 4734 4735 for (i = 0; i < npacking; i++) { 4736 const struct ixl_hmc_pack *pack = &packing[i]; 4737 unsigned int offset = pack->lsb / 8; 4738 unsigned int align = pack->lsb % 8; 4739 const uint8_t *in = src + pack->offset; 4740 uint8_t *out = dst + offset; 4741 int width = pack->width; 4742 unsigned int inbits = 0; 4743 4744 if (align) { 4745 inbits = (*in++) << align; 4746 *out++ |= (inbits & 0xff); 4747 inbits >>= 8; 4748 4749 width -= 8 - align; 4750 } 4751 4752 while (width >= 8) { 4753 inbits |= (*in++) << align; 4754 *out++ = (inbits & 0xff); 4755 inbits >>= 8; 4756 4757 width -= 8; 4758 } 4759 4760 if (width > 0) { 4761 inbits |= (*in) << align; 4762 *out |= (inbits & ((1 << width) - 1)); 4763 } 4764 } 4765 } 4766 4767 static struct ixl_aq_buf * 4768 ixl_aqb_alloc(struct ixl_softc *sc) 4769 { 4770 struct ixl_aq_buf *aqb; 4771 4772 aqb = malloc(sizeof(*aqb), M_DEVBUF, M_WAITOK); 4773 if (aqb == NULL) 4774 return NULL; 4775 4776 aqb->aqb_size = IXL_AQ_BUFLEN; 4777 4778 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 4779 aqb->aqb_size, 0, 4780 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 4781 goto free; 4782 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 4783 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 4784 BUS_DMA_WAITOK) != 0) 4785 goto destroy; 4786 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 4787 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 4788 goto dma_free; 4789 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 4790 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 4791 goto unmap; 4792 4793 return aqb; 4794 unmap: 4795 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 4796 dma_free: 4797 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 4798 destroy: 4799 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 4800 free: 4801 free(aqb, M_DEVBUF); 4802 4803 return NULL; 4804 } 4805 4806 static void 4807 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 4808 { 4809 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 4810 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 4811 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 4812 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 4813 free(aqb, M_DEVBUF); 4814 } 4815 4816 static int 4817 ixl_arq_fill(struct ixl_softc *sc) 4818 { 4819 struct ixl_aq_buf *aqb; 4820 struct ixl_aq_desc *arq, *iaq; 4821 unsigned int prod = sc->sc_arq_prod; 4822 unsigned int n; 4823 int post = 0; 4824 4825 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 4826 IXL_AQ_NUM); 4827 arq = IXL_DMA_KVA(&sc->sc_arq); 4828 4829 if (__predict_false(n <= 0)) 4830 return 0; 4831 4832 do { 4833 aqb = sc->sc_arq_live[prod]; 4834 iaq = &arq[prod]; 4835 4836 if (aqb == NULL) { 4837 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 4838 if (aqb != NULL) { 4839 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 4840 ixl_aq_buf, aqb_entry); 4841 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 4842 break; 4843 } 4844 4845 sc->sc_arq_live[prod] = aqb; 4846 memset(aqb->aqb_data, 0, aqb->aqb_size); 4847 4848 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 4849 aqb->aqb_size, BUS_DMASYNC_PREREAD); 4850 4851 iaq->iaq_flags = htole16(IXL_AQ_BUF | 4852 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 4853 IXL_AQ_LB : 0)); 4854 iaq->iaq_opcode = 0; 4855 iaq->iaq_datalen = htole16(aqb->aqb_size); 4856 iaq->iaq_retval = 0; 4857 iaq->iaq_cookie = 0; 4858 iaq->iaq_param[0] = 0; 4859 iaq->iaq_param[1] = 0; 4860 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 4861 } 4862 4863 prod++; 4864 prod &= IXL_AQ_MASK; 4865 4866 post = 1; 4867 4868 } while (--n); 4869 4870 if (post) { 4871 sc->sc_arq_prod = prod; 4872 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 4873 } 4874 4875 return post; 4876 } 4877 4878 static void 4879 ixl_arq_unfill(struct ixl_softc *sc) 4880 { 4881 struct ixl_aq_buf *aqb; 4882 unsigned int i; 4883 4884 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 4885 aqb = sc->sc_arq_live[i]; 4886 if (aqb == NULL) 4887 continue; 4888 4889 sc->sc_arq_live[i] = NULL; 4890 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 4891 BUS_DMASYNC_POSTREAD); 4892 ixl_aqb_free(sc, aqb); 4893 } 4894 4895 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 4896 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 4897 ixl_aq_buf, aqb_entry); 4898 ixl_aqb_free(sc, aqb); 4899 } 4900 } 4901 4902 static void 4903 ixl_clear_hw(struct ixl_softc *sc) 4904 { 4905 uint32_t num_queues, base_queue; 4906 uint32_t num_pf_int; 4907 uint32_t num_vf_int; 4908 uint32_t num_vfs; 4909 uint32_t i, j; 4910 uint32_t val; 4911 uint32_t eol = 0x7ff; 4912 4913 /* get number of interrupts, queues, and vfs */ 4914 val = ixl_rd(sc, I40E_GLPCI_CNF2); 4915 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 4916 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 4917 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 4918 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 4919 4920 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 4921 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 4922 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 4923 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 4924 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 4925 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 4926 num_queues = (j - base_queue) + 1; 4927 else 4928 num_queues = 0; 4929 4930 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 4931 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 4932 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 4933 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 4934 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 4935 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 4936 num_vfs = (j - i) + 1; 4937 else 4938 num_vfs = 0; 4939 4940 /* stop all the interrupts */ 4941 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 4942 ixl_flush(sc); 4943 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 4944 for (i = 0; i < num_pf_int - 2; i++) 4945 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 4946 ixl_flush(sc); 4947 4948 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 4949 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4950 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 4951 for (i = 0; i < num_pf_int - 2; i++) 4952 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 4953 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4954 for (i = 0; i < num_vfs; i++) 4955 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 4956 for (i = 0; i < num_vf_int - 2; i++) 4957 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 4958 4959 /* warn the HW of the coming Tx disables */ 4960 for (i = 0; i < num_queues; i++) { 4961 uint32_t abs_queue_idx = base_queue + i; 4962 uint32_t reg_block = 0; 4963 4964 if (abs_queue_idx >= 128) { 4965 reg_block = abs_queue_idx / 128; 4966 abs_queue_idx %= 128; 4967 } 4968 4969 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 4970 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 4971 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 4972 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 4973 4974 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 4975 } 4976 delaymsec(400); 4977 4978 /* stop all the queues */ 4979 for (i = 0; i < num_queues; i++) { 4980 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 4981 ixl_wr(sc, I40E_QTX_ENA(i), 0); 4982 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 4983 ixl_wr(sc, I40E_QRX_ENA(i), 0); 4984 } 4985 4986 /* short wait for all queue disables to settle */ 4987 delaymsec(50); 4988 } 4989 4990 static int 4991 ixl_pf_reset(struct ixl_softc *sc) 4992 { 4993 uint32_t cnt = 0; 4994 uint32_t cnt1 = 0; 4995 uint32_t reg = 0, reg0 = 0; 4996 uint32_t grst_del; 4997 4998 /* 4999 * Poll for Global Reset steady state in case of recent GRST. 5000 * The grst delay value is in 100ms units, and we'll wait a 5001 * couple counts longer to be sure we don't just miss the end. 5002 */ 5003 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 5004 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 5005 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 5006 5007 grst_del = grst_del * 20; 5008 5009 for (cnt = 0; cnt < grst_del; cnt++) { 5010 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 5011 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 5012 break; 5013 delaymsec(100); 5014 } 5015 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5016 aprint_error(", Global reset polling failed to complete\n"); 5017 return -1; 5018 } 5019 5020 /* Now Wait for the FW to be ready */ 5021 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 5022 reg = ixl_rd(sc, I40E_GLNVM_ULD); 5023 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5024 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 5025 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5026 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 5027 break; 5028 5029 delaymsec(10); 5030 } 5031 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5032 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 5033 aprint_error(", wait for FW Reset complete timed out " 5034 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 5035 return -1; 5036 } 5037 5038 /* 5039 * If there was a Global Reset in progress when we got here, 5040 * we don't need to do the PF Reset 5041 */ 5042 if (cnt == 0) { 5043 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5044 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 5045 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 5046 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5047 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 5048 break; 5049 delaymsec(1); 5050 5051 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 5052 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5053 aprint_error(", Core reset upcoming." 5054 " Skipping PF reset reset request\n"); 5055 return -1; 5056 } 5057 } 5058 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 5059 aprint_error(", PF reset polling failed to complete" 5060 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 5061 return -1; 5062 } 5063 } 5064 5065 return 0; 5066 } 5067 5068 static int 5069 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 5070 bus_size_t size, bus_size_t align) 5071 { 5072 ixm->ixm_size = size; 5073 5074 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 5075 ixm->ixm_size, 0, 5076 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 5077 &ixm->ixm_map) != 0) 5078 return 1; 5079 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 5080 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 5081 BUS_DMA_WAITOK) != 0) 5082 goto destroy; 5083 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 5084 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 5085 goto free; 5086 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 5087 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 5088 goto unmap; 5089 5090 memset(ixm->ixm_kva, 0, ixm->ixm_size); 5091 5092 return 0; 5093 unmap: 5094 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5095 free: 5096 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5097 destroy: 5098 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5099 return 1; 5100 } 5101 5102 static void 5103 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 5104 { 5105 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 5106 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5107 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5108 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5109 } 5110 5111 static int 5112 ixl_setup_vlan_hwfilter(struct ixl_softc *sc) 5113 { 5114 struct ethercom *ec = &sc->sc_ec; 5115 struct vlanid_list *vlanidp; 5116 int rv; 5117 5118 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5119 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5120 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5121 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5122 5123 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5124 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5125 if (rv != 0) 5126 return rv; 5127 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5128 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5129 if (rv != 0) 5130 return rv; 5131 5132 ETHER_LOCK(ec); 5133 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5134 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 5135 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5136 if (rv != 0) 5137 break; 5138 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 5139 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5140 if (rv != 0) 5141 break; 5142 } 5143 ETHER_UNLOCK(ec); 5144 5145 return rv; 5146 } 5147 5148 static void 5149 ixl_teardown_vlan_hwfilter(struct ixl_softc *sc) 5150 { 5151 struct vlanid_list *vlanidp; 5152 struct ethercom *ec = &sc->sc_ec; 5153 5154 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5155 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5156 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5157 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5158 5159 ETHER_LOCK(ec); 5160 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5161 ixl_remove_macvlan(sc, sc->sc_enaddr, 5162 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5163 ixl_remove_macvlan(sc, etherbroadcastaddr, 5164 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5165 } 5166 ETHER_UNLOCK(ec); 5167 5168 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5169 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5170 ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5171 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5172 } 5173 5174 static int 5175 ixl_update_macvlan(struct ixl_softc *sc) 5176 { 5177 int rv = 0; 5178 int next_ec_capenable = sc->sc_ec.ec_capenable; 5179 5180 if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 5181 rv = ixl_setup_vlan_hwfilter(sc); 5182 if (rv != 0) 5183 ixl_teardown_vlan_hwfilter(sc); 5184 } else { 5185 ixl_teardown_vlan_hwfilter(sc); 5186 } 5187 5188 return rv; 5189 } 5190 5191 static int 5192 ixl_ifflags_cb(struct ethercom *ec) 5193 { 5194 struct ifnet *ifp = &ec->ec_if; 5195 struct ixl_softc *sc = ifp->if_softc; 5196 int rv, change; 5197 5198 mutex_enter(&sc->sc_cfg_lock); 5199 5200 change = ec->ec_capenable ^ sc->sc_cur_ec_capenable; 5201 5202 if (ISSET(change, ETHERCAP_VLAN_HWTAGGING)) { 5203 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 5204 rv = ENETRESET; 5205 goto out; 5206 } 5207 5208 if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) { 5209 rv = ixl_update_macvlan(sc); 5210 if (rv == 0) { 5211 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 5212 } else { 5213 CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER); 5214 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 5215 } 5216 } 5217 5218 rv = ixl_iff(sc); 5219 out: 5220 mutex_exit(&sc->sc_cfg_lock); 5221 5222 return rv; 5223 } 5224 5225 static int 5226 ixl_set_link_status(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 5227 { 5228 const struct ixl_aq_link_status *status; 5229 const struct ixl_phy_type *itype; 5230 5231 uint64_t ifm_active = IFM_ETHER; 5232 uint64_t ifm_status = IFM_AVALID; 5233 int link_state = LINK_STATE_DOWN; 5234 uint64_t baudrate = 0; 5235 5236 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 5237 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) 5238 goto done; 5239 5240 ifm_active |= IFM_FDX; 5241 ifm_status |= IFM_ACTIVE; 5242 link_state = LINK_STATE_UP; 5243 5244 itype = ixl_search_phy_type(status->phy_type); 5245 if (itype != NULL) 5246 ifm_active |= itype->ifm_type; 5247 5248 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 5249 ifm_active |= IFM_ETH_TXPAUSE; 5250 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 5251 ifm_active |= IFM_ETH_RXPAUSE; 5252 5253 baudrate = ixl_search_link_speed(status->link_speed); 5254 5255 done: 5256 /* NET_ASSERT_LOCKED() except during attach */ 5257 sc->sc_media_active = ifm_active; 5258 sc->sc_media_status = ifm_status; 5259 5260 sc->sc_ec.ec_if.if_baudrate = baudrate; 5261 5262 return link_state; 5263 } 5264 5265 static int 5266 ixl_establish_intx(struct ixl_softc *sc) 5267 { 5268 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5269 pci_intr_handle_t *intr; 5270 char xnamebuf[32]; 5271 char intrbuf[PCI_INTRSTR_LEN]; 5272 char const *intrstr; 5273 5274 KASSERT(sc->sc_nintrs == 1); 5275 5276 intr = &sc->sc_ihp[0]; 5277 5278 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 5279 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 5280 device_xname(sc->sc_dev)); 5281 5282 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 5283 sc, xnamebuf); 5284 5285 if (sc->sc_ihs[0] == NULL) { 5286 aprint_error_dev(sc->sc_dev, 5287 "unable to establish interrupt at %s\n", intrstr); 5288 return -1; 5289 } 5290 5291 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 5292 return 0; 5293 } 5294 5295 static int 5296 ixl_establish_msix(struct ixl_softc *sc) 5297 { 5298 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5299 unsigned int vector = 0; 5300 unsigned int i; 5301 char xnamebuf[32]; 5302 char intrbuf[PCI_INTRSTR_LEN]; 5303 char const *intrstr; 5304 5305 /* the "other" intr is mapped to vector 0 */ 5306 vector = 0; 5307 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5308 intrbuf, sizeof(intrbuf)); 5309 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 5310 device_xname(sc->sc_dev)); 5311 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5312 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 5313 sc, xnamebuf); 5314 if (sc->sc_ihs[vector] == NULL) { 5315 aprint_error_dev(sc->sc_dev, 5316 "unable to establish interrupt at %s\n", intrstr); 5317 goto fail; 5318 } 5319 vector++; 5320 aprint_normal_dev(sc->sc_dev, "interrupt at %s\n", intrstr); 5321 5322 sc->sc_msix_vector_queue = vector; 5323 5324 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5325 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5326 intrbuf, sizeof(intrbuf)); 5327 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 5328 device_xname(sc->sc_dev), i); 5329 5330 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5331 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 5332 (void *)&sc->sc_qps[i], xnamebuf); 5333 5334 if (sc->sc_ihs[vector] == NULL) { 5335 aprint_error_dev(sc->sc_dev, 5336 "unable to establish interrupt at %s\n", intrstr); 5337 goto fail; 5338 } 5339 vector++; 5340 aprint_normal_dev(sc->sc_dev, 5341 "interrupt at %s\n", intrstr); 5342 } 5343 5344 return 0; 5345 fail: 5346 for (i = 0; i < vector; i++) { 5347 pci_intr_disestablish(pc, sc->sc_ihs[i]); 5348 } 5349 5350 sc->sc_msix_vector_queue = 0; 5351 sc->sc_msix_vector_queue = 0; 5352 5353 return -1; 5354 } 5355 5356 static void 5357 ixl_set_affinity_msix(struct ixl_softc *sc) 5358 { 5359 kcpuset_t *affinity; 5360 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5361 int affinity_to, r; 5362 unsigned int i, vector; 5363 char intrbuf[PCI_INTRSTR_LEN]; 5364 char const *intrstr; 5365 5366 affinity_to = 0; 5367 kcpuset_create(&affinity, false); 5368 5369 vector = sc->sc_msix_vector_queue; 5370 5371 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5372 affinity_to = i % ncpu; 5373 5374 kcpuset_zero(affinity); 5375 kcpuset_set(affinity, affinity_to); 5376 5377 intrstr = pci_intr_string(pc, sc->sc_ihp[vector + i], 5378 intrbuf, sizeof(intrbuf)); 5379 r = interrupt_distribute(sc->sc_ihs[vector + i], 5380 affinity, NULL); 5381 if (r == 0) { 5382 aprint_normal_dev(sc->sc_dev, 5383 "for TXRX%u interrupting at %s affinity to %u\n", 5384 i, intrstr, affinity_to); 5385 } else { 5386 aprint_normal_dev(sc->sc_dev, 5387 "for TXRX%u interrupting at %s\n", 5388 i, intrstr); 5389 } 5390 } 5391 5392 vector = 0; /* vector 0 means "other" interrupt */ 5393 affinity_to = (affinity_to + 1) % ncpu; 5394 kcpuset_zero(affinity); 5395 kcpuset_set(affinity, affinity_to); 5396 5397 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5398 intrbuf, sizeof(intrbuf)); 5399 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5400 if (r == 0) { 5401 aprint_normal_dev(sc->sc_dev, 5402 "for other interrupting at %s affinity to %u\n", 5403 intrstr, affinity_to); 5404 } else { 5405 aprint_normal_dev(sc->sc_dev, 5406 "for other interrupting at %s", intrstr); 5407 } 5408 5409 kcpuset_destroy(affinity); 5410 } 5411 5412 static void 5413 ixl_config_queue_intr(struct ixl_softc *sc) 5414 { 5415 unsigned int i, vector; 5416 5417 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5418 vector = sc->sc_msix_vector_queue; 5419 } else { 5420 vector = I40E_INTR_NOTX_INTR; 5421 5422 ixl_wr(sc, I40E_PFINT_LNKLST0, 5423 (I40E_INTR_NOTX_QUEUE << 5424 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5425 (I40E_QUEUE_TYPE_RX << 5426 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5427 } 5428 5429 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5430 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5431 ixl_flush(sc); 5432 5433 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5434 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5435 (I40E_QUEUE_TYPE_RX << 5436 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5437 5438 ixl_wr(sc, I40E_QINT_RQCTL(i), 5439 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5440 (I40E_ITR_INDEX_RX << 5441 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5442 (I40E_INTR_NOTX_RX_QUEUE << 5443 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5444 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5445 (I40E_QUEUE_TYPE_TX << 5446 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5447 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5448 5449 ixl_wr(sc, I40E_QINT_TQCTL(i), 5450 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5451 (I40E_ITR_INDEX_TX << 5452 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5453 (I40E_INTR_NOTX_TX_QUEUE << 5454 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5455 (I40E_QUEUE_TYPE_EOL << 5456 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5457 (I40E_QUEUE_TYPE_RX << 5458 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5459 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5460 5461 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) 5462 vector++; 5463 } 5464 ixl_flush(sc); 5465 5466 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), 0x7a); 5467 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), 0x7a); 5468 ixl_flush(sc); 5469 } 5470 5471 static void 5472 ixl_config_other_intr(struct ixl_softc *sc) 5473 { 5474 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5475 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5476 5477 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5478 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5479 I40E_PFINT_ICR0_ENA_GRST_MASK | 5480 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5481 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5482 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5483 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5484 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5485 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5486 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5487 5488 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5489 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5490 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5491 (I40E_ITR_INDEX_OTHER << 5492 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5493 ixl_flush(sc); 5494 } 5495 5496 static int 5497 ixl_setup_interrupts(struct ixl_softc *sc) 5498 { 5499 struct pci_attach_args *pa = &sc->sc_pa; 5500 pci_intr_type_t max_type, intr_type; 5501 int counts[PCI_INTR_TYPE_SIZE]; 5502 int error; 5503 unsigned int i; 5504 bool retry; 5505 5506 memset(counts, 0, sizeof(counts)); 5507 max_type = PCI_INTR_TYPE_MSIX; 5508 /* QPs + other interrupt */ 5509 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5510 counts[PCI_INTR_TYPE_INTX] = 1; 5511 5512 if (ixl_param_nomsix) 5513 counts[PCI_INTR_TYPE_MSIX] = 0; 5514 5515 do { 5516 retry = false; 5517 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5518 if (error != 0) { 5519 aprint_error_dev(sc->sc_dev, 5520 "couldn't map interrupt\n"); 5521 break; 5522 } 5523 for (i = 0; i < sc->sc_nintrs; i++) { 5524 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5525 PCI_INTR_MPSAFE, true); 5526 } 5527 5528 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5529 sc->sc_nintrs = counts[intr_type]; 5530 KASSERT(sc->sc_nintrs > 0); 5531 5532 sc->sc_ihs = kmem_alloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5533 KM_SLEEP); 5534 5535 if (intr_type == PCI_INTR_TYPE_MSIX) { 5536 error = ixl_establish_msix(sc); 5537 if (error) { 5538 counts[PCI_INTR_TYPE_MSIX] = 0; 5539 retry = true; 5540 } else { 5541 ixl_set_affinity_msix(sc); 5542 } 5543 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5544 error = ixl_establish_intx(sc); 5545 } else { 5546 error = -1; 5547 } 5548 5549 if (error) { 5550 kmem_free(sc->sc_ihs, 5551 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5552 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5553 } else { 5554 sc->sc_intrtype = intr_type; 5555 } 5556 } while (retry); 5557 5558 return error; 5559 } 5560 5561 static void 5562 ixl_teardown_interrupts(struct ixl_softc *sc) 5563 { 5564 struct pci_attach_args *pa = &sc->sc_pa; 5565 unsigned int i; 5566 5567 for (i = 0; i < sc->sc_nintrs; i++) { 5568 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 5569 } 5570 5571 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5572 5573 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5574 sc->sc_ihs = NULL; 5575 sc->sc_nintrs = 0; 5576 } 5577 5578 static int 5579 ixl_setup_stats(struct ixl_softc *sc) 5580 { 5581 struct ixl_queue_pair *qp; 5582 struct ixl_tx_ring *txr; 5583 struct ixl_rx_ring *rxr; 5584 struct ixl_stats_counters *isc; 5585 unsigned int i; 5586 5587 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5588 qp = &sc->sc_qps[i]; 5589 txr = qp->qp_txr; 5590 rxr = qp->qp_rxr; 5591 5592 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 5593 NULL, qp->qp_name, "m_defrag successed"); 5594 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 5595 NULL, qp->qp_name, "m_defrag_failed"); 5596 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 5597 NULL, qp->qp_name, "Dropped in pcq"); 5598 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 5599 NULL, qp->qp_name, "Deferred transmit"); 5600 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 5601 NULL, qp->qp_name, "Interrupt on queue"); 5602 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 5603 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5604 5605 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 5606 NULL, qp->qp_name, "MGETHDR failed"); 5607 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 5608 NULL, qp->qp_name, "MCLGET failed"); 5609 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 5610 EVCNT_TYPE_MISC, NULL, qp->qp_name, 5611 "bus_dmamap_load_mbuf failed"); 5612 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 5613 NULL, qp->qp_name, "Interrupt on queue"); 5614 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 5615 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5616 } 5617 5618 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 5619 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 5620 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 5621 NULL, device_xname(sc->sc_dev), "Link status event"); 5622 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 5623 NULL, device_xname(sc->sc_dev), "ECC error"); 5624 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 5625 NULL, device_xname(sc->sc_dev), "PCI exception"); 5626 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 5627 NULL, device_xname(sc->sc_dev), "Critical error"); 5628 5629 isc = &sc->sc_stats_counters; 5630 evcnt_attach_dynamic(&isc->isc_crc_errors, EVCNT_TYPE_MISC, 5631 NULL, device_xname(sc->sc_dev), "CRC errors"); 5632 evcnt_attach_dynamic(&isc->isc_illegal_bytes, EVCNT_TYPE_MISC, 5633 NULL, device_xname(sc->sc_dev), "Illegal bytes"); 5634 evcnt_attach_dynamic(&isc->isc_mac_local_faults, EVCNT_TYPE_MISC, 5635 NULL, device_xname(sc->sc_dev), "Mac local faults"); 5636 evcnt_attach_dynamic(&isc->isc_mac_remote_faults, EVCNT_TYPE_MISC, 5637 NULL, device_xname(sc->sc_dev), "Mac remote faults"); 5638 evcnt_attach_dynamic(&isc->isc_link_xon_rx, EVCNT_TYPE_MISC, 5639 NULL, device_xname(sc->sc_dev), "Rx xon"); 5640 evcnt_attach_dynamic(&isc->isc_link_xon_tx, EVCNT_TYPE_MISC, 5641 NULL, device_xname(sc->sc_dev), "Tx xon"); 5642 evcnt_attach_dynamic(&isc->isc_link_xoff_rx, EVCNT_TYPE_MISC, 5643 NULL, device_xname(sc->sc_dev), "Rx xoff"); 5644 evcnt_attach_dynamic(&isc->isc_link_xoff_tx, EVCNT_TYPE_MISC, 5645 NULL, device_xname(sc->sc_dev), "Tx xoff"); 5646 evcnt_attach_dynamic(&isc->isc_rx_fragments, EVCNT_TYPE_MISC, 5647 NULL, device_xname(sc->sc_dev), "Rx fragments"); 5648 evcnt_attach_dynamic(&isc->isc_rx_jabber, EVCNT_TYPE_MISC, 5649 NULL, device_xname(sc->sc_dev), "Rx jabber"); 5650 5651 evcnt_attach_dynamic(&isc->isc_rx_size_64, EVCNT_TYPE_MISC, 5652 NULL, device_xname(sc->sc_dev), "Rx size 64"); 5653 evcnt_attach_dynamic(&isc->isc_rx_size_127, EVCNT_TYPE_MISC, 5654 NULL, device_xname(sc->sc_dev), "Rx size 127"); 5655 evcnt_attach_dynamic(&isc->isc_rx_size_255, EVCNT_TYPE_MISC, 5656 NULL, device_xname(sc->sc_dev), "Rx size 255"); 5657 evcnt_attach_dynamic(&isc->isc_rx_size_511, EVCNT_TYPE_MISC, 5658 NULL, device_xname(sc->sc_dev), "Rx size 511"); 5659 evcnt_attach_dynamic(&isc->isc_rx_size_1023, EVCNT_TYPE_MISC, 5660 NULL, device_xname(sc->sc_dev), "Rx size 1023"); 5661 evcnt_attach_dynamic(&isc->isc_rx_size_1522, EVCNT_TYPE_MISC, 5662 NULL, device_xname(sc->sc_dev), "Rx size 1522"); 5663 evcnt_attach_dynamic(&isc->isc_rx_size_big, EVCNT_TYPE_MISC, 5664 NULL, device_xname(sc->sc_dev), "Rx jumbo packets"); 5665 evcnt_attach_dynamic(&isc->isc_rx_undersize, EVCNT_TYPE_MISC, 5666 NULL, device_xname(sc->sc_dev), "Rx under size"); 5667 evcnt_attach_dynamic(&isc->isc_rx_oversize, EVCNT_TYPE_MISC, 5668 NULL, device_xname(sc->sc_dev), "Rx over size"); 5669 5670 evcnt_attach_dynamic(&isc->isc_rx_bytes, EVCNT_TYPE_MISC, 5671 NULL, device_xname(sc->sc_dev), "Rx bytes / port"); 5672 evcnt_attach_dynamic(&isc->isc_rx_discards, EVCNT_TYPE_MISC, 5673 NULL, device_xname(sc->sc_dev), "Rx discards / port"); 5674 evcnt_attach_dynamic(&isc->isc_rx_unicast, EVCNT_TYPE_MISC, 5675 NULL, device_xname(sc->sc_dev), "Rx unicast / port"); 5676 evcnt_attach_dynamic(&isc->isc_rx_multicast, EVCNT_TYPE_MISC, 5677 NULL, device_xname(sc->sc_dev), "Rx multicast / port"); 5678 evcnt_attach_dynamic(&isc->isc_rx_broadcast, EVCNT_TYPE_MISC, 5679 NULL, device_xname(sc->sc_dev), "Rx broadcast / port"); 5680 5681 evcnt_attach_dynamic(&isc->isc_vsi_rx_bytes, EVCNT_TYPE_MISC, 5682 NULL, device_xname(sc->sc_dev), "Rx bytes / vsi"); 5683 evcnt_attach_dynamic(&isc->isc_vsi_rx_discards, EVCNT_TYPE_MISC, 5684 NULL, device_xname(sc->sc_dev), "Rx discard / vsi"); 5685 evcnt_attach_dynamic(&isc->isc_vsi_rx_unicast, EVCNT_TYPE_MISC, 5686 NULL, device_xname(sc->sc_dev), "Rx unicast / vsi"); 5687 evcnt_attach_dynamic(&isc->isc_vsi_rx_multicast, EVCNT_TYPE_MISC, 5688 NULL, device_xname(sc->sc_dev), "Rx multicast / vsi"); 5689 evcnt_attach_dynamic(&isc->isc_vsi_rx_broadcast, EVCNT_TYPE_MISC, 5690 NULL, device_xname(sc->sc_dev), "Rx broadcast / vsi"); 5691 5692 evcnt_attach_dynamic(&isc->isc_tx_size_64, EVCNT_TYPE_MISC, 5693 NULL, device_xname(sc->sc_dev), "Tx size 64"); 5694 evcnt_attach_dynamic(&isc->isc_tx_size_127, EVCNT_TYPE_MISC, 5695 NULL, device_xname(sc->sc_dev), "Tx size 127"); 5696 evcnt_attach_dynamic(&isc->isc_tx_size_255, EVCNT_TYPE_MISC, 5697 NULL, device_xname(sc->sc_dev), "Tx size 255"); 5698 evcnt_attach_dynamic(&isc->isc_tx_size_511, EVCNT_TYPE_MISC, 5699 NULL, device_xname(sc->sc_dev), "Tx size 511"); 5700 evcnt_attach_dynamic(&isc->isc_tx_size_1023, EVCNT_TYPE_MISC, 5701 NULL, device_xname(sc->sc_dev), "Tx size 1023"); 5702 evcnt_attach_dynamic(&isc->isc_tx_size_1522, EVCNT_TYPE_MISC, 5703 NULL, device_xname(sc->sc_dev), "Tx size 1522"); 5704 evcnt_attach_dynamic(&isc->isc_tx_size_big, EVCNT_TYPE_MISC, 5705 NULL, device_xname(sc->sc_dev), "Tx jumbo packets"); 5706 5707 evcnt_attach_dynamic(&isc->isc_tx_bytes, EVCNT_TYPE_MISC, 5708 NULL, device_xname(sc->sc_dev), "Tx bytes / port"); 5709 evcnt_attach_dynamic(&isc->isc_tx_dropped_link_down, EVCNT_TYPE_MISC, 5710 NULL, device_xname(sc->sc_dev), 5711 "Tx dropped due to link down / port"); 5712 evcnt_attach_dynamic(&isc->isc_tx_unicast, EVCNT_TYPE_MISC, 5713 NULL, device_xname(sc->sc_dev), "Tx unicast / port"); 5714 evcnt_attach_dynamic(&isc->isc_tx_multicast, EVCNT_TYPE_MISC, 5715 NULL, device_xname(sc->sc_dev), "Tx multicast / port"); 5716 evcnt_attach_dynamic(&isc->isc_tx_broadcast, EVCNT_TYPE_MISC, 5717 NULL, device_xname(sc->sc_dev), "Tx broadcast / port"); 5718 5719 evcnt_attach_dynamic(&isc->isc_vsi_tx_bytes, EVCNT_TYPE_MISC, 5720 NULL, device_xname(sc->sc_dev), "Tx bytes / vsi"); 5721 evcnt_attach_dynamic(&isc->isc_vsi_tx_errors, EVCNT_TYPE_MISC, 5722 NULL, device_xname(sc->sc_dev), "Tx errors / vsi"); 5723 evcnt_attach_dynamic(&isc->isc_vsi_tx_unicast, EVCNT_TYPE_MISC, 5724 NULL, device_xname(sc->sc_dev), "Tx unicast / vsi"); 5725 evcnt_attach_dynamic(&isc->isc_vsi_tx_multicast, EVCNT_TYPE_MISC, 5726 NULL, device_xname(sc->sc_dev), "Tx multicast / vsi"); 5727 evcnt_attach_dynamic(&isc->isc_vsi_tx_broadcast, EVCNT_TYPE_MISC, 5728 NULL, device_xname(sc->sc_dev), "Tx broadcast / vsi"); 5729 5730 sc->sc_stats_intval = ixl_param_stats_interval; 5731 callout_init(&sc->sc_stats_callout, CALLOUT_MPSAFE); 5732 callout_setfunc(&sc->sc_stats_callout, ixl_stats_callout, sc); 5733 ixl_work_set(&sc->sc_stats_task, ixl_stats_update, sc); 5734 5735 return 0; 5736 } 5737 5738 static void 5739 ixl_teardown_stats(struct ixl_softc *sc) 5740 { 5741 struct ixl_tx_ring *txr; 5742 struct ixl_rx_ring *rxr; 5743 struct ixl_stats_counters *isc; 5744 unsigned int i; 5745 5746 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5747 txr = sc->sc_qps[i].qp_txr; 5748 rxr = sc->sc_qps[i].qp_rxr; 5749 5750 evcnt_detach(&txr->txr_defragged); 5751 evcnt_detach(&txr->txr_defrag_failed); 5752 evcnt_detach(&txr->txr_pcqdrop); 5753 evcnt_detach(&txr->txr_transmitdef); 5754 evcnt_detach(&txr->txr_intr); 5755 evcnt_detach(&txr->txr_defer); 5756 5757 evcnt_detach(&rxr->rxr_mgethdr_failed); 5758 evcnt_detach(&rxr->rxr_mgetcl_failed); 5759 evcnt_detach(&rxr->rxr_mbuf_load_failed); 5760 evcnt_detach(&rxr->rxr_intr); 5761 evcnt_detach(&rxr->rxr_defer); 5762 } 5763 5764 isc = &sc->sc_stats_counters; 5765 evcnt_detach(&isc->isc_crc_errors); 5766 evcnt_detach(&isc->isc_illegal_bytes); 5767 evcnt_detach(&isc->isc_mac_local_faults); 5768 evcnt_detach(&isc->isc_mac_remote_faults); 5769 evcnt_detach(&isc->isc_link_xon_rx); 5770 evcnt_detach(&isc->isc_link_xon_tx); 5771 evcnt_detach(&isc->isc_link_xoff_rx); 5772 evcnt_detach(&isc->isc_link_xoff_tx); 5773 evcnt_detach(&isc->isc_rx_fragments); 5774 evcnt_detach(&isc->isc_rx_jabber); 5775 evcnt_detach(&isc->isc_rx_bytes); 5776 evcnt_detach(&isc->isc_rx_discards); 5777 evcnt_detach(&isc->isc_rx_unicast); 5778 evcnt_detach(&isc->isc_rx_multicast); 5779 evcnt_detach(&isc->isc_rx_broadcast); 5780 evcnt_detach(&isc->isc_rx_size_64); 5781 evcnt_detach(&isc->isc_rx_size_127); 5782 evcnt_detach(&isc->isc_rx_size_255); 5783 evcnt_detach(&isc->isc_rx_size_511); 5784 evcnt_detach(&isc->isc_rx_size_1023); 5785 evcnt_detach(&isc->isc_rx_size_1522); 5786 evcnt_detach(&isc->isc_rx_size_big); 5787 evcnt_detach(&isc->isc_rx_undersize); 5788 evcnt_detach(&isc->isc_rx_oversize); 5789 evcnt_detach(&isc->isc_tx_bytes); 5790 evcnt_detach(&isc->isc_tx_dropped_link_down); 5791 evcnt_detach(&isc->isc_tx_unicast); 5792 evcnt_detach(&isc->isc_tx_multicast); 5793 evcnt_detach(&isc->isc_tx_broadcast); 5794 evcnt_detach(&isc->isc_tx_size_64); 5795 evcnt_detach(&isc->isc_tx_size_127); 5796 evcnt_detach(&isc->isc_tx_size_255); 5797 evcnt_detach(&isc->isc_tx_size_511); 5798 evcnt_detach(&isc->isc_tx_size_1023); 5799 evcnt_detach(&isc->isc_tx_size_1522); 5800 evcnt_detach(&isc->isc_tx_size_big); 5801 evcnt_detach(&isc->isc_vsi_rx_discards); 5802 evcnt_detach(&isc->isc_vsi_rx_bytes); 5803 evcnt_detach(&isc->isc_vsi_rx_unicast); 5804 evcnt_detach(&isc->isc_vsi_rx_multicast); 5805 evcnt_detach(&isc->isc_vsi_rx_broadcast); 5806 evcnt_detach(&isc->isc_vsi_tx_errors); 5807 evcnt_detach(&isc->isc_vsi_tx_bytes); 5808 evcnt_detach(&isc->isc_vsi_tx_unicast); 5809 evcnt_detach(&isc->isc_vsi_tx_multicast); 5810 evcnt_detach(&isc->isc_vsi_tx_broadcast); 5811 5812 evcnt_detach(&sc->sc_event_atq); 5813 evcnt_detach(&sc->sc_event_link); 5814 evcnt_detach(&sc->sc_event_ecc_err); 5815 evcnt_detach(&sc->sc_event_pci_exception); 5816 evcnt_detach(&sc->sc_event_crit_err); 5817 5818 callout_destroy(&sc->sc_stats_callout); 5819 } 5820 5821 static void 5822 ixl_stats_callout(void *xsc) 5823 { 5824 struct ixl_softc *sc = xsc; 5825 5826 ixl_work_add(sc->sc_workq, &sc->sc_stats_task); 5827 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 5828 } 5829 5830 static uint64_t 5831 ixl_stat_delta(struct ixl_softc *sc, uint32_t reg_hi, uint32_t reg_lo, 5832 uint64_t *offset, bool has_offset) 5833 { 5834 uint64_t value, delta; 5835 int bitwidth; 5836 5837 bitwidth = reg_hi == 0 ? 32 : 48; 5838 5839 value = ixl_rd(sc, reg_lo); 5840 5841 if (bitwidth > 32) { 5842 value |= ((uint64_t)ixl_rd(sc, reg_hi) << 32); 5843 } 5844 5845 if (__predict_true(has_offset)) { 5846 delta = value; 5847 if (value < *offset) 5848 delta += ((uint64_t)1 << bitwidth); 5849 delta -= *offset; 5850 } else { 5851 delta = 0; 5852 } 5853 atomic_swap_64(offset, value); 5854 5855 return delta; 5856 } 5857 5858 static void 5859 ixl_stats_update(void *xsc) 5860 { 5861 struct ixl_softc *sc = xsc; 5862 struct ixl_stats_counters *isc; 5863 uint64_t delta; 5864 5865 isc = &sc->sc_stats_counters; 5866 5867 /* errors */ 5868 delta = ixl_stat_delta(sc, 5869 0, I40E_GLPRT_CRCERRS(sc->sc_port), 5870 &isc->isc_crc_errors_offset, isc->isc_has_offset); 5871 atomic_add_64(&isc->isc_crc_errors.ev_count, delta); 5872 5873 delta = ixl_stat_delta(sc, 5874 0, I40E_GLPRT_ILLERRC(sc->sc_port), 5875 &isc->isc_illegal_bytes_offset, isc->isc_has_offset); 5876 atomic_add_64(&isc->isc_illegal_bytes.ev_count, delta); 5877 5878 /* rx */ 5879 delta = ixl_stat_delta(sc, 5880 I40E_GLPRT_GORCH(sc->sc_port), I40E_GLPRT_GORCL(sc->sc_port), 5881 &isc->isc_rx_bytes_offset, isc->isc_has_offset); 5882 atomic_add_64(&isc->isc_rx_bytes.ev_count, delta); 5883 5884 delta = ixl_stat_delta(sc, 5885 0, I40E_GLPRT_RDPC(sc->sc_port), 5886 &isc->isc_rx_discards_offset, isc->isc_has_offset); 5887 atomic_add_64(&isc->isc_rx_discards.ev_count, delta); 5888 5889 delta = ixl_stat_delta(sc, 5890 I40E_GLPRT_UPRCH(sc->sc_port), I40E_GLPRT_UPRCL(sc->sc_port), 5891 &isc->isc_rx_unicast_offset, isc->isc_has_offset); 5892 atomic_add_64(&isc->isc_rx_unicast.ev_count, delta); 5893 5894 delta = ixl_stat_delta(sc, 5895 I40E_GLPRT_MPRCH(sc->sc_port), I40E_GLPRT_MPRCL(sc->sc_port), 5896 &isc->isc_rx_multicast_offset, isc->isc_has_offset); 5897 atomic_add_64(&isc->isc_rx_multicast.ev_count, delta); 5898 5899 delta = ixl_stat_delta(sc, 5900 I40E_GLPRT_BPRCH(sc->sc_port), I40E_GLPRT_BPRCL(sc->sc_port), 5901 &isc->isc_rx_broadcast_offset, isc->isc_has_offset); 5902 atomic_add_64(&isc->isc_rx_broadcast.ev_count, delta); 5903 5904 /* Packet size stats rx */ 5905 delta = ixl_stat_delta(sc, 5906 I40E_GLPRT_PRC64H(sc->sc_port), I40E_GLPRT_PRC64L(sc->sc_port), 5907 &isc->isc_rx_size_64_offset, isc->isc_has_offset); 5908 atomic_add_64(&isc->isc_rx_size_64.ev_count, delta); 5909 5910 delta = ixl_stat_delta(sc, 5911 I40E_GLPRT_PRC127H(sc->sc_port), I40E_GLPRT_PRC127L(sc->sc_port), 5912 &isc->isc_rx_size_127_offset, isc->isc_has_offset); 5913 atomic_add_64(&isc->isc_rx_size_127.ev_count, delta); 5914 5915 delta = ixl_stat_delta(sc, 5916 I40E_GLPRT_PRC255H(sc->sc_port), I40E_GLPRT_PRC255L(sc->sc_port), 5917 &isc->isc_rx_size_255_offset, isc->isc_has_offset); 5918 atomic_add_64(&isc->isc_rx_size_255.ev_count, delta); 5919 5920 delta = ixl_stat_delta(sc, 5921 I40E_GLPRT_PRC511H(sc->sc_port), I40E_GLPRT_PRC511L(sc->sc_port), 5922 &isc->isc_rx_size_511_offset, isc->isc_has_offset); 5923 atomic_add_64(&isc->isc_rx_size_511.ev_count, delta); 5924 5925 delta = ixl_stat_delta(sc, 5926 I40E_GLPRT_PRC1023H(sc->sc_port), I40E_GLPRT_PRC1023L(sc->sc_port), 5927 &isc->isc_rx_size_1023_offset, isc->isc_has_offset); 5928 atomic_add_64(&isc->isc_rx_size_1023.ev_count, delta); 5929 5930 delta = ixl_stat_delta(sc, 5931 I40E_GLPRT_PRC1522H(sc->sc_port), I40E_GLPRT_PRC1522L(sc->sc_port), 5932 &isc->isc_rx_size_1522_offset, isc->isc_has_offset); 5933 atomic_add_64(&isc->isc_rx_size_1522.ev_count, delta); 5934 5935 delta = ixl_stat_delta(sc, 5936 I40E_GLPRT_PRC9522H(sc->sc_port), I40E_GLPRT_PRC9522L(sc->sc_port), 5937 &isc->isc_rx_size_big_offset, isc->isc_has_offset); 5938 atomic_add_64(&isc->isc_rx_size_big.ev_count, delta); 5939 5940 delta = ixl_stat_delta(sc, 5941 0, I40E_GLPRT_RUC(sc->sc_port), 5942 &isc->isc_rx_undersize_offset, isc->isc_has_offset); 5943 atomic_add_64(&isc->isc_rx_undersize.ev_count, delta); 5944 5945 delta = ixl_stat_delta(sc, 5946 0, I40E_GLPRT_ROC(sc->sc_port), 5947 &isc->isc_rx_oversize_offset, isc->isc_has_offset); 5948 atomic_add_64(&isc->isc_rx_oversize.ev_count, delta); 5949 5950 /* tx */ 5951 delta = ixl_stat_delta(sc, 5952 I40E_GLPRT_GOTCH(sc->sc_port), I40E_GLPRT_GOTCL(sc->sc_port), 5953 &isc->isc_tx_bytes_offset, isc->isc_has_offset); 5954 atomic_add_64(&isc->isc_tx_bytes.ev_count, delta); 5955 5956 delta = ixl_stat_delta(sc, 5957 0, I40E_GLPRT_TDOLD(sc->sc_port), 5958 &isc->isc_tx_dropped_link_down_offset, isc->isc_has_offset); 5959 atomic_add_64(&isc->isc_tx_dropped_link_down.ev_count, delta); 5960 5961 delta = ixl_stat_delta(sc, 5962 I40E_GLPRT_UPTCH(sc->sc_port), I40E_GLPRT_UPTCL(sc->sc_port), 5963 &isc->isc_tx_unicast_offset, isc->isc_has_offset); 5964 atomic_add_64(&isc->isc_tx_unicast.ev_count, delta); 5965 5966 delta = ixl_stat_delta(sc, 5967 I40E_GLPRT_MPTCH(sc->sc_port), I40E_GLPRT_MPTCL(sc->sc_port), 5968 &isc->isc_tx_multicast_offset, isc->isc_has_offset); 5969 atomic_add_64(&isc->isc_tx_multicast.ev_count, delta); 5970 5971 delta = ixl_stat_delta(sc, 5972 I40E_GLPRT_BPTCH(sc->sc_port), I40E_GLPRT_BPTCL(sc->sc_port), 5973 &isc->isc_tx_broadcast_offset, isc->isc_has_offset); 5974 atomic_add_64(&isc->isc_tx_broadcast.ev_count, delta); 5975 5976 /* Packet size stats tx */ 5977 delta = ixl_stat_delta(sc, 5978 I40E_GLPRT_PTC64L(sc->sc_port), I40E_GLPRT_PTC64L(sc->sc_port), 5979 &isc->isc_tx_size_64_offset, isc->isc_has_offset); 5980 atomic_add_64(&isc->isc_tx_size_64.ev_count, delta); 5981 5982 delta = ixl_stat_delta(sc, 5983 I40E_GLPRT_PTC127H(sc->sc_port), I40E_GLPRT_PTC127L(sc->sc_port), 5984 &isc->isc_tx_size_127_offset, isc->isc_has_offset); 5985 atomic_add_64(&isc->isc_tx_size_127.ev_count, delta); 5986 5987 delta = ixl_stat_delta(sc, 5988 I40E_GLPRT_PTC255H(sc->sc_port), I40E_GLPRT_PTC255L(sc->sc_port), 5989 &isc->isc_tx_size_255_offset, isc->isc_has_offset); 5990 atomic_add_64(&isc->isc_tx_size_255.ev_count, delta); 5991 5992 delta = ixl_stat_delta(sc, 5993 I40E_GLPRT_PTC511H(sc->sc_port), I40E_GLPRT_PTC511L(sc->sc_port), 5994 &isc->isc_tx_size_511_offset, isc->isc_has_offset); 5995 atomic_add_64(&isc->isc_tx_size_511.ev_count, delta); 5996 5997 delta = ixl_stat_delta(sc, 5998 I40E_GLPRT_PTC1023H(sc->sc_port), I40E_GLPRT_PTC1023L(sc->sc_port), 5999 &isc->isc_tx_size_1023_offset, isc->isc_has_offset); 6000 atomic_add_64(&isc->isc_tx_size_1023.ev_count, delta); 6001 6002 delta = ixl_stat_delta(sc, 6003 I40E_GLPRT_PTC1522H(sc->sc_port), I40E_GLPRT_PTC1522L(sc->sc_port), 6004 &isc->isc_tx_size_1522_offset, isc->isc_has_offset); 6005 atomic_add_64(&isc->isc_tx_size_1522.ev_count, delta); 6006 6007 delta = ixl_stat_delta(sc, 6008 I40E_GLPRT_PTC9522H(sc->sc_port), I40E_GLPRT_PTC9522L(sc->sc_port), 6009 &isc->isc_tx_size_big_offset, isc->isc_has_offset); 6010 atomic_add_64(&isc->isc_tx_size_big.ev_count, delta); 6011 6012 /* mac faults */ 6013 delta = ixl_stat_delta(sc, 6014 0, I40E_GLPRT_MLFC(sc->sc_port), 6015 &isc->isc_mac_local_faults_offset, isc->isc_has_offset); 6016 atomic_add_64(&isc->isc_mac_local_faults.ev_count, delta); 6017 6018 delta = ixl_stat_delta(sc, 6019 0, I40E_GLPRT_MRFC(sc->sc_port), 6020 &isc->isc_mac_remote_faults_offset, isc->isc_has_offset); 6021 atomic_add_64(&isc->isc_mac_remote_faults.ev_count, delta); 6022 6023 /* Flow control (LFC) stats */ 6024 delta = ixl_stat_delta(sc, 6025 0, I40E_GLPRT_LXONRXC(sc->sc_port), 6026 &isc->isc_link_xon_rx_offset, isc->isc_has_offset); 6027 atomic_add_64(&isc->isc_link_xon_rx.ev_count, delta); 6028 6029 delta = ixl_stat_delta(sc, 6030 0, I40E_GLPRT_LXONTXC(sc->sc_port), 6031 &isc->isc_link_xon_tx_offset, isc->isc_has_offset); 6032 atomic_add_64(&isc->isc_link_xon_tx.ev_count, delta); 6033 6034 delta = ixl_stat_delta(sc, 6035 0, I40E_GLPRT_LXOFFRXC(sc->sc_port), 6036 &isc->isc_link_xoff_rx_offset, isc->isc_has_offset); 6037 atomic_add_64(&isc->isc_link_xoff_rx.ev_count, delta); 6038 6039 delta = ixl_stat_delta(sc, 6040 0, I40E_GLPRT_LXOFFTXC(sc->sc_port), 6041 &isc->isc_link_xoff_tx_offset, isc->isc_has_offset); 6042 atomic_add_64(&isc->isc_link_xoff_tx.ev_count, delta); 6043 6044 /* fragments */ 6045 delta = ixl_stat_delta(sc, 6046 0, I40E_GLPRT_RFC(sc->sc_port), 6047 &isc->isc_rx_fragments_offset, isc->isc_has_offset); 6048 atomic_add_64(&isc->isc_rx_fragments.ev_count, delta); 6049 6050 delta = ixl_stat_delta(sc, 6051 0, I40E_GLPRT_RJC(sc->sc_port), 6052 &isc->isc_rx_jabber_offset, isc->isc_has_offset); 6053 atomic_add_64(&isc->isc_rx_jabber.ev_count, delta); 6054 6055 /* VSI rx counters */ 6056 delta = ixl_stat_delta(sc, 6057 0, I40E_GLV_RDPC(sc->sc_vsi_stat_counter_idx), 6058 &isc->isc_vsi_rx_discards_offset, isc->isc_has_offset); 6059 atomic_add_64(&isc->isc_vsi_rx_discards.ev_count, delta); 6060 6061 delta = ixl_stat_delta(sc, 6062 I40E_GLV_GORCH(sc->sc_vsi_stat_counter_idx), 6063 I40E_GLV_GORCL(sc->sc_vsi_stat_counter_idx), 6064 &isc->isc_vsi_rx_bytes_offset, isc->isc_has_offset); 6065 atomic_add_64(&isc->isc_vsi_rx_bytes.ev_count, delta); 6066 6067 delta = ixl_stat_delta(sc, 6068 I40E_GLV_UPRCH(sc->sc_vsi_stat_counter_idx), 6069 I40E_GLV_UPRCL(sc->sc_vsi_stat_counter_idx), 6070 &isc->isc_vsi_rx_unicast_offset, isc->isc_has_offset); 6071 atomic_add_64(&isc->isc_vsi_rx_unicast.ev_count, delta); 6072 6073 delta = ixl_stat_delta(sc, 6074 I40E_GLV_MPRCH(sc->sc_vsi_stat_counter_idx), 6075 I40E_GLV_MPRCL(sc->sc_vsi_stat_counter_idx), 6076 &isc->isc_vsi_rx_multicast_offset, isc->isc_has_offset); 6077 atomic_add_64(&isc->isc_vsi_rx_multicast.ev_count, delta); 6078 6079 delta = ixl_stat_delta(sc, 6080 I40E_GLV_BPRCH(sc->sc_vsi_stat_counter_idx), 6081 I40E_GLV_BPRCL(sc->sc_vsi_stat_counter_idx), 6082 &isc->isc_vsi_rx_broadcast_offset, isc->isc_has_offset); 6083 atomic_add_64(&isc->isc_vsi_rx_broadcast.ev_count, delta); 6084 6085 /* VSI tx counters */ 6086 delta = ixl_stat_delta(sc, 6087 0, I40E_GLV_TEPC(sc->sc_vsi_stat_counter_idx), 6088 &isc->isc_vsi_tx_errors_offset, isc->isc_has_offset); 6089 atomic_add_64(&isc->isc_vsi_tx_errors.ev_count, delta); 6090 6091 delta = ixl_stat_delta(sc, 6092 I40E_GLV_GOTCH(sc->sc_vsi_stat_counter_idx), 6093 I40E_GLV_GOTCL(sc->sc_vsi_stat_counter_idx), 6094 &isc->isc_vsi_tx_bytes_offset, isc->isc_has_offset); 6095 atomic_add_64(&isc->isc_vsi_tx_bytes.ev_count, delta); 6096 6097 delta = ixl_stat_delta(sc, 6098 I40E_GLV_UPTCH(sc->sc_vsi_stat_counter_idx), 6099 I40E_GLV_UPTCL(sc->sc_vsi_stat_counter_idx), 6100 &isc->isc_vsi_tx_unicast_offset, isc->isc_has_offset); 6101 atomic_add_64(&isc->isc_vsi_tx_unicast.ev_count, delta); 6102 6103 delta = ixl_stat_delta(sc, 6104 I40E_GLV_MPTCH(sc->sc_vsi_stat_counter_idx), 6105 I40E_GLV_MPTCL(sc->sc_vsi_stat_counter_idx), 6106 &isc->isc_vsi_tx_multicast_offset, isc->isc_has_offset); 6107 atomic_add_64(&isc->isc_vsi_tx_multicast.ev_count, delta); 6108 6109 delta = ixl_stat_delta(sc, 6110 I40E_GLV_BPTCH(sc->sc_vsi_stat_counter_idx), 6111 I40E_GLV_BPTCL(sc->sc_vsi_stat_counter_idx), 6112 &isc->isc_vsi_tx_broadcast_offset, isc->isc_has_offset); 6113 atomic_add_64(&isc->isc_vsi_tx_broadcast.ev_count, delta); 6114 } 6115 6116 static int 6117 ixl_setup_sysctls(struct ixl_softc *sc) 6118 { 6119 const char *devname; 6120 struct sysctllog **log; 6121 const struct sysctlnode *rnode, *rxnode, *txnode; 6122 int error; 6123 6124 log = &sc->sc_sysctllog; 6125 devname = device_xname(sc->sc_dev); 6126 6127 error = sysctl_createv(log, 0, NULL, &rnode, 6128 0, CTLTYPE_NODE, devname, 6129 SYSCTL_DESCR("ixl information and settings"), 6130 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 6131 if (error) 6132 goto out; 6133 6134 error = sysctl_createv(log, 0, &rnode, NULL, 6135 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 6136 SYSCTL_DESCR("Use workqueue for packet processing"), 6137 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 6138 if (error) 6139 goto out; 6140 6141 error = sysctl_createv(log, 0, &rnode, NULL, 6142 CTLFLAG_READONLY, CTLTYPE_INT, "stats_interval", 6143 SYSCTL_DESCR("Statistics collection interval in milliseconds"), 6144 NULL, 0, &sc->sc_stats_intval, 0, CTL_CREATE, CTL_EOL); 6145 6146 error = sysctl_createv(log, 0, &rnode, &rxnode, 6147 0, CTLTYPE_NODE, "rx", 6148 SYSCTL_DESCR("ixl information and settings for Rx"), 6149 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6150 if (error) 6151 goto out; 6152 6153 error = sysctl_createv(log, 0, &rxnode, NULL, 6154 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6155 SYSCTL_DESCR("max number of Rx packets" 6156 " to process for interrupt processing"), 6157 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6158 if (error) 6159 goto out; 6160 6161 error = sysctl_createv(log, 0, &rxnode, NULL, 6162 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6163 SYSCTL_DESCR("max number of Rx packets" 6164 " to process for deferred processing"), 6165 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 6166 if (error) 6167 goto out; 6168 6169 error = sysctl_createv(log, 0, &rnode, &txnode, 6170 0, CTLTYPE_NODE, "tx", 6171 SYSCTL_DESCR("ixl information and settings for Tx"), 6172 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6173 if (error) 6174 goto out; 6175 6176 error = sysctl_createv(log, 0, &txnode, NULL, 6177 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6178 SYSCTL_DESCR("max number of Tx packets" 6179 " to process for interrupt processing"), 6180 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6181 if (error) 6182 goto out; 6183 6184 error = sysctl_createv(log, 0, &txnode, NULL, 6185 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6186 SYSCTL_DESCR("max number of Tx packets" 6187 " to process for deferred processing"), 6188 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 6189 if (error) 6190 goto out; 6191 6192 out: 6193 if (error) { 6194 aprint_error_dev(sc->sc_dev, 6195 "unable to create sysctl node\n"); 6196 sysctl_teardown(log); 6197 } 6198 6199 return error; 6200 } 6201 6202 static void 6203 ixl_teardown_sysctls(struct ixl_softc *sc) 6204 { 6205 6206 sysctl_teardown(&sc->sc_sysctllog); 6207 } 6208 6209 static struct workqueue * 6210 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 6211 { 6212 struct workqueue *wq; 6213 int error; 6214 6215 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 6216 prio, ipl, flags); 6217 6218 if (error) 6219 return NULL; 6220 6221 return wq; 6222 } 6223 6224 static void 6225 ixl_workq_destroy(struct workqueue *wq) 6226 { 6227 6228 workqueue_destroy(wq); 6229 } 6230 6231 static void 6232 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 6233 { 6234 6235 memset(work, 0, sizeof(*work)); 6236 work->ixw_func = func; 6237 work->ixw_arg = arg; 6238 } 6239 6240 static void 6241 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 6242 { 6243 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 6244 return; 6245 6246 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 6247 } 6248 6249 static void 6250 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 6251 { 6252 6253 workqueue_wait(wq, &work->ixw_cookie); 6254 } 6255 6256 static void 6257 ixl_workq_work(struct work *wk, void *context) 6258 { 6259 struct ixl_work *work; 6260 6261 work = container_of(wk, struct ixl_work, ixw_cookie); 6262 6263 atomic_swap_uint(&work->ixw_added, 0); 6264 work->ixw_func(work->ixw_arg); 6265 } 6266 6267 static int 6268 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 6269 { 6270 struct ixl_aq_desc iaq; 6271 6272 memset(&iaq, 0, sizeof(iaq)); 6273 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 6274 iaq.iaq_param[1] = htole32(reg); 6275 6276 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6277 return ETIMEDOUT; 6278 6279 switch (htole16(iaq.iaq_retval)) { 6280 case IXL_AQ_RC_OK: 6281 /* success */ 6282 break; 6283 case IXL_AQ_RC_EACCES: 6284 return EPERM; 6285 case IXL_AQ_RC_EAGAIN: 6286 return EAGAIN; 6287 default: 6288 return EIO; 6289 } 6290 6291 *rv = htole32(iaq.iaq_param[3]); 6292 return 0; 6293 } 6294 6295 static uint32_t 6296 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 6297 { 6298 uint32_t val; 6299 int rv, retry, retry_limit; 6300 6301 retry_limit = sc->sc_rxctl_atq ? 5 : 0; 6302 6303 for (retry = 0; retry < retry_limit; retry++) { 6304 rv = ixl_rx_ctl_read(sc, reg, &val); 6305 if (rv == 0) 6306 return val; 6307 else if (rv == EAGAIN) 6308 delaymsec(1); 6309 else 6310 break; 6311 } 6312 6313 val = ixl_rd(sc, reg); 6314 6315 return val; 6316 } 6317 6318 static int 6319 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6320 { 6321 struct ixl_aq_desc iaq; 6322 6323 memset(&iaq, 0, sizeof(iaq)); 6324 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 6325 iaq.iaq_param[1] = htole32(reg); 6326 iaq.iaq_param[3] = htole32(value); 6327 6328 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6329 return ETIMEDOUT; 6330 6331 switch (htole16(iaq.iaq_retval)) { 6332 case IXL_AQ_RC_OK: 6333 /* success */ 6334 break; 6335 case IXL_AQ_RC_EACCES: 6336 return EPERM; 6337 case IXL_AQ_RC_EAGAIN: 6338 return EAGAIN; 6339 default: 6340 return EIO; 6341 } 6342 6343 return 0; 6344 } 6345 6346 static void 6347 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6348 { 6349 int rv, retry, retry_limit; 6350 6351 retry_limit = sc->sc_rxctl_atq ? 5 : 0; 6352 6353 for (retry = 0; retry < retry_limit; retry++) { 6354 rv = ixl_rx_ctl_write(sc, reg, value); 6355 if (rv == 0) 6356 return; 6357 else if (rv == EAGAIN) 6358 delaymsec(1); 6359 else 6360 break; 6361 } 6362 6363 ixl_wr(sc, reg, value); 6364 } 6365 6366 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 6367 6368 #ifdef _MODULE 6369 #include "ioconf.c" 6370 #endif 6371 6372 #ifdef _MODULE 6373 static void 6374 ixl_parse_modprop(prop_dictionary_t dict) 6375 { 6376 prop_object_t obj; 6377 int64_t val; 6378 uint64_t uval; 6379 6380 if (dict == NULL) 6381 return; 6382 6383 obj = prop_dictionary_get(dict, "nomsix"); 6384 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_BOOL) { 6385 ixl_param_nomsix = prop_bool_true((prop_bool_t)obj); 6386 } 6387 6388 obj = prop_dictionary_get(dict, "stats_interval"); 6389 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 6390 val = prop_number_integer_value((prop_number_t)obj); 6391 6392 /* the range has no reason */ 6393 if (100 < val || val < 180000) { 6394 ixl_param_stats_interval = val; 6395 } 6396 } 6397 6398 obj = prop_dictionary_get(dict, "nqps_limit"); 6399 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 6400 val = prop_number_integer_value((prop_number_t)obj); 6401 6402 if (val <= INT32_MAX) 6403 ixl_param_nqps_limit = val; 6404 } 6405 6406 obj = prop_dictionary_get(dict, "rx_ndescs"); 6407 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 6408 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 6409 6410 if (uval > 8) 6411 ixl_param_rx_ndescs = uval; 6412 } 6413 6414 obj = prop_dictionary_get(dict, "tx_ndescs"); 6415 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 6416 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 6417 6418 if (uval > IXL_TX_PKT_DESCS) 6419 ixl_param_tx_ndescs = uval; 6420 } 6421 6422 } 6423 #endif 6424 6425 static int 6426 if_ixl_modcmd(modcmd_t cmd, void *opaque) 6427 { 6428 int error = 0; 6429 6430 #ifdef _MODULE 6431 switch (cmd) { 6432 case MODULE_CMD_INIT: 6433 ixl_parse_modprop((prop_dictionary_t)opaque); 6434 error = config_init_component(cfdriver_ioconf_if_ixl, 6435 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 6436 break; 6437 case MODULE_CMD_FINI: 6438 error = config_fini_component(cfdriver_ioconf_if_ixl, 6439 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 6440 break; 6441 default: 6442 error = ENOTTY; 6443 break; 6444 } 6445 #endif 6446 6447 return error; 6448 } 6449
Indexes created Sat Sep 20 22:09:52 GMT 2025