if_ixl.c revision 1.17
1 /* $NetBSD: if_ixl.c,v 1.17 2020/01/09 02:43:45 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 78 #ifdef _KERNEL_OPT 79 #include "opt_net_mpsafe.h" 80 #endif 81 82 #include <sys/param.h> 83 #include <sys/types.h> 84 85 #include <sys/cpu.h> 86 #include <sys/device.h> 87 #include <sys/evcnt.h> 88 #include <sys/interrupt.h> 89 #include <sys/kmem.h> 90 #include <sys/malloc.h> 91 #include <sys/module.h> 92 #include <sys/mutex.h> 93 #include <sys/pcq.h> 94 #include <sys/syslog.h> 95 #include <sys/workqueue.h> 96 97 #include <sys/bus.h> 98 99 #include <net/bpf.h> 100 #include <net/if.h> 101 #include <net/if_dl.h> 102 #include <net/if_media.h> 103 #include <net/if_ether.h> 104 #include <net/rss_config.h> 105 106 #include <dev/pci/pcivar.h> 107 #include <dev/pci/pcidevs.h> 108 109 #include <dev/pci/if_ixlreg.h> 110 #include <dev/pci/if_ixlvar.h> 111 112 struct ixl_softc; /* defined */ 113 114 #define I40E_PF_RESET_WAIT_COUNT 200 115 #define I40E_AQ_LARGE_BUF 512 116 117 /* bitfields for Tx queue mapping in QTX_CTL */ 118 #define I40E_QTX_CTL_VF_QUEUE 0x0 119 #define I40E_QTX_CTL_VM_QUEUE 0x1 120 #define I40E_QTX_CTL_PF_QUEUE 0x2 121 122 #define I40E_QUEUE_TYPE_EOL 0x7ff 123 #define I40E_INTR_NOTX_QUEUE 0 124 125 #define I40E_QUEUE_TYPE_RX 0x0 126 #define I40E_QUEUE_TYPE_TX 0x1 127 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 128 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 129 130 #define I40E_ITR_INDEX_RX 0x0 131 #define I40E_ITR_INDEX_TX 0x1 132 #define I40E_ITR_INDEX_OTHER 0x2 133 #define I40E_ITR_INDEX_NONE 0x3 134 135 #define I40E_INTR_NOTX_QUEUE 0 136 #define I40E_INTR_NOTX_INTR 0 137 #define I40E_INTR_NOTX_RX_QUEUE 0 138 #define I40E_INTR_NOTX_TX_QUEUE 1 139 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 140 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 141 142 #define BIT_ULL(a) (1ULL << (a)) 143 #define IXL_RSS_HENA_DEFAULT_BASE \ 144 (BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 145 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 146 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 147 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 148 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 149 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 150 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 151 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 152 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 153 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 154 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 155 #define IXL_RSS_HENA_DEFAULT_XL710 IXL_RSS_HENA_DEFAULT_BASE 156 #define IXL_RSS_HENA_DEFAULT_X722 (IXL_RSS_HENA_DEFAULT_XL710 | \ 157 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 158 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 159 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 160 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 161 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 162 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 163 #define I40E_HASH_LUT_SIZE_128 0 164 #define IXL_RSS_KEY_SIZE_REG 13 165 166 #define IXL_ICR0_CRIT_ERR_MASK \ 167 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 168 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 169 I40E_PFINT_ICR0_PE_CRITERR_MASK) 170 171 #define IXL_TX_PKT_DESCS 8 172 #define IXL_TX_QUEUE_ALIGN 128 173 #define IXL_RX_QUEUE_ALIGN 128 174 175 #define IXL_HARDMTU 9712 /* 9726 - ETHER_HDR_LEN */ 176 177 #define IXL_PCIREG PCI_MAPREG_START 178 179 #define IXL_ITR0 0x0 180 #define IXL_ITR1 0x1 181 #define IXL_ITR2 0x2 182 #define IXL_NOITR 0x3 183 184 #define IXL_AQ_NUM 256 185 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 186 #define IXL_AQ_ALIGN 64 /* lol */ 187 #define IXL_AQ_BUFLEN 4096 188 189 #define IXL_HMC_ROUNDUP 512 190 #define IXL_HMC_PGSIZE 4096 191 #define IXL_HMC_DVASZ sizeof(uint64_t) 192 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 193 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 194 #define IXL_HMC_PDVALID 1ULL 195 196 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 197 198 struct ixl_aq_regs { 199 bus_size_t atq_tail; 200 bus_size_t atq_head; 201 bus_size_t atq_len; 202 bus_size_t atq_bal; 203 bus_size_t atq_bah; 204 205 bus_size_t arq_tail; 206 bus_size_t arq_head; 207 bus_size_t arq_len; 208 bus_size_t arq_bal; 209 bus_size_t arq_bah; 210 211 uint32_t atq_len_enable; 212 uint32_t atq_tail_mask; 213 uint32_t atq_head_mask; 214 215 uint32_t arq_len_enable; 216 uint32_t arq_tail_mask; 217 uint32_t arq_head_mask; 218 }; 219 220 struct ixl_phy_type { 221 uint64_t phy_type; 222 uint64_t ifm_type; 223 }; 224 225 struct ixl_speed_type { 226 uint8_t dev_speed; 227 uint64_t net_speed; 228 }; 229 230 struct ixl_aq_buf { 231 SIMPLEQ_ENTRY(ixl_aq_buf) 232 aqb_entry; 233 void *aqb_data; 234 bus_dmamap_t aqb_map; 235 bus_dma_segment_t aqb_seg; 236 size_t aqb_size; 237 int aqb_nsegs; 238 }; 239 SIMPLEQ_HEAD(ixl_aq_bufs, ixl_aq_buf); 240 241 struct ixl_dmamem { 242 bus_dmamap_t ixm_map; 243 bus_dma_segment_t ixm_seg; 244 int ixm_nsegs; 245 size_t ixm_size; 246 void *ixm_kva; 247 }; 248 249 #define IXL_DMA_MAP(_ixm) ((_ixm)->ixm_map) 250 #define IXL_DMA_DVA(_ixm) ((_ixm)->ixm_map->dm_segs[0].ds_addr) 251 #define IXL_DMA_KVA(_ixm) ((void *)(_ixm)->ixm_kva) 252 #define IXL_DMA_LEN(_ixm) ((_ixm)->ixm_size) 253 254 struct ixl_hmc_entry { 255 uint64_t hmc_base; 256 uint32_t hmc_count; 257 uint64_t hmc_size; 258 }; 259 260 enum ixl_hmc_types { 261 IXL_HMC_LAN_TX = 0, 262 IXL_HMC_LAN_RX, 263 IXL_HMC_FCOE_CTX, 264 IXL_HMC_FCOE_FILTER, 265 IXL_HMC_COUNT 266 }; 267 268 struct ixl_hmc_pack { 269 uint16_t offset; 270 uint16_t width; 271 uint16_t lsb; 272 }; 273 274 /* 275 * these hmc objects have weird sizes and alignments, so these are abstract 276 * representations of them that are nice for c to populate. 277 * 278 * the packing code relies on little-endian values being stored in the fields, 279 * no high bits in the fields being set, and the fields must be packed in the 280 * same order as they are in the ctx structure. 281 */ 282 283 struct ixl_hmc_rxq { 284 uint16_t head; 285 uint8_t cpuid; 286 uint64_t base; 287 #define IXL_HMC_RXQ_BASE_UNIT 128 288 uint16_t qlen; 289 uint16_t dbuff; 290 #define IXL_HMC_RXQ_DBUFF_UNIT 128 291 uint8_t hbuff; 292 #define IXL_HMC_RXQ_HBUFF_UNIT 64 293 uint8_t dtype; 294 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 295 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 296 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 297 uint8_t dsize; 298 #define IXL_HMC_RXQ_DSIZE_16 0 299 #define IXL_HMC_RXQ_DSIZE_32 1 300 uint8_t crcstrip; 301 uint8_t fc_ena; 302 uint8_t l2sel; 303 uint8_t hsplit_0; 304 uint8_t hsplit_1; 305 uint8_t showiv; 306 uint16_t rxmax; 307 uint8_t tphrdesc_ena; 308 uint8_t tphwdesc_ena; 309 uint8_t tphdata_ena; 310 uint8_t tphhead_ena; 311 uint8_t lrxqthresh; 312 uint8_t prefena; 313 }; 314 315 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 316 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 317 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 318 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 319 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 320 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 321 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 322 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 323 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 324 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 325 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 326 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 327 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 328 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 329 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 330 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 331 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 332 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 333 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 334 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 335 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 336 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 337 }; 338 339 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 340 341 struct ixl_hmc_txq { 342 uint16_t head; 343 uint8_t new_context; 344 uint64_t base; 345 #define IXL_HMC_TXQ_BASE_UNIT 128 346 uint8_t fc_ena; 347 uint8_t timesync_ena; 348 uint8_t fd_ena; 349 uint8_t alt_vlan_ena; 350 uint16_t thead_wb; 351 uint8_t cpuid; 352 uint8_t head_wb_ena; 353 #define IXL_HMC_TXQ_DESC_WB 0 354 #define IXL_HMC_TXQ_HEAD_WB 1 355 uint16_t qlen; 356 uint8_t tphrdesc_ena; 357 uint8_t tphrpacket_ena; 358 uint8_t tphwdesc_ena; 359 uint64_t head_wb_addr; 360 uint32_t crc; 361 uint16_t rdylist; 362 uint8_t rdylist_act; 363 }; 364 365 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 366 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 367 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 368 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 369 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 370 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 371 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 372 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 373 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 374 /* line 1 */ 375 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 376 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 377 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 378 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 379 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 380 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 381 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 382 /* line 7 */ 383 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 384 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 385 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 386 }; 387 388 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 389 390 struct ixl_work { 391 struct work ixw_cookie; 392 void (*ixw_func)(void *); 393 void *ixw_arg; 394 unsigned int ixw_added; 395 }; 396 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 397 398 struct ixl_tx_map { 399 struct mbuf *txm_m; 400 bus_dmamap_t txm_map; 401 unsigned int txm_eop; 402 }; 403 404 struct ixl_tx_ring { 405 kmutex_t txr_lock; 406 struct ixl_softc *txr_sc; 407 408 unsigned int txr_prod; 409 unsigned int txr_cons; 410 411 struct ixl_tx_map *txr_maps; 412 struct ixl_dmamem txr_mem; 413 414 bus_size_t txr_tail; 415 unsigned int txr_qid; 416 pcq_t *txr_intrq; 417 void *txr_si; 418 419 uint64_t txr_oerrors; /* if_oerrors */ 420 uint64_t txr_opackets; /* if_opackets */ 421 uint64_t txr_obytes; /* if_obytes */ 422 uint64_t txr_omcasts; /* if_omcasts */ 423 424 struct evcnt txr_defragged; 425 struct evcnt txr_defrag_failed; 426 struct evcnt txr_pcqdrop; 427 struct evcnt txr_transmitdef; 428 struct evcnt txr_intr; 429 struct evcnt txr_defer; 430 }; 431 432 struct ixl_rx_map { 433 struct mbuf *rxm_m; 434 bus_dmamap_t rxm_map; 435 }; 436 437 struct ixl_rx_ring { 438 kmutex_t rxr_lock; 439 440 unsigned int rxr_prod; 441 unsigned int rxr_cons; 442 443 struct ixl_rx_map *rxr_maps; 444 struct ixl_dmamem rxr_mem; 445 446 struct mbuf *rxr_m_head; 447 struct mbuf **rxr_m_tail; 448 449 bus_size_t rxr_tail; 450 unsigned int rxr_qid; 451 452 uint64_t rxr_ipackets; /* if_ipackets */ 453 uint64_t rxr_ibytes; /* if_ibytes */ 454 uint64_t rxr_iqdrops; /* iqdrops */ 455 uint64_t rxr_ierrors; /* if_ierrors */ 456 457 struct evcnt rxr_mgethdr_failed; 458 struct evcnt rxr_mgetcl_failed; 459 struct evcnt rxr_mbuf_load_failed; 460 struct evcnt rxr_intr; 461 struct evcnt rxr_defer; 462 }; 463 464 struct ixl_queue_pair { 465 struct ixl_softc *qp_sc; 466 struct ixl_tx_ring *qp_txr; 467 struct ixl_rx_ring *qp_rxr; 468 469 char qp_name[16]; 470 471 void *qp_si; 472 struct ixl_work qp_task; 473 bool qp_workqueue; 474 }; 475 476 struct ixl_atq { 477 struct ixl_aq_desc iatq_desc; 478 void (*iatq_fn)(struct ixl_softc *, 479 const struct ixl_aq_desc *); 480 }; 481 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 482 483 struct ixl_product { 484 unsigned int vendor_id; 485 unsigned int product_id; 486 }; 487 488 struct ixl_stats_counters { 489 bool isc_has_offset; 490 struct evcnt isc_crc_errors; 491 uint64_t isc_crc_errors_offset; 492 struct evcnt isc_illegal_bytes; 493 uint64_t isc_illegal_bytes_offset; 494 struct evcnt isc_rx_bytes; 495 uint64_t isc_rx_bytes_offset; 496 struct evcnt isc_rx_discards; 497 uint64_t isc_rx_discards_offset; 498 struct evcnt isc_rx_unicast; 499 uint64_t isc_rx_unicast_offset; 500 struct evcnt isc_rx_multicast; 501 uint64_t isc_rx_multicast_offset; 502 struct evcnt isc_rx_broadcast; 503 uint64_t isc_rx_broadcast_offset; 504 struct evcnt isc_rx_size_64; 505 uint64_t isc_rx_size_64_offset; 506 struct evcnt isc_rx_size_127; 507 uint64_t isc_rx_size_127_offset; 508 struct evcnt isc_rx_size_255; 509 uint64_t isc_rx_size_255_offset; 510 struct evcnt isc_rx_size_511; 511 uint64_t isc_rx_size_511_offset; 512 struct evcnt isc_rx_size_1023; 513 uint64_t isc_rx_size_1023_offset; 514 struct evcnt isc_rx_size_1522; 515 uint64_t isc_rx_size_1522_offset; 516 struct evcnt isc_rx_size_big; 517 uint64_t isc_rx_size_big_offset; 518 struct evcnt isc_rx_undersize; 519 uint64_t isc_rx_undersize_offset; 520 struct evcnt isc_rx_oversize; 521 uint64_t isc_rx_oversize_offset; 522 struct evcnt isc_rx_fragments; 523 uint64_t isc_rx_fragments_offset; 524 struct evcnt isc_rx_jabber; 525 uint64_t isc_rx_jabber_offset; 526 struct evcnt isc_tx_bytes; 527 uint64_t isc_tx_bytes_offset; 528 struct evcnt isc_tx_dropped_link_down; 529 uint64_t isc_tx_dropped_link_down_offset; 530 struct evcnt isc_tx_unicast; 531 uint64_t isc_tx_unicast_offset; 532 struct evcnt isc_tx_multicast; 533 uint64_t isc_tx_multicast_offset; 534 struct evcnt isc_tx_broadcast; 535 uint64_t isc_tx_broadcast_offset; 536 struct evcnt isc_tx_size_64; 537 uint64_t isc_tx_size_64_offset; 538 struct evcnt isc_tx_size_127; 539 uint64_t isc_tx_size_127_offset; 540 struct evcnt isc_tx_size_255; 541 uint64_t isc_tx_size_255_offset; 542 struct evcnt isc_tx_size_511; 543 uint64_t isc_tx_size_511_offset; 544 struct evcnt isc_tx_size_1023; 545 uint64_t isc_tx_size_1023_offset; 546 struct evcnt isc_tx_size_1522; 547 uint64_t isc_tx_size_1522_offset; 548 struct evcnt isc_tx_size_big; 549 uint64_t isc_tx_size_big_offset; 550 struct evcnt isc_mac_local_faults; 551 uint64_t isc_mac_local_faults_offset; 552 struct evcnt isc_mac_remote_faults; 553 uint64_t isc_mac_remote_faults_offset; 554 struct evcnt isc_link_xon_rx; 555 uint64_t isc_link_xon_rx_offset; 556 struct evcnt isc_link_xon_tx; 557 uint64_t isc_link_xon_tx_offset; 558 struct evcnt isc_link_xoff_rx; 559 uint64_t isc_link_xoff_rx_offset; 560 struct evcnt isc_link_xoff_tx; 561 uint64_t isc_link_xoff_tx_offset; 562 }; 563 564 /* 565 * Locking notes: 566 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 567 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 568 * - more than one lock of them cannot be held at once. 569 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 570 * (a spin mutex). 571 * - the lock cannot held with txr_lock or rxr_lock. 572 * + a field named sc_arq_* is not protected by any lock. 573 * - operations for sc_arq_* is done in one context related to 574 * sc_arq_task. 575 * + other fields in ixl_softc is protected by sc_cfg_lock 576 * (an adaptive mutex) 577 * - It must be held before another lock is held, and It can be 578 * released after the other lock is released. 579 * */ 580 581 struct ixl_softc { 582 device_t sc_dev; 583 struct ethercom sc_ec; 584 bool sc_attached; 585 bool sc_dead; 586 bool sc_rxctl_atq; 587 uint32_t sc_port; 588 struct sysctllog *sc_sysctllog; 589 struct workqueue *sc_workq; 590 struct workqueue *sc_workq_txrx; 591 int sc_stats_intval; 592 callout_t sc_stats_callout; 593 struct ixl_work sc_stats_task; 594 struct ixl_stats_counters 595 sc_stats_counters; 596 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 597 struct ifmedia sc_media; 598 uint64_t sc_media_status; 599 uint64_t sc_media_active; 600 kmutex_t sc_cfg_lock; 601 enum i40e_mac_type sc_mac_type; 602 uint32_t sc_rss_table_size; 603 uint32_t sc_rss_table_entry_width; 604 bool sc_txrx_workqueue; 605 u_int sc_tx_process_limit; 606 u_int sc_rx_process_limit; 607 u_int sc_tx_intr_process_limit; 608 u_int sc_rx_intr_process_limit; 609 610 int sc_cur_ec_capenable; 611 612 struct pci_attach_args sc_pa; 613 pci_intr_handle_t *sc_ihp; 614 void **sc_ihs; 615 unsigned int sc_nintrs; 616 617 bus_dma_tag_t sc_dmat; 618 bus_space_tag_t sc_memt; 619 bus_space_handle_t sc_memh; 620 bus_size_t sc_mems; 621 622 uint8_t sc_pf_id; 623 uint16_t sc_uplink_seid; /* le */ 624 uint16_t sc_downlink_seid; /* le */ 625 uint16_t sc_vsi_number; /* le */ 626 uint16_t sc_seid; 627 unsigned int sc_base_queue; 628 629 pci_intr_type_t sc_intrtype; 630 unsigned int sc_msix_vector_queue; 631 632 struct ixl_dmamem sc_scratch; 633 634 const struct ixl_aq_regs * 635 sc_aq_regs; 636 637 kmutex_t sc_atq_lock; 638 kcondvar_t sc_atq_cv; 639 struct ixl_dmamem sc_atq; 640 unsigned int sc_atq_prod; 641 unsigned int sc_atq_cons; 642 643 struct ixl_dmamem sc_arq; 644 struct ixl_work sc_arq_task; 645 struct ixl_aq_bufs sc_arq_idle; 646 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 647 unsigned int sc_arq_prod; 648 unsigned int sc_arq_cons; 649 650 struct ixl_work sc_link_state_task; 651 struct ixl_atq sc_link_state_atq; 652 653 struct ixl_dmamem sc_hmc_sd; 654 struct ixl_dmamem sc_hmc_pd; 655 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 656 657 unsigned int sc_tx_ring_ndescs; 658 unsigned int sc_rx_ring_ndescs; 659 unsigned int sc_nqueue_pairs; 660 unsigned int sc_nqueue_pairs_max; 661 unsigned int sc_nqueue_pairs_device; 662 struct ixl_queue_pair *sc_qps; 663 664 struct evcnt sc_event_atq; 665 struct evcnt sc_event_link; 666 struct evcnt sc_event_ecc_err; 667 struct evcnt sc_event_pci_exception; 668 struct evcnt sc_event_crit_err; 669 }; 670 671 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 672 #define IXL_TX_PROCESS_LIMIT 256 673 #define IXL_RX_PROCESS_LIMIT 256 674 #define IXL_TX_INTR_PROCESS_LIMIT 256 675 #define IXL_RX_INTR_PROCESS_LIMIT 0U 676 677 #define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx| \ 678 IFCAP_CSUM_TCPv4_Rx| \ 679 IFCAP_CSUM_UDPv4_Rx| \ 680 IFCAP_CSUM_TCPv6_Rx| \ 681 IFCAP_CSUM_UDPv6_Rx) 682 683 #define delaymsec(_x) DELAY(1000 * (_x)) 684 #ifdef IXL_DEBUG 685 #define DDPRINTF(sc, fmt, args...) \ 686 do { \ 687 if ((sc) != NULL) { \ 688 device_printf( \ 689 ((struct ixl_softc *)(sc))->sc_dev, \ 690 ""); \ 691 } \ 692 printf("%s:\t" fmt, __func__, ##args); \ 693 } while (0) 694 #else 695 #define DDPRINTF(sc, fmt, args...) __nothing 696 #endif 697 #define IXL_NOMSIX false 698 #ifndef IXL_STATS_INTERVAL_MSEC 699 #define IXL_STATS_INTERVAL_MSEC 10000 700 #endif 701 702 static enum i40e_mac_type 703 ixl_mactype(pci_product_id_t); 704 static void ixl_clear_hw(struct ixl_softc *); 705 static int ixl_pf_reset(struct ixl_softc *); 706 707 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 708 bus_size_t, bus_size_t); 709 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 710 711 static int ixl_arq_fill(struct ixl_softc *); 712 static void ixl_arq_unfill(struct ixl_softc *); 713 714 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 715 unsigned int); 716 static void ixl_atq_set(struct ixl_atq *, 717 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 718 static int ixl_atq_post(struct ixl_softc *, struct ixl_atq *); 719 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 720 static void ixl_atq_done(struct ixl_softc *); 721 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 722 static int ixl_get_version(struct ixl_softc *); 723 static int ixl_get_hw_capabilities(struct ixl_softc *); 724 static int ixl_pxe_clear(struct ixl_softc *); 725 static int ixl_lldp_shut(struct ixl_softc *); 726 static int ixl_get_mac(struct ixl_softc *); 727 static int ixl_get_switch_config(struct ixl_softc *); 728 static int ixl_phy_mask_ints(struct ixl_softc *); 729 static int ixl_get_phy_types(struct ixl_softc *, uint64_t *); 730 static int ixl_restart_an(struct ixl_softc *); 731 static int ixl_hmc(struct ixl_softc *); 732 static void ixl_hmc_free(struct ixl_softc *); 733 static int ixl_get_vsi(struct ixl_softc *); 734 static int ixl_set_vsi(struct ixl_softc *); 735 static void ixl_set_filter_control(struct ixl_softc *); 736 static void ixl_get_link_status(void *); 737 static int ixl_get_link_status_poll(struct ixl_softc *); 738 static int ixl_set_link_status(struct ixl_softc *, 739 const struct ixl_aq_desc *); 740 static void ixl_config_rss(struct ixl_softc *); 741 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 742 uint16_t, uint16_t); 743 static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *, 744 uint16_t, uint16_t); 745 static void ixl_arq(void *); 746 static void ixl_hmc_pack(void *, const void *, 747 const struct ixl_hmc_pack *, unsigned int); 748 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 749 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 750 751 static int ixl_match(device_t, cfdata_t, void *); 752 static void ixl_attach(device_t, device_t, void *); 753 static int ixl_detach(device_t, int); 754 755 static void ixl_media_add(struct ixl_softc *, uint64_t); 756 static int ixl_media_change(struct ifnet *); 757 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 758 static void ixl_watchdog(struct ifnet *); 759 static int ixl_ioctl(struct ifnet *, u_long, void *); 760 static void ixl_start(struct ifnet *); 761 static int ixl_transmit(struct ifnet *, struct mbuf *); 762 static void ixl_deferred_transmit(void *); 763 static int ixl_intr(void *); 764 static int ixl_queue_intr(void *); 765 static int ixl_other_intr(void *); 766 static void ixl_handle_queue(void *); 767 static void ixl_sched_handle_queue(struct ixl_softc *, 768 struct ixl_queue_pair *); 769 static int ixl_init(struct ifnet *); 770 static int ixl_init_locked(struct ixl_softc *); 771 static void ixl_stop(struct ifnet *, int); 772 static void ixl_stop_locked(struct ixl_softc *); 773 static int ixl_iff(struct ixl_softc *); 774 static int ixl_ifflags_cb(struct ethercom *); 775 static int ixl_setup_interrupts(struct ixl_softc *); 776 static int ixl_establish_intx(struct ixl_softc *); 777 static int ixl_establish_msix(struct ixl_softc *); 778 static void ixl_set_affinity_msix(struct ixl_softc *); 779 static void ixl_enable_queue_intr(struct ixl_softc *, 780 struct ixl_queue_pair *); 781 static void ixl_disable_queue_intr(struct ixl_softc *, 782 struct ixl_queue_pair *); 783 static void ixl_enable_other_intr(struct ixl_softc *); 784 static void ixl_disable_other_intr(struct ixl_softc *); 785 static void ixl_config_queue_intr(struct ixl_softc *); 786 static void ixl_config_other_intr(struct ixl_softc *); 787 788 static struct ixl_tx_ring * 789 ixl_txr_alloc(struct ixl_softc *, unsigned int); 790 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 791 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 792 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 793 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 794 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 795 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 796 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 797 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 798 799 static struct ixl_rx_ring * 800 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 801 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 802 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 803 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 804 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 805 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 806 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 807 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 808 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 809 810 static struct workqueue * 811 ixl_workq_create(const char *, pri_t, int, int); 812 static void ixl_workq_destroy(struct workqueue *); 813 static int ixl_workqs_teardown(device_t); 814 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 815 static void ixl_work_add(struct workqueue *, struct ixl_work *); 816 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 817 static void ixl_workq_work(struct work *, void *); 818 static const struct ixl_product * 819 ixl_lookup(const struct pci_attach_args *pa); 820 static void ixl_link_state_update(struct ixl_softc *, 821 const struct ixl_aq_desc *); 822 static int ixl_vlan_cb(struct ethercom *, uint16_t, bool); 823 static int ixl_setup_vlan_hwfilter(struct ixl_softc *); 824 static void ixl_teardown_vlan_hwfilter(struct ixl_softc *); 825 static int ixl_update_macvlan(struct ixl_softc *); 826 static int ixl_setup_interrupts(struct ixl_softc *);; 827 static void ixl_teardown_interrupts(struct ixl_softc *); 828 static int ixl_setup_stats(struct ixl_softc *); 829 static void ixl_teardown_stats(struct ixl_softc *); 830 static void ixl_stats_callout(void *); 831 static void ixl_stats_update(void *); 832 static int ixl_setup_sysctls(struct ixl_softc *); 833 static void ixl_teardown_sysctls(struct ixl_softc *); 834 static int ixl_queue_pairs_alloc(struct ixl_softc *); 835 static void ixl_queue_pairs_free(struct ixl_softc *); 836 837 static const struct ixl_phy_type ixl_phy_type_map[] = { 838 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 839 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 840 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 841 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 842 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 843 { 1ULL << IXL_PHY_TYPE_XAUI | 844 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 845 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 846 { 1ULL << IXL_PHY_TYPE_XLAUI | 847 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 848 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 849 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 850 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 851 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 852 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 853 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 854 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 855 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 856 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 857 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 858 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 859 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 860 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 861 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 862 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 863 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 864 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 865 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 866 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 867 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 868 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 869 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 870 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 871 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_CR }, 872 }; 873 874 static const struct ixl_speed_type ixl_speed_type_map[] = { 875 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 876 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 877 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 878 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 879 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 880 }; 881 882 static const struct ixl_aq_regs ixl_pf_aq_regs = { 883 .atq_tail = I40E_PF_ATQT, 884 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 885 .atq_head = I40E_PF_ATQH, 886 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 887 .atq_len = I40E_PF_ATQLEN, 888 .atq_bal = I40E_PF_ATQBAL, 889 .atq_bah = I40E_PF_ATQBAH, 890 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 891 892 .arq_tail = I40E_PF_ARQT, 893 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 894 .arq_head = I40E_PF_ARQH, 895 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 896 .arq_len = I40E_PF_ARQLEN, 897 .arq_bal = I40E_PF_ARQBAL, 898 .arq_bah = I40E_PF_ARQBAH, 899 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 900 }; 901 902 #define ixl_rd(_s, _r) \ 903 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 904 #define ixl_wr(_s, _r, _v) \ 905 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 906 #define ixl_barrier(_s, _r, _l, _o) \ 907 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 908 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 909 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 910 911 static inline uint32_t 912 ixl_dmamem_hi(struct ixl_dmamem *ixm) 913 { 914 uint32_t retval; 915 uint64_t val; 916 917 if (sizeof(IXL_DMA_DVA(ixm)) > 4) { 918 val = (intptr_t)IXL_DMA_DVA(ixm); 919 retval = (uint32_t)(val >> 32); 920 } else { 921 retval = 0; 922 } 923 924 return retval; 925 } 926 927 static inline uint32_t 928 ixl_dmamem_lo(struct ixl_dmamem *ixm) 929 { 930 931 return (uint32_t)IXL_DMA_DVA(ixm); 932 } 933 934 static inline void 935 ixl_aq_dva(struct ixl_aq_desc *iaq, bus_addr_t addr) 936 { 937 uint64_t val; 938 939 if (sizeof(addr) > 4) { 940 val = (intptr_t)addr; 941 iaq->iaq_param[2] = htole32(val >> 32); 942 } else { 943 iaq->iaq_param[2] = htole32(0); 944 } 945 946 iaq->iaq_param[3] = htole32(addr); 947 } 948 949 static inline unsigned int 950 ixl_rxr_unrefreshed(unsigned int prod, unsigned int cons, unsigned int ndescs) 951 { 952 unsigned int num; 953 954 if (prod < cons) 955 num = cons - prod; 956 else 957 num = (ndescs - prod) + cons; 958 959 if (__predict_true(num > 0)) { 960 /* device cannot receive packets if all descripter is filled */ 961 num -= 1; 962 } 963 964 return num; 965 } 966 967 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 968 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 969 DVF_DETACH_SHUTDOWN); 970 971 static const struct ixl_product ixl_products[] = { 972 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 973 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 974 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 975 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 976 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 977 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 978 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T }, 979 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 980 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 981 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 982 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 983 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 984 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 985 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 986 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 987 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 988 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 989 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 990 /* required last entry */ 991 {0, 0} 992 }; 993 994 static const struct ixl_product * 995 ixl_lookup(const struct pci_attach_args *pa) 996 { 997 const struct ixl_product *ixlp; 998 999 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 1000 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 1001 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 1002 return ixlp; 1003 } 1004 1005 return NULL; 1006 } 1007 1008 static int 1009 ixl_match(device_t parent, cfdata_t match, void *aux) 1010 { 1011 const struct pci_attach_args *pa = aux; 1012 1013 return (ixl_lookup(pa) != NULL) ? 1 : 0; 1014 } 1015 1016 static void 1017 ixl_attach(device_t parent, device_t self, void *aux) 1018 { 1019 struct ixl_softc *sc; 1020 struct pci_attach_args *pa = aux; 1021 struct ifnet *ifp; 1022 pcireg_t memtype, reg; 1023 uint32_t firstq, port, ari, func; 1024 uint64_t phy_types = 0; 1025 char xnamebuf[32]; 1026 int tries, rv; 1027 1028 sc = device_private(self); 1029 sc->sc_dev = self; 1030 ifp = &sc->sc_ec.ec_if; 1031 1032 sc->sc_pa = *pa; 1033 sc->sc_dmat = (pci_dma64_available(pa)) ? 1034 pa->pa_dmat64 : pa->pa_dmat; 1035 sc->sc_aq_regs = &ixl_pf_aq_regs; 1036 1037 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 1038 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(reg)); 1039 1040 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 1041 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 1042 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 1043 aprint_error(": unable to map registers\n"); 1044 return; 1045 } 1046 1047 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 1048 1049 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 1050 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 1051 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 1052 sc->sc_base_queue = firstq; 1053 1054 ixl_clear_hw(sc); 1055 if (ixl_pf_reset(sc) == -1) { 1056 /* error printed by ixl pf_reset */ 1057 goto unmap; 1058 } 1059 1060 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 1061 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 1062 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 1063 sc->sc_port = port; 1064 aprint_normal(": port %u", sc->sc_port); 1065 1066 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 1067 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 1068 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 1069 1070 func = ixl_rd(sc, I40E_PF_FUNC_RID); 1071 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 1072 1073 /* initialise the adminq */ 1074 1075 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 1076 1077 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 1078 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1079 aprint_error("\n" "%s: unable to allocate atq\n", 1080 device_xname(self)); 1081 goto unmap; 1082 } 1083 1084 SIMPLEQ_INIT(&sc->sc_arq_idle); 1085 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 1086 sc->sc_arq_cons = 0; 1087 sc->sc_arq_prod = 0; 1088 1089 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 1090 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1091 aprint_error("\n" "%s: unable to allocate arq\n", 1092 device_xname(self)); 1093 goto free_atq; 1094 } 1095 1096 if (!ixl_arq_fill(sc)) { 1097 aprint_error("\n" "%s: unable to fill arq descriptors\n", 1098 device_xname(self)); 1099 goto free_arq; 1100 } 1101 1102 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1103 0, IXL_DMA_LEN(&sc->sc_atq), 1104 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1105 1106 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1107 0, IXL_DMA_LEN(&sc->sc_arq), 1108 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1109 1110 for (tries = 0; tries < 10; tries++) { 1111 sc->sc_atq_cons = 0; 1112 sc->sc_atq_prod = 0; 1113 1114 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1115 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1116 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1117 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1118 1119 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1120 1121 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1122 ixl_dmamem_lo(&sc->sc_atq)); 1123 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1124 ixl_dmamem_hi(&sc->sc_atq)); 1125 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1126 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1127 1128 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1129 ixl_dmamem_lo(&sc->sc_arq)); 1130 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1131 ixl_dmamem_hi(&sc->sc_arq)); 1132 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1133 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1134 1135 rv = ixl_get_version(sc); 1136 if (rv == 0) 1137 break; 1138 if (rv != ETIMEDOUT) { 1139 aprint_error(", unable to get firmware version\n"); 1140 goto shutdown; 1141 } 1142 1143 delaymsec(100); 1144 } 1145 1146 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1147 1148 if (sc->sc_mac_type == I40E_MAC_X722) 1149 sc->sc_nqueue_pairs_device = 128; 1150 else 1151 sc->sc_nqueue_pairs_device = 64; 1152 1153 rv = ixl_get_hw_capabilities(sc); 1154 if (rv != 0) { 1155 aprint_error(", GET HW CAPABILITIES %s\n", 1156 rv == ETIMEDOUT ? "timeout" : "error"); 1157 goto shutdown; 1158 } 1159 1160 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max = 1161 MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1162 sc->sc_tx_ring_ndescs = 1024; 1163 sc->sc_rx_ring_ndescs = 1024; 1164 1165 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1166 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1167 1168 if (ixl_get_mac(sc) != 0) { 1169 /* error printed by ixl_get_mac */ 1170 goto shutdown; 1171 } 1172 1173 aprint_normal("\n"); 1174 aprint_naive("\n"); 1175 1176 aprint_normal_dev(self, "Ethernet address %s\n", 1177 ether_sprintf(sc->sc_enaddr)); 1178 1179 rv = ixl_pxe_clear(sc); 1180 if (rv != 0) { 1181 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1182 rv == ETIMEDOUT ? "timeout" : "error"); 1183 } 1184 1185 ixl_set_filter_control(sc); 1186 1187 if (ixl_hmc(sc) != 0) { 1188 /* error printed by ixl_hmc */ 1189 goto shutdown; 1190 } 1191 1192 if (ixl_lldp_shut(sc) != 0) { 1193 /* error printed by ixl_lldp_shut */ 1194 goto free_hmc; 1195 } 1196 1197 if (ixl_phy_mask_ints(sc) != 0) { 1198 /* error printed by ixl_phy_mask_ints */ 1199 goto free_hmc; 1200 } 1201 1202 if (ixl_restart_an(sc) != 0) { 1203 /* error printed by ixl_restart_an */ 1204 goto free_hmc; 1205 } 1206 1207 if (ixl_get_switch_config(sc) != 0) { 1208 /* error printed by ixl_get_switch_config */ 1209 goto free_hmc; 1210 } 1211 1212 if (ixl_get_phy_types(sc, &phy_types) != 0) { 1213 /* error printed by ixl_get_phy_abilities */ 1214 goto free_hmc; 1215 } 1216 1217 rv = ixl_get_link_status_poll(sc); 1218 if (rv != 0) { 1219 aprint_error_dev(self, "GET LINK STATUS %s\n", 1220 rv == ETIMEDOUT ? "timeout" : "error"); 1221 goto free_hmc; 1222 } 1223 1224 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1225 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1226 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1227 goto free_hmc; 1228 } 1229 1230 rv = ixl_get_vsi(sc); 1231 if (rv != 0) { 1232 aprint_error_dev(self, "GET VSI %s %d\n", 1233 rv == ETIMEDOUT ? "timeout" : "error", rv); 1234 goto free_scratch; 1235 } 1236 1237 rv = ixl_set_vsi(sc); 1238 if (rv != 0) { 1239 aprint_error_dev(self, "UPDATE VSI error %s %d\n", 1240 rv == ETIMEDOUT ? "timeout" : "error", rv); 1241 goto free_scratch; 1242 } 1243 1244 if (ixl_queue_pairs_alloc(sc) != 0) { 1245 /* error printed by ixl_queue_pairs_alloc */ 1246 goto free_scratch; 1247 } 1248 1249 if (ixl_setup_interrupts(sc) != 0) { 1250 /* error printed by ixl_setup_interrupts */ 1251 goto free_queue_pairs; 1252 } 1253 1254 if (ixl_setup_stats(sc) != 0) { 1255 aprint_error_dev(self, "failed to setup event counters\n"); 1256 goto teardown_intrs; 1257 } 1258 1259 if (ixl_setup_sysctls(sc) != 0) { 1260 /* error printed by ixl_setup_sysctls */ 1261 goto teardown_stats; 1262 } 1263 1264 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1265 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1266 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1267 if (sc->sc_workq == NULL) 1268 goto teardown_sysctls; 1269 1270 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1271 sc->sc_workq_txrx = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1272 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1273 if (sc->sc_workq_txrx == NULL) 1274 goto teardown_wqs; 1275 1276 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1277 cv_init(&sc->sc_atq_cv, xnamebuf); 1278 1279 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1280 1281 ifp->if_softc = sc; 1282 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1283 ifp->if_extflags = IFEF_MPSAFE; 1284 ifp->if_ioctl = ixl_ioctl; 1285 ifp->if_start = ixl_start; 1286 ifp->if_transmit = ixl_transmit; 1287 ifp->if_watchdog = ixl_watchdog; 1288 ifp->if_init = ixl_init; 1289 ifp->if_stop = ixl_stop; 1290 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1291 IFQ_SET_READY(&ifp->if_snd); 1292 ifp->if_capabilities |= IXL_IFCAP_RXCSUM; 1293 #if 0 1294 ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_UDPv4_Tx; 1295 #endif 1296 ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb); 1297 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_MTU; 1298 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1299 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1300 1301 sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities; 1302 /* Disable VLAN_HWFILTER by default */ 1303 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1304 1305 sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable; 1306 1307 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1308 ifmedia_init(&sc->sc_media, IFM_IMASK, ixl_media_change, 1309 ixl_media_status); 1310 1311 ixl_media_add(sc, phy_types); 1312 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1313 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1314 1315 if_attach(ifp); 1316 if_deferred_start_init(ifp, NULL); 1317 ether_ifattach(ifp, sc->sc_enaddr); 1318 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1319 1320 (void)ixl_get_link_status_poll(sc); 1321 ixl_work_set(&sc->sc_link_state_task, ixl_get_link_status, sc); 1322 1323 ixl_config_other_intr(sc); 1324 ixl_enable_other_intr(sc); 1325 1326 /* remove default mac filter and replace it so we can see vlans */ 1327 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 1328 if (rv != ENOENT) { 1329 aprint_debug_dev(self, 1330 "unable to remove macvlan %u\n", rv); 1331 } 1332 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1333 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1334 if (rv != ENOENT) { 1335 aprint_debug_dev(self, 1336 "unable to remove macvlan, ignore vlan %u\n", rv); 1337 } 1338 1339 if (ixl_update_macvlan(sc) != 0) { 1340 aprint_debug_dev(self, 1341 "couldn't enable vlan hardware filter\n"); 1342 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1343 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1344 } 1345 1346 sc->sc_txrx_workqueue = true; 1347 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1348 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1349 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1350 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1351 1352 ixl_stats_update(sc); 1353 sc->sc_stats_counters.isc_has_offset = true; 1354 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 1355 1356 if (pmf_device_register(self, NULL, NULL) != true) 1357 aprint_debug_dev(self, "couldn't establish power handler\n"); 1358 sc->sc_attached = true; 1359 return; 1360 1361 teardown_wqs: 1362 config_finalize_register(self, ixl_workqs_teardown); 1363 teardown_sysctls: 1364 ixl_teardown_sysctls(sc); 1365 teardown_stats: 1366 ixl_teardown_stats(sc); 1367 teardown_intrs: 1368 ixl_teardown_interrupts(sc); 1369 free_queue_pairs: 1370 ixl_queue_pairs_free(sc); 1371 free_scratch: 1372 ixl_dmamem_free(sc, &sc->sc_scratch); 1373 free_hmc: 1374 ixl_hmc_free(sc); 1375 shutdown: 1376 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1377 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1378 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1379 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1380 1381 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1382 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1383 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1384 1385 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1386 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1387 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1388 1389 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1390 0, IXL_DMA_LEN(&sc->sc_arq), 1391 BUS_DMASYNC_POSTREAD); 1392 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1393 0, IXL_DMA_LEN(&sc->sc_atq), 1394 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1395 1396 ixl_arq_unfill(sc); 1397 free_arq: 1398 ixl_dmamem_free(sc, &sc->sc_arq); 1399 free_atq: 1400 ixl_dmamem_free(sc, &sc->sc_atq); 1401 unmap: 1402 mutex_destroy(&sc->sc_atq_lock); 1403 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1404 mutex_destroy(&sc->sc_cfg_lock); 1405 sc->sc_mems = 0; 1406 1407 sc->sc_attached = false; 1408 } 1409 1410 static int 1411 ixl_detach(device_t self, int flags) 1412 { 1413 struct ixl_softc *sc = device_private(self); 1414 struct ifnet *ifp = &sc->sc_ec.ec_if; 1415 1416 if (!sc->sc_attached) 1417 return 0; 1418 1419 ixl_stop(ifp, 1); 1420 1421 ixl_disable_other_intr(sc); 1422 1423 callout_stop(&sc->sc_stats_callout); 1424 ixl_work_wait(sc->sc_workq, &sc->sc_stats_task); 1425 1426 /* wait for ATQ handler */ 1427 mutex_enter(&sc->sc_atq_lock); 1428 mutex_exit(&sc->sc_atq_lock); 1429 1430 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1431 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1432 1433 if (sc->sc_workq != NULL) { 1434 ixl_workq_destroy(sc->sc_workq); 1435 sc->sc_workq = NULL; 1436 } 1437 1438 if (sc->sc_workq_txrx != NULL) { 1439 ixl_workq_destroy(sc->sc_workq_txrx); 1440 sc->sc_workq_txrx = NULL; 1441 } 1442 1443 ifmedia_delete_instance(&sc->sc_media, IFM_INST_ANY); 1444 ether_ifdetach(ifp); 1445 if_detach(ifp); 1446 1447 ixl_teardown_interrupts(sc); 1448 ixl_teardown_stats(sc); 1449 ixl_teardown_sysctls(sc); 1450 1451 ixl_queue_pairs_free(sc); 1452 1453 ixl_dmamem_free(sc, &sc->sc_scratch); 1454 ixl_hmc_free(sc); 1455 1456 /* shutdown */ 1457 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1458 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1459 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1460 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1461 1462 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1463 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1464 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1465 1466 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1467 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1468 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1469 1470 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1471 0, IXL_DMA_LEN(&sc->sc_arq), 1472 BUS_DMASYNC_POSTREAD); 1473 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1474 0, IXL_DMA_LEN(&sc->sc_atq), 1475 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1476 1477 ixl_arq_unfill(sc); 1478 1479 ixl_dmamem_free(sc, &sc->sc_arq); 1480 ixl_dmamem_free(sc, &sc->sc_atq); 1481 1482 cv_destroy(&sc->sc_atq_cv); 1483 mutex_destroy(&sc->sc_atq_lock); 1484 1485 if (sc->sc_mems != 0) { 1486 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1487 sc->sc_mems = 0; 1488 } 1489 1490 mutex_destroy(&sc->sc_cfg_lock); 1491 1492 return 0; 1493 } 1494 1495 static int 1496 ixl_workqs_teardown(device_t self) 1497 { 1498 struct ixl_softc *sc = device_private(self); 1499 1500 if (sc->sc_workq != NULL) { 1501 ixl_workq_destroy(sc->sc_workq); 1502 sc->sc_workq = NULL; 1503 } 1504 1505 if (sc->sc_workq_txrx != NULL) { 1506 ixl_workq_destroy(sc->sc_workq_txrx); 1507 sc->sc_workq_txrx = NULL; 1508 } 1509 1510 return 0; 1511 } 1512 1513 static int 1514 ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 1515 { 1516 struct ifnet *ifp = &ec->ec_if; 1517 struct ixl_softc *sc = ifp->if_softc; 1518 int rv; 1519 1520 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1521 return 0; 1522 } 1523 1524 if (set) { 1525 rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid, 1526 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1527 if (rv == 0) { 1528 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 1529 vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1530 } 1531 } else { 1532 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid, 1533 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1534 (void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid, 1535 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1536 } 1537 1538 return rv; 1539 } 1540 1541 static void 1542 ixl_media_add(struct ixl_softc *sc, uint64_t phy_types) 1543 { 1544 struct ifmedia *ifm = &sc->sc_media; 1545 const struct ixl_phy_type *itype; 1546 unsigned int i; 1547 1548 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1549 itype = &ixl_phy_type_map[i]; 1550 1551 if (ISSET(phy_types, itype->phy_type)) { 1552 ifmedia_add(ifm, 1553 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1554 1555 if (itype->ifm_type == IFM_100_TX) { 1556 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1557 0, NULL); 1558 } 1559 } 1560 } 1561 } 1562 1563 static void 1564 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1565 { 1566 struct ixl_softc *sc = ifp->if_softc; 1567 1568 ifmr->ifm_status = sc->sc_media_status; 1569 ifmr->ifm_active = sc->sc_media_active; 1570 1571 mutex_enter(&sc->sc_cfg_lock); 1572 if (ifp->if_link_state == LINK_STATE_UP) 1573 SET(ifmr->ifm_status, IFM_ACTIVE); 1574 mutex_exit(&sc->sc_cfg_lock); 1575 } 1576 1577 static int 1578 ixl_media_change(struct ifnet *ifp) 1579 { 1580 1581 return 0; 1582 } 1583 1584 static void 1585 ixl_watchdog(struct ifnet *ifp) 1586 { 1587 1588 } 1589 1590 static void 1591 ixl_del_all_multiaddr(struct ixl_softc *sc) 1592 { 1593 struct ethercom *ec = &sc->sc_ec; 1594 struct ether_multi *enm; 1595 struct ether_multistep step; 1596 1597 ETHER_LOCK(ec); 1598 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1599 ETHER_NEXT_MULTI(step, enm)) { 1600 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1601 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1602 } 1603 ETHER_UNLOCK(ec); 1604 } 1605 1606 static int 1607 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1608 { 1609 struct ifnet *ifp = &sc->sc_ec.ec_if; 1610 int rv; 1611 1612 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1613 return 0; 1614 1615 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1616 ixl_del_all_multiaddr(sc); 1617 SET(ifp->if_flags, IFF_ALLMULTI); 1618 return ENETRESET; 1619 } 1620 1621 /* multicast address can not use VLAN HWFILTER */ 1622 rv = ixl_add_macvlan(sc, addrlo, 0, 1623 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1624 1625 if (rv == ENOSPC) { 1626 ixl_del_all_multiaddr(sc); 1627 SET(ifp->if_flags, IFF_ALLMULTI); 1628 return ENETRESET; 1629 } 1630 1631 return rv; 1632 } 1633 1634 static int 1635 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1636 { 1637 struct ifnet *ifp = &sc->sc_ec.ec_if; 1638 struct ethercom *ec = &sc->sc_ec; 1639 struct ether_multi *enm, *enm_last; 1640 struct ether_multistep step; 1641 int error, rv = 0; 1642 1643 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1644 ixl_remove_macvlan(sc, addrlo, 0, 1645 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1646 return 0; 1647 } 1648 1649 ETHER_LOCK(ec); 1650 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1651 ETHER_NEXT_MULTI(step, enm)) { 1652 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1653 ETHER_ADDR_LEN) != 0) { 1654 goto out; 1655 } 1656 } 1657 1658 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1659 ETHER_NEXT_MULTI(step, enm)) { 1660 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1661 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1662 if (error != 0) 1663 break; 1664 } 1665 1666 if (enm != NULL) { 1667 enm_last = enm; 1668 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1669 ETHER_NEXT_MULTI(step, enm)) { 1670 if (enm == enm_last) 1671 break; 1672 1673 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1674 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1675 } 1676 } else { 1677 CLR(ifp->if_flags, IFF_ALLMULTI); 1678 rv = ENETRESET; 1679 } 1680 1681 out: 1682 ETHER_UNLOCK(ec); 1683 return rv; 1684 } 1685 1686 static int 1687 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1688 { 1689 struct ifreq *ifr = (struct ifreq *)data; 1690 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1691 struct ixl_tx_ring *txr; 1692 struct ixl_rx_ring *rxr; 1693 const struct sockaddr *sa; 1694 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1695 int s, error = 0; 1696 unsigned int i; 1697 1698 switch (cmd) { 1699 case SIOCADDMULTI: 1700 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1701 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1702 error = ether_multiaddr(sa, addrlo, addrhi); 1703 if (error != 0) 1704 return error; 1705 1706 error = ixl_add_multi(sc, addrlo, addrhi); 1707 if (error != 0 && error != ENETRESET) { 1708 ether_delmulti(sa, &sc->sc_ec); 1709 error = EIO; 1710 } 1711 } 1712 break; 1713 1714 case SIOCDELMULTI: 1715 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1716 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1717 error = ether_multiaddr(sa, addrlo, addrhi); 1718 if (error != 0) 1719 return error; 1720 1721 error = ixl_del_multi(sc, addrlo, addrhi); 1722 } 1723 break; 1724 1725 case SIOCGIFDATA: 1726 case SIOCZIFDATA: 1727 ifp->if_ipackets = 0; 1728 ifp->if_ibytes = 0; 1729 ifp->if_iqdrops = 0; 1730 ifp->if_ierrors = 0; 1731 ifp->if_opackets = 0; 1732 ifp->if_obytes = 0; 1733 ifp->if_omcasts = 0; 1734 1735 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 1736 txr = sc->sc_qps[i].qp_txr; 1737 rxr = sc->sc_qps[i].qp_rxr; 1738 1739 mutex_enter(&rxr->rxr_lock); 1740 ifp->if_ipackets += rxr->rxr_ipackets; 1741 ifp->if_ibytes += rxr->rxr_ibytes; 1742 ifp->if_iqdrops += rxr->rxr_iqdrops; 1743 ifp->if_ierrors += rxr->rxr_ierrors; 1744 if (cmd == SIOCZIFDATA) { 1745 rxr->rxr_ipackets = 0; 1746 rxr->rxr_ibytes = 0; 1747 rxr->rxr_iqdrops = 0; 1748 rxr->rxr_ierrors = 0; 1749 } 1750 mutex_exit(&rxr->rxr_lock); 1751 1752 mutex_enter(&txr->txr_lock); 1753 ifp->if_opackets += txr->txr_opackets; 1754 ifp->if_obytes += txr->txr_opackets; 1755 ifp->if_omcasts += txr->txr_omcasts; 1756 if (cmd == SIOCZIFDATA) { 1757 txr->txr_opackets = 0; 1758 txr->txr_opackets = 0; 1759 txr->txr_omcasts = 0; 1760 } 1761 mutex_exit(&txr->txr_lock); 1762 } 1763 /* FALLTHROUGH */ 1764 default: 1765 s = splnet(); 1766 error = ether_ioctl(ifp, cmd, data); 1767 splx(s); 1768 } 1769 1770 if (error == ENETRESET) 1771 error = ixl_iff(sc); 1772 1773 return error; 1774 } 1775 1776 static enum i40e_mac_type 1777 ixl_mactype(pci_product_id_t id) 1778 { 1779 1780 switch (id) { 1781 case PCI_PRODUCT_INTEL_XL710_SFP: 1782 case PCI_PRODUCT_INTEL_XL710_KX_B: 1783 case PCI_PRODUCT_INTEL_XL710_KX_C: 1784 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1785 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1786 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1787 case PCI_PRODUCT_INTEL_X710_10G_T: 1788 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1789 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1790 case PCI_PRODUCT_INTEL_X710_T4_10G: 1791 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1792 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1793 return I40E_MAC_XL710; 1794 1795 case PCI_PRODUCT_INTEL_X722_KX: 1796 case PCI_PRODUCT_INTEL_X722_QSFP: 1797 case PCI_PRODUCT_INTEL_X722_SFP: 1798 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1799 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1800 case PCI_PRODUCT_INTEL_X722_I_SFP: 1801 return I40E_MAC_X722; 1802 } 1803 1804 return I40E_MAC_GENERIC; 1805 } 1806 1807 static inline void * 1808 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1809 { 1810 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1811 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1812 1813 if (i >= e->hmc_count) 1814 return NULL; 1815 1816 kva += e->hmc_base; 1817 kva += i * e->hmc_size; 1818 1819 return kva; 1820 } 1821 1822 static inline size_t 1823 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1824 { 1825 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1826 1827 return e->hmc_size; 1828 } 1829 1830 static void 1831 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1832 { 1833 struct ixl_rx_ring *rxr = qp->qp_rxr; 1834 1835 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1836 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1837 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1838 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1839 ixl_flush(sc); 1840 } 1841 1842 static void 1843 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1844 { 1845 struct ixl_rx_ring *rxr = qp->qp_rxr; 1846 1847 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1848 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1849 ixl_flush(sc); 1850 } 1851 1852 static void 1853 ixl_enable_other_intr(struct ixl_softc *sc) 1854 { 1855 1856 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1857 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1858 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1859 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1860 ixl_flush(sc); 1861 } 1862 1863 static void 1864 ixl_disable_other_intr(struct ixl_softc *sc) 1865 { 1866 1867 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1868 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1869 ixl_flush(sc); 1870 } 1871 1872 static int 1873 ixl_reinit(struct ixl_softc *sc) 1874 { 1875 struct ixl_rx_ring *rxr; 1876 struct ixl_tx_ring *txr; 1877 unsigned int i; 1878 uint32_t reg; 1879 1880 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1881 1882 if (ixl_get_vsi(sc) != 0) 1883 return EIO; 1884 1885 if (ixl_set_vsi(sc) != 0) 1886 return EIO; 1887 1888 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1889 txr = sc->sc_qps[i].qp_txr; 1890 rxr = sc->sc_qps[i].qp_rxr; 1891 1892 txr->txr_cons = txr->txr_prod = 0; 1893 rxr->rxr_cons = rxr->rxr_prod = 0; 1894 1895 ixl_txr_config(sc, txr); 1896 ixl_rxr_config(sc, rxr); 1897 } 1898 1899 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1900 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 1901 1902 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1903 txr = sc->sc_qps[i].qp_txr; 1904 rxr = sc->sc_qps[i].qp_rxr; 1905 1906 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 1907 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 1908 ixl_flush(sc); 1909 1910 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 1911 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 1912 1913 /* ixl_rxfill() needs lock held */ 1914 mutex_enter(&rxr->rxr_lock); 1915 ixl_rxfill(sc, rxr); 1916 mutex_exit(&rxr->rxr_lock); 1917 1918 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 1919 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 1920 ixl_wr(sc, I40E_QRX_ENA(i), reg); 1921 if (ixl_rxr_enabled(sc, rxr) != 0) 1922 goto stop; 1923 1924 ixl_txr_qdis(sc, txr, 1); 1925 1926 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 1927 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 1928 ixl_wr(sc, I40E_QTX_ENA(i), reg); 1929 1930 if (ixl_txr_enabled(sc, txr) != 0) 1931 goto stop; 1932 } 1933 1934 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1935 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 1936 1937 return 0; 1938 1939 stop: 1940 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1941 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 1942 1943 return ETIMEDOUT; 1944 } 1945 1946 static int 1947 ixl_init_locked(struct ixl_softc *sc) 1948 { 1949 struct ifnet *ifp = &sc->sc_ec.ec_if; 1950 unsigned int i; 1951 int error, eccap_change; 1952 1953 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1954 1955 if (ISSET(ifp->if_flags, IFF_RUNNING)) 1956 ixl_stop_locked(sc); 1957 1958 if (sc->sc_dead) { 1959 return ENXIO; 1960 } 1961 1962 eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable; 1963 if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING)) 1964 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 1965 1966 if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) { 1967 if (ixl_update_macvlan(sc) == 0) { 1968 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 1969 } else { 1970 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1971 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1972 } 1973 } 1974 1975 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 1976 sc->sc_nqueue_pairs = 1; 1977 else 1978 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1979 1980 error = ixl_reinit(sc); 1981 if (error) { 1982 ixl_stop_locked(sc); 1983 return error; 1984 } 1985 1986 SET(ifp->if_flags, IFF_RUNNING); 1987 CLR(ifp->if_flags, IFF_OACTIVE); 1988 1989 (void)ixl_get_link_status(sc); 1990 1991 ixl_config_rss(sc); 1992 ixl_config_queue_intr(sc); 1993 1994 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1995 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 1996 } 1997 1998 error = ixl_iff(sc); 1999 if (error) { 2000 ixl_stop_locked(sc); 2001 return error; 2002 } 2003 2004 return 0; 2005 } 2006 2007 static int 2008 ixl_init(struct ifnet *ifp) 2009 { 2010 struct ixl_softc *sc = ifp->if_softc; 2011 int error; 2012 2013 mutex_enter(&sc->sc_cfg_lock); 2014 error = ixl_init_locked(sc); 2015 mutex_exit(&sc->sc_cfg_lock); 2016 2017 return error; 2018 } 2019 2020 static int 2021 ixl_iff(struct ixl_softc *sc) 2022 { 2023 struct ifnet *ifp = &sc->sc_ec.ec_if; 2024 struct ixl_atq iatq; 2025 struct ixl_aq_desc *iaq; 2026 struct ixl_aq_vsi_promisc_param *param; 2027 uint16_t flag_add, flag_del; 2028 int error; 2029 2030 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2031 return 0; 2032 2033 memset(&iatq, 0, sizeof(iatq)); 2034 2035 iaq = &iatq.iatq_desc; 2036 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 2037 2038 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 2039 param->flags = htole16(0); 2040 2041 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER) 2042 || ISSET(ifp->if_flags, IFF_PROMISC)) { 2043 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2044 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2045 } 2046 2047 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 2048 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2049 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2050 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 2051 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2052 } 2053 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2054 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2055 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2056 param->seid = sc->sc_seid; 2057 2058 error = ixl_atq_exec(sc, &iatq); 2059 if (error) 2060 return error; 2061 2062 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 2063 return EIO; 2064 2065 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 2066 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 2067 flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH; 2068 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH; 2069 } else { 2070 flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN; 2071 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN; 2072 } 2073 2074 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del); 2075 2076 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2077 ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add); 2078 } 2079 return 0; 2080 } 2081 2082 static void 2083 ixl_stop_rendezvous(struct ixl_softc *sc) 2084 { 2085 struct ixl_tx_ring *txr; 2086 struct ixl_rx_ring *rxr; 2087 unsigned int i; 2088 2089 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2090 txr = sc->sc_qps[i].qp_txr; 2091 rxr = sc->sc_qps[i].qp_rxr; 2092 2093 mutex_enter(&txr->txr_lock); 2094 mutex_exit(&txr->txr_lock); 2095 2096 mutex_enter(&rxr->rxr_lock); 2097 mutex_exit(&rxr->rxr_lock); 2098 2099 ixl_work_wait(sc->sc_workq_txrx, 2100 &sc->sc_qps[i].qp_task); 2101 } 2102 } 2103 2104 static void 2105 ixl_stop_locked(struct ixl_softc *sc) 2106 { 2107 struct ifnet *ifp = &sc->sc_ec.ec_if; 2108 struct ixl_rx_ring *rxr; 2109 struct ixl_tx_ring *txr; 2110 unsigned int i; 2111 uint32_t reg; 2112 2113 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2114 2115 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 2116 2117 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2118 txr = sc->sc_qps[i].qp_txr; 2119 rxr = sc->sc_qps[i].qp_rxr; 2120 2121 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 2122 2123 mutex_enter(&txr->txr_lock); 2124 ixl_txr_qdis(sc, txr, 0); 2125 /* XXX wait at least 400 usec for all tx queues in one go */ 2126 ixl_flush(sc); 2127 DELAY(500); 2128 2129 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2130 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2131 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2132 /* XXX wait 50ms from completaion of the TX queue disable*/ 2133 ixl_flush(sc); 2134 DELAY(50); 2135 2136 if (ixl_txr_disabled(sc, txr) != 0) { 2137 mutex_exit(&txr->txr_lock); 2138 goto die; 2139 } 2140 mutex_exit(&txr->txr_lock); 2141 2142 mutex_enter(&rxr->rxr_lock); 2143 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2144 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2145 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2146 /* XXX wait 50ms from completion of the RX queue disable */ 2147 ixl_flush(sc); 2148 DELAY(50); 2149 2150 if (ixl_rxr_disabled(sc, rxr) != 0) { 2151 mutex_exit(&rxr->rxr_lock); 2152 goto die; 2153 } 2154 mutex_exit(&rxr->rxr_lock); 2155 } 2156 2157 ixl_stop_rendezvous(sc); 2158 2159 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2160 txr = sc->sc_qps[i].qp_txr; 2161 rxr = sc->sc_qps[i].qp_rxr; 2162 2163 ixl_txr_unconfig(sc, txr); 2164 ixl_rxr_unconfig(sc, rxr); 2165 2166 ixl_txr_clean(sc, txr); 2167 ixl_rxr_clean(sc, rxr); 2168 } 2169 2170 return; 2171 die: 2172 sc->sc_dead = true; 2173 log(LOG_CRIT, "%s: failed to shut down rings", 2174 device_xname(sc->sc_dev)); 2175 return; 2176 } 2177 2178 static void 2179 ixl_stop(struct ifnet *ifp, int disable) 2180 { 2181 struct ixl_softc *sc = ifp->if_softc; 2182 2183 mutex_enter(&sc->sc_cfg_lock); 2184 ixl_stop_locked(sc); 2185 mutex_exit(&sc->sc_cfg_lock); 2186 } 2187 2188 static int 2189 ixl_queue_pairs_alloc(struct ixl_softc *sc) 2190 { 2191 struct ixl_queue_pair *qp; 2192 unsigned int i; 2193 size_t sz; 2194 2195 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2196 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 2197 2198 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2199 qp = &sc->sc_qps[i]; 2200 2201 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2202 ixl_handle_queue, qp); 2203 if (qp->qp_si == NULL) 2204 goto free; 2205 2206 qp->qp_txr = ixl_txr_alloc(sc, i); 2207 if (qp->qp_txr == NULL) 2208 goto free; 2209 2210 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2211 if (qp->qp_rxr == NULL) 2212 goto free; 2213 2214 qp->qp_sc = sc; 2215 ixl_work_set(&qp->qp_task, ixl_handle_queue, qp); 2216 snprintf(qp->qp_name, sizeof(qp->qp_name), 2217 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2218 } 2219 2220 return 0; 2221 free: 2222 if (sc->sc_qps != NULL) { 2223 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2224 qp = &sc->sc_qps[i]; 2225 2226 if (qp->qp_txr != NULL) 2227 ixl_txr_free(sc, qp->qp_txr); 2228 if (qp->qp_rxr != NULL) 2229 ixl_rxr_free(sc, qp->qp_rxr); 2230 if (qp->qp_si != NULL) 2231 softint_disestablish(qp->qp_si); 2232 } 2233 2234 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2235 kmem_free(sc->sc_qps, sz); 2236 sc->sc_qps = NULL; 2237 } 2238 2239 return -1; 2240 } 2241 2242 static void 2243 ixl_queue_pairs_free(struct ixl_softc *sc) 2244 { 2245 struct ixl_queue_pair *qp; 2246 unsigned int i; 2247 size_t sz; 2248 2249 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2250 qp = &sc->sc_qps[i]; 2251 ixl_txr_free(sc, qp->qp_txr); 2252 ixl_rxr_free(sc, qp->qp_rxr); 2253 softint_disestablish(qp->qp_si); 2254 } 2255 2256 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2257 kmem_free(sc->sc_qps, sz); 2258 sc->sc_qps = NULL; 2259 } 2260 2261 static struct ixl_tx_ring * 2262 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2263 { 2264 struct ixl_tx_ring *txr = NULL; 2265 struct ixl_tx_map *maps = NULL, *txm; 2266 unsigned int i; 2267 2268 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2269 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2270 KM_SLEEP); 2271 2272 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2273 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2274 IXL_TX_QUEUE_ALIGN) != 0) 2275 goto free; 2276 2277 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2278 txm = &maps[i]; 2279 2280 if (bus_dmamap_create(sc->sc_dmat, 2281 IXL_HARDMTU, IXL_TX_PKT_DESCS, IXL_HARDMTU, 0, 2282 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2283 goto uncreate; 2284 2285 txm->txm_eop = -1; 2286 txm->txm_m = NULL; 2287 } 2288 2289 txr->txr_cons = txr->txr_prod = 0; 2290 txr->txr_maps = maps; 2291 2292 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2293 if (txr->txr_intrq == NULL) 2294 goto uncreate; 2295 2296 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2297 ixl_deferred_transmit, txr); 2298 if (txr->txr_si == NULL) 2299 goto destroy_pcq; 2300 2301 txr->txr_tail = I40E_QTX_TAIL(qid); 2302 txr->txr_qid = qid; 2303 txr->txr_sc = sc; 2304 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2305 2306 return txr; 2307 2308 destroy_pcq: 2309 pcq_destroy(txr->txr_intrq); 2310 uncreate: 2311 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2312 txm = &maps[i]; 2313 2314 if (txm->txm_map == NULL) 2315 continue; 2316 2317 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2318 } 2319 2320 ixl_dmamem_free(sc, &txr->txr_mem); 2321 free: 2322 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2323 kmem_free(txr, sizeof(*txr)); 2324 2325 return NULL; 2326 } 2327 2328 static void 2329 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2330 { 2331 unsigned int qid; 2332 bus_size_t reg; 2333 uint32_t r; 2334 2335 qid = txr->txr_qid + sc->sc_base_queue; 2336 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2337 qid %= 128; 2338 2339 r = ixl_rd(sc, reg); 2340 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2341 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2342 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2343 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2344 ixl_wr(sc, reg, r); 2345 } 2346 2347 static void 2348 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2349 { 2350 struct ixl_hmc_txq txq; 2351 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2352 void *hmc; 2353 2354 memset(&txq, 0, sizeof(txq)); 2355 txq.head = htole16(txr->txr_cons); 2356 txq.new_context = 1; 2357 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2358 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2359 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2360 txq.tphrdesc_ena = 0; 2361 txq.tphrpacket_ena = 0; 2362 txq.tphwdesc_ena = 0; 2363 txq.rdylist = data->qs_handle[0]; 2364 2365 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2366 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2367 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2368 __arraycount(ixl_hmc_pack_txq)); 2369 } 2370 2371 static void 2372 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2373 { 2374 void *hmc; 2375 2376 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2377 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2378 } 2379 2380 static void 2381 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2382 { 2383 struct ixl_tx_map *maps, *txm; 2384 bus_dmamap_t map; 2385 unsigned int i; 2386 2387 maps = txr->txr_maps; 2388 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2389 txm = &maps[i]; 2390 2391 if (txm->txm_m == NULL) 2392 continue; 2393 2394 map = txm->txm_map; 2395 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2396 BUS_DMASYNC_POSTWRITE); 2397 bus_dmamap_unload(sc->sc_dmat, map); 2398 2399 m_freem(txm->txm_m); 2400 txm->txm_m = NULL; 2401 } 2402 } 2403 2404 static int 2405 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2406 { 2407 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2408 uint32_t reg; 2409 int i; 2410 2411 for (i = 0; i < 10; i++) { 2412 reg = ixl_rd(sc, ena); 2413 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2414 return 0; 2415 2416 delaymsec(10); 2417 } 2418 2419 return ETIMEDOUT; 2420 } 2421 2422 static int 2423 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2424 { 2425 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2426 uint32_t reg; 2427 int i; 2428 2429 KASSERT(mutex_owned(&txr->txr_lock)); 2430 2431 for (i = 0; i < 20; i++) { 2432 reg = ixl_rd(sc, ena); 2433 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2434 return 0; 2435 2436 delaymsec(10); 2437 } 2438 2439 return ETIMEDOUT; 2440 } 2441 2442 static void 2443 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2444 { 2445 struct ixl_tx_map *maps, *txm; 2446 struct mbuf *m; 2447 unsigned int i; 2448 2449 softint_disestablish(txr->txr_si); 2450 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2451 m_freem(m); 2452 pcq_destroy(txr->txr_intrq); 2453 2454 maps = txr->txr_maps; 2455 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2456 txm = &maps[i]; 2457 2458 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2459 } 2460 2461 ixl_dmamem_free(sc, &txr->txr_mem); 2462 mutex_destroy(&txr->txr_lock); 2463 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2464 kmem_free(txr, sizeof(*txr)); 2465 } 2466 2467 static inline int 2468 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2469 struct ixl_tx_ring *txr) 2470 { 2471 struct mbuf *m; 2472 int error; 2473 2474 KASSERT(mutex_owned(&txr->txr_lock)); 2475 2476 m = *m0; 2477 2478 error = bus_dmamap_load_mbuf(dmat, map, m, 2479 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2480 if (error != EFBIG) 2481 return error; 2482 2483 m = m_defrag(m, M_DONTWAIT); 2484 if (m != NULL) { 2485 *m0 = m; 2486 txr->txr_defragged.ev_count++; 2487 2488 error = bus_dmamap_load_mbuf(dmat, map, m, 2489 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2490 } else { 2491 txr->txr_defrag_failed.ev_count++; 2492 error = ENOBUFS; 2493 } 2494 2495 return error; 2496 } 2497 2498 static void 2499 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2500 bool is_transmit) 2501 { 2502 struct ixl_softc *sc = ifp->if_softc; 2503 struct ixl_tx_desc *ring, *txd; 2504 struct ixl_tx_map *txm; 2505 bus_dmamap_t map; 2506 struct mbuf *m; 2507 uint64_t cmd, cmd_vlan; 2508 unsigned int prod, free, last, i; 2509 unsigned int mask; 2510 int post = 0; 2511 2512 KASSERT(mutex_owned(&txr->txr_lock)); 2513 2514 if (ifp->if_link_state != LINK_STATE_UP 2515 || !ISSET(ifp->if_flags, IFF_RUNNING) 2516 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2517 if (!is_transmit) 2518 IFQ_PURGE(&ifp->if_snd); 2519 return; 2520 } 2521 2522 prod = txr->txr_prod; 2523 free = txr->txr_cons; 2524 if (free <= prod) 2525 free += sc->sc_tx_ring_ndescs; 2526 free -= prod; 2527 2528 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2529 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2530 2531 ring = IXL_DMA_KVA(&txr->txr_mem); 2532 mask = sc->sc_tx_ring_ndescs - 1; 2533 last = prod; 2534 cmd = 0; 2535 txd = NULL; 2536 2537 for (;;) { 2538 if (free <= IXL_TX_PKT_DESCS) { 2539 if (!is_transmit) 2540 SET(ifp->if_flags, IFF_OACTIVE); 2541 break; 2542 } 2543 2544 if (is_transmit) 2545 m = pcq_get(txr->txr_intrq); 2546 else 2547 IFQ_DEQUEUE(&ifp->if_snd, m); 2548 2549 if (m == NULL) 2550 break; 2551 2552 txm = &txr->txr_maps[prod]; 2553 map = txm->txm_map; 2554 2555 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2556 txr->txr_oerrors++; 2557 m_freem(m); 2558 continue; 2559 } 2560 2561 if (vlan_has_tag(m)) { 2562 cmd_vlan = (uint64_t)vlan_get_tag(m) << 2563 IXL_TX_DESC_L2TAG1_SHIFT; 2564 cmd_vlan |= IXL_TX_DESC_CMD_IL2TAG1; 2565 } else { 2566 cmd_vlan = 0; 2567 } 2568 2569 bus_dmamap_sync(sc->sc_dmat, map, 0, 2570 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2571 2572 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2573 txd = &ring[prod]; 2574 2575 cmd = (uint64_t)map->dm_segs[i].ds_len << 2576 IXL_TX_DESC_BSIZE_SHIFT; 2577 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2578 cmd |= cmd_vlan; 2579 2580 txd->addr = htole64(map->dm_segs[i].ds_addr); 2581 txd->cmd = htole64(cmd); 2582 2583 last = prod; 2584 2585 prod++; 2586 prod &= mask; 2587 } 2588 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2589 txd->cmd = htole64(cmd); 2590 2591 txm->txm_m = m; 2592 txm->txm_eop = last; 2593 2594 bpf_mtap(ifp, m, BPF_D_OUT); 2595 2596 free -= i; 2597 post = 1; 2598 } 2599 2600 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2601 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2602 2603 if (post) { 2604 txr->txr_prod = prod; 2605 ixl_wr(sc, txr->txr_tail, prod); 2606 } 2607 } 2608 2609 static int 2610 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2611 { 2612 struct ifnet *ifp = &sc->sc_ec.ec_if; 2613 struct ixl_tx_desc *ring, *txd; 2614 struct ixl_tx_map *txm; 2615 struct mbuf *m; 2616 bus_dmamap_t map; 2617 unsigned int cons, prod, last; 2618 unsigned int mask; 2619 uint64_t dtype; 2620 int done = 0, more = 0; 2621 2622 KASSERT(mutex_owned(&txr->txr_lock)); 2623 2624 prod = txr->txr_prod; 2625 cons = txr->txr_cons; 2626 2627 if (cons == prod) 2628 return 0; 2629 2630 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2631 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2632 2633 ring = IXL_DMA_KVA(&txr->txr_mem); 2634 mask = sc->sc_tx_ring_ndescs - 1; 2635 2636 do { 2637 if (txlimit-- <= 0) { 2638 more = 1; 2639 break; 2640 } 2641 2642 txm = &txr->txr_maps[cons]; 2643 last = txm->txm_eop; 2644 txd = &ring[last]; 2645 2646 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2647 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2648 break; 2649 2650 map = txm->txm_map; 2651 2652 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2653 BUS_DMASYNC_POSTWRITE); 2654 bus_dmamap_unload(sc->sc_dmat, map); 2655 2656 m = txm->txm_m; 2657 if (m != NULL) { 2658 txr->txr_opackets++; 2659 txr->txr_obytes += m->m_pkthdr.len; 2660 if (ISSET(m->m_flags, M_MCAST)) 2661 txr->txr_omcasts++; 2662 m_freem(m); 2663 } 2664 2665 txm->txm_m = NULL; 2666 txm->txm_eop = -1; 2667 2668 cons = last + 1; 2669 cons &= mask; 2670 done = 1; 2671 } while (cons != prod); 2672 2673 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2674 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2675 2676 txr->txr_cons = cons; 2677 2678 if (done) { 2679 softint_schedule(txr->txr_si); 2680 if (txr->txr_qid == 0) { 2681 CLR(ifp->if_flags, IFF_OACTIVE); 2682 if_schedule_deferred_start(ifp); 2683 } 2684 } 2685 2686 return more; 2687 } 2688 2689 static void 2690 ixl_start(struct ifnet *ifp) 2691 { 2692 struct ixl_softc *sc; 2693 struct ixl_tx_ring *txr; 2694 2695 sc = ifp->if_softc; 2696 txr = sc->sc_qps[0].qp_txr; 2697 2698 mutex_enter(&txr->txr_lock); 2699 ixl_tx_common_locked(ifp, txr, false); 2700 mutex_exit(&txr->txr_lock); 2701 } 2702 2703 static inline unsigned int 2704 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2705 { 2706 u_int cpuid; 2707 2708 cpuid = cpu_index(curcpu()); 2709 2710 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2711 } 2712 2713 static int 2714 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2715 { 2716 struct ixl_softc *sc; 2717 struct ixl_tx_ring *txr; 2718 unsigned int qid; 2719 2720 sc = ifp->if_softc; 2721 qid = ixl_select_txqueue(sc, m); 2722 2723 txr = sc->sc_qps[qid].qp_txr; 2724 2725 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2726 mutex_enter(&txr->txr_lock); 2727 txr->txr_pcqdrop.ev_count++; 2728 mutex_exit(&txr->txr_lock); 2729 2730 m_freem(m); 2731 return ENOBUFS; 2732 } 2733 2734 if (mutex_tryenter(&txr->txr_lock)) { 2735 ixl_tx_common_locked(ifp, txr, true); 2736 mutex_exit(&txr->txr_lock); 2737 } else { 2738 softint_schedule(txr->txr_si); 2739 } 2740 2741 return 0; 2742 } 2743 2744 static void 2745 ixl_deferred_transmit(void *xtxr) 2746 { 2747 struct ixl_tx_ring *txr = xtxr; 2748 struct ixl_softc *sc = txr->txr_sc; 2749 struct ifnet *ifp = &sc->sc_ec.ec_if; 2750 2751 mutex_enter(&txr->txr_lock); 2752 txr->txr_transmitdef.ev_count++; 2753 if (pcq_peek(txr->txr_intrq) != NULL) 2754 ixl_tx_common_locked(ifp, txr, true); 2755 mutex_exit(&txr->txr_lock); 2756 } 2757 2758 static struct ixl_rx_ring * 2759 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2760 { 2761 struct ixl_rx_ring *rxr = NULL; 2762 struct ixl_rx_map *maps = NULL, *rxm; 2763 unsigned int i; 2764 2765 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2766 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2767 KM_SLEEP); 2768 2769 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2770 sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs, 2771 IXL_RX_QUEUE_ALIGN) != 0) 2772 goto free; 2773 2774 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2775 rxm = &maps[i]; 2776 2777 if (bus_dmamap_create(sc->sc_dmat, 2778 IXL_HARDMTU, 1, IXL_HARDMTU, 0, 2779 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2780 goto uncreate; 2781 2782 rxm->rxm_m = NULL; 2783 } 2784 2785 rxr->rxr_cons = rxr->rxr_prod = 0; 2786 rxr->rxr_m_head = NULL; 2787 rxr->rxr_m_tail = &rxr->rxr_m_head; 2788 rxr->rxr_maps = maps; 2789 2790 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2791 rxr->rxr_qid = qid; 2792 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2793 2794 return rxr; 2795 2796 uncreate: 2797 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2798 rxm = &maps[i]; 2799 2800 if (rxm->rxm_map == NULL) 2801 continue; 2802 2803 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2804 } 2805 2806 ixl_dmamem_free(sc, &rxr->rxr_mem); 2807 free: 2808 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2809 kmem_free(rxr, sizeof(*rxr)); 2810 2811 return NULL; 2812 } 2813 2814 static void 2815 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2816 { 2817 struct ixl_rx_map *maps, *rxm; 2818 bus_dmamap_t map; 2819 unsigned int i; 2820 2821 maps = rxr->rxr_maps; 2822 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2823 rxm = &maps[i]; 2824 2825 if (rxm->rxm_m == NULL) 2826 continue; 2827 2828 map = rxm->rxm_map; 2829 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2830 BUS_DMASYNC_POSTWRITE); 2831 bus_dmamap_unload(sc->sc_dmat, map); 2832 2833 m_freem(rxm->rxm_m); 2834 rxm->rxm_m = NULL; 2835 } 2836 2837 m_freem(rxr->rxr_m_head); 2838 rxr->rxr_m_head = NULL; 2839 rxr->rxr_m_tail = &rxr->rxr_m_head; 2840 2841 rxr->rxr_prod = rxr->rxr_cons = 0; 2842 } 2843 2844 static int 2845 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2846 { 2847 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 2848 uint32_t reg; 2849 int i; 2850 2851 for (i = 0; i < 10; i++) { 2852 reg = ixl_rd(sc, ena); 2853 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 2854 return 0; 2855 2856 delaymsec(10); 2857 } 2858 2859 return ETIMEDOUT; 2860 } 2861 2862 static int 2863 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2864 { 2865 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 2866 uint32_t reg; 2867 int i; 2868 2869 KASSERT(mutex_owned(&rxr->rxr_lock)); 2870 2871 for (i = 0; i < 20; i++) { 2872 reg = ixl_rd(sc, ena); 2873 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 2874 return 0; 2875 2876 delaymsec(10); 2877 } 2878 2879 return ETIMEDOUT; 2880 } 2881 2882 static void 2883 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2884 { 2885 struct ixl_hmc_rxq rxq; 2886 void *hmc; 2887 2888 memset(&rxq, 0, sizeof(rxq)); 2889 2890 rxq.head = htole16(rxr->rxr_cons); 2891 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 2892 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 2893 rxq.dbuff = htole16(MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 2894 rxq.hbuff = 0; 2895 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 2896 rxq.dsize = IXL_HMC_RXQ_DSIZE_32; 2897 rxq.crcstrip = 1; 2898 rxq.l2sel = 1; 2899 rxq.showiv = 1; 2900 rxq.rxmax = htole16(IXL_HARDMTU); 2901 rxq.tphrdesc_ena = 0; 2902 rxq.tphwdesc_ena = 0; 2903 rxq.tphdata_ena = 0; 2904 rxq.tphhead_ena = 0; 2905 rxq.lrxqthresh = 0; 2906 rxq.prefena = 1; 2907 2908 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 2909 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 2910 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 2911 __arraycount(ixl_hmc_pack_rxq)); 2912 } 2913 2914 static void 2915 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2916 { 2917 void *hmc; 2918 2919 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 2920 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 2921 } 2922 2923 static void 2924 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2925 { 2926 struct ixl_rx_map *maps, *rxm; 2927 unsigned int i; 2928 2929 maps = rxr->rxr_maps; 2930 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2931 rxm = &maps[i]; 2932 2933 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2934 } 2935 2936 ixl_dmamem_free(sc, &rxr->rxr_mem); 2937 mutex_destroy(&rxr->rxr_lock); 2938 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2939 kmem_free(rxr, sizeof(*rxr)); 2940 } 2941 2942 static inline void 2943 ixl_rx_csum(struct mbuf *m, uint64_t qword) 2944 { 2945 int flags_mask; 2946 2947 if (!ISSET(qword, IXL_RX_DESC_L3L4P)) { 2948 /* No L3 or L4 checksum was calculated */ 2949 return; 2950 } 2951 2952 switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) { 2953 case IXL_RX_DESC_PTYPE_IPV4FRAG: 2954 case IXL_RX_DESC_PTYPE_IPV4: 2955 case IXL_RX_DESC_PTYPE_SCTPV4: 2956 case IXL_RX_DESC_PTYPE_ICMPV4: 2957 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 2958 break; 2959 case IXL_RX_DESC_PTYPE_TCPV4: 2960 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 2961 flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD; 2962 break; 2963 case IXL_RX_DESC_PTYPE_UDPV4: 2964 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 2965 flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD; 2966 break; 2967 case IXL_RX_DESC_PTYPE_TCPV6: 2968 flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD; 2969 break; 2970 case IXL_RX_DESC_PTYPE_UDPV6: 2971 flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD; 2972 break; 2973 default: 2974 flags_mask = 0; 2975 } 2976 2977 m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 | 2978 M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6)); 2979 2980 if (ISSET(qword, IXL_RX_DESC_IPE)) { 2981 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD); 2982 } 2983 2984 if (ISSET(qword, IXL_RX_DESC_L4E)) { 2985 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD); 2986 } 2987 } 2988 2989 static int 2990 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 2991 { 2992 struct ifnet *ifp = &sc->sc_ec.ec_if; 2993 struct ixl_rx_wb_desc_32 *ring, *rxd; 2994 struct ixl_rx_map *rxm; 2995 bus_dmamap_t map; 2996 unsigned int cons, prod; 2997 struct mbuf *m; 2998 uint64_t word, word0; 2999 unsigned int len; 3000 unsigned int mask; 3001 int done = 0, more = 0; 3002 3003 KASSERT(mutex_owned(&rxr->rxr_lock)); 3004 3005 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3006 return 0; 3007 3008 prod = rxr->rxr_prod; 3009 cons = rxr->rxr_cons; 3010 3011 if (cons == prod) 3012 return 0; 3013 3014 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3015 0, IXL_DMA_LEN(&rxr->rxr_mem), 3016 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3017 3018 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3019 mask = sc->sc_rx_ring_ndescs - 1; 3020 3021 do { 3022 if (rxlimit-- <= 0) { 3023 more = 1; 3024 break; 3025 } 3026 3027 rxd = &ring[cons]; 3028 3029 word = le64toh(rxd->qword1); 3030 3031 if (!ISSET(word, IXL_RX_DESC_DD)) 3032 break; 3033 3034 rxm = &rxr->rxr_maps[cons]; 3035 3036 map = rxm->rxm_map; 3037 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3038 BUS_DMASYNC_POSTREAD); 3039 bus_dmamap_unload(sc->sc_dmat, map); 3040 3041 m = rxm->rxm_m; 3042 rxm->rxm_m = NULL; 3043 3044 KASSERT(m != NULL); 3045 3046 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 3047 m->m_len = len; 3048 m->m_pkthdr.len = 0; 3049 3050 m->m_next = NULL; 3051 *rxr->rxr_m_tail = m; 3052 rxr->rxr_m_tail = &m->m_next; 3053 3054 m = rxr->rxr_m_head; 3055 m->m_pkthdr.len += len; 3056 3057 if (ISSET(word, IXL_RX_DESC_EOP)) { 3058 word0 = le64toh(rxd->qword0); 3059 3060 if (ISSET(word, IXL_RX_DESC_L2TAG1P)) { 3061 vlan_set_tag(m, 3062 __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK)); 3063 } 3064 3065 if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0) 3066 ixl_rx_csum(m, word); 3067 3068 if (!ISSET(word, 3069 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 3070 m_set_rcvif(m, ifp); 3071 rxr->rxr_ipackets++; 3072 rxr->rxr_ibytes += m->m_pkthdr.len; 3073 if_percpuq_enqueue(ifp->if_percpuq, m); 3074 } else { 3075 rxr->rxr_ierrors++; 3076 m_freem(m); 3077 } 3078 3079 rxr->rxr_m_head = NULL; 3080 rxr->rxr_m_tail = &rxr->rxr_m_head; 3081 } 3082 3083 cons++; 3084 cons &= mask; 3085 3086 done = 1; 3087 } while (cons != prod); 3088 3089 if (done) { 3090 rxr->rxr_cons = cons; 3091 if (ixl_rxfill(sc, rxr) == -1) 3092 rxr->rxr_iqdrops++; 3093 } 3094 3095 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3096 0, IXL_DMA_LEN(&rxr->rxr_mem), 3097 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3098 3099 return more; 3100 } 3101 3102 static int 3103 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3104 { 3105 struct ixl_rx_rd_desc_32 *ring, *rxd; 3106 struct ixl_rx_map *rxm; 3107 bus_dmamap_t map; 3108 struct mbuf *m; 3109 unsigned int prod; 3110 unsigned int slots; 3111 unsigned int mask; 3112 int post = 0, error = 0; 3113 3114 KASSERT(mutex_owned(&rxr->rxr_lock)); 3115 3116 prod = rxr->rxr_prod; 3117 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 3118 sc->sc_rx_ring_ndescs); 3119 3120 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3121 mask = sc->sc_rx_ring_ndescs - 1; 3122 3123 if (__predict_false(slots <= 0)) 3124 return -1; 3125 3126 do { 3127 rxm = &rxr->rxr_maps[prod]; 3128 3129 MGETHDR(m, M_DONTWAIT, MT_DATA); 3130 if (m == NULL) { 3131 rxr->rxr_mgethdr_failed.ev_count++; 3132 error = -1; 3133 break; 3134 } 3135 3136 MCLGET(m, M_DONTWAIT); 3137 if (!ISSET(m->m_flags, M_EXT)) { 3138 rxr->rxr_mgetcl_failed.ev_count++; 3139 error = -1; 3140 m_freem(m); 3141 break; 3142 } 3143 3144 m->m_len = m->m_pkthdr.len = MCLBYTES + ETHER_ALIGN; 3145 m_adj(m, ETHER_ALIGN); 3146 3147 map = rxm->rxm_map; 3148 3149 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 3150 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 3151 rxr->rxr_mbuf_load_failed.ev_count++; 3152 error = -1; 3153 m_freem(m); 3154 break; 3155 } 3156 3157 rxm->rxm_m = m; 3158 3159 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3160 BUS_DMASYNC_PREREAD); 3161 3162 rxd = &ring[prod]; 3163 3164 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 3165 rxd->haddr = htole64(0); 3166 3167 prod++; 3168 prod &= mask; 3169 3170 post = 1; 3171 3172 } while (--slots); 3173 3174 if (post) { 3175 rxr->rxr_prod = prod; 3176 ixl_wr(sc, rxr->rxr_tail, prod); 3177 } 3178 3179 return error; 3180 } 3181 3182 static inline int 3183 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 3184 u_int txlimit, struct evcnt *txevcnt, 3185 u_int rxlimit, struct evcnt *rxevcnt) 3186 { 3187 struct ixl_tx_ring *txr = qp->qp_txr; 3188 struct ixl_rx_ring *rxr = qp->qp_rxr; 3189 int txmore, rxmore; 3190 int rv; 3191 3192 KASSERT(!mutex_owned(&txr->txr_lock)); 3193 KASSERT(!mutex_owned(&rxr->rxr_lock)); 3194 3195 mutex_enter(&txr->txr_lock); 3196 txevcnt->ev_count++; 3197 txmore = ixl_txeof(sc, txr, txlimit); 3198 mutex_exit(&txr->txr_lock); 3199 3200 mutex_enter(&rxr->rxr_lock); 3201 rxevcnt->ev_count++; 3202 rxmore = ixl_rxeof(sc, rxr, rxlimit); 3203 mutex_exit(&rxr->rxr_lock); 3204 3205 rv = txmore | (rxmore << 1); 3206 3207 return rv; 3208 } 3209 3210 static void 3211 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 3212 { 3213 3214 if (qp->qp_workqueue) 3215 ixl_work_add(sc->sc_workq_txrx, &qp->qp_task); 3216 else 3217 softint_schedule(qp->qp_si); 3218 } 3219 3220 static int 3221 ixl_intr(void *xsc) 3222 { 3223 struct ixl_softc *sc = xsc; 3224 struct ixl_tx_ring *txr; 3225 struct ixl_rx_ring *rxr; 3226 uint32_t icr, rxintr, txintr; 3227 int rv = 0; 3228 unsigned int i; 3229 3230 KASSERT(sc != NULL); 3231 3232 ixl_enable_other_intr(sc); 3233 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3234 3235 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3236 atomic_inc_64(&sc->sc_event_atq.ev_count); 3237 ixl_atq_done(sc); 3238 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3239 rv = 1; 3240 } 3241 3242 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3243 atomic_inc_64(&sc->sc_event_link.ev_count); 3244 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3245 rv = 1; 3246 } 3247 3248 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 3249 txintr = icr & I40E_INTR_NOTX_TX_MASK; 3250 3251 if (txintr || rxintr) { 3252 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 3253 txr = sc->sc_qps[i].qp_txr; 3254 rxr = sc->sc_qps[i].qp_rxr; 3255 3256 ixl_handle_queue_common(sc, &sc->sc_qps[i], 3257 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 3258 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 3259 } 3260 rv = 1; 3261 } 3262 3263 return rv; 3264 } 3265 3266 static int 3267 ixl_queue_intr(void *xqp) 3268 { 3269 struct ixl_queue_pair *qp = xqp; 3270 struct ixl_tx_ring *txr = qp->qp_txr; 3271 struct ixl_rx_ring *rxr = qp->qp_rxr; 3272 struct ixl_softc *sc = qp->qp_sc; 3273 u_int txlimit, rxlimit; 3274 int more; 3275 3276 txlimit = sc->sc_tx_intr_process_limit; 3277 rxlimit = sc->sc_rx_intr_process_limit; 3278 qp->qp_workqueue = sc->sc_txrx_workqueue; 3279 3280 more = ixl_handle_queue_common(sc, qp, 3281 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3282 3283 if (more != 0) { 3284 ixl_sched_handle_queue(sc, qp); 3285 } else { 3286 /* for ALTQ */ 3287 if (txr->txr_qid == 0) 3288 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3289 softint_schedule(txr->txr_si); 3290 3291 ixl_enable_queue_intr(sc, qp); 3292 } 3293 3294 return 1; 3295 } 3296 3297 static void 3298 ixl_handle_queue(void *xqp) 3299 { 3300 struct ixl_queue_pair *qp = xqp; 3301 struct ixl_softc *sc = qp->qp_sc; 3302 struct ixl_tx_ring *txr = qp->qp_txr; 3303 struct ixl_rx_ring *rxr = qp->qp_rxr; 3304 u_int txlimit, rxlimit; 3305 int more; 3306 3307 txlimit = sc->sc_tx_process_limit; 3308 rxlimit = sc->sc_rx_process_limit; 3309 3310 more = ixl_handle_queue_common(sc, qp, 3311 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3312 3313 if (more != 0) 3314 ixl_sched_handle_queue(sc, qp); 3315 else 3316 ixl_enable_queue_intr(sc, qp); 3317 } 3318 3319 static inline void 3320 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3321 { 3322 uint32_t hmc_idx, hmc_isvf; 3323 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3324 3325 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3326 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3327 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3328 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3329 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3330 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3331 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3332 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3333 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3334 3335 device_printf(sc->sc_dev, 3336 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3337 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3338 } 3339 3340 static int 3341 ixl_other_intr(void *xsc) 3342 { 3343 struct ixl_softc *sc = xsc; 3344 uint32_t icr, mask, reg; 3345 int rv; 3346 3347 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3348 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3349 3350 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3351 atomic_inc_64(&sc->sc_event_atq.ev_count); 3352 ixl_atq_done(sc); 3353 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3354 rv = 1; 3355 } 3356 3357 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3358 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3359 device_printf(sc->sc_dev, "link stat changed\n"); 3360 3361 atomic_inc_64(&sc->sc_event_link.ev_count); 3362 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3363 rv = 1; 3364 } 3365 3366 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3367 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3368 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3369 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3370 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3371 3372 device_printf(sc->sc_dev, "GRST: %s\n", 3373 reg == I40E_RESET_CORER ? "CORER" : 3374 reg == I40E_RESET_GLOBR ? "GLOBR" : 3375 reg == I40E_RESET_EMPR ? "EMPR" : 3376 "POR"); 3377 } 3378 3379 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3380 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3381 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3382 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3383 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3384 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3385 3386 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3387 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3388 device_printf(sc->sc_dev, "critical error\n"); 3389 } 3390 3391 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3392 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3393 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3394 ixl_print_hmc_error(sc, reg); 3395 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3396 } 3397 3398 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3399 ixl_flush(sc); 3400 ixl_enable_other_intr(sc); 3401 return rv; 3402 } 3403 3404 static void 3405 ixl_get_link_status_done(struct ixl_softc *sc, 3406 const struct ixl_aq_desc *iaq) 3407 { 3408 3409 ixl_link_state_update(sc, iaq); 3410 } 3411 3412 static void 3413 ixl_get_link_status(void *xsc) 3414 { 3415 struct ixl_softc *sc = xsc; 3416 struct ixl_aq_desc *iaq; 3417 struct ixl_aq_link_param *param; 3418 3419 memset(&sc->sc_link_state_atq, 0, sizeof(sc->sc_link_state_atq)); 3420 iaq = &sc->sc_link_state_atq.iatq_desc; 3421 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3422 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3423 param->notify = IXL_AQ_LINK_NOTIFY; 3424 3425 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 3426 (void)ixl_atq_post(sc, &sc->sc_link_state_atq); 3427 } 3428 3429 static void 3430 ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3431 { 3432 struct ifnet *ifp = &sc->sc_ec.ec_if; 3433 int link_state; 3434 3435 KASSERT(kpreempt_disabled()); 3436 3437 link_state = ixl_set_link_status(sc, iaq); 3438 3439 if (ifp->if_link_state != link_state) 3440 if_link_state_change(ifp, link_state); 3441 3442 if (link_state != LINK_STATE_DOWN) { 3443 if_schedule_deferred_start(ifp); 3444 } 3445 } 3446 3447 static void 3448 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3449 const char *msg) 3450 { 3451 char buf[512]; 3452 size_t len; 3453 3454 len = sizeof(buf); 3455 buf[--len] = '\0'; 3456 3457 device_printf(sc->sc_dev, "%s\n", msg); 3458 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3459 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3460 buf, le16toh(iaq->iaq_opcode)); 3461 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3462 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3463 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3464 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3465 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3466 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3467 } 3468 3469 static void 3470 ixl_arq(void *xsc) 3471 { 3472 struct ixl_softc *sc = xsc; 3473 struct ixl_aq_desc *arq, *iaq; 3474 struct ixl_aq_buf *aqb; 3475 unsigned int cons = sc->sc_arq_cons; 3476 unsigned int prod; 3477 int done = 0; 3478 3479 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3480 sc->sc_aq_regs->arq_head_mask; 3481 3482 if (cons == prod) 3483 goto done; 3484 3485 arq = IXL_DMA_KVA(&sc->sc_arq); 3486 3487 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3488 0, IXL_DMA_LEN(&sc->sc_arq), 3489 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3490 3491 do { 3492 iaq = &arq[cons]; 3493 aqb = sc->sc_arq_live[cons]; 3494 3495 KASSERT(aqb != NULL); 3496 3497 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3498 BUS_DMASYNC_POSTREAD); 3499 3500 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3501 ixl_aq_dump(sc, iaq, "arq event"); 3502 3503 switch (iaq->iaq_opcode) { 3504 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3505 kpreempt_disable(); 3506 ixl_link_state_update(sc, iaq); 3507 kpreempt_enable(); 3508 break; 3509 } 3510 3511 memset(iaq, 0, sizeof(*iaq)); 3512 sc->sc_arq_live[cons] = NULL; 3513 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3514 3515 cons++; 3516 cons &= IXL_AQ_MASK; 3517 3518 done = 1; 3519 } while (cons != prod); 3520 3521 if (done) { 3522 sc->sc_arq_cons = cons; 3523 ixl_arq_fill(sc); 3524 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3525 0, IXL_DMA_LEN(&sc->sc_arq), 3526 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3527 } 3528 3529 done: 3530 ixl_enable_other_intr(sc); 3531 } 3532 3533 static void 3534 ixl_atq_set(struct ixl_atq *iatq, 3535 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3536 { 3537 3538 iatq->iatq_fn = fn; 3539 } 3540 3541 static int 3542 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3543 { 3544 struct ixl_aq_desc *atq, *slot; 3545 unsigned int prod, cons, prod_next; 3546 3547 /* assert locked */ 3548 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3549 3550 atq = IXL_DMA_KVA(&sc->sc_atq); 3551 prod = sc->sc_atq_prod; 3552 cons = sc->sc_atq_cons; 3553 prod_next = (prod +1) & IXL_AQ_MASK; 3554 3555 if (cons == prod_next) 3556 return ENOMEM; 3557 3558 slot = &atq[prod]; 3559 3560 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3561 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3562 3563 *slot = iatq->iatq_desc; 3564 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3565 3566 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3567 ixl_aq_dump(sc, slot, "atq command"); 3568 3569 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3570 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3571 3572 sc->sc_atq_prod = prod_next; 3573 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3574 3575 return 0; 3576 } 3577 3578 static int 3579 ixl_atq_post(struct ixl_softc *sc, struct ixl_atq *iatq) 3580 { 3581 int rv; 3582 3583 mutex_enter(&sc->sc_atq_lock); 3584 rv = ixl_atq_post_locked(sc, iatq); 3585 mutex_exit(&sc->sc_atq_lock); 3586 3587 return rv; 3588 } 3589 3590 static void 3591 ixl_atq_done_locked(struct ixl_softc *sc) 3592 { 3593 struct ixl_aq_desc *atq, *slot; 3594 struct ixl_atq *iatq; 3595 unsigned int cons; 3596 unsigned int prod; 3597 3598 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3599 3600 prod = sc->sc_atq_prod; 3601 cons = sc->sc_atq_cons; 3602 3603 if (prod == cons) 3604 return; 3605 3606 atq = IXL_DMA_KVA(&sc->sc_atq); 3607 3608 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3609 0, IXL_DMA_LEN(&sc->sc_atq), 3610 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3611 3612 do { 3613 slot = &atq[cons]; 3614 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3615 break; 3616 3617 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3618 iatq->iatq_desc = *slot; 3619 3620 memset(slot, 0, sizeof(*slot)); 3621 3622 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3623 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3624 3625 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3626 3627 cons++; 3628 cons &= IXL_AQ_MASK; 3629 } while (cons != prod); 3630 3631 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3632 0, IXL_DMA_LEN(&sc->sc_atq), 3633 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3634 3635 sc->sc_atq_cons = cons; 3636 } 3637 3638 static void 3639 ixl_atq_done(struct ixl_softc *sc) 3640 { 3641 3642 mutex_enter(&sc->sc_atq_lock); 3643 ixl_atq_done_locked(sc); 3644 mutex_exit(&sc->sc_atq_lock); 3645 } 3646 3647 static void 3648 ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3649 { 3650 3651 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3652 3653 cv_signal(&sc->sc_atq_cv); 3654 } 3655 3656 static int 3657 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3658 { 3659 int error; 3660 3661 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3662 3663 ixl_atq_set(iatq, ixl_wakeup); 3664 3665 mutex_enter(&sc->sc_atq_lock); 3666 error = ixl_atq_post_locked(sc, iatq); 3667 if (error) { 3668 mutex_exit(&sc->sc_atq_lock); 3669 return error; 3670 } 3671 3672 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3673 IXL_ATQ_EXEC_TIMEOUT); 3674 mutex_exit(&sc->sc_atq_lock); 3675 3676 return error; 3677 } 3678 3679 static int 3680 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3681 { 3682 struct ixl_aq_desc *atq, *slot; 3683 unsigned int prod; 3684 unsigned int t = 0; 3685 3686 mutex_enter(&sc->sc_atq_lock); 3687 3688 atq = IXL_DMA_KVA(&sc->sc_atq); 3689 prod = sc->sc_atq_prod; 3690 slot = atq + prod; 3691 3692 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3693 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3694 3695 *slot = *iaq; 3696 slot->iaq_flags |= htole16(IXL_AQ_SI); 3697 3698 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3699 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3700 3701 prod++; 3702 prod &= IXL_AQ_MASK; 3703 sc->sc_atq_prod = prod; 3704 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3705 3706 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3707 delaymsec(1); 3708 3709 if (t++ > tm) { 3710 mutex_exit(&sc->sc_atq_lock); 3711 return ETIMEDOUT; 3712 } 3713 } 3714 3715 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3716 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3717 *iaq = *slot; 3718 memset(slot, 0, sizeof(*slot)); 3719 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3720 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3721 3722 sc->sc_atq_cons = prod; 3723 3724 mutex_exit(&sc->sc_atq_lock); 3725 3726 return 0; 3727 } 3728 3729 static int 3730 ixl_get_version(struct ixl_softc *sc) 3731 { 3732 struct ixl_aq_desc iaq; 3733 uint32_t fwbuild, fwver, apiver; 3734 uint16_t api_maj_ver, api_min_ver; 3735 3736 memset(&iaq, 0, sizeof(iaq)); 3737 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3738 3739 iaq.iaq_retval = le16toh(23); 3740 3741 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3742 return ETIMEDOUT; 3743 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3744 return EIO; 3745 3746 fwbuild = le32toh(iaq.iaq_param[1]); 3747 fwver = le32toh(iaq.iaq_param[2]); 3748 apiver = le32toh(iaq.iaq_param[3]); 3749 3750 api_maj_ver = (uint16_t)apiver; 3751 api_min_ver = (uint16_t)(apiver >> 16); 3752 3753 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 3754 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 3755 3756 sc->sc_rxctl_atq = true; 3757 if (sc->sc_mac_type == I40E_MAC_X722) { 3758 if (api_maj_ver == 1 && api_min_ver < 5) { 3759 sc->sc_rxctl_atq = false; 3760 } 3761 } 3762 3763 return 0; 3764 } 3765 3766 static int 3767 ixl_pxe_clear(struct ixl_softc *sc) 3768 { 3769 struct ixl_aq_desc iaq; 3770 int rv; 3771 3772 memset(&iaq, 0, sizeof(iaq)); 3773 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 3774 iaq.iaq_param[0] = htole32(0x2); 3775 3776 rv = ixl_atq_poll(sc, &iaq, 250); 3777 3778 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 3779 3780 if (rv != 0) 3781 return ETIMEDOUT; 3782 3783 switch (iaq.iaq_retval) { 3784 case htole16(IXL_AQ_RC_OK): 3785 case htole16(IXL_AQ_RC_EEXIST): 3786 break; 3787 default: 3788 return EIO; 3789 } 3790 3791 return 0; 3792 } 3793 3794 static int 3795 ixl_lldp_shut(struct ixl_softc *sc) 3796 { 3797 struct ixl_aq_desc iaq; 3798 3799 memset(&iaq, 0, sizeof(iaq)); 3800 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 3801 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 3802 3803 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 3804 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 3805 return -1; 3806 } 3807 3808 switch (iaq.iaq_retval) { 3809 case htole16(IXL_AQ_RC_EMODE): 3810 case htole16(IXL_AQ_RC_EPERM): 3811 /* ignore silently */ 3812 default: 3813 break; 3814 } 3815 3816 return 0; 3817 } 3818 3819 static void 3820 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 3821 { 3822 uint16_t id; 3823 uint32_t number, logical_id; 3824 3825 id = le16toh(cap->cap_id); 3826 number = le32toh(cap->number); 3827 logical_id = le32toh(cap->logical_id); 3828 3829 switch (id) { 3830 case IXL_AQ_CAP_RSS: 3831 sc->sc_rss_table_size = number; 3832 sc->sc_rss_table_entry_width = logical_id; 3833 break; 3834 case IXL_AQ_CAP_RXQ: 3835 case IXL_AQ_CAP_TXQ: 3836 sc->sc_nqueue_pairs_device = MIN(number, 3837 sc->sc_nqueue_pairs_device); 3838 break; 3839 } 3840 } 3841 3842 static int 3843 ixl_get_hw_capabilities(struct ixl_softc *sc) 3844 { 3845 struct ixl_dmamem idm; 3846 struct ixl_aq_desc iaq; 3847 struct ixl_aq_capability *caps; 3848 size_t i, ncaps; 3849 bus_size_t caps_size; 3850 uint16_t status; 3851 int rv; 3852 3853 caps_size = sizeof(caps[0]) * 40; 3854 memset(&iaq, 0, sizeof(iaq)); 3855 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 3856 3857 do { 3858 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 3859 return -1; 3860 } 3861 3862 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3863 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3864 iaq.iaq_datalen = htole16(caps_size); 3865 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3866 3867 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 3868 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 3869 3870 rv = ixl_atq_poll(sc, &iaq, 250); 3871 3872 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 3873 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 3874 3875 if (rv != 0) { 3876 aprint_error(", HW capabilities timeout\n"); 3877 goto done; 3878 } 3879 3880 status = le16toh(iaq.iaq_retval); 3881 3882 if (status == IXL_AQ_RC_ENOMEM) { 3883 caps_size = le16toh(iaq.iaq_datalen); 3884 ixl_dmamem_free(sc, &idm); 3885 } 3886 } while (status == IXL_AQ_RC_ENOMEM); 3887 3888 if (status != IXL_AQ_RC_OK) { 3889 aprint_error(", HW capabilities error\n"); 3890 goto done; 3891 } 3892 3893 caps = IXL_DMA_KVA(&idm); 3894 ncaps = le16toh(iaq.iaq_param[1]); 3895 3896 for (i = 0; i < ncaps; i++) { 3897 ixl_parse_hw_capability(sc, &caps[i]); 3898 } 3899 3900 done: 3901 ixl_dmamem_free(sc, &idm); 3902 return rv; 3903 } 3904 3905 static int 3906 ixl_get_mac(struct ixl_softc *sc) 3907 { 3908 struct ixl_dmamem idm; 3909 struct ixl_aq_desc iaq; 3910 struct ixl_aq_mac_addresses *addrs; 3911 int rv; 3912 3913 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 3914 aprint_error(", unable to allocate mac addresses\n"); 3915 return -1; 3916 } 3917 3918 memset(&iaq, 0, sizeof(iaq)); 3919 iaq.iaq_flags = htole16(IXL_AQ_BUF); 3920 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 3921 iaq.iaq_datalen = htole16(sizeof(*addrs)); 3922 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3923 3924 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3925 BUS_DMASYNC_PREREAD); 3926 3927 rv = ixl_atq_poll(sc, &iaq, 250); 3928 3929 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3930 BUS_DMASYNC_POSTREAD); 3931 3932 if (rv != 0) { 3933 aprint_error(", MAC ADDRESS READ timeout\n"); 3934 rv = -1; 3935 goto done; 3936 } 3937 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3938 aprint_error(", MAC ADDRESS READ error\n"); 3939 rv = -1; 3940 goto done; 3941 } 3942 3943 addrs = IXL_DMA_KVA(&idm); 3944 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 3945 printf(", port address is not valid\n"); 3946 goto done; 3947 } 3948 3949 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 3950 rv = 0; 3951 3952 done: 3953 ixl_dmamem_free(sc, &idm); 3954 return rv; 3955 } 3956 3957 static int 3958 ixl_get_switch_config(struct ixl_softc *sc) 3959 { 3960 struct ixl_dmamem idm; 3961 struct ixl_aq_desc iaq; 3962 struct ixl_aq_switch_config *hdr; 3963 struct ixl_aq_switch_config_element *elms, *elm; 3964 unsigned int nelm, i; 3965 int rv; 3966 3967 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 3968 aprint_error_dev(sc->sc_dev, 3969 "unable to allocate switch config buffer\n"); 3970 return -1; 3971 } 3972 3973 memset(&iaq, 0, sizeof(iaq)); 3974 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3975 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3976 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 3977 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 3978 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3979 3980 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3981 BUS_DMASYNC_PREREAD); 3982 3983 rv = ixl_atq_poll(sc, &iaq, 250); 3984 3985 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3986 BUS_DMASYNC_POSTREAD); 3987 3988 if (rv != 0) { 3989 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 3990 rv = -1; 3991 goto done; 3992 } 3993 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3994 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 3995 rv = -1; 3996 goto done; 3997 } 3998 3999 hdr = IXL_DMA_KVA(&idm); 4000 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 4001 4002 nelm = le16toh(hdr->num_reported); 4003 if (nelm < 1) { 4004 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 4005 rv = -1; 4006 goto done; 4007 } 4008 4009 for (i = 0; i < nelm; i++) { 4010 elm = &elms[i]; 4011 4012 aprint_debug_dev(sc->sc_dev, 4013 "type %x revision %u seid %04x\n", 4014 elm->type, elm->revision, le16toh(elm->seid)); 4015 aprint_debug_dev(sc->sc_dev, 4016 "uplink %04x downlink %04x\n", 4017 le16toh(elm->uplink_seid), 4018 le16toh(elm->downlink_seid)); 4019 aprint_debug_dev(sc->sc_dev, 4020 "conntype %x scheduler %04x extra %04x\n", 4021 elm->connection_type, 4022 le16toh(elm->scheduler_id), 4023 le16toh(elm->element_info)); 4024 } 4025 4026 elm = &elms[0]; 4027 4028 sc->sc_uplink_seid = elm->uplink_seid; 4029 sc->sc_downlink_seid = elm->downlink_seid; 4030 sc->sc_seid = elm->seid; 4031 4032 if ((sc->sc_uplink_seid == htole16(0)) != 4033 (sc->sc_downlink_seid == htole16(0))) { 4034 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 4035 rv = -1; 4036 goto done; 4037 } 4038 4039 done: 4040 ixl_dmamem_free(sc, &idm); 4041 return rv; 4042 } 4043 4044 static int 4045 ixl_phy_mask_ints(struct ixl_softc *sc) 4046 { 4047 struct ixl_aq_desc iaq; 4048 4049 memset(&iaq, 0, sizeof(iaq)); 4050 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 4051 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 4052 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 4053 IXL_AQ_PHY_EV_MEDIA_NA)); 4054 4055 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4056 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 4057 return -1; 4058 } 4059 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4060 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 4061 return -1; 4062 } 4063 4064 return 0; 4065 } 4066 4067 static int 4068 ixl_get_phy_abilities(struct ixl_softc *sc,struct ixl_dmamem *idm) 4069 { 4070 struct ixl_aq_desc iaq; 4071 int rv; 4072 4073 memset(&iaq, 0, sizeof(iaq)); 4074 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4075 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4076 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 4077 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 4078 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 4079 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 4080 4081 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4082 BUS_DMASYNC_PREREAD); 4083 4084 rv = ixl_atq_poll(sc, &iaq, 250); 4085 4086 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4087 BUS_DMASYNC_POSTREAD); 4088 4089 if (rv != 0) 4090 return -1; 4091 4092 return le16toh(iaq.iaq_retval); 4093 } 4094 4095 static int 4096 ixl_get_phy_types(struct ixl_softc *sc, uint64_t *phy_types_ptr) 4097 { 4098 struct ixl_dmamem idm; 4099 struct ixl_aq_phy_abilities *phy; 4100 uint64_t phy_types; 4101 int rv; 4102 4103 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4104 aprint_error_dev(sc->sc_dev, 4105 "unable to allocate switch config buffer\n"); 4106 return -1; 4107 } 4108 4109 rv = ixl_get_phy_abilities(sc, &idm); 4110 switch (rv) { 4111 case -1: 4112 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 4113 goto done; 4114 case IXL_AQ_RC_OK: 4115 break; 4116 case IXL_AQ_RC_EIO: 4117 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 4118 break; 4119 default: 4120 aprint_error_dev(sc->sc_dev, 4121 "GET PHY ABILITIIES error %u\n", rv); 4122 goto done; 4123 } 4124 4125 phy = IXL_DMA_KVA(&idm); 4126 4127 phy_types = le32toh(phy->phy_type); 4128 phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 4129 4130 *phy_types_ptr = phy_types; 4131 4132 rv = 0; 4133 4134 done: 4135 ixl_dmamem_free(sc, &idm); 4136 return rv; 4137 } 4138 4139 static int 4140 ixl_get_link_status_poll(struct ixl_softc *sc) 4141 { 4142 struct ixl_aq_desc iaq; 4143 struct ixl_aq_link_param *param; 4144 int link; 4145 4146 memset(&iaq, 0, sizeof(iaq)); 4147 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 4148 param = (struct ixl_aq_link_param *)iaq.iaq_param; 4149 param->notify = IXL_AQ_LINK_NOTIFY; 4150 4151 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4152 return ETIMEDOUT; 4153 } 4154 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4155 return EIO; 4156 } 4157 4158 link = ixl_set_link_status(sc, &iaq); 4159 sc->sc_ec.ec_if.if_link_state = link; 4160 4161 return 0; 4162 } 4163 4164 static int 4165 ixl_get_vsi(struct ixl_softc *sc) 4166 { 4167 struct ixl_dmamem *vsi = &sc->sc_scratch; 4168 struct ixl_aq_desc iaq; 4169 struct ixl_aq_vsi_param *param; 4170 struct ixl_aq_vsi_reply *reply; 4171 int rv; 4172 4173 /* grumble, vsi info isn't "known" at compile time */ 4174 4175 memset(&iaq, 0, sizeof(iaq)); 4176 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4177 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4178 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 4179 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4180 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4181 4182 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4183 param->uplink_seid = sc->sc_seid; 4184 4185 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4186 BUS_DMASYNC_PREREAD); 4187 4188 rv = ixl_atq_poll(sc, &iaq, 250); 4189 4190 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4191 BUS_DMASYNC_POSTREAD); 4192 4193 if (rv != 0) { 4194 return ETIMEDOUT; 4195 } 4196 4197 switch (le16toh(iaq.iaq_retval)) { 4198 case IXL_AQ_RC_OK: 4199 break; 4200 case IXL_AQ_RC_ENOENT: 4201 return ENOENT; 4202 case IXL_AQ_RC_EACCES: 4203 return EACCES; 4204 default: 4205 return EIO; 4206 } 4207 4208 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 4209 sc->sc_vsi_number = reply->vsi_number; 4210 4211 return 0; 4212 } 4213 4214 static int 4215 ixl_set_vsi(struct ixl_softc *sc) 4216 { 4217 struct ixl_dmamem *vsi = &sc->sc_scratch; 4218 struct ixl_aq_desc iaq; 4219 struct ixl_aq_vsi_param *param; 4220 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 4221 unsigned int qnum; 4222 uint16_t val; 4223 int rv; 4224 4225 qnum = sc->sc_nqueue_pairs - 1; 4226 4227 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 4228 IXL_AQ_VSI_VALID_VLAN); 4229 4230 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 4231 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 4232 data->queue_mapping[0] = htole16(0); 4233 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 4234 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 4235 4236 val = le16toh(data->port_vlan_flags); 4237 CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK); 4238 SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL); 4239 4240 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { 4241 SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH); 4242 } else { 4243 SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING); 4244 } 4245 4246 data->port_vlan_flags = htole16(val); 4247 4248 /* grumble, vsi info isn't "known" at compile time */ 4249 4250 memset(&iaq, 0, sizeof(iaq)); 4251 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4252 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4253 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 4254 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4255 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4256 4257 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4258 param->uplink_seid = sc->sc_seid; 4259 4260 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4261 BUS_DMASYNC_PREWRITE); 4262 4263 rv = ixl_atq_poll(sc, &iaq, 250); 4264 4265 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4266 BUS_DMASYNC_POSTWRITE); 4267 4268 if (rv != 0) { 4269 return ETIMEDOUT; 4270 } 4271 4272 switch (le16toh(iaq.iaq_retval)) { 4273 case IXL_AQ_RC_OK: 4274 break; 4275 case IXL_AQ_RC_ENOENT: 4276 return ENOENT; 4277 case IXL_AQ_RC_EACCES: 4278 return EACCES; 4279 default: 4280 return EIO; 4281 } 4282 4283 return 0; 4284 } 4285 4286 static void 4287 ixl_set_filter_control(struct ixl_softc *sc) 4288 { 4289 uint32_t reg; 4290 4291 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4292 4293 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4294 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4295 4296 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4297 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4298 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4299 4300 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4301 } 4302 4303 static inline void 4304 ixl_get_default_rss_key(uint32_t *buf, size_t len) 4305 { 4306 size_t cplen; 4307 uint8_t rss_seed[RSS_KEYSIZE]; 4308 4309 rss_getkey(rss_seed); 4310 memset(buf, 0, len); 4311 4312 cplen = MIN(len, sizeof(rss_seed)); 4313 memcpy(buf, rss_seed, cplen); 4314 } 4315 4316 static void 4317 ixl_set_rss_key(struct ixl_softc *sc) 4318 { 4319 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4320 size_t i; 4321 4322 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4323 4324 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4325 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4326 } 4327 } 4328 4329 static void 4330 ixl_set_rss_pctype(struct ixl_softc *sc) 4331 { 4332 uint64_t set_hena = 0; 4333 uint32_t hena0, hena1; 4334 4335 if (sc->sc_mac_type == I40E_MAC_X722) 4336 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4337 else 4338 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4339 4340 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4341 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4342 4343 SET(hena0, set_hena); 4344 SET(hena1, set_hena >> 32); 4345 4346 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4347 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4348 } 4349 4350 static void 4351 ixl_set_rss_hlut(struct ixl_softc *sc) 4352 { 4353 unsigned int qid; 4354 uint8_t hlut_buf[512], lut_mask; 4355 uint32_t *hluts; 4356 size_t i, hluts_num; 4357 4358 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4359 4360 for (i = 0; i < sc->sc_rss_table_size; i++) { 4361 qid = i % sc->sc_nqueue_pairs; 4362 hlut_buf[i] = qid & lut_mask; 4363 } 4364 4365 hluts = (uint32_t *)hlut_buf; 4366 hluts_num = sc->sc_rss_table_size >> 2; 4367 for (i = 0; i < hluts_num; i++) { 4368 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4369 } 4370 ixl_flush(sc); 4371 } 4372 4373 static void 4374 ixl_config_rss(struct ixl_softc *sc) 4375 { 4376 4377 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4378 4379 ixl_set_rss_key(sc); 4380 ixl_set_rss_pctype(sc); 4381 ixl_set_rss_hlut(sc); 4382 } 4383 4384 static const struct ixl_phy_type * 4385 ixl_search_phy_type(uint8_t phy_type) 4386 { 4387 const struct ixl_phy_type *itype; 4388 uint64_t mask; 4389 unsigned int i; 4390 4391 if (phy_type >= 64) 4392 return NULL; 4393 4394 mask = 1ULL << phy_type; 4395 4396 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4397 itype = &ixl_phy_type_map[i]; 4398 4399 if (ISSET(itype->phy_type, mask)) 4400 return itype; 4401 } 4402 4403 return NULL; 4404 } 4405 4406 static uint64_t 4407 ixl_search_link_speed(uint8_t link_speed) 4408 { 4409 const struct ixl_speed_type *type; 4410 unsigned int i; 4411 4412 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4413 type = &ixl_speed_type_map[i]; 4414 4415 if (ISSET(type->dev_speed, link_speed)) 4416 return type->net_speed; 4417 } 4418 4419 return 0; 4420 } 4421 4422 static int 4423 ixl_restart_an(struct ixl_softc *sc) 4424 { 4425 struct ixl_aq_desc iaq; 4426 4427 memset(&iaq, 0, sizeof(iaq)); 4428 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4429 iaq.iaq_param[0] = 4430 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4431 4432 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4433 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4434 return -1; 4435 } 4436 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4437 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4438 return -1; 4439 } 4440 4441 return 0; 4442 } 4443 4444 static int 4445 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4446 uint16_t vlan, uint16_t flags) 4447 { 4448 struct ixl_aq_desc iaq; 4449 struct ixl_aq_add_macvlan *param; 4450 struct ixl_aq_add_macvlan_elem *elem; 4451 4452 memset(&iaq, 0, sizeof(iaq)); 4453 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4454 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4455 iaq.iaq_datalen = htole16(sizeof(*elem)); 4456 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4457 4458 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4459 param->num_addrs = htole16(1); 4460 param->seid0 = htole16(0x8000) | sc->sc_seid; 4461 param->seid1 = 0; 4462 param->seid2 = 0; 4463 4464 elem = IXL_DMA_KVA(&sc->sc_scratch); 4465 memset(elem, 0, sizeof(*elem)); 4466 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4467 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4468 elem->vlan = htole16(vlan); 4469 4470 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4471 return IXL_AQ_RC_EINVAL; 4472 } 4473 4474 switch (le16toh(iaq.iaq_retval)) { 4475 case IXL_AQ_RC_OK: 4476 break; 4477 case IXL_AQ_RC_ENOSPC: 4478 return ENOSPC; 4479 case IXL_AQ_RC_ENOENT: 4480 return ENOENT; 4481 case IXL_AQ_RC_EACCES: 4482 return EACCES; 4483 case IXL_AQ_RC_EEXIST: 4484 return EEXIST; 4485 case IXL_AQ_RC_EINVAL: 4486 return EINVAL; 4487 default: 4488 return EIO; 4489 } 4490 4491 return 0; 4492 } 4493 4494 static int 4495 ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4496 uint16_t vlan, uint16_t flags) 4497 { 4498 struct ixl_aq_desc iaq; 4499 struct ixl_aq_remove_macvlan *param; 4500 struct ixl_aq_remove_macvlan_elem *elem; 4501 4502 memset(&iaq, 0, sizeof(iaq)); 4503 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4504 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 4505 iaq.iaq_datalen = htole16(sizeof(*elem)); 4506 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4507 4508 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 4509 param->num_addrs = htole16(1); 4510 param->seid0 = htole16(0x8000) | sc->sc_seid; 4511 param->seid1 = 0; 4512 param->seid2 = 0; 4513 4514 elem = IXL_DMA_KVA(&sc->sc_scratch); 4515 memset(elem, 0, sizeof(*elem)); 4516 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4517 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 4518 elem->vlan = htole16(vlan); 4519 4520 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4521 return EINVAL; 4522 } 4523 4524 switch (le16toh(iaq.iaq_retval)) { 4525 case IXL_AQ_RC_OK: 4526 break; 4527 case IXL_AQ_RC_ENOENT: 4528 return ENOENT; 4529 case IXL_AQ_RC_EACCES: 4530 return EACCES; 4531 case IXL_AQ_RC_EINVAL: 4532 return EINVAL; 4533 default: 4534 return EIO; 4535 } 4536 4537 return 0; 4538 } 4539 4540 static int 4541 ixl_hmc(struct ixl_softc *sc) 4542 { 4543 struct { 4544 uint32_t count; 4545 uint32_t minsize; 4546 bus_size_t objsiz; 4547 bus_size_t setoff; 4548 bus_size_t setcnt; 4549 } regs[] = { 4550 { 4551 0, 4552 IXL_HMC_TXQ_MINSIZE, 4553 I40E_GLHMC_LANTXOBJSZ, 4554 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 4555 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 4556 }, 4557 { 4558 0, 4559 IXL_HMC_RXQ_MINSIZE, 4560 I40E_GLHMC_LANRXOBJSZ, 4561 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 4562 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 4563 }, 4564 { 4565 0, 4566 0, 4567 I40E_GLHMC_FCOEDDPOBJSZ, 4568 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 4569 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 4570 }, 4571 { 4572 0, 4573 0, 4574 I40E_GLHMC_FCOEFOBJSZ, 4575 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 4576 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 4577 }, 4578 }; 4579 struct ixl_hmc_entry *e; 4580 uint64_t size, dva; 4581 uint8_t *kva; 4582 uint64_t *sdpage; 4583 unsigned int i; 4584 int npages, tables; 4585 uint32_t reg; 4586 4587 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 4588 4589 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 4590 ixl_rd(sc, I40E_GLHMC_LANQMAX); 4591 4592 size = 0; 4593 for (i = 0; i < __arraycount(regs); i++) { 4594 e = &sc->sc_hmc_entries[i]; 4595 4596 e->hmc_count = regs[i].count; 4597 reg = ixl_rd(sc, regs[i].objsiz); 4598 e->hmc_size = BIT_ULL(0x3F & reg); 4599 e->hmc_base = size; 4600 4601 if ((e->hmc_size * 8) < regs[i].minsize) { 4602 aprint_error_dev(sc->sc_dev, 4603 "kernel hmc entry is too big\n"); 4604 return -1; 4605 } 4606 4607 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 4608 } 4609 size = roundup(size, IXL_HMC_PGSIZE); 4610 npages = size / IXL_HMC_PGSIZE; 4611 4612 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 4613 4614 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 4615 aprint_error_dev(sc->sc_dev, 4616 "unable to allocate hmc pd memory\n"); 4617 return -1; 4618 } 4619 4620 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 4621 IXL_HMC_PGSIZE) != 0) { 4622 aprint_error_dev(sc->sc_dev, 4623 "unable to allocate hmc sd memory\n"); 4624 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 4625 return -1; 4626 } 4627 4628 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 4629 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 4630 4631 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 4632 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 4633 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4634 4635 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 4636 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 4637 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 4638 4639 for (i = 0; (int)i < npages; i++) { 4640 *sdpage = htole64(dva | IXL_HMC_PDVALID); 4641 sdpage++; 4642 4643 dva += IXL_HMC_PGSIZE; 4644 } 4645 4646 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 4647 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 4648 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4649 4650 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 4651 for (i = 0; (int)i < tables; i++) { 4652 uint32_t count; 4653 4654 KASSERT(npages >= 0); 4655 4656 count = ((unsigned int)npages > IXL_HMC_PGS) ? 4657 IXL_HMC_PGS : (unsigned int)npages; 4658 4659 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 4660 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 4661 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 4662 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 4663 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 4664 ixl_wr(sc, I40E_PFHMC_SDCMD, 4665 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 4666 4667 npages -= IXL_HMC_PGS; 4668 dva += IXL_HMC_PGSIZE; 4669 } 4670 4671 for (i = 0; i < __arraycount(regs); i++) { 4672 e = &sc->sc_hmc_entries[i]; 4673 4674 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 4675 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 4676 } 4677 4678 return 0; 4679 } 4680 4681 static void 4682 ixl_hmc_free(struct ixl_softc *sc) 4683 { 4684 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 4685 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 4686 } 4687 4688 static void 4689 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 4690 unsigned int npacking) 4691 { 4692 uint8_t *dst = d; 4693 const uint8_t *src = s; 4694 unsigned int i; 4695 4696 for (i = 0; i < npacking; i++) { 4697 const struct ixl_hmc_pack *pack = &packing[i]; 4698 unsigned int offset = pack->lsb / 8; 4699 unsigned int align = pack->lsb % 8; 4700 const uint8_t *in = src + pack->offset; 4701 uint8_t *out = dst + offset; 4702 int width = pack->width; 4703 unsigned int inbits = 0; 4704 4705 if (align) { 4706 inbits = (*in++) << align; 4707 *out++ |= (inbits & 0xff); 4708 inbits >>= 8; 4709 4710 width -= 8 - align; 4711 } 4712 4713 while (width >= 8) { 4714 inbits |= (*in++) << align; 4715 *out++ = (inbits & 0xff); 4716 inbits >>= 8; 4717 4718 width -= 8; 4719 } 4720 4721 if (width > 0) { 4722 inbits |= (*in) << align; 4723 *out |= (inbits & ((1 << width) - 1)); 4724 } 4725 } 4726 } 4727 4728 static struct ixl_aq_buf * 4729 ixl_aqb_alloc(struct ixl_softc *sc) 4730 { 4731 struct ixl_aq_buf *aqb; 4732 4733 aqb = malloc(sizeof(*aqb), M_DEVBUF, M_WAITOK); 4734 if (aqb == NULL) 4735 return NULL; 4736 4737 aqb->aqb_size = IXL_AQ_BUFLEN; 4738 4739 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 4740 aqb->aqb_size, 0, 4741 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 4742 goto free; 4743 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 4744 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 4745 BUS_DMA_WAITOK) != 0) 4746 goto destroy; 4747 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 4748 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 4749 goto dma_free; 4750 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 4751 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 4752 goto unmap; 4753 4754 return aqb; 4755 unmap: 4756 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 4757 dma_free: 4758 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 4759 destroy: 4760 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 4761 free: 4762 free(aqb, M_DEVBUF); 4763 4764 return NULL; 4765 } 4766 4767 static void 4768 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 4769 { 4770 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 4771 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 4772 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 4773 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 4774 free(aqb, M_DEVBUF); 4775 } 4776 4777 static int 4778 ixl_arq_fill(struct ixl_softc *sc) 4779 { 4780 struct ixl_aq_buf *aqb; 4781 struct ixl_aq_desc *arq, *iaq; 4782 unsigned int prod = sc->sc_arq_prod; 4783 unsigned int n; 4784 int post = 0; 4785 4786 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 4787 IXL_AQ_NUM); 4788 arq = IXL_DMA_KVA(&sc->sc_arq); 4789 4790 if (__predict_false(n <= 0)) 4791 return 0; 4792 4793 do { 4794 aqb = sc->sc_arq_live[prod]; 4795 iaq = &arq[prod]; 4796 4797 if (aqb == NULL) { 4798 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 4799 if (aqb != NULL) { 4800 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 4801 ixl_aq_buf, aqb_entry); 4802 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 4803 break; 4804 } 4805 4806 sc->sc_arq_live[prod] = aqb; 4807 memset(aqb->aqb_data, 0, aqb->aqb_size); 4808 4809 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 4810 aqb->aqb_size, BUS_DMASYNC_PREREAD); 4811 4812 iaq->iaq_flags = htole16(IXL_AQ_BUF | 4813 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 4814 IXL_AQ_LB : 0)); 4815 iaq->iaq_opcode = 0; 4816 iaq->iaq_datalen = htole16(aqb->aqb_size); 4817 iaq->iaq_retval = 0; 4818 iaq->iaq_cookie = 0; 4819 iaq->iaq_param[0] = 0; 4820 iaq->iaq_param[1] = 0; 4821 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 4822 } 4823 4824 prod++; 4825 prod &= IXL_AQ_MASK; 4826 4827 post = 1; 4828 4829 } while (--n); 4830 4831 if (post) { 4832 sc->sc_arq_prod = prod; 4833 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 4834 } 4835 4836 return post; 4837 } 4838 4839 static void 4840 ixl_arq_unfill(struct ixl_softc *sc) 4841 { 4842 struct ixl_aq_buf *aqb; 4843 unsigned int i; 4844 4845 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 4846 aqb = sc->sc_arq_live[i]; 4847 if (aqb == NULL) 4848 continue; 4849 4850 sc->sc_arq_live[i] = NULL; 4851 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 4852 BUS_DMASYNC_POSTREAD); 4853 ixl_aqb_free(sc, aqb); 4854 } 4855 4856 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 4857 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 4858 ixl_aq_buf, aqb_entry); 4859 ixl_aqb_free(sc, aqb); 4860 } 4861 } 4862 4863 static void 4864 ixl_clear_hw(struct ixl_softc *sc) 4865 { 4866 uint32_t num_queues, base_queue; 4867 uint32_t num_pf_int; 4868 uint32_t num_vf_int; 4869 uint32_t num_vfs; 4870 uint32_t i, j; 4871 uint32_t val; 4872 uint32_t eol = 0x7ff; 4873 4874 /* get number of interrupts, queues, and vfs */ 4875 val = ixl_rd(sc, I40E_GLPCI_CNF2); 4876 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 4877 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 4878 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 4879 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 4880 4881 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 4882 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 4883 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 4884 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 4885 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 4886 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 4887 num_queues = (j - base_queue) + 1; 4888 else 4889 num_queues = 0; 4890 4891 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 4892 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 4893 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 4894 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 4895 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 4896 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 4897 num_vfs = (j - i) + 1; 4898 else 4899 num_vfs = 0; 4900 4901 /* stop all the interrupts */ 4902 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 4903 ixl_flush(sc); 4904 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 4905 for (i = 0; i < num_pf_int - 2; i++) 4906 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 4907 ixl_flush(sc); 4908 4909 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 4910 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4911 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 4912 for (i = 0; i < num_pf_int - 2; i++) 4913 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 4914 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4915 for (i = 0; i < num_vfs; i++) 4916 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 4917 for (i = 0; i < num_vf_int - 2; i++) 4918 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 4919 4920 /* warn the HW of the coming Tx disables */ 4921 for (i = 0; i < num_queues; i++) { 4922 uint32_t abs_queue_idx = base_queue + i; 4923 uint32_t reg_block = 0; 4924 4925 if (abs_queue_idx >= 128) { 4926 reg_block = abs_queue_idx / 128; 4927 abs_queue_idx %= 128; 4928 } 4929 4930 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 4931 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 4932 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 4933 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 4934 4935 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 4936 } 4937 delaymsec(400); 4938 4939 /* stop all the queues */ 4940 for (i = 0; i < num_queues; i++) { 4941 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 4942 ixl_wr(sc, I40E_QTX_ENA(i), 0); 4943 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 4944 ixl_wr(sc, I40E_QRX_ENA(i), 0); 4945 } 4946 4947 /* short wait for all queue disables to settle */ 4948 delaymsec(50); 4949 } 4950 4951 static int 4952 ixl_pf_reset(struct ixl_softc *sc) 4953 { 4954 uint32_t cnt = 0; 4955 uint32_t cnt1 = 0; 4956 uint32_t reg = 0, reg0 = 0; 4957 uint32_t grst_del; 4958 4959 /* 4960 * Poll for Global Reset steady state in case of recent GRST. 4961 * The grst delay value is in 100ms units, and we'll wait a 4962 * couple counts longer to be sure we don't just miss the end. 4963 */ 4964 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 4965 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 4966 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 4967 4968 grst_del = grst_del * 20; 4969 4970 for (cnt = 0; cnt < grst_del; cnt++) { 4971 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 4972 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 4973 break; 4974 delaymsec(100); 4975 } 4976 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 4977 aprint_error(", Global reset polling failed to complete\n"); 4978 return -1; 4979 } 4980 4981 /* Now Wait for the FW to be ready */ 4982 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 4983 reg = ixl_rd(sc, I40E_GLNVM_ULD); 4984 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4985 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 4986 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4987 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 4988 break; 4989 4990 delaymsec(10); 4991 } 4992 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4993 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 4994 aprint_error(", wait for FW Reset complete timed out " 4995 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 4996 return -1; 4997 } 4998 4999 /* 5000 * If there was a Global Reset in progress when we got here, 5001 * we don't need to do the PF Reset 5002 */ 5003 if (cnt == 0) { 5004 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5005 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 5006 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 5007 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5008 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 5009 break; 5010 delaymsec(1); 5011 5012 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 5013 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5014 aprint_error(", Core reset upcoming." 5015 " Skipping PF reset reset request\n"); 5016 return -1; 5017 } 5018 } 5019 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 5020 aprint_error(", PF reset polling failed to complete" 5021 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 5022 return -1; 5023 } 5024 } 5025 5026 return 0; 5027 } 5028 5029 static int 5030 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 5031 bus_size_t size, bus_size_t align) 5032 { 5033 ixm->ixm_size = size; 5034 5035 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 5036 ixm->ixm_size, 0, 5037 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 5038 &ixm->ixm_map) != 0) 5039 return 1; 5040 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 5041 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 5042 BUS_DMA_WAITOK) != 0) 5043 goto destroy; 5044 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 5045 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 5046 goto free; 5047 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 5048 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 5049 goto unmap; 5050 5051 memset(ixm->ixm_kva, 0, ixm->ixm_size); 5052 5053 return 0; 5054 unmap: 5055 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5056 free: 5057 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5058 destroy: 5059 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5060 return 1; 5061 } 5062 5063 static void 5064 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 5065 { 5066 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 5067 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5068 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5069 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5070 } 5071 5072 static int 5073 ixl_setup_vlan_hwfilter(struct ixl_softc *sc) 5074 { 5075 struct ethercom *ec = &sc->sc_ec; 5076 struct vlanid_list *vlanidp; 5077 int rv; 5078 5079 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5080 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5081 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5082 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5083 5084 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5085 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5086 if (rv != 0) 5087 return rv; 5088 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5089 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5090 if (rv != 0) 5091 return rv; 5092 5093 ETHER_LOCK(ec); 5094 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5095 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 5096 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5097 if (rv != 0) 5098 break; 5099 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 5100 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5101 if (rv != 0) 5102 break; 5103 } 5104 ETHER_UNLOCK(ec); 5105 5106 return rv; 5107 } 5108 5109 static void 5110 ixl_teardown_vlan_hwfilter(struct ixl_softc *sc) 5111 { 5112 struct vlanid_list *vlanidp; 5113 struct ethercom *ec = &sc->sc_ec; 5114 5115 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5116 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5117 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5118 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5119 5120 ETHER_LOCK(ec); 5121 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5122 ixl_remove_macvlan(sc, sc->sc_enaddr, 5123 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5124 ixl_remove_macvlan(sc, etherbroadcastaddr, 5125 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5126 } 5127 ETHER_UNLOCK(ec); 5128 5129 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5130 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5131 ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5132 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5133 } 5134 5135 static int 5136 ixl_update_macvlan(struct ixl_softc *sc) 5137 { 5138 int rv = 0; 5139 int next_ec_capenable = sc->sc_ec.ec_capenable; 5140 5141 if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 5142 rv = ixl_setup_vlan_hwfilter(sc); 5143 if (rv != 0) 5144 ixl_teardown_vlan_hwfilter(sc); 5145 } else { 5146 ixl_teardown_vlan_hwfilter(sc); 5147 } 5148 5149 return rv; 5150 } 5151 5152 static int 5153 ixl_ifflags_cb(struct ethercom *ec) 5154 { 5155 struct ifnet *ifp = &ec->ec_if; 5156 struct ixl_softc *sc = ifp->if_softc; 5157 int rv, change; 5158 5159 mutex_enter(&sc->sc_cfg_lock); 5160 5161 change = ec->ec_capenable ^ sc->sc_cur_ec_capenable; 5162 5163 if (ISSET(change, ETHERCAP_VLAN_HWTAGGING)) { 5164 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 5165 rv = ENETRESET; 5166 goto out; 5167 } 5168 5169 if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) { 5170 rv = ixl_update_macvlan(sc); 5171 if (rv == 0) { 5172 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 5173 } else { 5174 CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER); 5175 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 5176 } 5177 } 5178 5179 rv = ixl_iff(sc); 5180 out: 5181 mutex_exit(&sc->sc_cfg_lock); 5182 5183 return rv; 5184 } 5185 5186 static int 5187 ixl_set_link_status(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 5188 { 5189 const struct ixl_aq_link_status *status; 5190 const struct ixl_phy_type *itype; 5191 5192 uint64_t ifm_active = IFM_ETHER; 5193 uint64_t ifm_status = IFM_AVALID; 5194 int link_state = LINK_STATE_DOWN; 5195 uint64_t baudrate = 0; 5196 5197 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 5198 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) 5199 goto done; 5200 5201 ifm_active |= IFM_FDX; 5202 ifm_status |= IFM_ACTIVE; 5203 link_state = LINK_STATE_UP; 5204 5205 itype = ixl_search_phy_type(status->phy_type); 5206 if (itype != NULL) 5207 ifm_active |= itype->ifm_type; 5208 5209 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 5210 ifm_active |= IFM_ETH_TXPAUSE; 5211 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 5212 ifm_active |= IFM_ETH_RXPAUSE; 5213 5214 baudrate = ixl_search_link_speed(status->link_speed); 5215 5216 done: 5217 /* NET_ASSERT_LOCKED() except during attach */ 5218 sc->sc_media_active = ifm_active; 5219 sc->sc_media_status = ifm_status; 5220 5221 sc->sc_ec.ec_if.if_baudrate = baudrate; 5222 5223 return link_state; 5224 } 5225 5226 static int 5227 ixl_establish_intx(struct ixl_softc *sc) 5228 { 5229 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5230 pci_intr_handle_t *intr; 5231 char xnamebuf[32]; 5232 char intrbuf[PCI_INTRSTR_LEN]; 5233 char const *intrstr; 5234 5235 KASSERT(sc->sc_nintrs == 1); 5236 5237 intr = &sc->sc_ihp[0]; 5238 5239 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 5240 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 5241 device_xname(sc->sc_dev)); 5242 5243 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 5244 sc, xnamebuf); 5245 5246 if (sc->sc_ihs[0] == NULL) { 5247 aprint_error_dev(sc->sc_dev, 5248 "unable to establish interrupt at %s\n", intrstr); 5249 return -1; 5250 } 5251 5252 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 5253 return 0; 5254 } 5255 5256 static int 5257 ixl_establish_msix(struct ixl_softc *sc) 5258 { 5259 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5260 unsigned int vector = 0; 5261 unsigned int i; 5262 char xnamebuf[32]; 5263 char intrbuf[PCI_INTRSTR_LEN]; 5264 char const *intrstr; 5265 5266 /* the "other" intr is mapped to vector 0 */ 5267 vector = 0; 5268 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5269 intrbuf, sizeof(intrbuf)); 5270 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 5271 device_xname(sc->sc_dev)); 5272 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5273 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 5274 sc, xnamebuf); 5275 if (sc->sc_ihs[vector] == NULL) { 5276 aprint_error_dev(sc->sc_dev, 5277 "unable to establish interrupt at %s\n", intrstr); 5278 goto fail; 5279 } 5280 vector++; 5281 aprint_normal_dev(sc->sc_dev, "interrupt at %s\n", intrstr); 5282 5283 sc->sc_msix_vector_queue = vector; 5284 5285 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5286 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5287 intrbuf, sizeof(intrbuf)); 5288 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 5289 device_xname(sc->sc_dev), i); 5290 5291 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5292 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 5293 (void *)&sc->sc_qps[i], xnamebuf); 5294 5295 if (sc->sc_ihs[vector] == NULL) { 5296 aprint_error_dev(sc->sc_dev, 5297 "unable to establish interrupt at %s\n", intrstr); 5298 goto fail; 5299 } 5300 vector++; 5301 aprint_normal_dev(sc->sc_dev, 5302 "interrupt at %s\n", intrstr); 5303 } 5304 5305 return 0; 5306 fail: 5307 for (i = 0; i < vector; i++) { 5308 pci_intr_disestablish(pc, sc->sc_ihs[i]); 5309 } 5310 5311 sc->sc_msix_vector_queue = 0; 5312 sc->sc_msix_vector_queue = 0; 5313 5314 return -1; 5315 } 5316 5317 static void 5318 ixl_set_affinity_msix(struct ixl_softc *sc) 5319 { 5320 kcpuset_t *affinity; 5321 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5322 int affinity_to, r; 5323 unsigned int i, vector; 5324 char intrbuf[PCI_INTRSTR_LEN]; 5325 char const *intrstr; 5326 5327 affinity_to = 0; 5328 kcpuset_create(&affinity, false); 5329 5330 vector = sc->sc_msix_vector_queue; 5331 5332 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5333 affinity_to = i % ncpu; 5334 5335 kcpuset_zero(affinity); 5336 kcpuset_set(affinity, affinity_to); 5337 5338 intrstr = pci_intr_string(pc, sc->sc_ihp[vector + i], 5339 intrbuf, sizeof(intrbuf)); 5340 r = interrupt_distribute(sc->sc_ihs[vector + i], 5341 affinity, NULL); 5342 if (r == 0) { 5343 aprint_normal_dev(sc->sc_dev, 5344 "for TXRX%u interrupting at %s affinity to %u\n", 5345 i, intrstr, affinity_to); 5346 } else { 5347 aprint_normal_dev(sc->sc_dev, 5348 "for TXRX%u interrupting at %s\n", 5349 i, intrstr); 5350 } 5351 } 5352 5353 vector = 0; /* vector 0 means "other" interrupt */ 5354 affinity_to = (affinity_to + 1) % ncpu; 5355 kcpuset_zero(affinity); 5356 kcpuset_set(affinity, affinity_to); 5357 5358 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5359 intrbuf, sizeof(intrbuf)); 5360 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5361 if (r == 0) { 5362 aprint_normal_dev(sc->sc_dev, 5363 "for other interrupting at %s affinity to %u\n", 5364 intrstr, affinity_to); 5365 } else { 5366 aprint_normal_dev(sc->sc_dev, 5367 "for other interrupting at %s", intrstr); 5368 } 5369 5370 kcpuset_destroy(affinity); 5371 } 5372 5373 static void 5374 ixl_config_queue_intr(struct ixl_softc *sc) 5375 { 5376 unsigned int i, vector; 5377 5378 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5379 vector = sc->sc_msix_vector_queue; 5380 } else { 5381 vector = I40E_INTR_NOTX_INTR; 5382 5383 ixl_wr(sc, I40E_PFINT_LNKLST0, 5384 (I40E_INTR_NOTX_QUEUE << 5385 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5386 (I40E_QUEUE_TYPE_RX << 5387 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5388 } 5389 5390 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5391 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5392 ixl_flush(sc); 5393 5394 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5395 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5396 (I40E_QUEUE_TYPE_RX << 5397 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5398 5399 ixl_wr(sc, I40E_QINT_RQCTL(i), 5400 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5401 (I40E_ITR_INDEX_RX << 5402 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5403 (I40E_INTR_NOTX_RX_QUEUE << 5404 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5405 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5406 (I40E_QUEUE_TYPE_TX << 5407 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5408 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5409 5410 ixl_wr(sc, I40E_QINT_TQCTL(i), 5411 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5412 (I40E_ITR_INDEX_TX << 5413 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5414 (I40E_INTR_NOTX_TX_QUEUE << 5415 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5416 (I40E_QUEUE_TYPE_EOL << 5417 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5418 (I40E_QUEUE_TYPE_RX << 5419 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5420 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5421 5422 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) 5423 vector++; 5424 } 5425 ixl_flush(sc); 5426 5427 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), 0x7a); 5428 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), 0x7a); 5429 ixl_flush(sc); 5430 } 5431 5432 static void 5433 ixl_config_other_intr(struct ixl_softc *sc) 5434 { 5435 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5436 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5437 5438 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5439 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5440 I40E_PFINT_ICR0_ENA_GRST_MASK | 5441 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5442 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5443 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5444 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5445 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5446 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5447 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5448 5449 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5450 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5451 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5452 (I40E_ITR_INDEX_OTHER << 5453 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5454 ixl_flush(sc); 5455 } 5456 5457 static int 5458 ixl_setup_interrupts(struct ixl_softc *sc) 5459 { 5460 struct pci_attach_args *pa = &sc->sc_pa; 5461 pci_intr_type_t max_type, intr_type; 5462 int counts[PCI_INTR_TYPE_SIZE]; 5463 int error; 5464 unsigned int i; 5465 bool retry, nomsix = IXL_NOMSIX; 5466 5467 memset(counts, 0, sizeof(counts)); 5468 max_type = PCI_INTR_TYPE_MSIX; 5469 /* QPs + other interrupt */ 5470 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5471 counts[PCI_INTR_TYPE_INTX] = 1; 5472 5473 if (nomsix) 5474 counts[PCI_INTR_TYPE_MSIX] = 0; 5475 5476 do { 5477 retry = false; 5478 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5479 if (error != 0) { 5480 aprint_error_dev(sc->sc_dev, 5481 "couldn't map interrupt\n"); 5482 break; 5483 } 5484 for (i = 0; i < sc->sc_nintrs; i++) { 5485 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5486 PCI_INTR_MPSAFE, true); 5487 } 5488 5489 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5490 sc->sc_nintrs = counts[intr_type]; 5491 KASSERT(sc->sc_nintrs > 0); 5492 5493 sc->sc_ihs = kmem_alloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5494 KM_SLEEP); 5495 5496 if (intr_type == PCI_INTR_TYPE_MSIX) { 5497 error = ixl_establish_msix(sc); 5498 if (error) { 5499 counts[PCI_INTR_TYPE_MSIX] = 0; 5500 retry = true; 5501 } else { 5502 ixl_set_affinity_msix(sc); 5503 } 5504 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5505 error = ixl_establish_intx(sc); 5506 } else { 5507 error = -1; 5508 } 5509 5510 if (error) { 5511 kmem_free(sc->sc_ihs, 5512 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5513 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5514 } else { 5515 sc->sc_intrtype = intr_type; 5516 } 5517 } while (retry); 5518 5519 return error; 5520 } 5521 5522 static void 5523 ixl_teardown_interrupts(struct ixl_softc *sc) 5524 { 5525 struct pci_attach_args *pa = &sc->sc_pa; 5526 unsigned int i; 5527 5528 for (i = 0; i < sc->sc_nintrs; i++) { 5529 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 5530 } 5531 5532 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5533 5534 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5535 sc->sc_ihs = NULL; 5536 sc->sc_nintrs = 0; 5537 } 5538 5539 static int 5540 ixl_setup_stats(struct ixl_softc *sc) 5541 { 5542 struct ixl_queue_pair *qp; 5543 struct ixl_tx_ring *txr; 5544 struct ixl_rx_ring *rxr; 5545 struct ixl_stats_counters *isc; 5546 unsigned int i; 5547 5548 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5549 qp = &sc->sc_qps[i]; 5550 txr = qp->qp_txr; 5551 rxr = qp->qp_rxr; 5552 5553 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 5554 NULL, qp->qp_name, "m_defrag successed"); 5555 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 5556 NULL, qp->qp_name, "m_defrag_failed"); 5557 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 5558 NULL, qp->qp_name, "Dropped in pcq"); 5559 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 5560 NULL, qp->qp_name, "Deferred transmit"); 5561 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 5562 NULL, qp->qp_name, "Interrupt on queue"); 5563 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 5564 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5565 5566 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 5567 NULL, qp->qp_name, "MGETHDR failed"); 5568 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 5569 NULL, qp->qp_name, "MCLGET failed"); 5570 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 5571 EVCNT_TYPE_MISC, NULL, qp->qp_name, 5572 "bus_dmamap_load_mbuf failed"); 5573 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 5574 NULL, qp->qp_name, "Interrupt on queue"); 5575 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 5576 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5577 } 5578 5579 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 5580 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 5581 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 5582 NULL, device_xname(sc->sc_dev), "Link status event"); 5583 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 5584 NULL, device_xname(sc->sc_dev), "ECC error"); 5585 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 5586 NULL, device_xname(sc->sc_dev), "PCI exception"); 5587 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 5588 NULL, device_xname(sc->sc_dev), "Critical error"); 5589 5590 isc = &sc->sc_stats_counters; 5591 evcnt_attach_dynamic(&isc->isc_crc_errors, EVCNT_TYPE_MISC, 5592 NULL, device_xname(sc->sc_dev), "CRC errors"); 5593 evcnt_attach_dynamic(&isc->isc_illegal_bytes, EVCNT_TYPE_MISC, 5594 NULL, device_xname(sc->sc_dev), "Illegal bytes"); 5595 evcnt_attach_dynamic(&isc->isc_mac_local_faults, EVCNT_TYPE_MISC, 5596 NULL, device_xname(sc->sc_dev), "Mac local faults"); 5597 evcnt_attach_dynamic(&isc->isc_mac_remote_faults, EVCNT_TYPE_MISC, 5598 NULL, device_xname(sc->sc_dev), "Mac remote faults"); 5599 evcnt_attach_dynamic(&isc->isc_link_xon_rx, EVCNT_TYPE_MISC, 5600 NULL, device_xname(sc->sc_dev), "Rx xon"); 5601 evcnt_attach_dynamic(&isc->isc_link_xon_tx, EVCNT_TYPE_MISC, 5602 NULL, device_xname(sc->sc_dev), "Tx xon"); 5603 evcnt_attach_dynamic(&isc->isc_link_xoff_rx, EVCNT_TYPE_MISC, 5604 NULL, device_xname(sc->sc_dev), "Rx xoff"); 5605 evcnt_attach_dynamic(&isc->isc_link_xoff_tx, EVCNT_TYPE_MISC, 5606 NULL, device_xname(sc->sc_dev), "Tx xoff"); 5607 evcnt_attach_dynamic(&isc->isc_rx_fragments, EVCNT_TYPE_MISC, 5608 NULL, device_xname(sc->sc_dev), "Rx fragments"); 5609 evcnt_attach_dynamic(&isc->isc_rx_jabber, EVCNT_TYPE_MISC, 5610 NULL, device_xname(sc->sc_dev), "Rx jabber"); 5611 5612 evcnt_attach_dynamic(&isc->isc_rx_size_64, EVCNT_TYPE_MISC, 5613 NULL, device_xname(sc->sc_dev), "Rx size 64"); 5614 evcnt_attach_dynamic(&isc->isc_rx_size_127, EVCNT_TYPE_MISC, 5615 NULL, device_xname(sc->sc_dev), "Rx size 127"); 5616 evcnt_attach_dynamic(&isc->isc_rx_size_255, EVCNT_TYPE_MISC, 5617 NULL, device_xname(sc->sc_dev), "Rx size 255"); 5618 evcnt_attach_dynamic(&isc->isc_rx_size_511, EVCNT_TYPE_MISC, 5619 NULL, device_xname(sc->sc_dev), "Rx size 511"); 5620 evcnt_attach_dynamic(&isc->isc_rx_size_1023, EVCNT_TYPE_MISC, 5621 NULL, device_xname(sc->sc_dev), "Rx size 1023"); 5622 evcnt_attach_dynamic(&isc->isc_rx_size_1522, EVCNT_TYPE_MISC, 5623 NULL, device_xname(sc->sc_dev), "Rx size 1522"); 5624 evcnt_attach_dynamic(&isc->isc_rx_size_big, EVCNT_TYPE_MISC, 5625 NULL, device_xname(sc->sc_dev), "Rx jumbo packets"); 5626 evcnt_attach_dynamic(&isc->isc_rx_undersize, EVCNT_TYPE_MISC, 5627 NULL, device_xname(sc->sc_dev), "Rx under size"); 5628 evcnt_attach_dynamic(&isc->isc_rx_oversize, EVCNT_TYPE_MISC, 5629 NULL, device_xname(sc->sc_dev), "Rx over size"); 5630 5631 evcnt_attach_dynamic(&isc->isc_rx_bytes, EVCNT_TYPE_MISC, 5632 NULL, device_xname(sc->sc_dev), "Rx bytes / port"); 5633 evcnt_attach_dynamic(&isc->isc_rx_discards, EVCNT_TYPE_MISC, 5634 NULL, device_xname(sc->sc_dev), "Rx discards / port"); 5635 evcnt_attach_dynamic(&isc->isc_rx_unicast, EVCNT_TYPE_MISC, 5636 NULL, device_xname(sc->sc_dev), "Rx unicast / port"); 5637 evcnt_attach_dynamic(&isc->isc_rx_multicast, EVCNT_TYPE_MISC, 5638 NULL, device_xname(sc->sc_dev), "Rx multicast / port"); 5639 evcnt_attach_dynamic(&isc->isc_rx_broadcast, EVCNT_TYPE_MISC, 5640 NULL, device_xname(sc->sc_dev), "Rx broadcast / port"); 5641 5642 evcnt_attach_dynamic(&isc->isc_tx_size_64, EVCNT_TYPE_MISC, 5643 NULL, device_xname(sc->sc_dev), "Tx size 64"); 5644 evcnt_attach_dynamic(&isc->isc_tx_size_127, EVCNT_TYPE_MISC, 5645 NULL, device_xname(sc->sc_dev), "Tx size 127"); 5646 evcnt_attach_dynamic(&isc->isc_tx_size_255, EVCNT_TYPE_MISC, 5647 NULL, device_xname(sc->sc_dev), "Tx size 255"); 5648 evcnt_attach_dynamic(&isc->isc_tx_size_511, EVCNT_TYPE_MISC, 5649 NULL, device_xname(sc->sc_dev), "Tx size 511"); 5650 evcnt_attach_dynamic(&isc->isc_tx_size_1023, EVCNT_TYPE_MISC, 5651 NULL, device_xname(sc->sc_dev), "Tx size 1023"); 5652 evcnt_attach_dynamic(&isc->isc_tx_size_1522, EVCNT_TYPE_MISC, 5653 NULL, device_xname(sc->sc_dev), "Tx size 1522"); 5654 evcnt_attach_dynamic(&isc->isc_tx_size_big, EVCNT_TYPE_MISC, 5655 NULL, device_xname(sc->sc_dev), "Tx jumbo packets"); 5656 5657 evcnt_attach_dynamic(&isc->isc_tx_bytes, EVCNT_TYPE_MISC, 5658 NULL, device_xname(sc->sc_dev), "Tx bytes / port"); 5659 evcnt_attach_dynamic(&isc->isc_tx_dropped_link_down, EVCNT_TYPE_MISC, 5660 NULL, device_xname(sc->sc_dev), 5661 "Tx dropped due to link down / port"); 5662 evcnt_attach_dynamic(&isc->isc_tx_unicast, EVCNT_TYPE_MISC, 5663 NULL, device_xname(sc->sc_dev), "Tx unicast / port"); 5664 evcnt_attach_dynamic(&isc->isc_tx_multicast, EVCNT_TYPE_MISC, 5665 NULL, device_xname(sc->sc_dev), "Tx multicast / port"); 5666 evcnt_attach_dynamic(&isc->isc_tx_broadcast, EVCNT_TYPE_MISC, 5667 NULL, device_xname(sc->sc_dev), "Tx broadcast / port"); 5668 5669 sc->sc_stats_intval = IXL_STATS_INTERVAL_MSEC; 5670 callout_init(&sc->sc_stats_callout, CALLOUT_MPSAFE); 5671 callout_setfunc(&sc->sc_stats_callout, ixl_stats_callout, sc); 5672 ixl_work_set(&sc->sc_stats_task, ixl_stats_update, sc); 5673 5674 return 0; 5675 } 5676 5677 static void 5678 ixl_teardown_stats(struct ixl_softc *sc) 5679 { 5680 struct ixl_tx_ring *txr; 5681 struct ixl_rx_ring *rxr; 5682 struct ixl_stats_counters *isc; 5683 unsigned int i; 5684 5685 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5686 txr = sc->sc_qps[i].qp_txr; 5687 rxr = sc->sc_qps[i].qp_rxr; 5688 5689 evcnt_detach(&txr->txr_defragged); 5690 evcnt_detach(&txr->txr_defrag_failed); 5691 evcnt_detach(&txr->txr_pcqdrop); 5692 evcnt_detach(&txr->txr_transmitdef); 5693 evcnt_detach(&txr->txr_intr); 5694 evcnt_detach(&txr->txr_defer); 5695 5696 evcnt_detach(&rxr->rxr_mgethdr_failed); 5697 evcnt_detach(&rxr->rxr_mgetcl_failed); 5698 evcnt_detach(&rxr->rxr_mbuf_load_failed); 5699 evcnt_detach(&rxr->rxr_intr); 5700 evcnt_detach(&rxr->rxr_defer); 5701 } 5702 5703 isc = &sc->sc_stats_counters; 5704 evcnt_detach(&isc->isc_crc_errors); 5705 evcnt_detach(&isc->isc_illegal_bytes); 5706 evcnt_detach(&isc->isc_mac_local_faults); 5707 evcnt_detach(&isc->isc_mac_remote_faults); 5708 evcnt_detach(&isc->isc_link_xon_rx); 5709 evcnt_detach(&isc->isc_link_xon_tx); 5710 evcnt_detach(&isc->isc_link_xoff_rx); 5711 evcnt_detach(&isc->isc_link_xoff_tx); 5712 evcnt_detach(&isc->isc_rx_fragments); 5713 evcnt_detach(&isc->isc_rx_jabber); 5714 evcnt_detach(&isc->isc_rx_bytes); 5715 evcnt_detach(&isc->isc_rx_discards); 5716 evcnt_detach(&isc->isc_rx_unicast); 5717 evcnt_detach(&isc->isc_rx_multicast); 5718 evcnt_detach(&isc->isc_rx_broadcast); 5719 evcnt_detach(&isc->isc_rx_size_64); 5720 evcnt_detach(&isc->isc_rx_size_127); 5721 evcnt_detach(&isc->isc_rx_size_255); 5722 evcnt_detach(&isc->isc_rx_size_511); 5723 evcnt_detach(&isc->isc_rx_size_1023); 5724 evcnt_detach(&isc->isc_rx_size_1522); 5725 evcnt_detach(&isc->isc_rx_size_big); 5726 evcnt_detach(&isc->isc_rx_undersize); 5727 evcnt_detach(&isc->isc_rx_oversize); 5728 evcnt_detach(&isc->isc_tx_bytes); 5729 evcnt_detach(&isc->isc_tx_dropped_link_down); 5730 evcnt_detach(&isc->isc_tx_unicast); 5731 evcnt_detach(&isc->isc_tx_multicast); 5732 evcnt_detach(&isc->isc_tx_broadcast); 5733 evcnt_detach(&isc->isc_tx_size_64); 5734 evcnt_detach(&isc->isc_tx_size_127); 5735 evcnt_detach(&isc->isc_tx_size_255); 5736 evcnt_detach(&isc->isc_tx_size_511); 5737 evcnt_detach(&isc->isc_tx_size_1023); 5738 evcnt_detach(&isc->isc_tx_size_1522); 5739 evcnt_detach(&isc->isc_tx_size_big); 5740 5741 evcnt_detach(&sc->sc_event_atq); 5742 evcnt_detach(&sc->sc_event_link); 5743 evcnt_detach(&sc->sc_event_ecc_err); 5744 evcnt_detach(&sc->sc_event_pci_exception); 5745 evcnt_detach(&sc->sc_event_crit_err); 5746 5747 callout_destroy(&sc->sc_stats_callout); 5748 } 5749 5750 static void 5751 ixl_stats_callout(void *xsc) 5752 { 5753 struct ixl_softc *sc = xsc; 5754 5755 ixl_work_add(sc->sc_workq, &sc->sc_stats_task); 5756 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 5757 } 5758 5759 static uint64_t 5760 ixl_stat_delta(struct ixl_softc *sc, uint32_t reg_hi, uint32_t reg_lo, 5761 uint64_t *offset, bool has_offset) 5762 { 5763 uint64_t value, delta; 5764 int bitwidth; 5765 5766 bitwidth = reg_hi == 0 ? 32 : 48; 5767 5768 value = ixl_rd(sc, reg_lo); 5769 5770 if (bitwidth > 32) { 5771 value |= ((uint64_t)ixl_rd(sc, reg_hi) << 32); 5772 } 5773 5774 if (__predict_true(has_offset)) { 5775 delta = value; 5776 if (value < *offset) 5777 delta += ((uint64_t)1 << bitwidth); 5778 delta -= *offset; 5779 } else { 5780 delta = 0; 5781 } 5782 atomic_swap_64(offset, value); 5783 5784 return delta; 5785 } 5786 5787 static void 5788 ixl_stats_update(void *xsc) 5789 { 5790 struct ixl_softc *sc = xsc; 5791 struct ixl_stats_counters *isc; 5792 uint64_t delta; 5793 5794 isc = &sc->sc_stats_counters; 5795 5796 /* errors */ 5797 delta = ixl_stat_delta(sc, 5798 0, I40E_GLPRT_CRCERRS(sc->sc_port), 5799 &isc->isc_crc_errors_offset, isc->isc_has_offset); 5800 atomic_add_64(&isc->isc_crc_errors.ev_count, delta); 5801 5802 delta = ixl_stat_delta(sc, 5803 0, I40E_GLPRT_ILLERRC(sc->sc_port), 5804 &isc->isc_illegal_bytes_offset, isc->isc_has_offset); 5805 atomic_add_64(&isc->isc_illegal_bytes.ev_count, delta); 5806 5807 /* rx */ 5808 delta = ixl_stat_delta(sc, 5809 I40E_GLPRT_GORCH(sc->sc_port), I40E_GLPRT_GORCL(sc->sc_port), 5810 &isc->isc_rx_bytes_offset, isc->isc_has_offset); 5811 atomic_add_64(&isc->isc_rx_bytes.ev_count, delta); 5812 5813 delta = ixl_stat_delta(sc, 5814 0, I40E_GLPRT_RDPC(sc->sc_port), 5815 &isc->isc_rx_discards_offset, isc->isc_has_offset); 5816 atomic_add_64(&isc->isc_rx_discards.ev_count, delta); 5817 5818 delta = ixl_stat_delta(sc, 5819 I40E_GLPRT_UPRCH(sc->sc_port), I40E_GLPRT_UPRCL(sc->sc_port), 5820 &isc->isc_rx_unicast_offset, isc->isc_has_offset); 5821 atomic_add_64(&isc->isc_rx_unicast.ev_count, delta); 5822 5823 delta = ixl_stat_delta(sc, 5824 I40E_GLPRT_MPRCH(sc->sc_port), I40E_GLPRT_MPRCL(sc->sc_port), 5825 &isc->isc_rx_multicast_offset, isc->isc_has_offset); 5826 atomic_add_64(&isc->isc_rx_multicast.ev_count, delta); 5827 5828 delta = ixl_stat_delta(sc, 5829 I40E_GLPRT_BPRCH(sc->sc_port), I40E_GLPRT_BPRCL(sc->sc_port), 5830 &isc->isc_rx_broadcast_offset, isc->isc_has_offset); 5831 atomic_add_64(&isc->isc_rx_broadcast.ev_count, delta); 5832 5833 /* Packet size stats rx */ 5834 delta = ixl_stat_delta(sc, 5835 I40E_GLPRT_PRC64H(sc->sc_port), I40E_GLPRT_PRC64L(sc->sc_port), 5836 &isc->isc_rx_size_64_offset, isc->isc_has_offset); 5837 atomic_add_64(&isc->isc_rx_size_64.ev_count, delta); 5838 5839 delta = ixl_stat_delta(sc, 5840 I40E_GLPRT_PRC127H(sc->sc_port), I40E_GLPRT_PRC127L(sc->sc_port), 5841 &isc->isc_rx_size_127_offset, isc->isc_has_offset); 5842 atomic_add_64(&isc->isc_rx_size_127.ev_count, delta); 5843 5844 delta = ixl_stat_delta(sc, 5845 I40E_GLPRT_PRC255H(sc->sc_port), I40E_GLPRT_PRC255L(sc->sc_port), 5846 &isc->isc_rx_size_255_offset, isc->isc_has_offset); 5847 atomic_add_64(&isc->isc_rx_size_255.ev_count, delta); 5848 5849 delta = ixl_stat_delta(sc, 5850 I40E_GLPRT_PRC511H(sc->sc_port), I40E_GLPRT_PRC511L(sc->sc_port), 5851 &isc->isc_rx_size_511_offset, isc->isc_has_offset); 5852 atomic_add_64(&isc->isc_rx_size_511.ev_count, delta); 5853 5854 delta = ixl_stat_delta(sc, 5855 I40E_GLPRT_PRC1023H(sc->sc_port), I40E_GLPRT_PRC1023L(sc->sc_port), 5856 &isc->isc_rx_size_1023_offset, isc->isc_has_offset); 5857 atomic_add_64(&isc->isc_rx_size_1023.ev_count, delta); 5858 5859 delta = ixl_stat_delta(sc, 5860 I40E_GLPRT_PRC1522H(sc->sc_port), I40E_GLPRT_PRC1522L(sc->sc_port), 5861 &isc->isc_rx_size_1522_offset, isc->isc_has_offset); 5862 atomic_add_64(&isc->isc_rx_size_1522.ev_count, delta); 5863 5864 delta = ixl_stat_delta(sc, 5865 I40E_GLPRT_PRC9522H(sc->sc_port), I40E_GLPRT_PRC9522L(sc->sc_port), 5866 &isc->isc_rx_size_big_offset, isc->isc_has_offset); 5867 atomic_add_64(&isc->isc_rx_size_big.ev_count, delta); 5868 5869 delta = ixl_stat_delta(sc, 5870 0, I40E_GLPRT_RUC(sc->sc_port), 5871 &isc->isc_rx_undersize_offset, isc->isc_has_offset); 5872 atomic_add_64(&isc->isc_rx_undersize.ev_count, delta); 5873 5874 delta = ixl_stat_delta(sc, 5875 0, I40E_GLPRT_ROC(sc->sc_port), 5876 &isc->isc_rx_oversize_offset, isc->isc_has_offset); 5877 atomic_add_64(&isc->isc_rx_oversize.ev_count, delta); 5878 5879 /* tx */ 5880 delta = ixl_stat_delta(sc, 5881 I40E_GLPRT_GOTCH(sc->sc_port), I40E_GLPRT_GOTCL(sc->sc_port), 5882 &isc->isc_tx_bytes_offset, isc->isc_has_offset); 5883 atomic_add_64(&isc->isc_tx_bytes.ev_count, delta); 5884 5885 delta = ixl_stat_delta(sc, 5886 0, I40E_GLPRT_TDOLD(sc->sc_port), 5887 &isc->isc_tx_dropped_link_down_offset, isc->isc_has_offset); 5888 atomic_add_64(&isc->isc_tx_dropped_link_down.ev_count, delta); 5889 5890 delta = ixl_stat_delta(sc, 5891 I40E_GLPRT_UPTCH(sc->sc_port), I40E_GLPRT_UPTCL(sc->sc_port), 5892 &isc->isc_tx_unicast_offset, isc->isc_has_offset); 5893 atomic_add_64(&isc->isc_tx_unicast.ev_count, delta); 5894 5895 delta = ixl_stat_delta(sc, 5896 I40E_GLPRT_MPTCH(sc->sc_port), I40E_GLPRT_MPTCL(sc->sc_port), 5897 &isc->isc_tx_multicast_offset, isc->isc_has_offset); 5898 atomic_add_64(&isc->isc_tx_multicast.ev_count, delta); 5899 5900 delta = ixl_stat_delta(sc, 5901 I40E_GLPRT_BPTCH(sc->sc_port), I40E_GLPRT_BPTCL(sc->sc_port), 5902 &isc->isc_tx_broadcast_offset, isc->isc_has_offset); 5903 atomic_add_64(&isc->isc_tx_broadcast.ev_count, delta); 5904 5905 /* Packet size stats tx */ 5906 delta = ixl_stat_delta(sc, 5907 I40E_GLPRT_PTC64L(sc->sc_port), I40E_GLPRT_PTC64L(sc->sc_port), 5908 &isc->isc_tx_size_64_offset, isc->isc_has_offset); 5909 atomic_add_64(&isc->isc_tx_size_64.ev_count, delta); 5910 5911 delta = ixl_stat_delta(sc, 5912 I40E_GLPRT_PTC127H(sc->sc_port), I40E_GLPRT_PTC127L(sc->sc_port), 5913 &isc->isc_tx_size_127_offset, isc->isc_has_offset); 5914 atomic_add_64(&isc->isc_tx_size_127.ev_count, delta); 5915 5916 delta = ixl_stat_delta(sc, 5917 I40E_GLPRT_PTC255H(sc->sc_port), I40E_GLPRT_PTC255L(sc->sc_port), 5918 &isc->isc_tx_size_255_offset, isc->isc_has_offset); 5919 atomic_add_64(&isc->isc_tx_size_255.ev_count, delta); 5920 5921 delta = ixl_stat_delta(sc, 5922 I40E_GLPRT_PTC511H(sc->sc_port), I40E_GLPRT_PTC511L(sc->sc_port), 5923 &isc->isc_tx_size_511_offset, isc->isc_has_offset); 5924 atomic_add_64(&isc->isc_tx_size_511.ev_count, delta); 5925 5926 delta = ixl_stat_delta(sc, 5927 I40E_GLPRT_PTC1023H(sc->sc_port), I40E_GLPRT_PTC1023L(sc->sc_port), 5928 &isc->isc_tx_size_1023_offset, isc->isc_has_offset); 5929 atomic_add_64(&isc->isc_tx_size_1023.ev_count, delta); 5930 5931 delta = ixl_stat_delta(sc, 5932 I40E_GLPRT_PTC1522H(sc->sc_port), I40E_GLPRT_PTC1522L(sc->sc_port), 5933 &isc->isc_tx_size_1522_offset, isc->isc_has_offset); 5934 atomic_add_64(&isc->isc_tx_size_1522.ev_count, delta); 5935 5936 delta = ixl_stat_delta(sc, 5937 I40E_GLPRT_PTC9522H(sc->sc_port), I40E_GLPRT_PTC9522L(sc->sc_port), 5938 &isc->isc_tx_size_big_offset, isc->isc_has_offset); 5939 atomic_add_64(&isc->isc_tx_size_big.ev_count, delta); 5940 5941 /* mac faults */ 5942 delta = ixl_stat_delta(sc, 5943 0, I40E_GLPRT_MLFC(sc->sc_port), 5944 &isc->isc_mac_local_faults_offset, isc->isc_has_offset); 5945 atomic_add_64(&isc->isc_mac_local_faults.ev_count, delta); 5946 5947 delta = ixl_stat_delta(sc, 5948 0, I40E_GLPRT_MRFC(sc->sc_port), 5949 &isc->isc_mac_remote_faults_offset, isc->isc_has_offset); 5950 atomic_add_64(&isc->isc_mac_remote_faults.ev_count, delta); 5951 5952 /* Flow control (LFC) stats */ 5953 delta = ixl_stat_delta(sc, 5954 0, I40E_GLPRT_LXONRXC(sc->sc_port), 5955 &isc->isc_link_xon_rx_offset, isc->isc_has_offset); 5956 atomic_add_64(&isc->isc_link_xon_rx.ev_count, delta); 5957 5958 delta = ixl_stat_delta(sc, 5959 0, I40E_GLPRT_LXONTXC(sc->sc_port), 5960 &isc->isc_link_xon_tx_offset, isc->isc_has_offset); 5961 atomic_add_64(&isc->isc_link_xon_tx.ev_count, delta); 5962 5963 delta = ixl_stat_delta(sc, 5964 0, I40E_GLPRT_LXOFFRXC(sc->sc_port), 5965 &isc->isc_link_xoff_rx_offset, isc->isc_has_offset); 5966 atomic_add_64(&isc->isc_link_xoff_rx.ev_count, delta); 5967 5968 delta = ixl_stat_delta(sc, 5969 0, I40E_GLPRT_LXOFFTXC(sc->sc_port), 5970 &isc->isc_link_xoff_tx_offset, isc->isc_has_offset); 5971 atomic_add_64(&isc->isc_link_xoff_tx.ev_count, delta); 5972 5973 /* fragments */ 5974 delta = ixl_stat_delta(sc, 5975 0, I40E_GLPRT_RFC(sc->sc_port), 5976 &isc->isc_rx_fragments_offset, isc->isc_has_offset); 5977 atomic_add_64(&isc->isc_rx_fragments.ev_count, delta); 5978 5979 delta = ixl_stat_delta(sc, 5980 0, I40E_GLPRT_RJC(sc->sc_port), 5981 &isc->isc_rx_jabber_offset, isc->isc_has_offset); 5982 atomic_add_64(&isc->isc_rx_jabber.ev_count, delta); 5983 } 5984 5985 static int 5986 ixl_setup_sysctls(struct ixl_softc *sc) 5987 { 5988 const char *devname; 5989 struct sysctllog **log; 5990 const struct sysctlnode *rnode, *rxnode, *txnode; 5991 int error; 5992 5993 log = &sc->sc_sysctllog; 5994 devname = device_xname(sc->sc_dev); 5995 5996 error = sysctl_createv(log, 0, NULL, &rnode, 5997 0, CTLTYPE_NODE, devname, 5998 SYSCTL_DESCR("ixl information and settings"), 5999 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 6000 if (error) 6001 goto out; 6002 6003 error = sysctl_createv(log, 0, &rnode, NULL, 6004 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 6005 SYSCTL_DESCR("Use workqueue for packet processing"), 6006 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 6007 if (error) 6008 goto out; 6009 6010 error = sysctl_createv(log, 0, &rnode, NULL, 6011 CTLFLAG_READONLY, CTLTYPE_INT, "stats_interval", 6012 SYSCTL_DESCR("Statistics collection interval in milliseconds"), 6013 NULL, 0, &sc->sc_stats_intval, 0, CTL_CREATE, CTL_EOL); 6014 6015 error = sysctl_createv(log, 0, &rnode, &rxnode, 6016 0, CTLTYPE_NODE, "rx", 6017 SYSCTL_DESCR("ixl information and settings for Rx"), 6018 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6019 if (error) 6020 goto out; 6021 6022 error = sysctl_createv(log, 0, &rxnode, NULL, 6023 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6024 SYSCTL_DESCR("max number of Rx packets" 6025 " to process for interrupt processing"), 6026 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6027 if (error) 6028 goto out; 6029 6030 error = sysctl_createv(log, 0, &rxnode, NULL, 6031 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6032 SYSCTL_DESCR("max number of Rx packets" 6033 " to process for deferred processing"), 6034 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 6035 if (error) 6036 goto out; 6037 6038 error = sysctl_createv(log, 0, &rnode, &txnode, 6039 0, CTLTYPE_NODE, "tx", 6040 SYSCTL_DESCR("ixl information and settings for Tx"), 6041 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6042 if (error) 6043 goto out; 6044 6045 error = sysctl_createv(log, 0, &txnode, NULL, 6046 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6047 SYSCTL_DESCR("max number of Tx packets" 6048 " to process for interrupt processing"), 6049 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6050 if (error) 6051 goto out; 6052 6053 error = sysctl_createv(log, 0, &txnode, NULL, 6054 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6055 SYSCTL_DESCR("max number of Tx packets" 6056 " to process for deferred processing"), 6057 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 6058 if (error) 6059 goto out; 6060 6061 out: 6062 if (error) { 6063 aprint_error_dev(sc->sc_dev, 6064 "unable to create sysctl node\n"); 6065 sysctl_teardown(log); 6066 } 6067 6068 return error; 6069 } 6070 6071 static void 6072 ixl_teardown_sysctls(struct ixl_softc *sc) 6073 { 6074 6075 sysctl_teardown(&sc->sc_sysctllog); 6076 } 6077 6078 static struct workqueue * 6079 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 6080 { 6081 struct workqueue *wq; 6082 int error; 6083 6084 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 6085 prio, ipl, flags); 6086 6087 if (error) 6088 return NULL; 6089 6090 return wq; 6091 } 6092 6093 static void 6094 ixl_workq_destroy(struct workqueue *wq) 6095 { 6096 6097 workqueue_destroy(wq); 6098 } 6099 6100 static void 6101 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 6102 { 6103 6104 memset(work, 0, sizeof(*work)); 6105 work->ixw_func = func; 6106 work->ixw_arg = arg; 6107 } 6108 6109 static void 6110 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 6111 { 6112 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 6113 return; 6114 6115 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 6116 } 6117 6118 static void 6119 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 6120 { 6121 6122 workqueue_wait(wq, &work->ixw_cookie); 6123 } 6124 6125 static void 6126 ixl_workq_work(struct work *wk, void *context) 6127 { 6128 struct ixl_work *work; 6129 6130 work = container_of(wk, struct ixl_work, ixw_cookie); 6131 6132 atomic_swap_uint(&work->ixw_added, 0); 6133 work->ixw_func(work->ixw_arg); 6134 } 6135 6136 static int 6137 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 6138 { 6139 struct ixl_aq_desc iaq; 6140 6141 memset(&iaq, 0, sizeof(iaq)); 6142 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 6143 iaq.iaq_param[1] = htole32(reg); 6144 6145 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6146 return ETIMEDOUT; 6147 6148 switch (htole16(iaq.iaq_retval)) { 6149 case IXL_AQ_RC_OK: 6150 /* success */ 6151 break; 6152 case IXL_AQ_RC_EACCES: 6153 return EPERM; 6154 case IXL_AQ_RC_EAGAIN: 6155 return EAGAIN; 6156 default: 6157 return EIO; 6158 } 6159 6160 *rv = htole32(iaq.iaq_param[3]); 6161 return 0; 6162 } 6163 6164 static uint32_t 6165 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 6166 { 6167 uint32_t val; 6168 int rv, retry, retry_limit; 6169 6170 retry_limit = sc->sc_rxctl_atq ? 5 : 0; 6171 6172 for (retry = 0; retry < retry_limit; retry++) { 6173 rv = ixl_rx_ctl_read(sc, reg, &val); 6174 if (rv == 0) 6175 return val; 6176 else if (rv == EAGAIN) 6177 delaymsec(1); 6178 else 6179 break; 6180 } 6181 6182 val = ixl_rd(sc, reg); 6183 6184 return val; 6185 } 6186 6187 static int 6188 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6189 { 6190 struct ixl_aq_desc iaq; 6191 6192 memset(&iaq, 0, sizeof(iaq)); 6193 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 6194 iaq.iaq_param[1] = htole32(reg); 6195 iaq.iaq_param[3] = htole32(value); 6196 6197 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6198 return ETIMEDOUT; 6199 6200 switch (htole16(iaq.iaq_retval)) { 6201 case IXL_AQ_RC_OK: 6202 /* success */ 6203 break; 6204 case IXL_AQ_RC_EACCES: 6205 return EPERM; 6206 case IXL_AQ_RC_EAGAIN: 6207 return EAGAIN; 6208 default: 6209 return EIO; 6210 } 6211 6212 return 0; 6213 } 6214 6215 static void 6216 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6217 { 6218 int rv, retry, retry_limit; 6219 6220 retry_limit = sc->sc_rxctl_atq ? 5 : 0; 6221 6222 for (retry = 0; retry < retry_limit; retry++) { 6223 rv = ixl_rx_ctl_write(sc, reg, value); 6224 if (rv == 0) 6225 return; 6226 else if (rv == EAGAIN) 6227 delaymsec(1); 6228 else 6229 break; 6230 } 6231 6232 ixl_wr(sc, reg, value); 6233 } 6234 6235 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 6236 6237 #ifdef _MODULE 6238 #include "ioconf.c" 6239 #endif 6240 6241 static int 6242 if_ixl_modcmd(modcmd_t cmd, void *opaque) 6243 { 6244 int error = 0; 6245 6246 #ifdef _MODULE 6247 switch (cmd) { 6248 case MODULE_CMD_INIT: 6249 error = config_init_component(cfdriver_ioconf_if_ixl, 6250 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 6251 break; 6252 case MODULE_CMD_FINI: 6253 error = config_fini_component(cfdriver_ioconf_if_ixl, 6254 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 6255 break; 6256 default: 6257 error = ENOTTY; 6258 break; 6259 } 6260 #endif 6261 6262 return error; 6263 } 6264
Indexes created Sat Sep 20 22:09:52 GMT 2025