if_ixl.c revision 1.44
1 /* $NetBSD: if_ixl.c,v 1.44 2020/02/25 07:05:57 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 __KERNEL_RCSID(0, "$NetBSD: if_ixl.c,v 1.44 2020/02/25 07:05:57 yamaguchi Exp $"); 78 79 #ifdef _KERNEL_OPT 80 #include "opt_net_mpsafe.h" 81 #include "opt_if_ixl.h" 82 #endif 83 84 #include <sys/param.h> 85 #include <sys/types.h> 86 87 #include <sys/cpu.h> 88 #include <sys/device.h> 89 #include <sys/evcnt.h> 90 #include <sys/interrupt.h> 91 #include <sys/kmem.h> 92 #include <sys/malloc.h> 93 #include <sys/module.h> 94 #include <sys/mutex.h> 95 #include <sys/pcq.h> 96 #include <sys/syslog.h> 97 #include <sys/workqueue.h> 98 99 #include <sys/bus.h> 100 101 #include <net/bpf.h> 102 #include <net/if.h> 103 #include <net/if_dl.h> 104 #include <net/if_media.h> 105 #include <net/if_ether.h> 106 #include <net/rss_config.h> 107 108 #include <netinet/tcp.h> /* for struct tcphdr */ 109 #include <netinet/udp.h> /* for struct udphdr */ 110 111 #include <dev/pci/pcivar.h> 112 #include <dev/pci/pcidevs.h> 113 114 #include <dev/pci/if_ixlreg.h> 115 #include <dev/pci/if_ixlvar.h> 116 117 #include <prop/proplib.h> 118 119 struct ixl_softc; /* defined */ 120 121 #define I40E_PF_RESET_WAIT_COUNT 200 122 #define I40E_AQ_LARGE_BUF 512 123 124 /* bitfields for Tx queue mapping in QTX_CTL */ 125 #define I40E_QTX_CTL_VF_QUEUE 0x0 126 #define I40E_QTX_CTL_VM_QUEUE 0x1 127 #define I40E_QTX_CTL_PF_QUEUE 0x2 128 129 #define I40E_QUEUE_TYPE_EOL 0x7ff 130 #define I40E_INTR_NOTX_QUEUE 0 131 132 #define I40E_QUEUE_TYPE_RX 0x0 133 #define I40E_QUEUE_TYPE_TX 0x1 134 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 135 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 136 137 #define I40E_ITR_INDEX_RX 0x0 138 #define I40E_ITR_INDEX_TX 0x1 139 #define I40E_ITR_INDEX_OTHER 0x2 140 #define I40E_ITR_INDEX_NONE 0x3 141 142 #define I40E_INTR_NOTX_QUEUE 0 143 #define I40E_INTR_NOTX_INTR 0 144 #define I40E_INTR_NOTX_RX_QUEUE 0 145 #define I40E_INTR_NOTX_TX_QUEUE 1 146 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 147 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 148 149 #define BIT_ULL(a) (1ULL << (a)) 150 #define IXL_RSS_HENA_DEFAULT_BASE \ 151 (BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 152 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 153 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 154 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 155 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 156 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 157 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 158 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 159 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 160 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 161 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 162 #define IXL_RSS_HENA_DEFAULT_XL710 IXL_RSS_HENA_DEFAULT_BASE 163 #define IXL_RSS_HENA_DEFAULT_X722 (IXL_RSS_HENA_DEFAULT_XL710 | \ 164 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 165 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 166 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 167 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 168 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 169 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 170 #define I40E_HASH_LUT_SIZE_128 0 171 #define IXL_RSS_KEY_SIZE_REG 13 172 173 #define IXL_ICR0_CRIT_ERR_MASK \ 174 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 175 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 176 I40E_PFINT_ICR0_PE_CRITERR_MASK) 177 178 #define IXL_TX_PKT_DESCS 8 179 #define IXL_TX_PKT_MAXSIZE (MCLBYTES * IXL_TX_PKT_DESCS) 180 #define IXL_TX_QUEUE_ALIGN 128 181 #define IXL_RX_QUEUE_ALIGN 128 182 183 #define IXL_MCLBYTES (MCLBYTES - ETHER_ALIGN) 184 #define IXL_MTU_ETHERLEN ETHER_HDR_LEN \ 185 + ETHER_CRC_LEN 186 #if 0 187 #define IXL_MAX_MTU (9728 - IXL_MTU_ETHERLEN) 188 #else 189 /* (dbuff * 5) - ETHER_HDR_LEN - ETHER_CRC_LEN */ 190 #define IXL_MAX_MTU (9600 - IXL_MTU_ETHERLEN) 191 #endif 192 #define IXL_MIN_MTU (ETHER_MIN_LEN - ETHER_CRC_LEN) 193 194 #define IXL_PCIREG PCI_MAPREG_START 195 196 #define IXL_ITR0 0x0 197 #define IXL_ITR1 0x1 198 #define IXL_ITR2 0x2 199 #define IXL_NOITR 0x3 200 201 #define IXL_AQ_NUM 256 202 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 203 #define IXL_AQ_ALIGN 64 /* lol */ 204 #define IXL_AQ_BUFLEN 4096 205 206 #define IXL_HMC_ROUNDUP 512 207 #define IXL_HMC_PGSIZE 4096 208 #define IXL_HMC_DVASZ sizeof(uint64_t) 209 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 210 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 211 #define IXL_HMC_PDVALID 1ULL 212 213 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 214 215 #define IXL_SRRD_SRCTL_ATTEMPTS 100000 216 217 struct ixl_aq_regs { 218 bus_size_t atq_tail; 219 bus_size_t atq_head; 220 bus_size_t atq_len; 221 bus_size_t atq_bal; 222 bus_size_t atq_bah; 223 224 bus_size_t arq_tail; 225 bus_size_t arq_head; 226 bus_size_t arq_len; 227 bus_size_t arq_bal; 228 bus_size_t arq_bah; 229 230 uint32_t atq_len_enable; 231 uint32_t atq_tail_mask; 232 uint32_t atq_head_mask; 233 234 uint32_t arq_len_enable; 235 uint32_t arq_tail_mask; 236 uint32_t arq_head_mask; 237 }; 238 239 struct ixl_phy_type { 240 uint64_t phy_type; 241 uint64_t ifm_type; 242 }; 243 244 struct ixl_speed_type { 245 uint8_t dev_speed; 246 uint64_t net_speed; 247 }; 248 249 struct ixl_aq_buf { 250 SIMPLEQ_ENTRY(ixl_aq_buf) 251 aqb_entry; 252 void *aqb_data; 253 bus_dmamap_t aqb_map; 254 bus_dma_segment_t aqb_seg; 255 size_t aqb_size; 256 int aqb_nsegs; 257 }; 258 SIMPLEQ_HEAD(ixl_aq_bufs, ixl_aq_buf); 259 260 struct ixl_dmamem { 261 bus_dmamap_t ixm_map; 262 bus_dma_segment_t ixm_seg; 263 int ixm_nsegs; 264 size_t ixm_size; 265 void *ixm_kva; 266 }; 267 268 #define IXL_DMA_MAP(_ixm) ((_ixm)->ixm_map) 269 #define IXL_DMA_DVA(_ixm) ((_ixm)->ixm_map->dm_segs[0].ds_addr) 270 #define IXL_DMA_KVA(_ixm) ((void *)(_ixm)->ixm_kva) 271 #define IXL_DMA_LEN(_ixm) ((_ixm)->ixm_size) 272 273 struct ixl_hmc_entry { 274 uint64_t hmc_base; 275 uint32_t hmc_count; 276 uint64_t hmc_size; 277 }; 278 279 enum ixl_hmc_types { 280 IXL_HMC_LAN_TX = 0, 281 IXL_HMC_LAN_RX, 282 IXL_HMC_FCOE_CTX, 283 IXL_HMC_FCOE_FILTER, 284 IXL_HMC_COUNT 285 }; 286 287 struct ixl_hmc_pack { 288 uint16_t offset; 289 uint16_t width; 290 uint16_t lsb; 291 }; 292 293 /* 294 * these hmc objects have weird sizes and alignments, so these are abstract 295 * representations of them that are nice for c to populate. 296 * 297 * the packing code relies on little-endian values being stored in the fields, 298 * no high bits in the fields being set, and the fields must be packed in the 299 * same order as they are in the ctx structure. 300 */ 301 302 struct ixl_hmc_rxq { 303 uint16_t head; 304 uint8_t cpuid; 305 uint64_t base; 306 #define IXL_HMC_RXQ_BASE_UNIT 128 307 uint16_t qlen; 308 uint16_t dbuff; 309 #define IXL_HMC_RXQ_DBUFF_UNIT 128 310 uint8_t hbuff; 311 #define IXL_HMC_RXQ_HBUFF_UNIT 64 312 uint8_t dtype; 313 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 314 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 315 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 316 uint8_t dsize; 317 #define IXL_HMC_RXQ_DSIZE_16 0 318 #define IXL_HMC_RXQ_DSIZE_32 1 319 uint8_t crcstrip; 320 uint8_t fc_ena; 321 uint8_t l2sel; 322 uint8_t hsplit_0; 323 uint8_t hsplit_1; 324 uint8_t showiv; 325 uint16_t rxmax; 326 uint8_t tphrdesc_ena; 327 uint8_t tphwdesc_ena; 328 uint8_t tphdata_ena; 329 uint8_t tphhead_ena; 330 uint8_t lrxqthresh; 331 uint8_t prefena; 332 }; 333 334 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 335 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 336 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 337 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 338 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 339 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 340 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 341 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 342 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 343 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 344 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 345 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 346 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 347 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 348 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 349 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 350 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 351 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 352 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 353 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 354 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 355 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 356 }; 357 358 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 359 360 struct ixl_hmc_txq { 361 uint16_t head; 362 uint8_t new_context; 363 uint64_t base; 364 #define IXL_HMC_TXQ_BASE_UNIT 128 365 uint8_t fc_ena; 366 uint8_t timesync_ena; 367 uint8_t fd_ena; 368 uint8_t alt_vlan_ena; 369 uint16_t thead_wb; 370 uint8_t cpuid; 371 uint8_t head_wb_ena; 372 #define IXL_HMC_TXQ_DESC_WB 0 373 #define IXL_HMC_TXQ_HEAD_WB 1 374 uint16_t qlen; 375 uint8_t tphrdesc_ena; 376 uint8_t tphrpacket_ena; 377 uint8_t tphwdesc_ena; 378 uint64_t head_wb_addr; 379 uint32_t crc; 380 uint16_t rdylist; 381 uint8_t rdylist_act; 382 }; 383 384 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 385 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 386 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 387 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 388 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 389 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 390 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 391 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 392 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 393 /* line 1 */ 394 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 395 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 396 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 397 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 398 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 399 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 400 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 401 /* line 7 */ 402 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 403 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 404 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 405 }; 406 407 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 408 409 struct ixl_work { 410 struct work ixw_cookie; 411 void (*ixw_func)(void *); 412 void *ixw_arg; 413 unsigned int ixw_added; 414 }; 415 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 416 417 struct ixl_tx_map { 418 struct mbuf *txm_m; 419 bus_dmamap_t txm_map; 420 unsigned int txm_eop; 421 }; 422 423 struct ixl_tx_ring { 424 kmutex_t txr_lock; 425 struct ixl_softc *txr_sc; 426 427 unsigned int txr_prod; 428 unsigned int txr_cons; 429 430 struct ixl_tx_map *txr_maps; 431 struct ixl_dmamem txr_mem; 432 433 bus_size_t txr_tail; 434 unsigned int txr_qid; 435 pcq_t *txr_intrq; 436 void *txr_si; 437 438 struct evcnt txr_defragged; 439 struct evcnt txr_defrag_failed; 440 struct evcnt txr_pcqdrop; 441 struct evcnt txr_transmitdef; 442 struct evcnt txr_intr; 443 struct evcnt txr_defer; 444 }; 445 446 struct ixl_rx_map { 447 struct mbuf *rxm_m; 448 bus_dmamap_t rxm_map; 449 }; 450 451 struct ixl_rx_ring { 452 kmutex_t rxr_lock; 453 454 unsigned int rxr_prod; 455 unsigned int rxr_cons; 456 457 struct ixl_rx_map *rxr_maps; 458 struct ixl_dmamem rxr_mem; 459 460 struct mbuf *rxr_m_head; 461 struct mbuf **rxr_m_tail; 462 463 bus_size_t rxr_tail; 464 unsigned int rxr_qid; 465 466 struct evcnt rxr_mgethdr_failed; 467 struct evcnt rxr_mgetcl_failed; 468 struct evcnt rxr_mbuf_load_failed; 469 struct evcnt rxr_intr; 470 struct evcnt rxr_defer; 471 }; 472 473 struct ixl_queue_pair { 474 struct ixl_softc *qp_sc; 475 struct ixl_tx_ring *qp_txr; 476 struct ixl_rx_ring *qp_rxr; 477 478 char qp_name[16]; 479 480 void *qp_si; 481 struct ixl_work qp_task; 482 bool qp_workqueue; 483 }; 484 485 struct ixl_atq { 486 struct ixl_aq_desc iatq_desc; 487 void (*iatq_fn)(struct ixl_softc *, 488 const struct ixl_aq_desc *); 489 }; 490 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 491 492 struct ixl_product { 493 unsigned int vendor_id; 494 unsigned int product_id; 495 }; 496 497 struct ixl_stats_counters { 498 bool isc_has_offset; 499 struct evcnt isc_crc_errors; 500 uint64_t isc_crc_errors_offset; 501 struct evcnt isc_illegal_bytes; 502 uint64_t isc_illegal_bytes_offset; 503 struct evcnt isc_rx_bytes; 504 uint64_t isc_rx_bytes_offset; 505 struct evcnt isc_rx_discards; 506 uint64_t isc_rx_discards_offset; 507 struct evcnt isc_rx_unicast; 508 uint64_t isc_rx_unicast_offset; 509 struct evcnt isc_rx_multicast; 510 uint64_t isc_rx_multicast_offset; 511 struct evcnt isc_rx_broadcast; 512 uint64_t isc_rx_broadcast_offset; 513 struct evcnt isc_rx_size_64; 514 uint64_t isc_rx_size_64_offset; 515 struct evcnt isc_rx_size_127; 516 uint64_t isc_rx_size_127_offset; 517 struct evcnt isc_rx_size_255; 518 uint64_t isc_rx_size_255_offset; 519 struct evcnt isc_rx_size_511; 520 uint64_t isc_rx_size_511_offset; 521 struct evcnt isc_rx_size_1023; 522 uint64_t isc_rx_size_1023_offset; 523 struct evcnt isc_rx_size_1522; 524 uint64_t isc_rx_size_1522_offset; 525 struct evcnt isc_rx_size_big; 526 uint64_t isc_rx_size_big_offset; 527 struct evcnt isc_rx_undersize; 528 uint64_t isc_rx_undersize_offset; 529 struct evcnt isc_rx_oversize; 530 uint64_t isc_rx_oversize_offset; 531 struct evcnt isc_rx_fragments; 532 uint64_t isc_rx_fragments_offset; 533 struct evcnt isc_rx_jabber; 534 uint64_t isc_rx_jabber_offset; 535 struct evcnt isc_tx_bytes; 536 uint64_t isc_tx_bytes_offset; 537 struct evcnt isc_tx_dropped_link_down; 538 uint64_t isc_tx_dropped_link_down_offset; 539 struct evcnt isc_tx_unicast; 540 uint64_t isc_tx_unicast_offset; 541 struct evcnt isc_tx_multicast; 542 uint64_t isc_tx_multicast_offset; 543 struct evcnt isc_tx_broadcast; 544 uint64_t isc_tx_broadcast_offset; 545 struct evcnt isc_tx_size_64; 546 uint64_t isc_tx_size_64_offset; 547 struct evcnt isc_tx_size_127; 548 uint64_t isc_tx_size_127_offset; 549 struct evcnt isc_tx_size_255; 550 uint64_t isc_tx_size_255_offset; 551 struct evcnt isc_tx_size_511; 552 uint64_t isc_tx_size_511_offset; 553 struct evcnt isc_tx_size_1023; 554 uint64_t isc_tx_size_1023_offset; 555 struct evcnt isc_tx_size_1522; 556 uint64_t isc_tx_size_1522_offset; 557 struct evcnt isc_tx_size_big; 558 uint64_t isc_tx_size_big_offset; 559 struct evcnt isc_mac_local_faults; 560 uint64_t isc_mac_local_faults_offset; 561 struct evcnt isc_mac_remote_faults; 562 uint64_t isc_mac_remote_faults_offset; 563 struct evcnt isc_link_xon_rx; 564 uint64_t isc_link_xon_rx_offset; 565 struct evcnt isc_link_xon_tx; 566 uint64_t isc_link_xon_tx_offset; 567 struct evcnt isc_link_xoff_rx; 568 uint64_t isc_link_xoff_rx_offset; 569 struct evcnt isc_link_xoff_tx; 570 uint64_t isc_link_xoff_tx_offset; 571 struct evcnt isc_vsi_rx_discards; 572 uint64_t isc_vsi_rx_discards_offset; 573 struct evcnt isc_vsi_rx_bytes; 574 uint64_t isc_vsi_rx_bytes_offset; 575 struct evcnt isc_vsi_rx_unicast; 576 uint64_t isc_vsi_rx_unicast_offset; 577 struct evcnt isc_vsi_rx_multicast; 578 uint64_t isc_vsi_rx_multicast_offset; 579 struct evcnt isc_vsi_rx_broadcast; 580 uint64_t isc_vsi_rx_broadcast_offset; 581 struct evcnt isc_vsi_tx_errors; 582 uint64_t isc_vsi_tx_errors_offset; 583 struct evcnt isc_vsi_tx_bytes; 584 uint64_t isc_vsi_tx_bytes_offset; 585 struct evcnt isc_vsi_tx_unicast; 586 uint64_t isc_vsi_tx_unicast_offset; 587 struct evcnt isc_vsi_tx_multicast; 588 uint64_t isc_vsi_tx_multicast_offset; 589 struct evcnt isc_vsi_tx_broadcast; 590 uint64_t isc_vsi_tx_broadcast_offset; 591 }; 592 593 /* 594 * Locking notes: 595 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 596 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 597 * - more than one lock of them cannot be held at once. 598 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 599 * (a spin mutex). 600 * - the lock cannot held with txr_lock or rxr_lock. 601 * + a field named sc_arq_* is not protected by any lock. 602 * - operations for sc_arq_* is done in one context related to 603 * sc_arq_task. 604 * + other fields in ixl_softc is protected by sc_cfg_lock 605 * (an adaptive mutex) 606 * - It must be held before another lock is held, and It can be 607 * released after the other lock is released. 608 * */ 609 610 struct ixl_softc { 611 device_t sc_dev; 612 struct ethercom sc_ec; 613 bool sc_attached; 614 bool sc_dead; 615 uint32_t sc_port; 616 struct sysctllog *sc_sysctllog; 617 struct workqueue *sc_workq; 618 struct workqueue *sc_workq_txrx; 619 int sc_stats_intval; 620 callout_t sc_stats_callout; 621 struct ixl_work sc_stats_task; 622 struct ixl_stats_counters 623 sc_stats_counters; 624 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 625 struct ifmedia sc_media; 626 uint64_t sc_media_status; 627 uint64_t sc_media_active; 628 uint64_t sc_phy_types; 629 uint8_t sc_phy_abilities; 630 uint8_t sc_phy_linkspeed; 631 uint8_t sc_phy_fec_cfg; 632 uint16_t sc_eee_cap; 633 uint32_t sc_eeer_val; 634 uint8_t sc_d3_lpan; 635 kmutex_t sc_cfg_lock; 636 enum i40e_mac_type sc_mac_type; 637 uint32_t sc_rss_table_size; 638 uint32_t sc_rss_table_entry_width; 639 bool sc_txrx_workqueue; 640 u_int sc_tx_process_limit; 641 u_int sc_rx_process_limit; 642 u_int sc_tx_intr_process_limit; 643 u_int sc_rx_intr_process_limit; 644 645 int sc_cur_ec_capenable; 646 647 struct pci_attach_args sc_pa; 648 pci_intr_handle_t *sc_ihp; 649 void **sc_ihs; 650 unsigned int sc_nintrs; 651 652 bus_dma_tag_t sc_dmat; 653 bus_space_tag_t sc_memt; 654 bus_space_handle_t sc_memh; 655 bus_size_t sc_mems; 656 657 uint8_t sc_pf_id; 658 uint16_t sc_uplink_seid; /* le */ 659 uint16_t sc_downlink_seid; /* le */ 660 uint16_t sc_vsi_number; 661 uint16_t sc_vsi_stat_counter_idx; 662 uint16_t sc_seid; 663 unsigned int sc_base_queue; 664 665 pci_intr_type_t sc_intrtype; 666 unsigned int sc_msix_vector_queue; 667 668 struct ixl_dmamem sc_scratch; 669 struct ixl_dmamem sc_aqbuf; 670 671 const struct ixl_aq_regs * 672 sc_aq_regs; 673 uint32_t sc_aq_flags; 674 #define IXL_SC_AQ_FLAG_RXCTL __BIT(0) 675 #define IXL_SC_AQ_FLAG_NVMLOCK __BIT(1) 676 #define IXL_SC_AQ_FLAG_NVMREAD __BIT(2) 677 #define IXL_SC_AQ_FLAG_RSS __BIT(3) 678 679 kmutex_t sc_atq_lock; 680 kcondvar_t sc_atq_cv; 681 struct ixl_dmamem sc_atq; 682 unsigned int sc_atq_prod; 683 unsigned int sc_atq_cons; 684 685 struct ixl_dmamem sc_arq; 686 struct ixl_work sc_arq_task; 687 struct ixl_aq_bufs sc_arq_idle; 688 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 689 unsigned int sc_arq_prod; 690 unsigned int sc_arq_cons; 691 692 struct ixl_work sc_link_state_task; 693 struct ixl_atq sc_link_state_atq; 694 695 struct ixl_dmamem sc_hmc_sd; 696 struct ixl_dmamem sc_hmc_pd; 697 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 698 699 unsigned int sc_tx_ring_ndescs; 700 unsigned int sc_rx_ring_ndescs; 701 unsigned int sc_nqueue_pairs; 702 unsigned int sc_nqueue_pairs_max; 703 unsigned int sc_nqueue_pairs_device; 704 struct ixl_queue_pair *sc_qps; 705 706 struct evcnt sc_event_atq; 707 struct evcnt sc_event_link; 708 struct evcnt sc_event_ecc_err; 709 struct evcnt sc_event_pci_exception; 710 struct evcnt sc_event_crit_err; 711 }; 712 713 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 714 #define IXL_TX_PROCESS_LIMIT 256 715 #define IXL_RX_PROCESS_LIMIT 256 716 #define IXL_TX_INTR_PROCESS_LIMIT 256 717 #define IXL_RX_INTR_PROCESS_LIMIT 0U 718 719 #define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx | \ 720 IFCAP_CSUM_TCPv4_Rx | \ 721 IFCAP_CSUM_UDPv4_Rx | \ 722 IFCAP_CSUM_TCPv6_Rx | \ 723 IFCAP_CSUM_UDPv6_Rx) 724 #define IXL_IFCAP_TXCSUM (IFCAP_CSUM_IPv4_Tx | \ 725 IFCAP_CSUM_TCPv4_Tx | \ 726 IFCAP_CSUM_UDPv4_Tx | \ 727 IFCAP_CSUM_TCPv6_Tx | \ 728 IFCAP_CSUM_UDPv6_Tx) 729 #define IXL_CSUM_ALL_OFFLOAD (M_CSUM_IPv4 | \ 730 M_CSUM_TCPv4 | M_CSUM_TCPv6 | \ 731 M_CSUM_UDPv4 | M_CSUM_UDPv6) 732 733 #define delaymsec(_x) DELAY(1000 * (_x)) 734 #ifdef IXL_DEBUG 735 #define DDPRINTF(sc, fmt, args...) \ 736 do { \ 737 if ((sc) != NULL) { \ 738 device_printf( \ 739 ((struct ixl_softc *)(sc))->sc_dev, \ 740 ""); \ 741 } \ 742 printf("%s:\t" fmt, __func__, ##args); \ 743 } while (0) 744 #else 745 #define DDPRINTF(sc, fmt, args...) __nothing 746 #endif 747 #ifndef IXL_STATS_INTERVAL_MSEC 748 #define IXL_STATS_INTERVAL_MSEC 10000 749 #endif 750 #ifndef IXL_QUEUE_NUM 751 #define IXL_QUEUE_NUM 0 752 #endif 753 754 static bool ixl_param_nomsix = false; 755 static int ixl_param_stats_interval = IXL_STATS_INTERVAL_MSEC; 756 static int ixl_param_nqps_limit = IXL_QUEUE_NUM; 757 static unsigned int ixl_param_tx_ndescs = 1024; 758 static unsigned int ixl_param_rx_ndescs = 1024; 759 760 static enum i40e_mac_type 761 ixl_mactype(pci_product_id_t); 762 static void ixl_clear_hw(struct ixl_softc *); 763 static int ixl_pf_reset(struct ixl_softc *); 764 765 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 766 bus_size_t, bus_size_t); 767 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 768 769 static int ixl_arq_fill(struct ixl_softc *); 770 static void ixl_arq_unfill(struct ixl_softc *); 771 772 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 773 unsigned int); 774 static void ixl_atq_set(struct ixl_atq *, 775 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 776 static int ixl_atq_post(struct ixl_softc *, struct ixl_atq *); 777 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 778 static void ixl_atq_done(struct ixl_softc *); 779 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 780 static int ixl_get_version(struct ixl_softc *); 781 static int ixl_get_nvm_version(struct ixl_softc *); 782 static int ixl_get_hw_capabilities(struct ixl_softc *); 783 static int ixl_pxe_clear(struct ixl_softc *); 784 static int ixl_lldp_shut(struct ixl_softc *); 785 static int ixl_get_mac(struct ixl_softc *); 786 static int ixl_get_switch_config(struct ixl_softc *); 787 static int ixl_phy_mask_ints(struct ixl_softc *); 788 static int ixl_get_phy_info(struct ixl_softc *); 789 static int ixl_set_phy_config(struct ixl_softc *, uint8_t, uint8_t, bool); 790 static int ixl_set_phy_autoselect(struct ixl_softc *); 791 static int ixl_restart_an(struct ixl_softc *); 792 static int ixl_hmc(struct ixl_softc *); 793 static void ixl_hmc_free(struct ixl_softc *); 794 static int ixl_get_vsi(struct ixl_softc *); 795 static int ixl_set_vsi(struct ixl_softc *); 796 static void ixl_set_filter_control(struct ixl_softc *); 797 static void ixl_get_link_status(void *); 798 static int ixl_get_link_status_poll(struct ixl_softc *, int *); 799 static int ixl_set_link_status(struct ixl_softc *, 800 const struct ixl_aq_desc *); 801 static uint64_t ixl_search_link_speed(uint8_t); 802 static uint8_t ixl_search_baudrate(uint64_t); 803 static void ixl_config_rss(struct ixl_softc *); 804 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 805 uint16_t, uint16_t); 806 static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *, 807 uint16_t, uint16_t); 808 static void ixl_arq(void *); 809 static void ixl_hmc_pack(void *, const void *, 810 const struct ixl_hmc_pack *, unsigned int); 811 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 812 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 813 static int ixl_rd16_nvm(struct ixl_softc *, uint16_t, uint16_t *); 814 815 static int ixl_match(device_t, cfdata_t, void *); 816 static void ixl_attach(device_t, device_t, void *); 817 static int ixl_detach(device_t, int); 818 819 static void ixl_media_add(struct ixl_softc *); 820 static int ixl_media_change(struct ifnet *); 821 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 822 static void ixl_watchdog(struct ifnet *); 823 static int ixl_ioctl(struct ifnet *, u_long, void *); 824 static void ixl_start(struct ifnet *); 825 static int ixl_transmit(struct ifnet *, struct mbuf *); 826 static void ixl_deferred_transmit(void *); 827 static int ixl_intr(void *); 828 static int ixl_queue_intr(void *); 829 static int ixl_other_intr(void *); 830 static void ixl_handle_queue(void *); 831 static void ixl_sched_handle_queue(struct ixl_softc *, 832 struct ixl_queue_pair *); 833 static int ixl_init(struct ifnet *); 834 static int ixl_init_locked(struct ixl_softc *); 835 static void ixl_stop(struct ifnet *, int); 836 static void ixl_stop_locked(struct ixl_softc *); 837 static int ixl_iff(struct ixl_softc *); 838 static int ixl_ifflags_cb(struct ethercom *); 839 static int ixl_setup_interrupts(struct ixl_softc *); 840 static int ixl_establish_intx(struct ixl_softc *); 841 static int ixl_establish_msix(struct ixl_softc *); 842 static void ixl_enable_queue_intr(struct ixl_softc *, 843 struct ixl_queue_pair *); 844 static void ixl_disable_queue_intr(struct ixl_softc *, 845 struct ixl_queue_pair *); 846 static void ixl_enable_other_intr(struct ixl_softc *); 847 static void ixl_disable_other_intr(struct ixl_softc *); 848 static void ixl_config_queue_intr(struct ixl_softc *); 849 static void ixl_config_other_intr(struct ixl_softc *); 850 851 static struct ixl_tx_ring * 852 ixl_txr_alloc(struct ixl_softc *, unsigned int); 853 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 854 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 855 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 856 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 857 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 858 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 859 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 860 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 861 862 static struct ixl_rx_ring * 863 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 864 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 865 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 866 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 867 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 868 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 869 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 870 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 871 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 872 873 static struct workqueue * 874 ixl_workq_create(const char *, pri_t, int, int); 875 static void ixl_workq_destroy(struct workqueue *); 876 static int ixl_workqs_teardown(device_t); 877 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 878 static void ixl_work_add(struct workqueue *, struct ixl_work *); 879 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 880 static void ixl_workq_work(struct work *, void *); 881 static const struct ixl_product * 882 ixl_lookup(const struct pci_attach_args *pa); 883 static void ixl_link_state_update(struct ixl_softc *, 884 const struct ixl_aq_desc *); 885 static int ixl_vlan_cb(struct ethercom *, uint16_t, bool); 886 static int ixl_setup_vlan_hwfilter(struct ixl_softc *); 887 static void ixl_teardown_vlan_hwfilter(struct ixl_softc *); 888 static int ixl_update_macvlan(struct ixl_softc *); 889 static int ixl_setup_interrupts(struct ixl_softc *);; 890 static void ixl_teardown_interrupts(struct ixl_softc *); 891 static int ixl_setup_stats(struct ixl_softc *); 892 static void ixl_teardown_stats(struct ixl_softc *); 893 static void ixl_stats_callout(void *); 894 static void ixl_stats_update(void *); 895 static int ixl_setup_sysctls(struct ixl_softc *); 896 static void ixl_teardown_sysctls(struct ixl_softc *); 897 static int ixl_queue_pairs_alloc(struct ixl_softc *); 898 static void ixl_queue_pairs_free(struct ixl_softc *); 899 900 static const struct ixl_phy_type ixl_phy_type_map[] = { 901 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 902 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 903 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 904 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 905 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 906 { 1ULL << IXL_PHY_TYPE_XAUI | 907 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 908 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 909 { 1ULL << IXL_PHY_TYPE_XLAUI | 910 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 911 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 912 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 913 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 914 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 915 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 916 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 917 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 918 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 919 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 920 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 921 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 922 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 923 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 924 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 925 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 926 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 927 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 928 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 929 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 930 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 931 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 932 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 933 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 934 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_CR }, 935 }; 936 937 static const struct ixl_speed_type ixl_speed_type_map[] = { 938 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 939 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 940 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 941 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 942 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 943 }; 944 945 static const struct ixl_aq_regs ixl_pf_aq_regs = { 946 .atq_tail = I40E_PF_ATQT, 947 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 948 .atq_head = I40E_PF_ATQH, 949 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 950 .atq_len = I40E_PF_ATQLEN, 951 .atq_bal = I40E_PF_ATQBAL, 952 .atq_bah = I40E_PF_ATQBAH, 953 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 954 955 .arq_tail = I40E_PF_ARQT, 956 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 957 .arq_head = I40E_PF_ARQH, 958 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 959 .arq_len = I40E_PF_ARQLEN, 960 .arq_bal = I40E_PF_ARQBAL, 961 .arq_bah = I40E_PF_ARQBAH, 962 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 963 }; 964 965 #define ixl_rd(_s, _r) \ 966 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 967 #define ixl_wr(_s, _r, _v) \ 968 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 969 #define ixl_barrier(_s, _r, _l, _o) \ 970 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 971 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 972 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 973 974 static inline uint32_t 975 ixl_dmamem_hi(struct ixl_dmamem *ixm) 976 { 977 uint32_t retval; 978 uint64_t val; 979 980 if (sizeof(IXL_DMA_DVA(ixm)) > 4) { 981 val = (intptr_t)IXL_DMA_DVA(ixm); 982 retval = (uint32_t)(val >> 32); 983 } else { 984 retval = 0; 985 } 986 987 return retval; 988 } 989 990 static inline uint32_t 991 ixl_dmamem_lo(struct ixl_dmamem *ixm) 992 { 993 994 return (uint32_t)IXL_DMA_DVA(ixm); 995 } 996 997 static inline void 998 ixl_aq_dva(struct ixl_aq_desc *iaq, bus_addr_t addr) 999 { 1000 uint64_t val; 1001 1002 if (sizeof(addr) > 4) { 1003 val = (intptr_t)addr; 1004 iaq->iaq_param[2] = htole32(val >> 32); 1005 } else { 1006 iaq->iaq_param[2] = htole32(0); 1007 } 1008 1009 iaq->iaq_param[3] = htole32(addr); 1010 } 1011 1012 static inline unsigned int 1013 ixl_rxr_unrefreshed(unsigned int prod, unsigned int cons, unsigned int ndescs) 1014 { 1015 unsigned int num; 1016 1017 if (prod < cons) 1018 num = cons - prod; 1019 else 1020 num = (ndescs - prod) + cons; 1021 1022 if (__predict_true(num > 0)) { 1023 /* device cannot receive packets if all descripter is filled */ 1024 num -= 1; 1025 } 1026 1027 return num; 1028 } 1029 1030 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 1031 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 1032 DVF_DETACH_SHUTDOWN); 1033 1034 static const struct ixl_product ixl_products[] = { 1035 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 1036 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 1037 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 1038 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 1039 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 1040 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 1041 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T }, 1042 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 1043 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 1044 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 1045 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 1046 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 1047 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 1048 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 1049 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 1050 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 1051 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 1052 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 1053 /* required last entry */ 1054 {0, 0} 1055 }; 1056 1057 static const struct ixl_product * 1058 ixl_lookup(const struct pci_attach_args *pa) 1059 { 1060 const struct ixl_product *ixlp; 1061 1062 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 1063 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 1064 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 1065 return ixlp; 1066 } 1067 1068 return NULL; 1069 } 1070 1071 static int 1072 ixl_match(device_t parent, cfdata_t match, void *aux) 1073 { 1074 const struct pci_attach_args *pa = aux; 1075 1076 return (ixl_lookup(pa) != NULL) ? 1 : 0; 1077 } 1078 1079 static void 1080 ixl_attach(device_t parent, device_t self, void *aux) 1081 { 1082 struct ixl_softc *sc; 1083 struct pci_attach_args *pa = aux; 1084 struct ifnet *ifp; 1085 pcireg_t memtype; 1086 uint32_t firstq, port, ari, func; 1087 char xnamebuf[32]; 1088 int tries, rv, link; 1089 1090 sc = device_private(self); 1091 sc->sc_dev = self; 1092 ifp = &sc->sc_ec.ec_if; 1093 1094 sc->sc_pa = *pa; 1095 sc->sc_dmat = (pci_dma64_available(pa)) ? 1096 pa->pa_dmat64 : pa->pa_dmat; 1097 sc->sc_aq_regs = &ixl_pf_aq_regs; 1098 1099 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(pa->pa_id)); 1100 1101 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 1102 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 1103 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 1104 aprint_error(": unable to map registers\n"); 1105 return; 1106 } 1107 1108 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 1109 1110 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 1111 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 1112 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 1113 sc->sc_base_queue = firstq; 1114 1115 ixl_clear_hw(sc); 1116 if (ixl_pf_reset(sc) == -1) { 1117 /* error printed by ixl pf_reset */ 1118 goto unmap; 1119 } 1120 1121 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 1122 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 1123 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 1124 sc->sc_port = port; 1125 aprint_normal(": port %u", sc->sc_port); 1126 1127 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 1128 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 1129 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 1130 1131 func = ixl_rd(sc, I40E_PF_FUNC_RID); 1132 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 1133 1134 /* initialise the adminq */ 1135 1136 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 1137 1138 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 1139 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1140 aprint_error("\n" "%s: unable to allocate atq\n", 1141 device_xname(self)); 1142 goto unmap; 1143 } 1144 1145 SIMPLEQ_INIT(&sc->sc_arq_idle); 1146 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 1147 sc->sc_arq_cons = 0; 1148 sc->sc_arq_prod = 0; 1149 1150 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 1151 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1152 aprint_error("\n" "%s: unable to allocate arq\n", 1153 device_xname(self)); 1154 goto free_atq; 1155 } 1156 1157 if (!ixl_arq_fill(sc)) { 1158 aprint_error("\n" "%s: unable to fill arq descriptors\n", 1159 device_xname(self)); 1160 goto free_arq; 1161 } 1162 1163 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1164 0, IXL_DMA_LEN(&sc->sc_atq), 1165 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1166 1167 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1168 0, IXL_DMA_LEN(&sc->sc_arq), 1169 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1170 1171 for (tries = 0; tries < 10; tries++) { 1172 sc->sc_atq_cons = 0; 1173 sc->sc_atq_prod = 0; 1174 1175 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1176 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1177 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1178 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1179 1180 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1181 1182 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1183 ixl_dmamem_lo(&sc->sc_atq)); 1184 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1185 ixl_dmamem_hi(&sc->sc_atq)); 1186 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1187 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1188 1189 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1190 ixl_dmamem_lo(&sc->sc_arq)); 1191 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1192 ixl_dmamem_hi(&sc->sc_arq)); 1193 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1194 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1195 1196 rv = ixl_get_version(sc); 1197 if (rv == 0) 1198 break; 1199 if (rv != ETIMEDOUT) { 1200 aprint_error(", unable to get firmware version\n"); 1201 goto shutdown; 1202 } 1203 1204 delaymsec(100); 1205 } 1206 1207 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1208 1209 if (ixl_dmamem_alloc(sc, &sc->sc_aqbuf, IXL_AQ_BUFLEN, 0) != 0) { 1210 aprint_error_dev(self, ", unable to allocate nvm buffer\n"); 1211 goto shutdown; 1212 } 1213 1214 ixl_get_nvm_version(sc); 1215 1216 if (sc->sc_mac_type == I40E_MAC_X722) 1217 sc->sc_nqueue_pairs_device = 128; 1218 else 1219 sc->sc_nqueue_pairs_device = 64; 1220 1221 rv = ixl_get_hw_capabilities(sc); 1222 if (rv != 0) { 1223 aprint_error(", GET HW CAPABILITIES %s\n", 1224 rv == ETIMEDOUT ? "timeout" : "error"); 1225 goto free_aqbuf; 1226 } 1227 1228 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1229 if (ixl_param_nqps_limit > 0) { 1230 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_max, 1231 ixl_param_nqps_limit); 1232 } 1233 1234 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1235 sc->sc_tx_ring_ndescs = ixl_param_tx_ndescs; 1236 sc->sc_rx_ring_ndescs = ixl_param_rx_ndescs; 1237 1238 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1239 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1240 1241 if (ixl_get_mac(sc) != 0) { 1242 /* error printed by ixl_get_mac */ 1243 goto free_aqbuf; 1244 } 1245 1246 aprint_normal("\n"); 1247 aprint_naive("\n"); 1248 1249 aprint_normal_dev(self, "Ethernet address %s\n", 1250 ether_sprintf(sc->sc_enaddr)); 1251 1252 rv = ixl_pxe_clear(sc); 1253 if (rv != 0) { 1254 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1255 rv == ETIMEDOUT ? "timeout" : "error"); 1256 } 1257 1258 ixl_set_filter_control(sc); 1259 1260 if (ixl_hmc(sc) != 0) { 1261 /* error printed by ixl_hmc */ 1262 goto free_aqbuf; 1263 } 1264 1265 if (ixl_lldp_shut(sc) != 0) { 1266 /* error printed by ixl_lldp_shut */ 1267 goto free_hmc; 1268 } 1269 1270 if (ixl_phy_mask_ints(sc) != 0) { 1271 /* error printed by ixl_phy_mask_ints */ 1272 goto free_hmc; 1273 } 1274 1275 if (ixl_restart_an(sc) != 0) { 1276 /* error printed by ixl_restart_an */ 1277 goto free_hmc; 1278 } 1279 1280 if (ixl_get_switch_config(sc) != 0) { 1281 /* error printed by ixl_get_switch_config */ 1282 goto free_hmc; 1283 } 1284 1285 rv = ixl_get_link_status_poll(sc, NULL); 1286 if (rv != 0) { 1287 aprint_error_dev(self, "GET LINK STATUS %s\n", 1288 rv == ETIMEDOUT ? "timeout" : "error"); 1289 goto free_hmc; 1290 } 1291 1292 /* 1293 * The FW often returns EIO in "Get PHY Abilities" command 1294 * if there is no delay 1295 */ 1296 DELAY(500); 1297 if (ixl_get_phy_info(sc) != 0) { 1298 /* error printed by ixl_get_phy_info */ 1299 goto free_hmc; 1300 } 1301 1302 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1303 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1304 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1305 goto free_hmc; 1306 } 1307 1308 rv = ixl_get_vsi(sc); 1309 if (rv != 0) { 1310 aprint_error_dev(self, "GET VSI %s %d\n", 1311 rv == ETIMEDOUT ? "timeout" : "error", rv); 1312 goto free_scratch; 1313 } 1314 1315 rv = ixl_set_vsi(sc); 1316 if (rv != 0) { 1317 aprint_error_dev(self, "UPDATE VSI error %s %d\n", 1318 rv == ETIMEDOUT ? "timeout" : "error", rv); 1319 goto free_scratch; 1320 } 1321 1322 if (ixl_queue_pairs_alloc(sc) != 0) { 1323 /* error printed by ixl_queue_pairs_alloc */ 1324 goto free_scratch; 1325 } 1326 1327 if (ixl_setup_interrupts(sc) != 0) { 1328 /* error printed by ixl_setup_interrupts */ 1329 goto free_queue_pairs; 1330 } 1331 1332 if (ixl_setup_stats(sc) != 0) { 1333 aprint_error_dev(self, "failed to setup event counters\n"); 1334 goto teardown_intrs; 1335 } 1336 1337 if (ixl_setup_sysctls(sc) != 0) { 1338 /* error printed by ixl_setup_sysctls */ 1339 goto teardown_stats; 1340 } 1341 1342 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1343 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1344 IPL_NET, WQ_MPSAFE); 1345 if (sc->sc_workq == NULL) 1346 goto teardown_sysctls; 1347 1348 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1349 sc->sc_workq_txrx = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1350 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1351 if (sc->sc_workq_txrx == NULL) 1352 goto teardown_wqs; 1353 1354 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1355 cv_init(&sc->sc_atq_cv, xnamebuf); 1356 1357 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1358 1359 ifp->if_softc = sc; 1360 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1361 ifp->if_extflags = IFEF_MPSAFE; 1362 ifp->if_ioctl = ixl_ioctl; 1363 ifp->if_start = ixl_start; 1364 ifp->if_transmit = ixl_transmit; 1365 ifp->if_watchdog = ixl_watchdog; 1366 ifp->if_init = ixl_init; 1367 ifp->if_stop = ixl_stop; 1368 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1369 IFQ_SET_READY(&ifp->if_snd); 1370 ifp->if_capabilities |= IXL_IFCAP_RXCSUM; 1371 ifp->if_capabilities |= IXL_IFCAP_TXCSUM; 1372 #if 0 1373 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1374 #endif 1375 ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb); 1376 sc->sc_ec.ec_capabilities |= ETHERCAP_JUMBO_MTU; 1377 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1378 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1379 1380 sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities; 1381 /* Disable VLAN_HWFILTER by default */ 1382 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1383 1384 sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable; 1385 1386 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1387 ifmedia_init(&sc->sc_media, IFM_IMASK, ixl_media_change, 1388 ixl_media_status); 1389 1390 ixl_media_add(sc); 1391 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1392 if (ISSET(sc->sc_phy_abilities, 1393 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1394 ifmedia_add(&sc->sc_media, 1395 IFM_ETHER | IFM_AUTO | IFM_FLOW, 0, NULL); 1396 } 1397 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_NONE, 0, NULL); 1398 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1399 1400 if_attach(ifp); 1401 if_deferred_start_init(ifp, NULL); 1402 ether_ifattach(ifp, sc->sc_enaddr); 1403 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1404 1405 rv = ixl_get_link_status_poll(sc, &link); 1406 if (rv != 0) 1407 link = LINK_STATE_UNKNOWN; 1408 if_link_state_change(ifp, link); 1409 1410 ixl_work_set(&sc->sc_link_state_task, ixl_get_link_status, sc); 1411 1412 ixl_config_other_intr(sc); 1413 ixl_enable_other_intr(sc); 1414 1415 ixl_set_phy_autoselect(sc); 1416 1417 /* remove default mac filter and replace it so we can see vlans */ 1418 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 1419 if (rv != ENOENT) { 1420 aprint_debug_dev(self, 1421 "unable to remove macvlan %u\n", rv); 1422 } 1423 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1424 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1425 if (rv != ENOENT) { 1426 aprint_debug_dev(self, 1427 "unable to remove macvlan, ignore vlan %u\n", rv); 1428 } 1429 1430 if (ixl_update_macvlan(sc) != 0) { 1431 aprint_debug_dev(self, 1432 "couldn't enable vlan hardware filter\n"); 1433 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1434 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1435 } 1436 1437 sc->sc_txrx_workqueue = true; 1438 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1439 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1440 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1441 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1442 1443 ixl_stats_update(sc); 1444 sc->sc_stats_counters.isc_has_offset = true; 1445 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 1446 1447 if (pmf_device_register(self, NULL, NULL) != true) 1448 aprint_debug_dev(self, "couldn't establish power handler\n"); 1449 sc->sc_attached = true; 1450 return; 1451 1452 teardown_wqs: 1453 config_finalize_register(self, ixl_workqs_teardown); 1454 teardown_sysctls: 1455 ixl_teardown_sysctls(sc); 1456 teardown_stats: 1457 ixl_teardown_stats(sc); 1458 teardown_intrs: 1459 ixl_teardown_interrupts(sc); 1460 free_queue_pairs: 1461 ixl_queue_pairs_free(sc); 1462 free_scratch: 1463 ixl_dmamem_free(sc, &sc->sc_scratch); 1464 free_hmc: 1465 ixl_hmc_free(sc); 1466 free_aqbuf: 1467 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1468 shutdown: 1469 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1470 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1471 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1472 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1473 1474 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1475 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1476 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1477 1478 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1479 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1480 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1481 1482 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1483 0, IXL_DMA_LEN(&sc->sc_arq), 1484 BUS_DMASYNC_POSTREAD); 1485 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1486 0, IXL_DMA_LEN(&sc->sc_atq), 1487 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1488 1489 ixl_arq_unfill(sc); 1490 free_arq: 1491 ixl_dmamem_free(sc, &sc->sc_arq); 1492 free_atq: 1493 ixl_dmamem_free(sc, &sc->sc_atq); 1494 unmap: 1495 mutex_destroy(&sc->sc_atq_lock); 1496 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1497 mutex_destroy(&sc->sc_cfg_lock); 1498 sc->sc_mems = 0; 1499 1500 sc->sc_attached = false; 1501 } 1502 1503 static int 1504 ixl_detach(device_t self, int flags) 1505 { 1506 struct ixl_softc *sc = device_private(self); 1507 struct ifnet *ifp = &sc->sc_ec.ec_if; 1508 1509 if (!sc->sc_attached) 1510 return 0; 1511 1512 ixl_stop(ifp, 1); 1513 1514 ixl_disable_other_intr(sc); 1515 1516 callout_stop(&sc->sc_stats_callout); 1517 ixl_work_wait(sc->sc_workq, &sc->sc_stats_task); 1518 1519 /* wait for ATQ handler */ 1520 mutex_enter(&sc->sc_atq_lock); 1521 mutex_exit(&sc->sc_atq_lock); 1522 1523 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1524 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1525 1526 if (sc->sc_workq != NULL) { 1527 ixl_workq_destroy(sc->sc_workq); 1528 sc->sc_workq = NULL; 1529 } 1530 1531 if (sc->sc_workq_txrx != NULL) { 1532 ixl_workq_destroy(sc->sc_workq_txrx); 1533 sc->sc_workq_txrx = NULL; 1534 } 1535 1536 ether_ifdetach(ifp); 1537 if_detach(ifp); 1538 ifmedia_fini(&sc->sc_media); 1539 1540 ixl_teardown_interrupts(sc); 1541 ixl_teardown_stats(sc); 1542 ixl_teardown_sysctls(sc); 1543 1544 ixl_queue_pairs_free(sc); 1545 1546 ixl_dmamem_free(sc, &sc->sc_scratch); 1547 ixl_hmc_free(sc); 1548 1549 /* shutdown */ 1550 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1551 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1552 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1553 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1554 1555 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1556 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1557 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1558 1559 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1560 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1561 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1562 1563 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1564 0, IXL_DMA_LEN(&sc->sc_arq), 1565 BUS_DMASYNC_POSTREAD); 1566 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1567 0, IXL_DMA_LEN(&sc->sc_atq), 1568 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1569 1570 ixl_arq_unfill(sc); 1571 1572 ixl_dmamem_free(sc, &sc->sc_arq); 1573 ixl_dmamem_free(sc, &sc->sc_atq); 1574 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1575 1576 cv_destroy(&sc->sc_atq_cv); 1577 mutex_destroy(&sc->sc_atq_lock); 1578 1579 if (sc->sc_mems != 0) { 1580 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1581 sc->sc_mems = 0; 1582 } 1583 1584 mutex_destroy(&sc->sc_cfg_lock); 1585 1586 return 0; 1587 } 1588 1589 static int 1590 ixl_workqs_teardown(device_t self) 1591 { 1592 struct ixl_softc *sc = device_private(self); 1593 1594 if (sc->sc_workq != NULL) { 1595 ixl_workq_destroy(sc->sc_workq); 1596 sc->sc_workq = NULL; 1597 } 1598 1599 if (sc->sc_workq_txrx != NULL) { 1600 ixl_workq_destroy(sc->sc_workq_txrx); 1601 sc->sc_workq_txrx = NULL; 1602 } 1603 1604 return 0; 1605 } 1606 1607 static int 1608 ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 1609 { 1610 struct ifnet *ifp = &ec->ec_if; 1611 struct ixl_softc *sc = ifp->if_softc; 1612 int rv; 1613 1614 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1615 return 0; 1616 } 1617 1618 if (set) { 1619 rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid, 1620 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1621 if (rv == 0) { 1622 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 1623 vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1624 } 1625 } else { 1626 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid, 1627 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1628 (void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid, 1629 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1630 } 1631 1632 return rv; 1633 } 1634 1635 static void 1636 ixl_media_add(struct ixl_softc *sc) 1637 { 1638 struct ifmedia *ifm = &sc->sc_media; 1639 const struct ixl_phy_type *itype; 1640 unsigned int i; 1641 bool flow; 1642 1643 if (ISSET(sc->sc_phy_abilities, 1644 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1645 flow = true; 1646 } else { 1647 flow = false; 1648 } 1649 1650 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1651 itype = &ixl_phy_type_map[i]; 1652 1653 if (ISSET(sc->sc_phy_types, itype->phy_type)) { 1654 ifmedia_add(ifm, 1655 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1656 1657 if (flow) { 1658 ifmedia_add(ifm, 1659 IFM_ETHER | IFM_FDX | IFM_FLOW | 1660 itype->ifm_type, 0, NULL); 1661 } 1662 1663 if (itype->ifm_type != IFM_100_TX) 1664 continue; 1665 1666 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1667 0, NULL); 1668 if (flow) { 1669 ifmedia_add(ifm, 1670 IFM_ETHER | IFM_FLOW | itype->ifm_type, 1671 0, NULL); 1672 } 1673 } 1674 } 1675 } 1676 1677 static void 1678 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1679 { 1680 struct ixl_softc *sc = ifp->if_softc; 1681 1682 ifmr->ifm_status = sc->sc_media_status; 1683 ifmr->ifm_active = sc->sc_media_active; 1684 1685 mutex_enter(&sc->sc_cfg_lock); 1686 if (ifp->if_link_state == LINK_STATE_UP) 1687 SET(ifmr->ifm_status, IFM_ACTIVE); 1688 mutex_exit(&sc->sc_cfg_lock); 1689 } 1690 1691 static int 1692 ixl_media_change(struct ifnet *ifp) 1693 { 1694 struct ixl_softc *sc = ifp->if_softc; 1695 struct ifmedia *ifm = &sc->sc_media; 1696 uint64_t ifm_active = sc->sc_media_active; 1697 uint8_t link_speed, abilities; 1698 1699 switch (IFM_SUBTYPE(ifm_active)) { 1700 case IFM_1000_SGMII: 1701 case IFM_1000_KX: 1702 case IFM_10G_KX4: 1703 case IFM_10G_KR: 1704 case IFM_40G_KR4: 1705 case IFM_20G_KR2: 1706 case IFM_25G_KR: 1707 /* backplanes */ 1708 return EINVAL; 1709 } 1710 1711 abilities = IXL_PHY_ABILITY_AUTONEGO | IXL_PHY_ABILITY_LINKUP; 1712 1713 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1714 case IFM_AUTO: 1715 link_speed = sc->sc_phy_linkspeed; 1716 break; 1717 case IFM_NONE: 1718 link_speed = 0; 1719 CLR(abilities, IXL_PHY_ABILITY_LINKUP); 1720 break; 1721 default: 1722 link_speed = ixl_search_baudrate( 1723 ifmedia_baudrate(ifm->ifm_media)); 1724 } 1725 1726 if (ISSET(abilities, IXL_PHY_ABILITY_LINKUP)) { 1727 if (ISSET(link_speed, sc->sc_phy_linkspeed) == 0) 1728 return EINVAL; 1729 } 1730 1731 if (ifm->ifm_media & IFM_FLOW) { 1732 abilities |= sc->sc_phy_abilities & 1733 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX); 1734 } 1735 1736 return ixl_set_phy_config(sc, link_speed, abilities, false); 1737 } 1738 1739 static void 1740 ixl_watchdog(struct ifnet *ifp) 1741 { 1742 1743 } 1744 1745 static void 1746 ixl_del_all_multiaddr(struct ixl_softc *sc) 1747 { 1748 struct ethercom *ec = &sc->sc_ec; 1749 struct ether_multi *enm; 1750 struct ether_multistep step; 1751 1752 ETHER_LOCK(ec); 1753 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1754 ETHER_NEXT_MULTI(step, enm)) { 1755 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1756 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1757 } 1758 ETHER_UNLOCK(ec); 1759 } 1760 1761 static int 1762 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1763 { 1764 struct ifnet *ifp = &sc->sc_ec.ec_if; 1765 int rv; 1766 1767 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1768 return 0; 1769 1770 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1771 ixl_del_all_multiaddr(sc); 1772 SET(ifp->if_flags, IFF_ALLMULTI); 1773 return ENETRESET; 1774 } 1775 1776 /* multicast address can not use VLAN HWFILTER */ 1777 rv = ixl_add_macvlan(sc, addrlo, 0, 1778 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1779 1780 if (rv == ENOSPC) { 1781 ixl_del_all_multiaddr(sc); 1782 SET(ifp->if_flags, IFF_ALLMULTI); 1783 return ENETRESET; 1784 } 1785 1786 return rv; 1787 } 1788 1789 static int 1790 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1791 { 1792 struct ifnet *ifp = &sc->sc_ec.ec_if; 1793 struct ethercom *ec = &sc->sc_ec; 1794 struct ether_multi *enm, *enm_last; 1795 struct ether_multistep step; 1796 int error, rv = 0; 1797 1798 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1799 ixl_remove_macvlan(sc, addrlo, 0, 1800 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1801 return 0; 1802 } 1803 1804 ETHER_LOCK(ec); 1805 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1806 ETHER_NEXT_MULTI(step, enm)) { 1807 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1808 ETHER_ADDR_LEN) != 0) { 1809 goto out; 1810 } 1811 } 1812 1813 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1814 ETHER_NEXT_MULTI(step, enm)) { 1815 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1816 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1817 if (error != 0) 1818 break; 1819 } 1820 1821 if (enm != NULL) { 1822 enm_last = enm; 1823 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1824 ETHER_NEXT_MULTI(step, enm)) { 1825 if (enm == enm_last) 1826 break; 1827 1828 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1829 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1830 } 1831 } else { 1832 CLR(ifp->if_flags, IFF_ALLMULTI); 1833 rv = ENETRESET; 1834 } 1835 1836 out: 1837 ETHER_UNLOCK(ec); 1838 return rv; 1839 } 1840 1841 static int 1842 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1843 { 1844 struct ifreq *ifr = (struct ifreq *)data; 1845 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1846 const struct sockaddr *sa; 1847 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1848 int s, error = 0; 1849 unsigned int nmtu; 1850 1851 switch (cmd) { 1852 case SIOCSIFMTU: 1853 nmtu = ifr->ifr_mtu; 1854 1855 if (nmtu < IXL_MIN_MTU || nmtu > IXL_MAX_MTU) { 1856 error = EINVAL; 1857 break; 1858 } 1859 if (ifp->if_mtu != nmtu) { 1860 s = splnet(); 1861 error = ether_ioctl(ifp, cmd, data); 1862 splx(s); 1863 if (error == ENETRESET) 1864 error = ixl_init(ifp); 1865 } 1866 break; 1867 case SIOCADDMULTI: 1868 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1869 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1870 error = ether_multiaddr(sa, addrlo, addrhi); 1871 if (error != 0) 1872 return error; 1873 1874 error = ixl_add_multi(sc, addrlo, addrhi); 1875 if (error != 0 && error != ENETRESET) { 1876 ether_delmulti(sa, &sc->sc_ec); 1877 error = EIO; 1878 } 1879 } 1880 break; 1881 1882 case SIOCDELMULTI: 1883 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1884 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1885 error = ether_multiaddr(sa, addrlo, addrhi); 1886 if (error != 0) 1887 return error; 1888 1889 error = ixl_del_multi(sc, addrlo, addrhi); 1890 } 1891 break; 1892 1893 default: 1894 s = splnet(); 1895 error = ether_ioctl(ifp, cmd, data); 1896 splx(s); 1897 } 1898 1899 if (error == ENETRESET) 1900 error = ixl_iff(sc); 1901 1902 return error; 1903 } 1904 1905 static enum i40e_mac_type 1906 ixl_mactype(pci_product_id_t id) 1907 { 1908 1909 switch (id) { 1910 case PCI_PRODUCT_INTEL_XL710_SFP: 1911 case PCI_PRODUCT_INTEL_XL710_KX_B: 1912 case PCI_PRODUCT_INTEL_XL710_KX_C: 1913 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1914 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1915 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1916 case PCI_PRODUCT_INTEL_X710_10G_T: 1917 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1918 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1919 case PCI_PRODUCT_INTEL_X710_T4_10G: 1920 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1921 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1922 return I40E_MAC_XL710; 1923 1924 case PCI_PRODUCT_INTEL_X722_KX: 1925 case PCI_PRODUCT_INTEL_X722_QSFP: 1926 case PCI_PRODUCT_INTEL_X722_SFP: 1927 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1928 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1929 case PCI_PRODUCT_INTEL_X722_I_SFP: 1930 return I40E_MAC_X722; 1931 } 1932 1933 return I40E_MAC_GENERIC; 1934 } 1935 1936 static inline void * 1937 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1938 { 1939 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1940 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1941 1942 if (i >= e->hmc_count) 1943 return NULL; 1944 1945 kva += e->hmc_base; 1946 kva += i * e->hmc_size; 1947 1948 return kva; 1949 } 1950 1951 static inline size_t 1952 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1953 { 1954 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1955 1956 return e->hmc_size; 1957 } 1958 1959 static void 1960 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1961 { 1962 struct ixl_rx_ring *rxr = qp->qp_rxr; 1963 1964 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1965 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1966 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1967 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1968 ixl_flush(sc); 1969 } 1970 1971 static void 1972 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1973 { 1974 struct ixl_rx_ring *rxr = qp->qp_rxr; 1975 1976 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1977 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1978 ixl_flush(sc); 1979 } 1980 1981 static void 1982 ixl_enable_other_intr(struct ixl_softc *sc) 1983 { 1984 1985 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1986 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1987 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1988 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1989 ixl_flush(sc); 1990 } 1991 1992 static void 1993 ixl_disable_other_intr(struct ixl_softc *sc) 1994 { 1995 1996 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1997 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1998 ixl_flush(sc); 1999 } 2000 2001 static int 2002 ixl_reinit(struct ixl_softc *sc) 2003 { 2004 struct ixl_rx_ring *rxr; 2005 struct ixl_tx_ring *txr; 2006 unsigned int i; 2007 uint32_t reg; 2008 2009 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2010 2011 if (ixl_get_vsi(sc) != 0) 2012 return EIO; 2013 2014 if (ixl_set_vsi(sc) != 0) 2015 return EIO; 2016 2017 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2018 txr = sc->sc_qps[i].qp_txr; 2019 rxr = sc->sc_qps[i].qp_rxr; 2020 2021 txr->txr_cons = txr->txr_prod = 0; 2022 rxr->rxr_cons = rxr->rxr_prod = 0; 2023 2024 ixl_txr_config(sc, txr); 2025 ixl_rxr_config(sc, rxr); 2026 } 2027 2028 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2029 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 2030 2031 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2032 txr = sc->sc_qps[i].qp_txr; 2033 rxr = sc->sc_qps[i].qp_rxr; 2034 2035 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 2036 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 2037 ixl_flush(sc); 2038 2039 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 2040 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 2041 2042 /* ixl_rxfill() needs lock held */ 2043 mutex_enter(&rxr->rxr_lock); 2044 ixl_rxfill(sc, rxr); 2045 mutex_exit(&rxr->rxr_lock); 2046 2047 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2048 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2049 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2050 if (ixl_rxr_enabled(sc, rxr) != 0) 2051 goto stop; 2052 2053 ixl_txr_qdis(sc, txr, 1); 2054 2055 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2056 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2057 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2058 2059 if (ixl_txr_enabled(sc, txr) != 0) 2060 goto stop; 2061 } 2062 2063 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2064 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2065 2066 return 0; 2067 2068 stop: 2069 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2070 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2071 2072 return ETIMEDOUT; 2073 } 2074 2075 static int 2076 ixl_init_locked(struct ixl_softc *sc) 2077 { 2078 struct ifnet *ifp = &sc->sc_ec.ec_if; 2079 unsigned int i; 2080 int error, eccap_change; 2081 2082 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2083 2084 if (ISSET(ifp->if_flags, IFF_RUNNING)) 2085 ixl_stop_locked(sc); 2086 2087 if (sc->sc_dead) { 2088 return ENXIO; 2089 } 2090 2091 eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable; 2092 if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING)) 2093 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 2094 2095 if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) { 2096 if (ixl_update_macvlan(sc) == 0) { 2097 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 2098 } else { 2099 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 2100 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 2101 } 2102 } 2103 2104 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 2105 sc->sc_nqueue_pairs = 1; 2106 else 2107 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 2108 2109 error = ixl_reinit(sc); 2110 if (error) { 2111 ixl_stop_locked(sc); 2112 return error; 2113 } 2114 2115 SET(ifp->if_flags, IFF_RUNNING); 2116 CLR(ifp->if_flags, IFF_OACTIVE); 2117 2118 (void)ixl_get_link_status(sc); 2119 2120 ixl_config_rss(sc); 2121 ixl_config_queue_intr(sc); 2122 2123 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2124 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 2125 } 2126 2127 error = ixl_iff(sc); 2128 if (error) { 2129 ixl_stop_locked(sc); 2130 return error; 2131 } 2132 2133 return 0; 2134 } 2135 2136 static int 2137 ixl_init(struct ifnet *ifp) 2138 { 2139 struct ixl_softc *sc = ifp->if_softc; 2140 int error; 2141 2142 mutex_enter(&sc->sc_cfg_lock); 2143 error = ixl_init_locked(sc); 2144 mutex_exit(&sc->sc_cfg_lock); 2145 2146 return error; 2147 } 2148 2149 static int 2150 ixl_iff(struct ixl_softc *sc) 2151 { 2152 struct ifnet *ifp = &sc->sc_ec.ec_if; 2153 struct ixl_atq iatq; 2154 struct ixl_aq_desc *iaq; 2155 struct ixl_aq_vsi_promisc_param *param; 2156 uint16_t flag_add, flag_del; 2157 int error; 2158 2159 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2160 return 0; 2161 2162 memset(&iatq, 0, sizeof(iatq)); 2163 2164 iaq = &iatq.iatq_desc; 2165 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 2166 2167 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 2168 param->flags = htole16(0); 2169 2170 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER) 2171 || ISSET(ifp->if_flags, IFF_PROMISC)) { 2172 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2173 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2174 } 2175 2176 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 2177 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2178 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2179 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 2180 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2181 } 2182 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2183 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2184 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2185 param->seid = sc->sc_seid; 2186 2187 error = ixl_atq_exec(sc, &iatq); 2188 if (error) 2189 return error; 2190 2191 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 2192 return EIO; 2193 2194 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 2195 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 2196 flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH; 2197 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH; 2198 } else { 2199 flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN; 2200 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN; 2201 } 2202 2203 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del); 2204 2205 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2206 ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add); 2207 } 2208 return 0; 2209 } 2210 2211 static void 2212 ixl_stop_rendezvous(struct ixl_softc *sc) 2213 { 2214 struct ixl_tx_ring *txr; 2215 struct ixl_rx_ring *rxr; 2216 unsigned int i; 2217 2218 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2219 txr = sc->sc_qps[i].qp_txr; 2220 rxr = sc->sc_qps[i].qp_rxr; 2221 2222 mutex_enter(&txr->txr_lock); 2223 mutex_exit(&txr->txr_lock); 2224 2225 mutex_enter(&rxr->rxr_lock); 2226 mutex_exit(&rxr->rxr_lock); 2227 2228 ixl_work_wait(sc->sc_workq_txrx, 2229 &sc->sc_qps[i].qp_task); 2230 } 2231 } 2232 2233 static void 2234 ixl_stop_locked(struct ixl_softc *sc) 2235 { 2236 struct ifnet *ifp = &sc->sc_ec.ec_if; 2237 struct ixl_rx_ring *rxr; 2238 struct ixl_tx_ring *txr; 2239 unsigned int i; 2240 uint32_t reg; 2241 2242 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2243 2244 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 2245 2246 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2247 txr = sc->sc_qps[i].qp_txr; 2248 rxr = sc->sc_qps[i].qp_rxr; 2249 2250 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 2251 2252 mutex_enter(&txr->txr_lock); 2253 ixl_txr_qdis(sc, txr, 0); 2254 /* XXX wait at least 400 usec for all tx queues in one go */ 2255 ixl_flush(sc); 2256 DELAY(500); 2257 2258 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2259 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2260 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2261 /* XXX wait 50ms from completaion of the TX queue disable*/ 2262 ixl_flush(sc); 2263 DELAY(50); 2264 2265 if (ixl_txr_disabled(sc, txr) != 0) { 2266 mutex_exit(&txr->txr_lock); 2267 goto die; 2268 } 2269 mutex_exit(&txr->txr_lock); 2270 2271 mutex_enter(&rxr->rxr_lock); 2272 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2273 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2274 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2275 /* XXX wait 50ms from completion of the RX queue disable */ 2276 ixl_flush(sc); 2277 DELAY(50); 2278 2279 if (ixl_rxr_disabled(sc, rxr) != 0) { 2280 mutex_exit(&rxr->rxr_lock); 2281 goto die; 2282 } 2283 mutex_exit(&rxr->rxr_lock); 2284 } 2285 2286 ixl_stop_rendezvous(sc); 2287 2288 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2289 txr = sc->sc_qps[i].qp_txr; 2290 rxr = sc->sc_qps[i].qp_rxr; 2291 2292 ixl_txr_unconfig(sc, txr); 2293 ixl_rxr_unconfig(sc, rxr); 2294 2295 ixl_txr_clean(sc, txr); 2296 ixl_rxr_clean(sc, rxr); 2297 } 2298 2299 return; 2300 die: 2301 sc->sc_dead = true; 2302 log(LOG_CRIT, "%s: failed to shut down rings", 2303 device_xname(sc->sc_dev)); 2304 return; 2305 } 2306 2307 static void 2308 ixl_stop(struct ifnet *ifp, int disable) 2309 { 2310 struct ixl_softc *sc = ifp->if_softc; 2311 2312 mutex_enter(&sc->sc_cfg_lock); 2313 ixl_stop_locked(sc); 2314 mutex_exit(&sc->sc_cfg_lock); 2315 } 2316 2317 static int 2318 ixl_queue_pairs_alloc(struct ixl_softc *sc) 2319 { 2320 struct ixl_queue_pair *qp; 2321 unsigned int i; 2322 size_t sz; 2323 2324 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2325 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 2326 2327 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2328 qp = &sc->sc_qps[i]; 2329 2330 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2331 ixl_handle_queue, qp); 2332 if (qp->qp_si == NULL) 2333 goto free; 2334 2335 qp->qp_txr = ixl_txr_alloc(sc, i); 2336 if (qp->qp_txr == NULL) 2337 goto free; 2338 2339 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2340 if (qp->qp_rxr == NULL) 2341 goto free; 2342 2343 qp->qp_sc = sc; 2344 ixl_work_set(&qp->qp_task, ixl_handle_queue, qp); 2345 snprintf(qp->qp_name, sizeof(qp->qp_name), 2346 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2347 } 2348 2349 return 0; 2350 free: 2351 if (sc->sc_qps != NULL) { 2352 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2353 qp = &sc->sc_qps[i]; 2354 2355 if (qp->qp_txr != NULL) 2356 ixl_txr_free(sc, qp->qp_txr); 2357 if (qp->qp_rxr != NULL) 2358 ixl_rxr_free(sc, qp->qp_rxr); 2359 if (qp->qp_si != NULL) 2360 softint_disestablish(qp->qp_si); 2361 } 2362 2363 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2364 kmem_free(sc->sc_qps, sz); 2365 sc->sc_qps = NULL; 2366 } 2367 2368 return -1; 2369 } 2370 2371 static void 2372 ixl_queue_pairs_free(struct ixl_softc *sc) 2373 { 2374 struct ixl_queue_pair *qp; 2375 unsigned int i; 2376 size_t sz; 2377 2378 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2379 qp = &sc->sc_qps[i]; 2380 ixl_txr_free(sc, qp->qp_txr); 2381 ixl_rxr_free(sc, qp->qp_rxr); 2382 softint_disestablish(qp->qp_si); 2383 } 2384 2385 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2386 kmem_free(sc->sc_qps, sz); 2387 sc->sc_qps = NULL; 2388 } 2389 2390 static struct ixl_tx_ring * 2391 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2392 { 2393 struct ixl_tx_ring *txr = NULL; 2394 struct ixl_tx_map *maps = NULL, *txm; 2395 unsigned int i; 2396 2397 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2398 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2399 KM_SLEEP); 2400 2401 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2402 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2403 IXL_TX_QUEUE_ALIGN) != 0) 2404 goto free; 2405 2406 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2407 txm = &maps[i]; 2408 2409 if (bus_dmamap_create(sc->sc_dmat, IXL_TX_PKT_MAXSIZE, 2410 IXL_TX_PKT_DESCS, IXL_TX_PKT_MAXSIZE, 0, 2411 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2412 goto uncreate; 2413 2414 txm->txm_eop = -1; 2415 txm->txm_m = NULL; 2416 } 2417 2418 txr->txr_cons = txr->txr_prod = 0; 2419 txr->txr_maps = maps; 2420 2421 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2422 if (txr->txr_intrq == NULL) 2423 goto uncreate; 2424 2425 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2426 ixl_deferred_transmit, txr); 2427 if (txr->txr_si == NULL) 2428 goto destroy_pcq; 2429 2430 txr->txr_tail = I40E_QTX_TAIL(qid); 2431 txr->txr_qid = qid; 2432 txr->txr_sc = sc; 2433 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2434 2435 return txr; 2436 2437 destroy_pcq: 2438 pcq_destroy(txr->txr_intrq); 2439 uncreate: 2440 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2441 txm = &maps[i]; 2442 2443 if (txm->txm_map == NULL) 2444 continue; 2445 2446 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2447 } 2448 2449 ixl_dmamem_free(sc, &txr->txr_mem); 2450 free: 2451 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2452 kmem_free(txr, sizeof(*txr)); 2453 2454 return NULL; 2455 } 2456 2457 static void 2458 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2459 { 2460 unsigned int qid; 2461 bus_size_t reg; 2462 uint32_t r; 2463 2464 qid = txr->txr_qid + sc->sc_base_queue; 2465 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2466 qid %= 128; 2467 2468 r = ixl_rd(sc, reg); 2469 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2470 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2471 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2472 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2473 ixl_wr(sc, reg, r); 2474 } 2475 2476 static void 2477 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2478 { 2479 struct ixl_hmc_txq txq; 2480 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2481 void *hmc; 2482 2483 memset(&txq, 0, sizeof(txq)); 2484 txq.head = htole16(txr->txr_cons); 2485 txq.new_context = 1; 2486 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2487 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2488 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2489 txq.tphrdesc_ena = 0; 2490 txq.tphrpacket_ena = 0; 2491 txq.tphwdesc_ena = 0; 2492 txq.rdylist = data->qs_handle[0]; 2493 2494 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2495 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2496 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2497 __arraycount(ixl_hmc_pack_txq)); 2498 } 2499 2500 static void 2501 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2502 { 2503 void *hmc; 2504 2505 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2506 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2507 } 2508 2509 static void 2510 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2511 { 2512 struct ixl_tx_map *maps, *txm; 2513 bus_dmamap_t map; 2514 unsigned int i; 2515 2516 maps = txr->txr_maps; 2517 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2518 txm = &maps[i]; 2519 2520 if (txm->txm_m == NULL) 2521 continue; 2522 2523 map = txm->txm_map; 2524 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2525 BUS_DMASYNC_POSTWRITE); 2526 bus_dmamap_unload(sc->sc_dmat, map); 2527 2528 m_freem(txm->txm_m); 2529 txm->txm_m = NULL; 2530 } 2531 } 2532 2533 static int 2534 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2535 { 2536 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2537 uint32_t reg; 2538 int i; 2539 2540 for (i = 0; i < 10; i++) { 2541 reg = ixl_rd(sc, ena); 2542 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2543 return 0; 2544 2545 delaymsec(10); 2546 } 2547 2548 return ETIMEDOUT; 2549 } 2550 2551 static int 2552 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2553 { 2554 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2555 uint32_t reg; 2556 int i; 2557 2558 KASSERT(mutex_owned(&txr->txr_lock)); 2559 2560 for (i = 0; i < 20; i++) { 2561 reg = ixl_rd(sc, ena); 2562 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2563 return 0; 2564 2565 delaymsec(10); 2566 } 2567 2568 return ETIMEDOUT; 2569 } 2570 2571 static void 2572 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2573 { 2574 struct ixl_tx_map *maps, *txm; 2575 struct mbuf *m; 2576 unsigned int i; 2577 2578 softint_disestablish(txr->txr_si); 2579 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2580 m_freem(m); 2581 pcq_destroy(txr->txr_intrq); 2582 2583 maps = txr->txr_maps; 2584 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2585 txm = &maps[i]; 2586 2587 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2588 } 2589 2590 ixl_dmamem_free(sc, &txr->txr_mem); 2591 mutex_destroy(&txr->txr_lock); 2592 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2593 kmem_free(txr, sizeof(*txr)); 2594 } 2595 2596 static inline int 2597 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2598 struct ixl_tx_ring *txr) 2599 { 2600 struct mbuf *m; 2601 int error; 2602 2603 KASSERT(mutex_owned(&txr->txr_lock)); 2604 2605 m = *m0; 2606 2607 error = bus_dmamap_load_mbuf(dmat, map, m, 2608 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2609 if (error != EFBIG) 2610 return error; 2611 2612 m = m_defrag(m, M_DONTWAIT); 2613 if (m != NULL) { 2614 *m0 = m; 2615 txr->txr_defragged.ev_count++; 2616 2617 error = bus_dmamap_load_mbuf(dmat, map, m, 2618 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2619 } else { 2620 txr->txr_defrag_failed.ev_count++; 2621 error = ENOBUFS; 2622 } 2623 2624 return error; 2625 } 2626 2627 static inline int 2628 ixl_tx_setup_offloads(struct mbuf *m, uint64_t *cmd_txd) 2629 { 2630 struct ether_header *eh; 2631 size_t len; 2632 uint64_t cmd; 2633 2634 cmd = 0; 2635 2636 eh = mtod(m, struct ether_header *); 2637 switch (htons(eh->ether_type)) { 2638 case ETHERTYPE_IP: 2639 case ETHERTYPE_IPV6: 2640 len = ETHER_HDR_LEN; 2641 break; 2642 case ETHERTYPE_VLAN: 2643 len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2644 break; 2645 default: 2646 len = 0; 2647 } 2648 cmd |= ((len >> 1) << IXL_TX_DESC_MACLEN_SHIFT); 2649 2650 if (m->m_pkthdr.csum_flags & 2651 (M_CSUM_TSOv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) { 2652 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4; 2653 } 2654 if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) { 2655 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4_CSUM; 2656 } 2657 2658 if (m->m_pkthdr.csum_flags & 2659 (M_CSUM_TSOv6 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) { 2660 cmd |= IXL_TX_DESC_CMD_IIPT_IPV6; 2661 } 2662 2663 switch (cmd & IXL_TX_DESC_CMD_IIPT_MASK) { 2664 case IXL_TX_DESC_CMD_IIPT_IPV4: 2665 case IXL_TX_DESC_CMD_IIPT_IPV4_CSUM: 2666 len = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); 2667 break; 2668 case IXL_TX_DESC_CMD_IIPT_IPV6: 2669 len = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); 2670 break; 2671 default: 2672 len = 0; 2673 } 2674 cmd |= ((len >> 2) << IXL_TX_DESC_IPLEN_SHIFT); 2675 2676 if (m->m_pkthdr.csum_flags & 2677 (M_CSUM_TSOv4 | M_CSUM_TSOv6 | M_CSUM_TCPv4 | M_CSUM_TCPv6)) { 2678 len = sizeof(struct tcphdr); 2679 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_TCP; 2680 } else if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) { 2681 len = sizeof(struct udphdr); 2682 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_UDP; 2683 } else { 2684 len = 0; 2685 } 2686 cmd |= ((len >> 2) << IXL_TX_DESC_L4LEN_SHIFT); 2687 2688 *cmd_txd |= cmd; 2689 return 0; 2690 } 2691 2692 static void 2693 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2694 bool is_transmit) 2695 { 2696 struct ixl_softc *sc = ifp->if_softc; 2697 struct ixl_tx_desc *ring, *txd; 2698 struct ixl_tx_map *txm; 2699 bus_dmamap_t map; 2700 struct mbuf *m; 2701 uint64_t cmd, cmd_txd; 2702 unsigned int prod, free, last, i; 2703 unsigned int mask; 2704 int post = 0; 2705 2706 KASSERT(mutex_owned(&txr->txr_lock)); 2707 2708 if (ifp->if_link_state != LINK_STATE_UP 2709 || !ISSET(ifp->if_flags, IFF_RUNNING) 2710 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2711 if (!is_transmit) 2712 IFQ_PURGE(&ifp->if_snd); 2713 return; 2714 } 2715 2716 prod = txr->txr_prod; 2717 free = txr->txr_cons; 2718 if (free <= prod) 2719 free += sc->sc_tx_ring_ndescs; 2720 free -= prod; 2721 2722 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2723 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2724 2725 ring = IXL_DMA_KVA(&txr->txr_mem); 2726 mask = sc->sc_tx_ring_ndescs - 1; 2727 last = prod; 2728 cmd = 0; 2729 txd = NULL; 2730 2731 for (;;) { 2732 if (free <= IXL_TX_PKT_DESCS) { 2733 if (!is_transmit) 2734 SET(ifp->if_flags, IFF_OACTIVE); 2735 break; 2736 } 2737 2738 if (is_transmit) 2739 m = pcq_get(txr->txr_intrq); 2740 else 2741 IFQ_DEQUEUE(&ifp->if_snd, m); 2742 2743 if (m == NULL) 2744 break; 2745 2746 txm = &txr->txr_maps[prod]; 2747 map = txm->txm_map; 2748 2749 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2750 if_statinc(ifp, if_oerrors); 2751 m_freem(m); 2752 continue; 2753 } 2754 2755 cmd_txd = 0; 2756 if (m->m_pkthdr.csum_flags & IXL_CSUM_ALL_OFFLOAD) { 2757 ixl_tx_setup_offloads(m, &cmd_txd); 2758 } 2759 2760 if (vlan_has_tag(m)) { 2761 cmd_txd |= (uint64_t)vlan_get_tag(m) << 2762 IXL_TX_DESC_L2TAG1_SHIFT; 2763 cmd_txd |= IXL_TX_DESC_CMD_IL2TAG1; 2764 } 2765 2766 bus_dmamap_sync(sc->sc_dmat, map, 0, 2767 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2768 2769 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2770 txd = &ring[prod]; 2771 2772 cmd = (uint64_t)map->dm_segs[i].ds_len << 2773 IXL_TX_DESC_BSIZE_SHIFT; 2774 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2775 cmd |= cmd_txd; 2776 2777 txd->addr = htole64(map->dm_segs[i].ds_addr); 2778 txd->cmd = htole64(cmd); 2779 2780 last = prod; 2781 2782 prod++; 2783 prod &= mask; 2784 } 2785 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2786 txd->cmd = htole64(cmd); 2787 2788 txm->txm_m = m; 2789 txm->txm_eop = last; 2790 2791 bpf_mtap(ifp, m, BPF_D_OUT); 2792 2793 free -= i; 2794 post = 1; 2795 } 2796 2797 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2798 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2799 2800 if (post) { 2801 txr->txr_prod = prod; 2802 ixl_wr(sc, txr->txr_tail, prod); 2803 } 2804 } 2805 2806 static int 2807 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2808 { 2809 struct ifnet *ifp = &sc->sc_ec.ec_if; 2810 struct ixl_tx_desc *ring, *txd; 2811 struct ixl_tx_map *txm; 2812 struct mbuf *m; 2813 bus_dmamap_t map; 2814 unsigned int cons, prod, last; 2815 unsigned int mask; 2816 uint64_t dtype; 2817 int done = 0, more = 0; 2818 2819 KASSERT(mutex_owned(&txr->txr_lock)); 2820 2821 prod = txr->txr_prod; 2822 cons = txr->txr_cons; 2823 2824 if (cons == prod) 2825 return 0; 2826 2827 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2828 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2829 2830 ring = IXL_DMA_KVA(&txr->txr_mem); 2831 mask = sc->sc_tx_ring_ndescs - 1; 2832 2833 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2834 2835 do { 2836 if (txlimit-- <= 0) { 2837 more = 1; 2838 break; 2839 } 2840 2841 txm = &txr->txr_maps[cons]; 2842 last = txm->txm_eop; 2843 txd = &ring[last]; 2844 2845 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2846 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2847 break; 2848 2849 map = txm->txm_map; 2850 2851 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2852 BUS_DMASYNC_POSTWRITE); 2853 bus_dmamap_unload(sc->sc_dmat, map); 2854 2855 m = txm->txm_m; 2856 if (m != NULL) { 2857 if_statinc_ref(nsr, if_opackets); 2858 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 2859 if (ISSET(m->m_flags, M_MCAST)) 2860 if_statinc_ref(nsr, if_omcasts); 2861 m_freem(m); 2862 } 2863 2864 txm->txm_m = NULL; 2865 txm->txm_eop = -1; 2866 2867 cons = last + 1; 2868 cons &= mask; 2869 done = 1; 2870 } while (cons != prod); 2871 2872 IF_STAT_PUTREF(ifp); 2873 2874 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2875 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2876 2877 txr->txr_cons = cons; 2878 2879 if (done) { 2880 softint_schedule(txr->txr_si); 2881 if (txr->txr_qid == 0) { 2882 CLR(ifp->if_flags, IFF_OACTIVE); 2883 if_schedule_deferred_start(ifp); 2884 } 2885 } 2886 2887 return more; 2888 } 2889 2890 static void 2891 ixl_start(struct ifnet *ifp) 2892 { 2893 struct ixl_softc *sc; 2894 struct ixl_tx_ring *txr; 2895 2896 sc = ifp->if_softc; 2897 txr = sc->sc_qps[0].qp_txr; 2898 2899 mutex_enter(&txr->txr_lock); 2900 ixl_tx_common_locked(ifp, txr, false); 2901 mutex_exit(&txr->txr_lock); 2902 } 2903 2904 static inline unsigned int 2905 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2906 { 2907 u_int cpuid; 2908 2909 cpuid = cpu_index(curcpu()); 2910 2911 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2912 } 2913 2914 static int 2915 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2916 { 2917 struct ixl_softc *sc; 2918 struct ixl_tx_ring *txr; 2919 unsigned int qid; 2920 2921 sc = ifp->if_softc; 2922 qid = ixl_select_txqueue(sc, m); 2923 2924 txr = sc->sc_qps[qid].qp_txr; 2925 2926 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2927 mutex_enter(&txr->txr_lock); 2928 txr->txr_pcqdrop.ev_count++; 2929 mutex_exit(&txr->txr_lock); 2930 2931 m_freem(m); 2932 return ENOBUFS; 2933 } 2934 2935 if (mutex_tryenter(&txr->txr_lock)) { 2936 ixl_tx_common_locked(ifp, txr, true); 2937 mutex_exit(&txr->txr_lock); 2938 } else { 2939 kpreempt_disable(); 2940 softint_schedule(txr->txr_si); 2941 kpreempt_enable(); 2942 } 2943 2944 return 0; 2945 } 2946 2947 static void 2948 ixl_deferred_transmit(void *xtxr) 2949 { 2950 struct ixl_tx_ring *txr = xtxr; 2951 struct ixl_softc *sc = txr->txr_sc; 2952 struct ifnet *ifp = &sc->sc_ec.ec_if; 2953 2954 mutex_enter(&txr->txr_lock); 2955 txr->txr_transmitdef.ev_count++; 2956 if (pcq_peek(txr->txr_intrq) != NULL) 2957 ixl_tx_common_locked(ifp, txr, true); 2958 mutex_exit(&txr->txr_lock); 2959 } 2960 2961 static struct ixl_rx_ring * 2962 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2963 { 2964 struct ixl_rx_ring *rxr = NULL; 2965 struct ixl_rx_map *maps = NULL, *rxm; 2966 unsigned int i; 2967 2968 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2969 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2970 KM_SLEEP); 2971 2972 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2973 sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs, 2974 IXL_RX_QUEUE_ALIGN) != 0) 2975 goto free; 2976 2977 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2978 rxm = &maps[i]; 2979 2980 if (bus_dmamap_create(sc->sc_dmat, 2981 IXL_MCLBYTES, 1, IXL_MCLBYTES, 0, 2982 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2983 goto uncreate; 2984 2985 rxm->rxm_m = NULL; 2986 } 2987 2988 rxr->rxr_cons = rxr->rxr_prod = 0; 2989 rxr->rxr_m_head = NULL; 2990 rxr->rxr_m_tail = &rxr->rxr_m_head; 2991 rxr->rxr_maps = maps; 2992 2993 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2994 rxr->rxr_qid = qid; 2995 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2996 2997 return rxr; 2998 2999 uncreate: 3000 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3001 rxm = &maps[i]; 3002 3003 if (rxm->rxm_map == NULL) 3004 continue; 3005 3006 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 3007 } 3008 3009 ixl_dmamem_free(sc, &rxr->rxr_mem); 3010 free: 3011 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3012 kmem_free(rxr, sizeof(*rxr)); 3013 3014 return NULL; 3015 } 3016 3017 static void 3018 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3019 { 3020 struct ixl_rx_map *maps, *rxm; 3021 bus_dmamap_t map; 3022 unsigned int i; 3023 3024 maps = rxr->rxr_maps; 3025 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3026 rxm = &maps[i]; 3027 3028 if (rxm->rxm_m == NULL) 3029 continue; 3030 3031 map = rxm->rxm_map; 3032 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3033 BUS_DMASYNC_POSTWRITE); 3034 bus_dmamap_unload(sc->sc_dmat, map); 3035 3036 m_freem(rxm->rxm_m); 3037 rxm->rxm_m = NULL; 3038 } 3039 3040 m_freem(rxr->rxr_m_head); 3041 rxr->rxr_m_head = NULL; 3042 rxr->rxr_m_tail = &rxr->rxr_m_head; 3043 3044 rxr->rxr_prod = rxr->rxr_cons = 0; 3045 } 3046 3047 static int 3048 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3049 { 3050 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3051 uint32_t reg; 3052 int i; 3053 3054 for (i = 0; i < 10; i++) { 3055 reg = ixl_rd(sc, ena); 3056 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 3057 return 0; 3058 3059 delaymsec(10); 3060 } 3061 3062 return ETIMEDOUT; 3063 } 3064 3065 static int 3066 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3067 { 3068 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3069 uint32_t reg; 3070 int i; 3071 3072 KASSERT(mutex_owned(&rxr->rxr_lock)); 3073 3074 for (i = 0; i < 20; i++) { 3075 reg = ixl_rd(sc, ena); 3076 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 3077 return 0; 3078 3079 delaymsec(10); 3080 } 3081 3082 return ETIMEDOUT; 3083 } 3084 3085 static void 3086 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3087 { 3088 struct ixl_hmc_rxq rxq; 3089 struct ifnet *ifp = &sc->sc_ec.ec_if; 3090 uint16_t rxmax; 3091 void *hmc; 3092 3093 memset(&rxq, 0, sizeof(rxq)); 3094 rxmax = ifp->if_mtu + IXL_MTU_ETHERLEN; 3095 3096 rxq.head = htole16(rxr->rxr_cons); 3097 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 3098 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 3099 rxq.dbuff = htole16(IXL_MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 3100 rxq.hbuff = 0; 3101 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 3102 rxq.dsize = IXL_HMC_RXQ_DSIZE_32; 3103 rxq.crcstrip = 1; 3104 rxq.l2sel = 1; 3105 rxq.showiv = 1; 3106 rxq.rxmax = htole16(rxmax); 3107 rxq.tphrdesc_ena = 0; 3108 rxq.tphwdesc_ena = 0; 3109 rxq.tphdata_ena = 0; 3110 rxq.tphhead_ena = 0; 3111 rxq.lrxqthresh = 0; 3112 rxq.prefena = 1; 3113 3114 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3115 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3116 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 3117 __arraycount(ixl_hmc_pack_rxq)); 3118 } 3119 3120 static void 3121 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3122 { 3123 void *hmc; 3124 3125 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3126 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3127 } 3128 3129 static void 3130 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3131 { 3132 struct ixl_rx_map *maps, *rxm; 3133 unsigned int i; 3134 3135 maps = rxr->rxr_maps; 3136 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3137 rxm = &maps[i]; 3138 3139 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 3140 } 3141 3142 ixl_dmamem_free(sc, &rxr->rxr_mem); 3143 mutex_destroy(&rxr->rxr_lock); 3144 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3145 kmem_free(rxr, sizeof(*rxr)); 3146 } 3147 3148 static inline void 3149 ixl_rx_csum(struct mbuf *m, uint64_t qword) 3150 { 3151 int flags_mask; 3152 3153 if (!ISSET(qword, IXL_RX_DESC_L3L4P)) { 3154 /* No L3 or L4 checksum was calculated */ 3155 return; 3156 } 3157 3158 switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) { 3159 case IXL_RX_DESC_PTYPE_IPV4FRAG: 3160 case IXL_RX_DESC_PTYPE_IPV4: 3161 case IXL_RX_DESC_PTYPE_SCTPV4: 3162 case IXL_RX_DESC_PTYPE_ICMPV4: 3163 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3164 break; 3165 case IXL_RX_DESC_PTYPE_TCPV4: 3166 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3167 flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD; 3168 break; 3169 case IXL_RX_DESC_PTYPE_UDPV4: 3170 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3171 flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD; 3172 break; 3173 case IXL_RX_DESC_PTYPE_TCPV6: 3174 flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD; 3175 break; 3176 case IXL_RX_DESC_PTYPE_UDPV6: 3177 flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD; 3178 break; 3179 default: 3180 flags_mask = 0; 3181 } 3182 3183 m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 | 3184 M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6)); 3185 3186 if (ISSET(qword, IXL_RX_DESC_IPE)) { 3187 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD); 3188 } 3189 3190 if (ISSET(qword, IXL_RX_DESC_L4E)) { 3191 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD); 3192 } 3193 } 3194 3195 static int 3196 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 3197 { 3198 struct ifnet *ifp = &sc->sc_ec.ec_if; 3199 struct ixl_rx_wb_desc_32 *ring, *rxd; 3200 struct ixl_rx_map *rxm; 3201 bus_dmamap_t map; 3202 unsigned int cons, prod; 3203 struct mbuf *m; 3204 uint64_t word, word0; 3205 unsigned int len; 3206 unsigned int mask; 3207 int done = 0, more = 0; 3208 3209 KASSERT(mutex_owned(&rxr->rxr_lock)); 3210 3211 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3212 return 0; 3213 3214 prod = rxr->rxr_prod; 3215 cons = rxr->rxr_cons; 3216 3217 if (cons == prod) 3218 return 0; 3219 3220 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3221 0, IXL_DMA_LEN(&rxr->rxr_mem), 3222 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3223 3224 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3225 mask = sc->sc_rx_ring_ndescs - 1; 3226 3227 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 3228 3229 do { 3230 if (rxlimit-- <= 0) { 3231 more = 1; 3232 break; 3233 } 3234 3235 rxd = &ring[cons]; 3236 3237 word = le64toh(rxd->qword1); 3238 3239 if (!ISSET(word, IXL_RX_DESC_DD)) 3240 break; 3241 3242 rxm = &rxr->rxr_maps[cons]; 3243 3244 map = rxm->rxm_map; 3245 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3246 BUS_DMASYNC_POSTREAD); 3247 bus_dmamap_unload(sc->sc_dmat, map); 3248 3249 m = rxm->rxm_m; 3250 rxm->rxm_m = NULL; 3251 3252 KASSERT(m != NULL); 3253 3254 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 3255 m->m_len = len; 3256 m->m_pkthdr.len = 0; 3257 3258 m->m_next = NULL; 3259 *rxr->rxr_m_tail = m; 3260 rxr->rxr_m_tail = &m->m_next; 3261 3262 m = rxr->rxr_m_head; 3263 m->m_pkthdr.len += len; 3264 3265 if (ISSET(word, IXL_RX_DESC_EOP)) { 3266 word0 = le64toh(rxd->qword0); 3267 3268 if (ISSET(word, IXL_RX_DESC_L2TAG1P)) { 3269 vlan_set_tag(m, 3270 __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK)); 3271 } 3272 3273 if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0) 3274 ixl_rx_csum(m, word); 3275 3276 if (!ISSET(word, 3277 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 3278 m_set_rcvif(m, ifp); 3279 if_statinc_ref(nsr, if_ipackets); 3280 if_statadd_ref(nsr, if_ibytes, 3281 m->m_pkthdr.len); 3282 if_percpuq_enqueue(ifp->if_percpuq, m); 3283 } else { 3284 if_statinc_ref(nsr, if_ierrors); 3285 m_freem(m); 3286 } 3287 3288 rxr->rxr_m_head = NULL; 3289 rxr->rxr_m_tail = &rxr->rxr_m_head; 3290 } 3291 3292 cons++; 3293 cons &= mask; 3294 3295 done = 1; 3296 } while (cons != prod); 3297 3298 if (done) { 3299 rxr->rxr_cons = cons; 3300 if (ixl_rxfill(sc, rxr) == -1) 3301 if_statinc_ref(nsr, if_iqdrops); 3302 } 3303 3304 IF_STAT_PUTREF(ifp); 3305 3306 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3307 0, IXL_DMA_LEN(&rxr->rxr_mem), 3308 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3309 3310 return more; 3311 } 3312 3313 static int 3314 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3315 { 3316 struct ixl_rx_rd_desc_32 *ring, *rxd; 3317 struct ixl_rx_map *rxm; 3318 bus_dmamap_t map; 3319 struct mbuf *m; 3320 unsigned int prod; 3321 unsigned int slots; 3322 unsigned int mask; 3323 int post = 0, error = 0; 3324 3325 KASSERT(mutex_owned(&rxr->rxr_lock)); 3326 3327 prod = rxr->rxr_prod; 3328 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 3329 sc->sc_rx_ring_ndescs); 3330 3331 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3332 mask = sc->sc_rx_ring_ndescs - 1; 3333 3334 if (__predict_false(slots <= 0)) 3335 return -1; 3336 3337 do { 3338 rxm = &rxr->rxr_maps[prod]; 3339 3340 MGETHDR(m, M_DONTWAIT, MT_DATA); 3341 if (m == NULL) { 3342 rxr->rxr_mgethdr_failed.ev_count++; 3343 error = -1; 3344 break; 3345 } 3346 3347 MCLGET(m, M_DONTWAIT); 3348 if (!ISSET(m->m_flags, M_EXT)) { 3349 rxr->rxr_mgetcl_failed.ev_count++; 3350 error = -1; 3351 m_freem(m); 3352 break; 3353 } 3354 3355 m->m_len = m->m_pkthdr.len = MCLBYTES; 3356 m_adj(m, ETHER_ALIGN); 3357 3358 map = rxm->rxm_map; 3359 3360 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 3361 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 3362 rxr->rxr_mbuf_load_failed.ev_count++; 3363 error = -1; 3364 m_freem(m); 3365 break; 3366 } 3367 3368 rxm->rxm_m = m; 3369 3370 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3371 BUS_DMASYNC_PREREAD); 3372 3373 rxd = &ring[prod]; 3374 3375 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 3376 rxd->haddr = htole64(0); 3377 3378 prod++; 3379 prod &= mask; 3380 3381 post = 1; 3382 3383 } while (--slots); 3384 3385 if (post) { 3386 rxr->rxr_prod = prod; 3387 ixl_wr(sc, rxr->rxr_tail, prod); 3388 } 3389 3390 return error; 3391 } 3392 3393 static inline int 3394 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 3395 u_int txlimit, struct evcnt *txevcnt, 3396 u_int rxlimit, struct evcnt *rxevcnt) 3397 { 3398 struct ixl_tx_ring *txr = qp->qp_txr; 3399 struct ixl_rx_ring *rxr = qp->qp_rxr; 3400 int txmore, rxmore; 3401 int rv; 3402 3403 mutex_enter(&txr->txr_lock); 3404 txevcnt->ev_count++; 3405 txmore = ixl_txeof(sc, txr, txlimit); 3406 mutex_exit(&txr->txr_lock); 3407 3408 mutex_enter(&rxr->rxr_lock); 3409 rxevcnt->ev_count++; 3410 rxmore = ixl_rxeof(sc, rxr, rxlimit); 3411 mutex_exit(&rxr->rxr_lock); 3412 3413 rv = txmore | (rxmore << 1); 3414 3415 return rv; 3416 } 3417 3418 static void 3419 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 3420 { 3421 3422 if (qp->qp_workqueue) 3423 ixl_work_add(sc->sc_workq_txrx, &qp->qp_task); 3424 else 3425 softint_schedule(qp->qp_si); 3426 } 3427 3428 static int 3429 ixl_intr(void *xsc) 3430 { 3431 struct ixl_softc *sc = xsc; 3432 struct ixl_tx_ring *txr; 3433 struct ixl_rx_ring *rxr; 3434 uint32_t icr, rxintr, txintr; 3435 int rv = 0; 3436 unsigned int i; 3437 3438 KASSERT(sc != NULL); 3439 3440 ixl_enable_other_intr(sc); 3441 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3442 3443 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3444 atomic_inc_64(&sc->sc_event_atq.ev_count); 3445 ixl_atq_done(sc); 3446 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3447 rv = 1; 3448 } 3449 3450 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3451 atomic_inc_64(&sc->sc_event_link.ev_count); 3452 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3453 rv = 1; 3454 } 3455 3456 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 3457 txintr = icr & I40E_INTR_NOTX_TX_MASK; 3458 3459 if (txintr || rxintr) { 3460 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 3461 txr = sc->sc_qps[i].qp_txr; 3462 rxr = sc->sc_qps[i].qp_rxr; 3463 3464 ixl_handle_queue_common(sc, &sc->sc_qps[i], 3465 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 3466 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 3467 } 3468 rv = 1; 3469 } 3470 3471 return rv; 3472 } 3473 3474 static int 3475 ixl_queue_intr(void *xqp) 3476 { 3477 struct ixl_queue_pair *qp = xqp; 3478 struct ixl_tx_ring *txr = qp->qp_txr; 3479 struct ixl_rx_ring *rxr = qp->qp_rxr; 3480 struct ixl_softc *sc = qp->qp_sc; 3481 u_int txlimit, rxlimit; 3482 int more; 3483 3484 txlimit = sc->sc_tx_intr_process_limit; 3485 rxlimit = sc->sc_rx_intr_process_limit; 3486 qp->qp_workqueue = sc->sc_txrx_workqueue; 3487 3488 more = ixl_handle_queue_common(sc, qp, 3489 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3490 3491 if (more != 0) { 3492 ixl_sched_handle_queue(sc, qp); 3493 } else { 3494 /* for ALTQ */ 3495 if (txr->txr_qid == 0) 3496 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3497 softint_schedule(txr->txr_si); 3498 3499 ixl_enable_queue_intr(sc, qp); 3500 } 3501 3502 return 1; 3503 } 3504 3505 static void 3506 ixl_handle_queue(void *xqp) 3507 { 3508 struct ixl_queue_pair *qp = xqp; 3509 struct ixl_softc *sc = qp->qp_sc; 3510 struct ixl_tx_ring *txr = qp->qp_txr; 3511 struct ixl_rx_ring *rxr = qp->qp_rxr; 3512 u_int txlimit, rxlimit; 3513 int more; 3514 3515 txlimit = sc->sc_tx_process_limit; 3516 rxlimit = sc->sc_rx_process_limit; 3517 3518 more = ixl_handle_queue_common(sc, qp, 3519 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3520 3521 if (more != 0) 3522 ixl_sched_handle_queue(sc, qp); 3523 else 3524 ixl_enable_queue_intr(sc, qp); 3525 } 3526 3527 static inline void 3528 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3529 { 3530 uint32_t hmc_idx, hmc_isvf; 3531 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3532 3533 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3534 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3535 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3536 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3537 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3538 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3539 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3540 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3541 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3542 3543 device_printf(sc->sc_dev, 3544 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3545 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3546 } 3547 3548 static int 3549 ixl_other_intr(void *xsc) 3550 { 3551 struct ixl_softc *sc = xsc; 3552 uint32_t icr, mask, reg; 3553 int rv; 3554 3555 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3556 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3557 3558 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3559 atomic_inc_64(&sc->sc_event_atq.ev_count); 3560 ixl_atq_done(sc); 3561 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3562 rv = 1; 3563 } 3564 3565 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3566 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3567 device_printf(sc->sc_dev, "link stat changed\n"); 3568 3569 atomic_inc_64(&sc->sc_event_link.ev_count); 3570 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3571 rv = 1; 3572 } 3573 3574 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3575 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3576 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3577 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3578 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3579 3580 device_printf(sc->sc_dev, "GRST: %s\n", 3581 reg == I40E_RESET_CORER ? "CORER" : 3582 reg == I40E_RESET_GLOBR ? "GLOBR" : 3583 reg == I40E_RESET_EMPR ? "EMPR" : 3584 "POR"); 3585 } 3586 3587 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3588 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3589 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3590 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3591 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3592 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3593 3594 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3595 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3596 device_printf(sc->sc_dev, "critical error\n"); 3597 } 3598 3599 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3600 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3601 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3602 ixl_print_hmc_error(sc, reg); 3603 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3604 } 3605 3606 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3607 ixl_flush(sc); 3608 ixl_enable_other_intr(sc); 3609 return rv; 3610 } 3611 3612 static void 3613 ixl_get_link_status_done(struct ixl_softc *sc, 3614 const struct ixl_aq_desc *iaq) 3615 { 3616 3617 ixl_link_state_update(sc, iaq); 3618 } 3619 3620 static void 3621 ixl_get_link_status(void *xsc) 3622 { 3623 struct ixl_softc *sc = xsc; 3624 struct ixl_aq_desc *iaq; 3625 struct ixl_aq_link_param *param; 3626 3627 memset(&sc->sc_link_state_atq, 0, sizeof(sc->sc_link_state_atq)); 3628 iaq = &sc->sc_link_state_atq.iatq_desc; 3629 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3630 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3631 param->notify = IXL_AQ_LINK_NOTIFY; 3632 3633 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 3634 (void)ixl_atq_post(sc, &sc->sc_link_state_atq); 3635 } 3636 3637 static void 3638 ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3639 { 3640 struct ifnet *ifp = &sc->sc_ec.ec_if; 3641 int link_state; 3642 3643 KASSERT(kpreempt_disabled()); 3644 3645 link_state = ixl_set_link_status(sc, iaq); 3646 3647 if (ifp->if_link_state != link_state) 3648 if_link_state_change(ifp, link_state); 3649 3650 if (link_state != LINK_STATE_DOWN) { 3651 if_schedule_deferred_start(ifp); 3652 } 3653 } 3654 3655 static void 3656 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3657 const char *msg) 3658 { 3659 char buf[512]; 3660 size_t len; 3661 3662 len = sizeof(buf); 3663 buf[--len] = '\0'; 3664 3665 device_printf(sc->sc_dev, "%s\n", msg); 3666 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3667 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3668 buf, le16toh(iaq->iaq_opcode)); 3669 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3670 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3671 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3672 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3673 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3674 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3675 } 3676 3677 static void 3678 ixl_arq(void *xsc) 3679 { 3680 struct ixl_softc *sc = xsc; 3681 struct ixl_aq_desc *arq, *iaq; 3682 struct ixl_aq_buf *aqb; 3683 unsigned int cons = sc->sc_arq_cons; 3684 unsigned int prod; 3685 int done = 0; 3686 3687 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3688 sc->sc_aq_regs->arq_head_mask; 3689 3690 if (cons == prod) 3691 goto done; 3692 3693 arq = IXL_DMA_KVA(&sc->sc_arq); 3694 3695 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3696 0, IXL_DMA_LEN(&sc->sc_arq), 3697 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3698 3699 do { 3700 iaq = &arq[cons]; 3701 aqb = sc->sc_arq_live[cons]; 3702 3703 KASSERT(aqb != NULL); 3704 3705 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3706 BUS_DMASYNC_POSTREAD); 3707 3708 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3709 ixl_aq_dump(sc, iaq, "arq event"); 3710 3711 switch (iaq->iaq_opcode) { 3712 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3713 kpreempt_disable(); 3714 ixl_link_state_update(sc, iaq); 3715 kpreempt_enable(); 3716 break; 3717 } 3718 3719 memset(iaq, 0, sizeof(*iaq)); 3720 sc->sc_arq_live[cons] = NULL; 3721 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3722 3723 cons++; 3724 cons &= IXL_AQ_MASK; 3725 3726 done = 1; 3727 } while (cons != prod); 3728 3729 if (done) { 3730 sc->sc_arq_cons = cons; 3731 ixl_arq_fill(sc); 3732 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3733 0, IXL_DMA_LEN(&sc->sc_arq), 3734 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3735 } 3736 3737 done: 3738 ixl_enable_other_intr(sc); 3739 } 3740 3741 static void 3742 ixl_atq_set(struct ixl_atq *iatq, 3743 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3744 { 3745 3746 iatq->iatq_fn = fn; 3747 } 3748 3749 static int 3750 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3751 { 3752 struct ixl_aq_desc *atq, *slot; 3753 unsigned int prod, cons, prod_next; 3754 3755 /* assert locked */ 3756 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3757 3758 atq = IXL_DMA_KVA(&sc->sc_atq); 3759 prod = sc->sc_atq_prod; 3760 cons = sc->sc_atq_cons; 3761 prod_next = (prod +1) & IXL_AQ_MASK; 3762 3763 if (cons == prod_next) 3764 return ENOMEM; 3765 3766 slot = &atq[prod]; 3767 3768 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3769 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3770 3771 *slot = iatq->iatq_desc; 3772 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3773 3774 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3775 ixl_aq_dump(sc, slot, "atq command"); 3776 3777 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3778 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3779 3780 sc->sc_atq_prod = prod_next; 3781 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3782 3783 return 0; 3784 } 3785 3786 static int 3787 ixl_atq_post(struct ixl_softc *sc, struct ixl_atq *iatq) 3788 { 3789 int rv; 3790 3791 mutex_enter(&sc->sc_atq_lock); 3792 rv = ixl_atq_post_locked(sc, iatq); 3793 mutex_exit(&sc->sc_atq_lock); 3794 3795 return rv; 3796 } 3797 3798 static void 3799 ixl_atq_done_locked(struct ixl_softc *sc) 3800 { 3801 struct ixl_aq_desc *atq, *slot; 3802 struct ixl_atq *iatq; 3803 unsigned int cons; 3804 unsigned int prod; 3805 3806 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3807 3808 prod = sc->sc_atq_prod; 3809 cons = sc->sc_atq_cons; 3810 3811 if (prod == cons) 3812 return; 3813 3814 atq = IXL_DMA_KVA(&sc->sc_atq); 3815 3816 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3817 0, IXL_DMA_LEN(&sc->sc_atq), 3818 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3819 3820 do { 3821 slot = &atq[cons]; 3822 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3823 break; 3824 3825 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3826 iatq->iatq_desc = *slot; 3827 3828 memset(slot, 0, sizeof(*slot)); 3829 3830 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3831 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3832 3833 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3834 3835 cons++; 3836 cons &= IXL_AQ_MASK; 3837 } while (cons != prod); 3838 3839 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3840 0, IXL_DMA_LEN(&sc->sc_atq), 3841 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3842 3843 sc->sc_atq_cons = cons; 3844 } 3845 3846 static void 3847 ixl_atq_done(struct ixl_softc *sc) 3848 { 3849 3850 mutex_enter(&sc->sc_atq_lock); 3851 ixl_atq_done_locked(sc); 3852 mutex_exit(&sc->sc_atq_lock); 3853 } 3854 3855 static void 3856 ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3857 { 3858 3859 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3860 3861 cv_signal(&sc->sc_atq_cv); 3862 } 3863 3864 static int 3865 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3866 { 3867 int error; 3868 3869 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3870 3871 ixl_atq_set(iatq, ixl_wakeup); 3872 3873 mutex_enter(&sc->sc_atq_lock); 3874 error = ixl_atq_post_locked(sc, iatq); 3875 if (error) { 3876 mutex_exit(&sc->sc_atq_lock); 3877 return error; 3878 } 3879 3880 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3881 IXL_ATQ_EXEC_TIMEOUT); 3882 mutex_exit(&sc->sc_atq_lock); 3883 3884 return error; 3885 } 3886 3887 static int 3888 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3889 { 3890 struct ixl_aq_desc *atq, *slot; 3891 unsigned int prod; 3892 unsigned int t = 0; 3893 3894 mutex_enter(&sc->sc_atq_lock); 3895 3896 atq = IXL_DMA_KVA(&sc->sc_atq); 3897 prod = sc->sc_atq_prod; 3898 slot = atq + prod; 3899 3900 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3901 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3902 3903 *slot = *iaq; 3904 slot->iaq_flags |= htole16(IXL_AQ_SI); 3905 3906 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3907 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3908 3909 prod++; 3910 prod &= IXL_AQ_MASK; 3911 sc->sc_atq_prod = prod; 3912 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3913 3914 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3915 delaymsec(1); 3916 3917 if (t++ > tm) { 3918 mutex_exit(&sc->sc_atq_lock); 3919 return ETIMEDOUT; 3920 } 3921 } 3922 3923 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3924 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3925 *iaq = *slot; 3926 memset(slot, 0, sizeof(*slot)); 3927 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3928 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3929 3930 sc->sc_atq_cons = prod; 3931 3932 mutex_exit(&sc->sc_atq_lock); 3933 3934 return 0; 3935 } 3936 3937 static int 3938 ixl_get_version(struct ixl_softc *sc) 3939 { 3940 struct ixl_aq_desc iaq; 3941 uint32_t fwbuild, fwver, apiver; 3942 uint16_t api_maj_ver, api_min_ver; 3943 3944 memset(&iaq, 0, sizeof(iaq)); 3945 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3946 3947 iaq.iaq_retval = le16toh(23); 3948 3949 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3950 return ETIMEDOUT; 3951 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3952 return EIO; 3953 3954 fwbuild = le32toh(iaq.iaq_param[1]); 3955 fwver = le32toh(iaq.iaq_param[2]); 3956 apiver = le32toh(iaq.iaq_param[3]); 3957 3958 api_maj_ver = (uint16_t)apiver; 3959 api_min_ver = (uint16_t)(apiver >> 16); 3960 3961 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 3962 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 3963 3964 if (sc->sc_mac_type == I40E_MAC_X722) { 3965 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK | 3966 IXL_SC_AQ_FLAG_NVMREAD); 3967 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 3968 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS); 3969 } 3970 3971 #define IXL_API_VER(maj, min) (((uint32_t)(maj) << 16) | (min)) 3972 if (IXL_API_VER(api_maj_ver, api_min_ver) >= IXL_API_VER(1, 5)) { 3973 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 3974 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK); 3975 } 3976 #undef IXL_API_VER 3977 3978 return 0; 3979 } 3980 3981 static int 3982 ixl_get_nvm_version(struct ixl_softc *sc) 3983 { 3984 uint16_t nvmver, cfg_ptr, eetrack_hi, eetrack_lo, oem_hi, oem_lo; 3985 uint32_t eetrack, oem; 3986 uint16_t nvm_maj_ver, nvm_min_ver, oem_build; 3987 uint8_t oem_ver, oem_patch; 3988 3989 nvmver = cfg_ptr = eetrack_hi = eetrack_lo = oem_hi = oem_lo = 0; 3990 ixl_rd16_nvm(sc, I40E_SR_NVM_DEV_STARTER_VERSION, &nvmver); 3991 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_HI, &eetrack_hi); 3992 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_LO, &eetrack_lo); 3993 ixl_rd16_nvm(sc, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr); 3994 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF, &oem_hi); 3995 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF + 1, &oem_lo); 3996 3997 nvm_maj_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_HI_MASK); 3998 nvm_min_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_LO_MASK); 3999 eetrack = ((uint32_t)eetrack_hi << 16) | eetrack_lo; 4000 oem = ((uint32_t)oem_hi << 16) | oem_lo; 4001 oem_ver = __SHIFTOUT(oem, IXL_NVM_OEMVERSION_MASK); 4002 oem_build = __SHIFTOUT(oem, IXL_NVM_OEMBUILD_MASK); 4003 oem_patch = __SHIFTOUT(oem, IXL_NVM_OEMPATCH_MASK); 4004 4005 aprint_normal(" nvm %x.%02x etid %08x oem %d.%d.%d", 4006 nvm_maj_ver, nvm_min_ver, eetrack, 4007 oem_ver, oem_build, oem_patch); 4008 4009 return 0; 4010 } 4011 4012 static int 4013 ixl_pxe_clear(struct ixl_softc *sc) 4014 { 4015 struct ixl_aq_desc iaq; 4016 int rv; 4017 4018 memset(&iaq, 0, sizeof(iaq)); 4019 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 4020 iaq.iaq_param[0] = htole32(0x2); 4021 4022 rv = ixl_atq_poll(sc, &iaq, 250); 4023 4024 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 4025 4026 if (rv != 0) 4027 return ETIMEDOUT; 4028 4029 switch (iaq.iaq_retval) { 4030 case htole16(IXL_AQ_RC_OK): 4031 case htole16(IXL_AQ_RC_EEXIST): 4032 break; 4033 default: 4034 return EIO; 4035 } 4036 4037 return 0; 4038 } 4039 4040 static int 4041 ixl_lldp_shut(struct ixl_softc *sc) 4042 { 4043 struct ixl_aq_desc iaq; 4044 4045 memset(&iaq, 0, sizeof(iaq)); 4046 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 4047 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 4048 4049 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4050 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 4051 return -1; 4052 } 4053 4054 switch (iaq.iaq_retval) { 4055 case htole16(IXL_AQ_RC_EMODE): 4056 case htole16(IXL_AQ_RC_EPERM): 4057 /* ignore silently */ 4058 default: 4059 break; 4060 } 4061 4062 return 0; 4063 } 4064 4065 static void 4066 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 4067 { 4068 uint16_t id; 4069 uint32_t number, logical_id; 4070 4071 id = le16toh(cap->cap_id); 4072 number = le32toh(cap->number); 4073 logical_id = le32toh(cap->logical_id); 4074 4075 switch (id) { 4076 case IXL_AQ_CAP_RSS: 4077 sc->sc_rss_table_size = number; 4078 sc->sc_rss_table_entry_width = logical_id; 4079 break; 4080 case IXL_AQ_CAP_RXQ: 4081 case IXL_AQ_CAP_TXQ: 4082 sc->sc_nqueue_pairs_device = MIN(number, 4083 sc->sc_nqueue_pairs_device); 4084 break; 4085 } 4086 } 4087 4088 static int 4089 ixl_get_hw_capabilities(struct ixl_softc *sc) 4090 { 4091 struct ixl_dmamem idm; 4092 struct ixl_aq_desc iaq; 4093 struct ixl_aq_capability *caps; 4094 size_t i, ncaps; 4095 bus_size_t caps_size; 4096 uint16_t status; 4097 int rv; 4098 4099 caps_size = sizeof(caps[0]) * 40; 4100 memset(&iaq, 0, sizeof(iaq)); 4101 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 4102 4103 do { 4104 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 4105 return -1; 4106 } 4107 4108 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4109 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4110 iaq.iaq_datalen = htole16(caps_size); 4111 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4112 4113 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4114 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 4115 4116 rv = ixl_atq_poll(sc, &iaq, 250); 4117 4118 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4119 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 4120 4121 if (rv != 0) { 4122 aprint_error(", HW capabilities timeout\n"); 4123 goto done; 4124 } 4125 4126 status = le16toh(iaq.iaq_retval); 4127 4128 if (status == IXL_AQ_RC_ENOMEM) { 4129 caps_size = le16toh(iaq.iaq_datalen); 4130 ixl_dmamem_free(sc, &idm); 4131 } 4132 } while (status == IXL_AQ_RC_ENOMEM); 4133 4134 if (status != IXL_AQ_RC_OK) { 4135 aprint_error(", HW capabilities error\n"); 4136 goto done; 4137 } 4138 4139 caps = IXL_DMA_KVA(&idm); 4140 ncaps = le16toh(iaq.iaq_param[1]); 4141 4142 for (i = 0; i < ncaps; i++) { 4143 ixl_parse_hw_capability(sc, &caps[i]); 4144 } 4145 4146 done: 4147 ixl_dmamem_free(sc, &idm); 4148 return rv; 4149 } 4150 4151 static int 4152 ixl_get_mac(struct ixl_softc *sc) 4153 { 4154 struct ixl_dmamem idm; 4155 struct ixl_aq_desc iaq; 4156 struct ixl_aq_mac_addresses *addrs; 4157 int rv; 4158 4159 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 4160 aprint_error(", unable to allocate mac addresses\n"); 4161 return -1; 4162 } 4163 4164 memset(&iaq, 0, sizeof(iaq)); 4165 iaq.iaq_flags = htole16(IXL_AQ_BUF); 4166 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 4167 iaq.iaq_datalen = htole16(sizeof(*addrs)); 4168 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4169 4170 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4171 BUS_DMASYNC_PREREAD); 4172 4173 rv = ixl_atq_poll(sc, &iaq, 250); 4174 4175 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4176 BUS_DMASYNC_POSTREAD); 4177 4178 if (rv != 0) { 4179 aprint_error(", MAC ADDRESS READ timeout\n"); 4180 rv = -1; 4181 goto done; 4182 } 4183 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4184 aprint_error(", MAC ADDRESS READ error\n"); 4185 rv = -1; 4186 goto done; 4187 } 4188 4189 addrs = IXL_DMA_KVA(&idm); 4190 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 4191 printf(", port address is not valid\n"); 4192 goto done; 4193 } 4194 4195 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 4196 rv = 0; 4197 4198 done: 4199 ixl_dmamem_free(sc, &idm); 4200 return rv; 4201 } 4202 4203 static int 4204 ixl_get_switch_config(struct ixl_softc *sc) 4205 { 4206 struct ixl_dmamem idm; 4207 struct ixl_aq_desc iaq; 4208 struct ixl_aq_switch_config *hdr; 4209 struct ixl_aq_switch_config_element *elms, *elm; 4210 unsigned int nelm, i; 4211 int rv; 4212 4213 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4214 aprint_error_dev(sc->sc_dev, 4215 "unable to allocate switch config buffer\n"); 4216 return -1; 4217 } 4218 4219 memset(&iaq, 0, sizeof(iaq)); 4220 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4221 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4222 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 4223 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 4224 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4225 4226 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4227 BUS_DMASYNC_PREREAD); 4228 4229 rv = ixl_atq_poll(sc, &iaq, 250); 4230 4231 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4232 BUS_DMASYNC_POSTREAD); 4233 4234 if (rv != 0) { 4235 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 4236 rv = -1; 4237 goto done; 4238 } 4239 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4240 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 4241 rv = -1; 4242 goto done; 4243 } 4244 4245 hdr = IXL_DMA_KVA(&idm); 4246 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 4247 4248 nelm = le16toh(hdr->num_reported); 4249 if (nelm < 1) { 4250 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 4251 rv = -1; 4252 goto done; 4253 } 4254 4255 for (i = 0; i < nelm; i++) { 4256 elm = &elms[i]; 4257 4258 aprint_debug_dev(sc->sc_dev, 4259 "type %x revision %u seid %04x\n", 4260 elm->type, elm->revision, le16toh(elm->seid)); 4261 aprint_debug_dev(sc->sc_dev, 4262 "uplink %04x downlink %04x\n", 4263 le16toh(elm->uplink_seid), 4264 le16toh(elm->downlink_seid)); 4265 aprint_debug_dev(sc->sc_dev, 4266 "conntype %x scheduler %04x extra %04x\n", 4267 elm->connection_type, 4268 le16toh(elm->scheduler_id), 4269 le16toh(elm->element_info)); 4270 } 4271 4272 elm = &elms[0]; 4273 4274 sc->sc_uplink_seid = elm->uplink_seid; 4275 sc->sc_downlink_seid = elm->downlink_seid; 4276 sc->sc_seid = elm->seid; 4277 4278 if ((sc->sc_uplink_seid == htole16(0)) != 4279 (sc->sc_downlink_seid == htole16(0))) { 4280 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 4281 rv = -1; 4282 goto done; 4283 } 4284 4285 done: 4286 ixl_dmamem_free(sc, &idm); 4287 return rv; 4288 } 4289 4290 static int 4291 ixl_phy_mask_ints(struct ixl_softc *sc) 4292 { 4293 struct ixl_aq_desc iaq; 4294 4295 memset(&iaq, 0, sizeof(iaq)); 4296 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 4297 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 4298 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 4299 IXL_AQ_PHY_EV_MEDIA_NA)); 4300 4301 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4302 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 4303 return -1; 4304 } 4305 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4306 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 4307 return -1; 4308 } 4309 4310 return 0; 4311 } 4312 4313 static int 4314 ixl_get_phy_abilities(struct ixl_softc *sc,struct ixl_dmamem *idm) 4315 { 4316 struct ixl_aq_desc iaq; 4317 int rv; 4318 4319 memset(&iaq, 0, sizeof(iaq)); 4320 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4321 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4322 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 4323 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 4324 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 4325 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 4326 4327 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4328 BUS_DMASYNC_PREREAD); 4329 4330 rv = ixl_atq_poll(sc, &iaq, 250); 4331 4332 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4333 BUS_DMASYNC_POSTREAD); 4334 4335 if (rv != 0) 4336 return -1; 4337 4338 return le16toh(iaq.iaq_retval); 4339 } 4340 4341 static int 4342 ixl_get_phy_info(struct ixl_softc *sc) 4343 { 4344 struct ixl_dmamem idm; 4345 struct ixl_aq_phy_abilities *phy; 4346 int rv; 4347 4348 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4349 aprint_error_dev(sc->sc_dev, 4350 "unable to allocate phy abilities buffer\n"); 4351 return -1; 4352 } 4353 4354 rv = ixl_get_phy_abilities(sc, &idm); 4355 switch (rv) { 4356 case -1: 4357 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 4358 goto done; 4359 case IXL_AQ_RC_OK: 4360 break; 4361 case IXL_AQ_RC_EIO: 4362 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 4363 goto done; 4364 default: 4365 aprint_error_dev(sc->sc_dev, 4366 "GET PHY ABILITIIES error %u\n", rv); 4367 goto done; 4368 } 4369 4370 phy = IXL_DMA_KVA(&idm); 4371 4372 sc->sc_phy_types = le32toh(phy->phy_type); 4373 sc->sc_phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 4374 4375 sc->sc_phy_abilities = phy->abilities; 4376 sc->sc_phy_linkspeed = phy->link_speed; 4377 sc->sc_phy_fec_cfg = phy->fec_cfg_curr_mod_ext_info & 4378 (IXL_AQ_ENABLE_FEC_KR | IXL_AQ_ENABLE_FEC_RS | 4379 IXL_AQ_REQUEST_FEC_KR | IXL_AQ_REQUEST_FEC_RS); 4380 sc->sc_eee_cap = phy->eee_capability; 4381 sc->sc_eeer_val = phy->eeer_val; 4382 sc->sc_d3_lpan = phy->d3_lpan; 4383 4384 rv = 0; 4385 4386 done: 4387 ixl_dmamem_free(sc, &idm); 4388 return rv; 4389 } 4390 4391 static int 4392 ixl_set_phy_config(struct ixl_softc *sc, 4393 uint8_t link_speed, uint8_t abilities, bool polling) 4394 { 4395 struct ixl_aq_phy_param *param; 4396 struct ixl_atq iatq; 4397 struct ixl_aq_desc *iaq; 4398 int error; 4399 4400 memset(&iatq, 0, sizeof(iatq)); 4401 4402 iaq = &iatq.iatq_desc; 4403 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_CONFIG); 4404 param = (struct ixl_aq_phy_param *)&iaq->iaq_param; 4405 param->phy_types = htole32((uint32_t)sc->sc_phy_types); 4406 param->phy_type_ext = (uint8_t)(sc->sc_phy_types >> 32); 4407 param->link_speed = link_speed; 4408 param->abilities = abilities | IXL_AQ_PHY_ABILITY_AUTO_LINK; 4409 param->fec_cfg = sc->sc_phy_fec_cfg; 4410 param->eee_capability = sc->sc_eee_cap; 4411 param->eeer_val = sc->sc_eeer_val; 4412 param->d3_lpan = sc->sc_d3_lpan; 4413 4414 if (polling) 4415 error = ixl_atq_poll(sc, iaq, 250); 4416 else 4417 error = ixl_atq_exec(sc, &iatq); 4418 4419 if (error != 0) 4420 return error; 4421 4422 switch (le16toh(iaq->iaq_retval)) { 4423 case IXL_AQ_RC_OK: 4424 break; 4425 case IXL_AQ_RC_EPERM: 4426 return EPERM; 4427 default: 4428 return EIO; 4429 } 4430 4431 return 0; 4432 } 4433 4434 static int 4435 ixl_set_phy_autoselect(struct ixl_softc *sc) 4436 { 4437 uint8_t link_speed, abilities; 4438 4439 link_speed = sc->sc_phy_linkspeed; 4440 abilities = IXL_PHY_ABILITY_LINKUP | IXL_PHY_ABILITY_AUTONEGO; 4441 4442 return ixl_set_phy_config(sc, link_speed, abilities, true); 4443 } 4444 4445 static int 4446 ixl_get_link_status_poll(struct ixl_softc *sc, int *l) 4447 { 4448 struct ixl_aq_desc iaq; 4449 struct ixl_aq_link_param *param; 4450 int link; 4451 4452 memset(&iaq, 0, sizeof(iaq)); 4453 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 4454 param = (struct ixl_aq_link_param *)iaq.iaq_param; 4455 param->notify = IXL_AQ_LINK_NOTIFY; 4456 4457 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4458 return ETIMEDOUT; 4459 } 4460 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4461 return EIO; 4462 } 4463 4464 link = ixl_set_link_status(sc, &iaq); 4465 4466 if (l != NULL) 4467 *l = link; 4468 4469 return 0; 4470 } 4471 4472 static int 4473 ixl_get_vsi(struct ixl_softc *sc) 4474 { 4475 struct ixl_dmamem *vsi = &sc->sc_scratch; 4476 struct ixl_aq_desc iaq; 4477 struct ixl_aq_vsi_param *param; 4478 struct ixl_aq_vsi_reply *reply; 4479 struct ixl_aq_vsi_data *data; 4480 int rv; 4481 4482 /* grumble, vsi info isn't "known" at compile time */ 4483 4484 memset(&iaq, 0, sizeof(iaq)); 4485 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4486 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4487 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 4488 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4489 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4490 4491 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4492 param->uplink_seid = sc->sc_seid; 4493 4494 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4495 BUS_DMASYNC_PREREAD); 4496 4497 rv = ixl_atq_poll(sc, &iaq, 250); 4498 4499 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4500 BUS_DMASYNC_POSTREAD); 4501 4502 if (rv != 0) { 4503 return ETIMEDOUT; 4504 } 4505 4506 switch (le16toh(iaq.iaq_retval)) { 4507 case IXL_AQ_RC_OK: 4508 break; 4509 case IXL_AQ_RC_ENOENT: 4510 return ENOENT; 4511 case IXL_AQ_RC_EACCES: 4512 return EACCES; 4513 default: 4514 return EIO; 4515 } 4516 4517 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 4518 sc->sc_vsi_number = le16toh(reply->vsi_number); 4519 data = IXL_DMA_KVA(vsi); 4520 sc->sc_vsi_stat_counter_idx = le16toh(data->stat_counter_idx); 4521 4522 return 0; 4523 } 4524 4525 static int 4526 ixl_set_vsi(struct ixl_softc *sc) 4527 { 4528 struct ixl_dmamem *vsi = &sc->sc_scratch; 4529 struct ixl_aq_desc iaq; 4530 struct ixl_aq_vsi_param *param; 4531 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 4532 unsigned int qnum; 4533 uint16_t val; 4534 int rv; 4535 4536 qnum = sc->sc_nqueue_pairs - 1; 4537 4538 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 4539 IXL_AQ_VSI_VALID_VLAN); 4540 4541 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 4542 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 4543 data->queue_mapping[0] = htole16(0); 4544 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 4545 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 4546 4547 val = le16toh(data->port_vlan_flags); 4548 CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK); 4549 SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL); 4550 4551 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { 4552 SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH); 4553 } else { 4554 SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING); 4555 } 4556 4557 data->port_vlan_flags = htole16(val); 4558 4559 /* grumble, vsi info isn't "known" at compile time */ 4560 4561 memset(&iaq, 0, sizeof(iaq)); 4562 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4563 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4564 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 4565 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4566 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4567 4568 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4569 param->uplink_seid = sc->sc_seid; 4570 4571 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4572 BUS_DMASYNC_PREWRITE); 4573 4574 rv = ixl_atq_poll(sc, &iaq, 250); 4575 4576 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4577 BUS_DMASYNC_POSTWRITE); 4578 4579 if (rv != 0) { 4580 return ETIMEDOUT; 4581 } 4582 4583 switch (le16toh(iaq.iaq_retval)) { 4584 case IXL_AQ_RC_OK: 4585 break; 4586 case IXL_AQ_RC_ENOENT: 4587 return ENOENT; 4588 case IXL_AQ_RC_EACCES: 4589 return EACCES; 4590 default: 4591 return EIO; 4592 } 4593 4594 return 0; 4595 } 4596 4597 static void 4598 ixl_set_filter_control(struct ixl_softc *sc) 4599 { 4600 uint32_t reg; 4601 4602 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4603 4604 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4605 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4606 4607 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4608 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4609 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4610 4611 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4612 } 4613 4614 static inline void 4615 ixl_get_default_rss_key(uint32_t *buf, size_t len) 4616 { 4617 size_t cplen; 4618 uint8_t rss_seed[RSS_KEYSIZE]; 4619 4620 rss_getkey(rss_seed); 4621 memset(buf, 0, len); 4622 4623 cplen = MIN(len, sizeof(rss_seed)); 4624 memcpy(buf, rss_seed, cplen); 4625 } 4626 4627 static int 4628 ixl_set_rss_key(struct ixl_softc *sc, uint8_t *key, size_t keylen) 4629 { 4630 struct ixl_dmamem *idm; 4631 struct ixl_atq iatq; 4632 struct ixl_aq_desc *iaq; 4633 struct ixl_aq_rss_key_param *param; 4634 struct ixl_aq_rss_key_data *data; 4635 size_t len, datalen, stdlen, extlen; 4636 uint16_t vsi_id; 4637 int rv; 4638 4639 memset(&iatq, 0, sizeof(iatq)); 4640 iaq = &iatq.iatq_desc; 4641 idm = &sc->sc_aqbuf; 4642 4643 datalen = sizeof(*data); 4644 4645 /*XXX The buf size has to be less than the size of the register */ 4646 datalen = MIN(IXL_RSS_KEY_SIZE_REG * sizeof(uint32_t), datalen); 4647 4648 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4649 (datalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4650 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_KEY); 4651 iaq->iaq_datalen = htole16(datalen); 4652 4653 param = (struct ixl_aq_rss_key_param *)iaq->iaq_param; 4654 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSKEY_VSI_ID_SHIFT) | 4655 IXL_AQ_RSSKEY_VSI_VALID; 4656 param->vsi_id = htole16(vsi_id); 4657 4658 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4659 data = IXL_DMA_KVA(idm); 4660 4661 len = MIN(keylen, datalen); 4662 stdlen = MIN(sizeof(data->standard_rss_key), len); 4663 memcpy(data->standard_rss_key, key, stdlen); 4664 len = (len > stdlen) ? (len - stdlen) : 0; 4665 4666 extlen = MIN(sizeof(data->extended_hash_key), len); 4667 extlen = (stdlen < keylen) ? 0 : keylen - stdlen; 4668 memcpy(data->extended_hash_key, key + stdlen, extlen); 4669 4670 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4671 4672 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4673 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4674 4675 rv = ixl_atq_exec(sc, &iatq); 4676 4677 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4678 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4679 4680 if (rv != 0) { 4681 return ETIMEDOUT; 4682 } 4683 4684 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4685 return EIO; 4686 } 4687 4688 return 0; 4689 } 4690 4691 static int 4692 ixl_set_rss_lut(struct ixl_softc *sc, uint8_t *lut, size_t lutlen) 4693 { 4694 struct ixl_dmamem *idm; 4695 struct ixl_atq iatq; 4696 struct ixl_aq_desc *iaq; 4697 struct ixl_aq_rss_lut_param *param; 4698 uint16_t vsi_id; 4699 uint8_t *data; 4700 size_t dmalen; 4701 int rv; 4702 4703 memset(&iatq, 0, sizeof(iatq)); 4704 iaq = &iatq.iatq_desc; 4705 idm = &sc->sc_aqbuf; 4706 4707 dmalen = MIN(lutlen, IXL_DMA_LEN(idm)); 4708 4709 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4710 (dmalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4711 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_LUT); 4712 iaq->iaq_datalen = htole16(dmalen); 4713 4714 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4715 data = IXL_DMA_KVA(idm); 4716 memcpy(data, lut, dmalen); 4717 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4718 4719 param = (struct ixl_aq_rss_lut_param *)iaq->iaq_param; 4720 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSLUT_VSI_ID_SHIFT) | 4721 IXL_AQ_RSSLUT_VSI_VALID; 4722 param->vsi_id = htole16(vsi_id); 4723 param->flags = htole16(IXL_AQ_RSSLUT_TABLE_TYPE_PF << 4724 IXL_AQ_RSSLUT_TABLE_TYPE_SHIFT); 4725 4726 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4727 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4728 4729 rv = ixl_atq_exec(sc, &iatq); 4730 4731 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4732 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4733 4734 if (rv != 0) { 4735 return ETIMEDOUT; 4736 } 4737 4738 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4739 return EIO; 4740 } 4741 4742 return 0; 4743 } 4744 4745 static int 4746 ixl_register_rss_key(struct ixl_softc *sc) 4747 { 4748 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4749 int rv; 4750 size_t i; 4751 4752 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4753 4754 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)){ 4755 rv = ixl_set_rss_key(sc, (uint8_t*)rss_seed, 4756 sizeof(rss_seed)); 4757 } else { 4758 rv = 0; 4759 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4760 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4761 } 4762 } 4763 4764 return rv; 4765 } 4766 4767 static void 4768 ixl_register_rss_pctype(struct ixl_softc *sc) 4769 { 4770 uint64_t set_hena = 0; 4771 uint32_t hena0, hena1; 4772 4773 if (sc->sc_mac_type == I40E_MAC_X722) 4774 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4775 else 4776 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4777 4778 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4779 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4780 4781 SET(hena0, set_hena); 4782 SET(hena1, set_hena >> 32); 4783 4784 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4785 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4786 } 4787 4788 static int 4789 ixl_register_rss_hlut(struct ixl_softc *sc) 4790 { 4791 unsigned int qid; 4792 uint8_t hlut_buf[512], lut_mask; 4793 uint32_t *hluts; 4794 size_t i, hluts_num; 4795 int rv; 4796 4797 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4798 4799 for (i = 0; i < sc->sc_rss_table_size; i++) { 4800 qid = i % sc->sc_nqueue_pairs; 4801 hlut_buf[i] = qid & lut_mask; 4802 } 4803 4804 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4805 rv = ixl_set_rss_lut(sc, hlut_buf, sizeof(hlut_buf)); 4806 } else { 4807 rv = 0; 4808 hluts = (uint32_t *)hlut_buf; 4809 hluts_num = sc->sc_rss_table_size >> 2; 4810 for (i = 0; i < hluts_num; i++) { 4811 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4812 } 4813 ixl_flush(sc); 4814 } 4815 4816 return rv; 4817 } 4818 4819 static void 4820 ixl_config_rss(struct ixl_softc *sc) 4821 { 4822 4823 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4824 4825 ixl_register_rss_key(sc); 4826 ixl_register_rss_pctype(sc); 4827 ixl_register_rss_hlut(sc); 4828 } 4829 4830 static const struct ixl_phy_type * 4831 ixl_search_phy_type(uint8_t phy_type) 4832 { 4833 const struct ixl_phy_type *itype; 4834 uint64_t mask; 4835 unsigned int i; 4836 4837 if (phy_type >= 64) 4838 return NULL; 4839 4840 mask = 1ULL << phy_type; 4841 4842 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4843 itype = &ixl_phy_type_map[i]; 4844 4845 if (ISSET(itype->phy_type, mask)) 4846 return itype; 4847 } 4848 4849 return NULL; 4850 } 4851 4852 static uint64_t 4853 ixl_search_link_speed(uint8_t link_speed) 4854 { 4855 const struct ixl_speed_type *type; 4856 unsigned int i; 4857 4858 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4859 type = &ixl_speed_type_map[i]; 4860 4861 if (ISSET(type->dev_speed, link_speed)) 4862 return type->net_speed; 4863 } 4864 4865 return 0; 4866 } 4867 4868 static uint8_t 4869 ixl_search_baudrate(uint64_t baudrate) 4870 { 4871 const struct ixl_speed_type *type; 4872 unsigned int i; 4873 4874 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4875 type = &ixl_speed_type_map[i]; 4876 4877 if (type->net_speed == baudrate) { 4878 return type->dev_speed; 4879 } 4880 } 4881 4882 return 0; 4883 } 4884 4885 static int 4886 ixl_restart_an(struct ixl_softc *sc) 4887 { 4888 struct ixl_aq_desc iaq; 4889 4890 memset(&iaq, 0, sizeof(iaq)); 4891 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4892 iaq.iaq_param[0] = 4893 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4894 4895 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4896 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4897 return -1; 4898 } 4899 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4900 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4901 return -1; 4902 } 4903 4904 return 0; 4905 } 4906 4907 static int 4908 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4909 uint16_t vlan, uint16_t flags) 4910 { 4911 struct ixl_aq_desc iaq; 4912 struct ixl_aq_add_macvlan *param; 4913 struct ixl_aq_add_macvlan_elem *elem; 4914 4915 memset(&iaq, 0, sizeof(iaq)); 4916 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4917 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4918 iaq.iaq_datalen = htole16(sizeof(*elem)); 4919 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4920 4921 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4922 param->num_addrs = htole16(1); 4923 param->seid0 = htole16(0x8000) | sc->sc_seid; 4924 param->seid1 = 0; 4925 param->seid2 = 0; 4926 4927 elem = IXL_DMA_KVA(&sc->sc_scratch); 4928 memset(elem, 0, sizeof(*elem)); 4929 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4930 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4931 elem->vlan = htole16(vlan); 4932 4933 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4934 return IXL_AQ_RC_EINVAL; 4935 } 4936 4937 switch (le16toh(iaq.iaq_retval)) { 4938 case IXL_AQ_RC_OK: 4939 break; 4940 case IXL_AQ_RC_ENOSPC: 4941 return ENOSPC; 4942 case IXL_AQ_RC_ENOENT: 4943 return ENOENT; 4944 case IXL_AQ_RC_EACCES: 4945 return EACCES; 4946 case IXL_AQ_RC_EEXIST: 4947 return EEXIST; 4948 case IXL_AQ_RC_EINVAL: 4949 return EINVAL; 4950 default: 4951 return EIO; 4952 } 4953 4954 return 0; 4955 } 4956 4957 static int 4958 ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4959 uint16_t vlan, uint16_t flags) 4960 { 4961 struct ixl_aq_desc iaq; 4962 struct ixl_aq_remove_macvlan *param; 4963 struct ixl_aq_remove_macvlan_elem *elem; 4964 4965 memset(&iaq, 0, sizeof(iaq)); 4966 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4967 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 4968 iaq.iaq_datalen = htole16(sizeof(*elem)); 4969 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4970 4971 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 4972 param->num_addrs = htole16(1); 4973 param->seid0 = htole16(0x8000) | sc->sc_seid; 4974 param->seid1 = 0; 4975 param->seid2 = 0; 4976 4977 elem = IXL_DMA_KVA(&sc->sc_scratch); 4978 memset(elem, 0, sizeof(*elem)); 4979 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4980 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 4981 elem->vlan = htole16(vlan); 4982 4983 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4984 return EINVAL; 4985 } 4986 4987 switch (le16toh(iaq.iaq_retval)) { 4988 case IXL_AQ_RC_OK: 4989 break; 4990 case IXL_AQ_RC_ENOENT: 4991 return ENOENT; 4992 case IXL_AQ_RC_EACCES: 4993 return EACCES; 4994 case IXL_AQ_RC_EINVAL: 4995 return EINVAL; 4996 default: 4997 return EIO; 4998 } 4999 5000 return 0; 5001 } 5002 5003 static int 5004 ixl_hmc(struct ixl_softc *sc) 5005 { 5006 struct { 5007 uint32_t count; 5008 uint32_t minsize; 5009 bus_size_t objsiz; 5010 bus_size_t setoff; 5011 bus_size_t setcnt; 5012 } regs[] = { 5013 { 5014 0, 5015 IXL_HMC_TXQ_MINSIZE, 5016 I40E_GLHMC_LANTXOBJSZ, 5017 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 5018 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 5019 }, 5020 { 5021 0, 5022 IXL_HMC_RXQ_MINSIZE, 5023 I40E_GLHMC_LANRXOBJSZ, 5024 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 5025 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 5026 }, 5027 { 5028 0, 5029 0, 5030 I40E_GLHMC_FCOEDDPOBJSZ, 5031 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 5032 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 5033 }, 5034 { 5035 0, 5036 0, 5037 I40E_GLHMC_FCOEFOBJSZ, 5038 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 5039 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 5040 }, 5041 }; 5042 struct ixl_hmc_entry *e; 5043 uint64_t size, dva; 5044 uint8_t *kva; 5045 uint64_t *sdpage; 5046 unsigned int i; 5047 int npages, tables; 5048 uint32_t reg; 5049 5050 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 5051 5052 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 5053 ixl_rd(sc, I40E_GLHMC_LANQMAX); 5054 5055 size = 0; 5056 for (i = 0; i < __arraycount(regs); i++) { 5057 e = &sc->sc_hmc_entries[i]; 5058 5059 e->hmc_count = regs[i].count; 5060 reg = ixl_rd(sc, regs[i].objsiz); 5061 e->hmc_size = BIT_ULL(0x3F & reg); 5062 e->hmc_base = size; 5063 5064 if ((e->hmc_size * 8) < regs[i].minsize) { 5065 aprint_error_dev(sc->sc_dev, 5066 "kernel hmc entry is too big\n"); 5067 return -1; 5068 } 5069 5070 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 5071 } 5072 size = roundup(size, IXL_HMC_PGSIZE); 5073 npages = size / IXL_HMC_PGSIZE; 5074 5075 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 5076 5077 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 5078 aprint_error_dev(sc->sc_dev, 5079 "unable to allocate hmc pd memory\n"); 5080 return -1; 5081 } 5082 5083 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 5084 IXL_HMC_PGSIZE) != 0) { 5085 aprint_error_dev(sc->sc_dev, 5086 "unable to allocate hmc sd memory\n"); 5087 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5088 return -1; 5089 } 5090 5091 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 5092 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 5093 5094 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 5095 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 5096 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5097 5098 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 5099 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 5100 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 5101 5102 for (i = 0; (int)i < npages; i++) { 5103 *sdpage = htole64(dva | IXL_HMC_PDVALID); 5104 sdpage++; 5105 5106 dva += IXL_HMC_PGSIZE; 5107 } 5108 5109 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 5110 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 5111 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5112 5113 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 5114 for (i = 0; (int)i < tables; i++) { 5115 uint32_t count; 5116 5117 KASSERT(npages >= 0); 5118 5119 count = ((unsigned int)npages > IXL_HMC_PGS) ? 5120 IXL_HMC_PGS : (unsigned int)npages; 5121 5122 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 5123 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 5124 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 5125 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 5126 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 5127 ixl_wr(sc, I40E_PFHMC_SDCMD, 5128 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 5129 5130 npages -= IXL_HMC_PGS; 5131 dva += IXL_HMC_PGSIZE; 5132 } 5133 5134 for (i = 0; i < __arraycount(regs); i++) { 5135 e = &sc->sc_hmc_entries[i]; 5136 5137 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 5138 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 5139 } 5140 5141 return 0; 5142 } 5143 5144 static void 5145 ixl_hmc_free(struct ixl_softc *sc) 5146 { 5147 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 5148 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5149 } 5150 5151 static void 5152 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 5153 unsigned int npacking) 5154 { 5155 uint8_t *dst = d; 5156 const uint8_t *src = s; 5157 unsigned int i; 5158 5159 for (i = 0; i < npacking; i++) { 5160 const struct ixl_hmc_pack *pack = &packing[i]; 5161 unsigned int offset = pack->lsb / 8; 5162 unsigned int align = pack->lsb % 8; 5163 const uint8_t *in = src + pack->offset; 5164 uint8_t *out = dst + offset; 5165 int width = pack->width; 5166 unsigned int inbits = 0; 5167 5168 if (align) { 5169 inbits = (*in++) << align; 5170 *out++ |= (inbits & 0xff); 5171 inbits >>= 8; 5172 5173 width -= 8 - align; 5174 } 5175 5176 while (width >= 8) { 5177 inbits |= (*in++) << align; 5178 *out++ = (inbits & 0xff); 5179 inbits >>= 8; 5180 5181 width -= 8; 5182 } 5183 5184 if (width > 0) { 5185 inbits |= (*in) << align; 5186 *out |= (inbits & ((1 << width) - 1)); 5187 } 5188 } 5189 } 5190 5191 static struct ixl_aq_buf * 5192 ixl_aqb_alloc(struct ixl_softc *sc) 5193 { 5194 struct ixl_aq_buf *aqb; 5195 5196 aqb = malloc(sizeof(*aqb), M_DEVBUF, M_WAITOK); 5197 if (aqb == NULL) 5198 return NULL; 5199 5200 aqb->aqb_size = IXL_AQ_BUFLEN; 5201 5202 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 5203 aqb->aqb_size, 0, 5204 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 5205 goto free; 5206 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 5207 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 5208 BUS_DMA_WAITOK) != 0) 5209 goto destroy; 5210 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 5211 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 5212 goto dma_free; 5213 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 5214 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 5215 goto unmap; 5216 5217 return aqb; 5218 unmap: 5219 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5220 dma_free: 5221 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5222 destroy: 5223 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5224 free: 5225 free(aqb, M_DEVBUF); 5226 5227 return NULL; 5228 } 5229 5230 static void 5231 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 5232 { 5233 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 5234 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5235 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5236 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5237 free(aqb, M_DEVBUF); 5238 } 5239 5240 static int 5241 ixl_arq_fill(struct ixl_softc *sc) 5242 { 5243 struct ixl_aq_buf *aqb; 5244 struct ixl_aq_desc *arq, *iaq; 5245 unsigned int prod = sc->sc_arq_prod; 5246 unsigned int n; 5247 int post = 0; 5248 5249 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 5250 IXL_AQ_NUM); 5251 arq = IXL_DMA_KVA(&sc->sc_arq); 5252 5253 if (__predict_false(n <= 0)) 5254 return 0; 5255 5256 do { 5257 aqb = sc->sc_arq_live[prod]; 5258 iaq = &arq[prod]; 5259 5260 if (aqb == NULL) { 5261 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 5262 if (aqb != NULL) { 5263 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5264 ixl_aq_buf, aqb_entry); 5265 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 5266 break; 5267 } 5268 5269 sc->sc_arq_live[prod] = aqb; 5270 memset(aqb->aqb_data, 0, aqb->aqb_size); 5271 5272 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 5273 aqb->aqb_size, BUS_DMASYNC_PREREAD); 5274 5275 iaq->iaq_flags = htole16(IXL_AQ_BUF | 5276 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 5277 IXL_AQ_LB : 0)); 5278 iaq->iaq_opcode = 0; 5279 iaq->iaq_datalen = htole16(aqb->aqb_size); 5280 iaq->iaq_retval = 0; 5281 iaq->iaq_cookie = 0; 5282 iaq->iaq_param[0] = 0; 5283 iaq->iaq_param[1] = 0; 5284 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 5285 } 5286 5287 prod++; 5288 prod &= IXL_AQ_MASK; 5289 5290 post = 1; 5291 5292 } while (--n); 5293 5294 if (post) { 5295 sc->sc_arq_prod = prod; 5296 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 5297 } 5298 5299 return post; 5300 } 5301 5302 static void 5303 ixl_arq_unfill(struct ixl_softc *sc) 5304 { 5305 struct ixl_aq_buf *aqb; 5306 unsigned int i; 5307 5308 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 5309 aqb = sc->sc_arq_live[i]; 5310 if (aqb == NULL) 5311 continue; 5312 5313 sc->sc_arq_live[i] = NULL; 5314 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 5315 BUS_DMASYNC_POSTREAD); 5316 ixl_aqb_free(sc, aqb); 5317 } 5318 5319 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 5320 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5321 ixl_aq_buf, aqb_entry); 5322 ixl_aqb_free(sc, aqb); 5323 } 5324 } 5325 5326 static void 5327 ixl_clear_hw(struct ixl_softc *sc) 5328 { 5329 uint32_t num_queues, base_queue; 5330 uint32_t num_pf_int; 5331 uint32_t num_vf_int; 5332 uint32_t num_vfs; 5333 uint32_t i, j; 5334 uint32_t val; 5335 uint32_t eol = 0x7ff; 5336 5337 /* get number of interrupts, queues, and vfs */ 5338 val = ixl_rd(sc, I40E_GLPCI_CNF2); 5339 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 5340 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 5341 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 5342 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 5343 5344 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 5345 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 5346 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 5347 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 5348 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 5349 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 5350 num_queues = (j - base_queue) + 1; 5351 else 5352 num_queues = 0; 5353 5354 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 5355 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 5356 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 5357 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 5358 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 5359 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 5360 num_vfs = (j - i) + 1; 5361 else 5362 num_vfs = 0; 5363 5364 /* stop all the interrupts */ 5365 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5366 ixl_flush(sc); 5367 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 5368 for (i = 0; i < num_pf_int - 2; i++) 5369 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 5370 ixl_flush(sc); 5371 5372 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 5373 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5374 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 5375 for (i = 0; i < num_pf_int - 2; i++) 5376 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 5377 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5378 for (i = 0; i < num_vfs; i++) 5379 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 5380 for (i = 0; i < num_vf_int - 2; i++) 5381 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 5382 5383 /* warn the HW of the coming Tx disables */ 5384 for (i = 0; i < num_queues; i++) { 5385 uint32_t abs_queue_idx = base_queue + i; 5386 uint32_t reg_block = 0; 5387 5388 if (abs_queue_idx >= 128) { 5389 reg_block = abs_queue_idx / 128; 5390 abs_queue_idx %= 128; 5391 } 5392 5393 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 5394 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 5395 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 5396 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 5397 5398 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 5399 } 5400 delaymsec(400); 5401 5402 /* stop all the queues */ 5403 for (i = 0; i < num_queues; i++) { 5404 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 5405 ixl_wr(sc, I40E_QTX_ENA(i), 0); 5406 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 5407 ixl_wr(sc, I40E_QRX_ENA(i), 0); 5408 } 5409 5410 /* short wait for all queue disables to settle */ 5411 delaymsec(50); 5412 } 5413 5414 static int 5415 ixl_pf_reset(struct ixl_softc *sc) 5416 { 5417 uint32_t cnt = 0; 5418 uint32_t cnt1 = 0; 5419 uint32_t reg = 0, reg0 = 0; 5420 uint32_t grst_del; 5421 5422 /* 5423 * Poll for Global Reset steady state in case of recent GRST. 5424 * The grst delay value is in 100ms units, and we'll wait a 5425 * couple counts longer to be sure we don't just miss the end. 5426 */ 5427 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 5428 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 5429 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 5430 5431 grst_del = grst_del * 20; 5432 5433 for (cnt = 0; cnt < grst_del; cnt++) { 5434 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 5435 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 5436 break; 5437 delaymsec(100); 5438 } 5439 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5440 aprint_error(", Global reset polling failed to complete\n"); 5441 return -1; 5442 } 5443 5444 /* Now Wait for the FW to be ready */ 5445 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 5446 reg = ixl_rd(sc, I40E_GLNVM_ULD); 5447 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5448 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 5449 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5450 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 5451 break; 5452 5453 delaymsec(10); 5454 } 5455 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5456 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 5457 aprint_error(", wait for FW Reset complete timed out " 5458 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 5459 return -1; 5460 } 5461 5462 /* 5463 * If there was a Global Reset in progress when we got here, 5464 * we don't need to do the PF Reset 5465 */ 5466 if (cnt == 0) { 5467 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5468 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 5469 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 5470 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5471 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 5472 break; 5473 delaymsec(1); 5474 5475 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 5476 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5477 aprint_error(", Core reset upcoming." 5478 " Skipping PF reset reset request\n"); 5479 return -1; 5480 } 5481 } 5482 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 5483 aprint_error(", PF reset polling failed to complete" 5484 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 5485 return -1; 5486 } 5487 } 5488 5489 return 0; 5490 } 5491 5492 static int 5493 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 5494 bus_size_t size, bus_size_t align) 5495 { 5496 ixm->ixm_size = size; 5497 5498 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 5499 ixm->ixm_size, 0, 5500 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 5501 &ixm->ixm_map) != 0) 5502 return 1; 5503 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 5504 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 5505 BUS_DMA_WAITOK) != 0) 5506 goto destroy; 5507 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 5508 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 5509 goto free; 5510 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 5511 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 5512 goto unmap; 5513 5514 memset(ixm->ixm_kva, 0, ixm->ixm_size); 5515 5516 return 0; 5517 unmap: 5518 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5519 free: 5520 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5521 destroy: 5522 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5523 return 1; 5524 } 5525 5526 static void 5527 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 5528 { 5529 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 5530 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5531 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5532 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5533 } 5534 5535 static int 5536 ixl_setup_vlan_hwfilter(struct ixl_softc *sc) 5537 { 5538 struct ethercom *ec = &sc->sc_ec; 5539 struct vlanid_list *vlanidp; 5540 int rv; 5541 5542 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5543 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5544 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5545 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5546 5547 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5548 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5549 if (rv != 0) 5550 return rv; 5551 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5552 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5553 if (rv != 0) 5554 return rv; 5555 5556 ETHER_LOCK(ec); 5557 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5558 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 5559 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5560 if (rv != 0) 5561 break; 5562 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 5563 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5564 if (rv != 0) 5565 break; 5566 } 5567 ETHER_UNLOCK(ec); 5568 5569 return rv; 5570 } 5571 5572 static void 5573 ixl_teardown_vlan_hwfilter(struct ixl_softc *sc) 5574 { 5575 struct vlanid_list *vlanidp; 5576 struct ethercom *ec = &sc->sc_ec; 5577 5578 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5579 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5580 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5581 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5582 5583 ETHER_LOCK(ec); 5584 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5585 ixl_remove_macvlan(sc, sc->sc_enaddr, 5586 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5587 ixl_remove_macvlan(sc, etherbroadcastaddr, 5588 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5589 } 5590 ETHER_UNLOCK(ec); 5591 5592 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5593 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5594 ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5595 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5596 } 5597 5598 static int 5599 ixl_update_macvlan(struct ixl_softc *sc) 5600 { 5601 int rv = 0; 5602 int next_ec_capenable = sc->sc_ec.ec_capenable; 5603 5604 if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 5605 rv = ixl_setup_vlan_hwfilter(sc); 5606 if (rv != 0) 5607 ixl_teardown_vlan_hwfilter(sc); 5608 } else { 5609 ixl_teardown_vlan_hwfilter(sc); 5610 } 5611 5612 return rv; 5613 } 5614 5615 static int 5616 ixl_ifflags_cb(struct ethercom *ec) 5617 { 5618 struct ifnet *ifp = &ec->ec_if; 5619 struct ixl_softc *sc = ifp->if_softc; 5620 int rv, change; 5621 5622 mutex_enter(&sc->sc_cfg_lock); 5623 5624 change = ec->ec_capenable ^ sc->sc_cur_ec_capenable; 5625 5626 if (ISSET(change, ETHERCAP_VLAN_HWTAGGING)) { 5627 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 5628 rv = ENETRESET; 5629 goto out; 5630 } 5631 5632 if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) { 5633 rv = ixl_update_macvlan(sc); 5634 if (rv == 0) { 5635 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 5636 } else { 5637 CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER); 5638 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 5639 } 5640 } 5641 5642 rv = ixl_iff(sc); 5643 out: 5644 mutex_exit(&sc->sc_cfg_lock); 5645 5646 return rv; 5647 } 5648 5649 static int 5650 ixl_set_link_status(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 5651 { 5652 const struct ixl_aq_link_status *status; 5653 const struct ixl_phy_type *itype; 5654 5655 uint64_t ifm_active = IFM_ETHER; 5656 uint64_t ifm_status = IFM_AVALID; 5657 int link_state = LINK_STATE_DOWN; 5658 uint64_t baudrate = 0; 5659 5660 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 5661 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) { 5662 ifm_active |= IFM_NONE; 5663 goto done; 5664 } 5665 5666 ifm_active |= IFM_FDX; 5667 ifm_status |= IFM_ACTIVE; 5668 link_state = LINK_STATE_UP; 5669 5670 itype = ixl_search_phy_type(status->phy_type); 5671 if (itype != NULL) 5672 ifm_active |= itype->ifm_type; 5673 5674 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 5675 ifm_active |= IFM_ETH_TXPAUSE; 5676 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 5677 ifm_active |= IFM_ETH_RXPAUSE; 5678 5679 baudrate = ixl_search_link_speed(status->link_speed); 5680 5681 done: 5682 /* NET_ASSERT_LOCKED() except during attach */ 5683 sc->sc_media_active = ifm_active; 5684 sc->sc_media_status = ifm_status; 5685 5686 sc->sc_ec.ec_if.if_baudrate = baudrate; 5687 5688 return link_state; 5689 } 5690 5691 static int 5692 ixl_establish_intx(struct ixl_softc *sc) 5693 { 5694 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5695 pci_intr_handle_t *intr; 5696 char xnamebuf[32]; 5697 char intrbuf[PCI_INTRSTR_LEN]; 5698 char const *intrstr; 5699 5700 KASSERT(sc->sc_nintrs == 1); 5701 5702 intr = &sc->sc_ihp[0]; 5703 5704 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 5705 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 5706 device_xname(sc->sc_dev)); 5707 5708 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 5709 sc, xnamebuf); 5710 5711 if (sc->sc_ihs[0] == NULL) { 5712 aprint_error_dev(sc->sc_dev, 5713 "unable to establish interrupt at %s\n", intrstr); 5714 return -1; 5715 } 5716 5717 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 5718 return 0; 5719 } 5720 5721 static int 5722 ixl_establish_msix(struct ixl_softc *sc) 5723 { 5724 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5725 kcpuset_t *affinity; 5726 unsigned int vector = 0; 5727 unsigned int i; 5728 int affinity_to, r; 5729 char xnamebuf[32]; 5730 char intrbuf[PCI_INTRSTR_LEN]; 5731 char const *intrstr; 5732 5733 kcpuset_create(&affinity, false); 5734 5735 /* the "other" intr is mapped to vector 0 */ 5736 vector = 0; 5737 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5738 intrbuf, sizeof(intrbuf)); 5739 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 5740 device_xname(sc->sc_dev)); 5741 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5742 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 5743 sc, xnamebuf); 5744 if (sc->sc_ihs[vector] == NULL) { 5745 aprint_error_dev(sc->sc_dev, 5746 "unable to establish interrupt at %s\n", intrstr); 5747 goto fail; 5748 } 5749 5750 aprint_normal_dev(sc->sc_dev, "other interrupt at %s", intrstr); 5751 5752 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5753 affinity_to = (affinity_to + sc->sc_nqueue_pairs_max) % ncpu; 5754 5755 kcpuset_zero(affinity); 5756 kcpuset_set(affinity, affinity_to); 5757 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5758 if (r == 0) { 5759 aprint_normal(", affinity to %u", affinity_to); 5760 } 5761 aprint_normal("\n"); 5762 vector++; 5763 5764 sc->sc_msix_vector_queue = vector; 5765 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5766 5767 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5768 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5769 intrbuf, sizeof(intrbuf)); 5770 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 5771 device_xname(sc->sc_dev), i); 5772 5773 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5774 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 5775 (void *)&sc->sc_qps[i], xnamebuf); 5776 5777 if (sc->sc_ihs[vector] == NULL) { 5778 aprint_error_dev(sc->sc_dev, 5779 "unable to establish interrupt at %s\n", intrstr); 5780 goto fail; 5781 } 5782 5783 aprint_normal_dev(sc->sc_dev, 5784 "for TXRX%d interrupt at %s",i , intrstr); 5785 5786 kcpuset_zero(affinity); 5787 kcpuset_set(affinity, affinity_to); 5788 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5789 if (r == 0) { 5790 aprint_normal(", affinity to %u", affinity_to); 5791 affinity_to = (affinity_to + 1) % ncpu; 5792 } 5793 aprint_normal("\n"); 5794 vector++; 5795 } 5796 5797 kcpuset_destroy(affinity); 5798 5799 return 0; 5800 fail: 5801 for (i = 0; i < vector; i++) { 5802 pci_intr_disestablish(pc, sc->sc_ihs[i]); 5803 } 5804 5805 sc->sc_msix_vector_queue = 0; 5806 sc->sc_msix_vector_queue = 0; 5807 kcpuset_destroy(affinity); 5808 5809 return -1; 5810 } 5811 5812 static void 5813 ixl_config_queue_intr(struct ixl_softc *sc) 5814 { 5815 unsigned int i, vector; 5816 5817 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5818 vector = sc->sc_msix_vector_queue; 5819 } else { 5820 vector = I40E_INTR_NOTX_INTR; 5821 5822 ixl_wr(sc, I40E_PFINT_LNKLST0, 5823 (I40E_INTR_NOTX_QUEUE << 5824 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5825 (I40E_QUEUE_TYPE_RX << 5826 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5827 } 5828 5829 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5830 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5831 ixl_flush(sc); 5832 5833 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5834 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5835 (I40E_QUEUE_TYPE_RX << 5836 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5837 5838 ixl_wr(sc, I40E_QINT_RQCTL(i), 5839 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5840 (I40E_ITR_INDEX_RX << 5841 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5842 (I40E_INTR_NOTX_RX_QUEUE << 5843 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5844 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5845 (I40E_QUEUE_TYPE_TX << 5846 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5847 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5848 5849 ixl_wr(sc, I40E_QINT_TQCTL(i), 5850 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5851 (I40E_ITR_INDEX_TX << 5852 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5853 (I40E_INTR_NOTX_TX_QUEUE << 5854 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5855 (I40E_QUEUE_TYPE_EOL << 5856 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5857 (I40E_QUEUE_TYPE_RX << 5858 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5859 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5860 5861 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) 5862 vector++; 5863 } 5864 ixl_flush(sc); 5865 5866 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), 0x7a); 5867 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), 0x7a); 5868 ixl_flush(sc); 5869 } 5870 5871 static void 5872 ixl_config_other_intr(struct ixl_softc *sc) 5873 { 5874 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5875 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5876 5877 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5878 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5879 I40E_PFINT_ICR0_ENA_GRST_MASK | 5880 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5881 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5882 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5883 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5884 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5885 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5886 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5887 5888 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5889 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5890 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5891 (I40E_ITR_INDEX_OTHER << 5892 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5893 ixl_flush(sc); 5894 } 5895 5896 static int 5897 ixl_setup_interrupts(struct ixl_softc *sc) 5898 { 5899 struct pci_attach_args *pa = &sc->sc_pa; 5900 pci_intr_type_t max_type, intr_type; 5901 int counts[PCI_INTR_TYPE_SIZE]; 5902 int error; 5903 unsigned int i; 5904 bool retry; 5905 5906 memset(counts, 0, sizeof(counts)); 5907 max_type = PCI_INTR_TYPE_MSIX; 5908 /* QPs + other interrupt */ 5909 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5910 counts[PCI_INTR_TYPE_INTX] = 1; 5911 5912 if (ixl_param_nomsix) 5913 counts[PCI_INTR_TYPE_MSIX] = 0; 5914 5915 do { 5916 retry = false; 5917 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5918 if (error != 0) { 5919 aprint_error_dev(sc->sc_dev, 5920 "couldn't map interrupt\n"); 5921 break; 5922 } 5923 5924 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5925 sc->sc_nintrs = counts[intr_type]; 5926 KASSERT(sc->sc_nintrs > 0); 5927 5928 for (i = 0; i < sc->sc_nintrs; i++) { 5929 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5930 PCI_INTR_MPSAFE, true); 5931 } 5932 5933 sc->sc_ihs = kmem_alloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5934 KM_SLEEP); 5935 5936 if (intr_type == PCI_INTR_TYPE_MSIX) { 5937 error = ixl_establish_msix(sc); 5938 if (error) { 5939 counts[PCI_INTR_TYPE_MSIX] = 0; 5940 retry = true; 5941 } 5942 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5943 error = ixl_establish_intx(sc); 5944 } else { 5945 error = -1; 5946 } 5947 5948 if (error) { 5949 kmem_free(sc->sc_ihs, 5950 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5951 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5952 } else { 5953 sc->sc_intrtype = intr_type; 5954 } 5955 } while (retry); 5956 5957 return error; 5958 } 5959 5960 static void 5961 ixl_teardown_interrupts(struct ixl_softc *sc) 5962 { 5963 struct pci_attach_args *pa = &sc->sc_pa; 5964 unsigned int i; 5965 5966 for (i = 0; i < sc->sc_nintrs; i++) { 5967 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 5968 } 5969 5970 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5971 5972 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5973 sc->sc_ihs = NULL; 5974 sc->sc_nintrs = 0; 5975 } 5976 5977 static int 5978 ixl_setup_stats(struct ixl_softc *sc) 5979 { 5980 struct ixl_queue_pair *qp; 5981 struct ixl_tx_ring *txr; 5982 struct ixl_rx_ring *rxr; 5983 struct ixl_stats_counters *isc; 5984 unsigned int i; 5985 5986 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5987 qp = &sc->sc_qps[i]; 5988 txr = qp->qp_txr; 5989 rxr = qp->qp_rxr; 5990 5991 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 5992 NULL, qp->qp_name, "m_defrag successed"); 5993 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 5994 NULL, qp->qp_name, "m_defrag_failed"); 5995 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 5996 NULL, qp->qp_name, "Dropped in pcq"); 5997 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 5998 NULL, qp->qp_name, "Deferred transmit"); 5999 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 6000 NULL, qp->qp_name, "Interrupt on queue"); 6001 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 6002 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6003 6004 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 6005 NULL, qp->qp_name, "MGETHDR failed"); 6006 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 6007 NULL, qp->qp_name, "MCLGET failed"); 6008 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 6009 EVCNT_TYPE_MISC, NULL, qp->qp_name, 6010 "bus_dmamap_load_mbuf failed"); 6011 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 6012 NULL, qp->qp_name, "Interrupt on queue"); 6013 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 6014 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6015 } 6016 6017 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 6018 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 6019 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 6020 NULL, device_xname(sc->sc_dev), "Link status event"); 6021 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 6022 NULL, device_xname(sc->sc_dev), "ECC error"); 6023 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 6024 NULL, device_xname(sc->sc_dev), "PCI exception"); 6025 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 6026 NULL, device_xname(sc->sc_dev), "Critical error"); 6027 6028 isc = &sc->sc_stats_counters; 6029 evcnt_attach_dynamic(&isc->isc_crc_errors, EVCNT_TYPE_MISC, 6030 NULL, device_xname(sc->sc_dev), "CRC errors"); 6031 evcnt_attach_dynamic(&isc->isc_illegal_bytes, EVCNT_TYPE_MISC, 6032 NULL, device_xname(sc->sc_dev), "Illegal bytes"); 6033 evcnt_attach_dynamic(&isc->isc_mac_local_faults, EVCNT_TYPE_MISC, 6034 NULL, device_xname(sc->sc_dev), "Mac local faults"); 6035 evcnt_attach_dynamic(&isc->isc_mac_remote_faults, EVCNT_TYPE_MISC, 6036 NULL, device_xname(sc->sc_dev), "Mac remote faults"); 6037 evcnt_attach_dynamic(&isc->isc_link_xon_rx, EVCNT_TYPE_MISC, 6038 NULL, device_xname(sc->sc_dev), "Rx xon"); 6039 evcnt_attach_dynamic(&isc->isc_link_xon_tx, EVCNT_TYPE_MISC, 6040 NULL, device_xname(sc->sc_dev), "Tx xon"); 6041 evcnt_attach_dynamic(&isc->isc_link_xoff_rx, EVCNT_TYPE_MISC, 6042 NULL, device_xname(sc->sc_dev), "Rx xoff"); 6043 evcnt_attach_dynamic(&isc->isc_link_xoff_tx, EVCNT_TYPE_MISC, 6044 NULL, device_xname(sc->sc_dev), "Tx xoff"); 6045 evcnt_attach_dynamic(&isc->isc_rx_fragments, EVCNT_TYPE_MISC, 6046 NULL, device_xname(sc->sc_dev), "Rx fragments"); 6047 evcnt_attach_dynamic(&isc->isc_rx_jabber, EVCNT_TYPE_MISC, 6048 NULL, device_xname(sc->sc_dev), "Rx jabber"); 6049 6050 evcnt_attach_dynamic(&isc->isc_rx_size_64, EVCNT_TYPE_MISC, 6051 NULL, device_xname(sc->sc_dev), "Rx size 64"); 6052 evcnt_attach_dynamic(&isc->isc_rx_size_127, EVCNT_TYPE_MISC, 6053 NULL, device_xname(sc->sc_dev), "Rx size 127"); 6054 evcnt_attach_dynamic(&isc->isc_rx_size_255, EVCNT_TYPE_MISC, 6055 NULL, device_xname(sc->sc_dev), "Rx size 255"); 6056 evcnt_attach_dynamic(&isc->isc_rx_size_511, EVCNT_TYPE_MISC, 6057 NULL, device_xname(sc->sc_dev), "Rx size 511"); 6058 evcnt_attach_dynamic(&isc->isc_rx_size_1023, EVCNT_TYPE_MISC, 6059 NULL, device_xname(sc->sc_dev), "Rx size 1023"); 6060 evcnt_attach_dynamic(&isc->isc_rx_size_1522, EVCNT_TYPE_MISC, 6061 NULL, device_xname(sc->sc_dev), "Rx size 1522"); 6062 evcnt_attach_dynamic(&isc->isc_rx_size_big, EVCNT_TYPE_MISC, 6063 NULL, device_xname(sc->sc_dev), "Rx jumbo packets"); 6064 evcnt_attach_dynamic(&isc->isc_rx_undersize, EVCNT_TYPE_MISC, 6065 NULL, device_xname(sc->sc_dev), "Rx under size"); 6066 evcnt_attach_dynamic(&isc->isc_rx_oversize, EVCNT_TYPE_MISC, 6067 NULL, device_xname(sc->sc_dev), "Rx over size"); 6068 6069 evcnt_attach_dynamic(&isc->isc_rx_bytes, EVCNT_TYPE_MISC, 6070 NULL, device_xname(sc->sc_dev), "Rx bytes / port"); 6071 evcnt_attach_dynamic(&isc->isc_rx_discards, EVCNT_TYPE_MISC, 6072 NULL, device_xname(sc->sc_dev), "Rx discards / port"); 6073 evcnt_attach_dynamic(&isc->isc_rx_unicast, EVCNT_TYPE_MISC, 6074 NULL, device_xname(sc->sc_dev), "Rx unicast / port"); 6075 evcnt_attach_dynamic(&isc->isc_rx_multicast, EVCNT_TYPE_MISC, 6076 NULL, device_xname(sc->sc_dev), "Rx multicast / port"); 6077 evcnt_attach_dynamic(&isc->isc_rx_broadcast, EVCNT_TYPE_MISC, 6078 NULL, device_xname(sc->sc_dev), "Rx broadcast / port"); 6079 6080 evcnt_attach_dynamic(&isc->isc_vsi_rx_bytes, EVCNT_TYPE_MISC, 6081 NULL, device_xname(sc->sc_dev), "Rx bytes / vsi"); 6082 evcnt_attach_dynamic(&isc->isc_vsi_rx_discards, EVCNT_TYPE_MISC, 6083 NULL, device_xname(sc->sc_dev), "Rx discard / vsi"); 6084 evcnt_attach_dynamic(&isc->isc_vsi_rx_unicast, EVCNT_TYPE_MISC, 6085 NULL, device_xname(sc->sc_dev), "Rx unicast / vsi"); 6086 evcnt_attach_dynamic(&isc->isc_vsi_rx_multicast, EVCNT_TYPE_MISC, 6087 NULL, device_xname(sc->sc_dev), "Rx multicast / vsi"); 6088 evcnt_attach_dynamic(&isc->isc_vsi_rx_broadcast, EVCNT_TYPE_MISC, 6089 NULL, device_xname(sc->sc_dev), "Rx broadcast / vsi"); 6090 6091 evcnt_attach_dynamic(&isc->isc_tx_size_64, EVCNT_TYPE_MISC, 6092 NULL, device_xname(sc->sc_dev), "Tx size 64"); 6093 evcnt_attach_dynamic(&isc->isc_tx_size_127, EVCNT_TYPE_MISC, 6094 NULL, device_xname(sc->sc_dev), "Tx size 127"); 6095 evcnt_attach_dynamic(&isc->isc_tx_size_255, EVCNT_TYPE_MISC, 6096 NULL, device_xname(sc->sc_dev), "Tx size 255"); 6097 evcnt_attach_dynamic(&isc->isc_tx_size_511, EVCNT_TYPE_MISC, 6098 NULL, device_xname(sc->sc_dev), "Tx size 511"); 6099 evcnt_attach_dynamic(&isc->isc_tx_size_1023, EVCNT_TYPE_MISC, 6100 NULL, device_xname(sc->sc_dev), "Tx size 1023"); 6101 evcnt_attach_dynamic(&isc->isc_tx_size_1522, EVCNT_TYPE_MISC, 6102 NULL, device_xname(sc->sc_dev), "Tx size 1522"); 6103 evcnt_attach_dynamic(&isc->isc_tx_size_big, EVCNT_TYPE_MISC, 6104 NULL, device_xname(sc->sc_dev), "Tx jumbo packets"); 6105 6106 evcnt_attach_dynamic(&isc->isc_tx_bytes, EVCNT_TYPE_MISC, 6107 NULL, device_xname(sc->sc_dev), "Tx bytes / port"); 6108 evcnt_attach_dynamic(&isc->isc_tx_dropped_link_down, EVCNT_TYPE_MISC, 6109 NULL, device_xname(sc->sc_dev), 6110 "Tx dropped due to link down / port"); 6111 evcnt_attach_dynamic(&isc->isc_tx_unicast, EVCNT_TYPE_MISC, 6112 NULL, device_xname(sc->sc_dev), "Tx unicast / port"); 6113 evcnt_attach_dynamic(&isc->isc_tx_multicast, EVCNT_TYPE_MISC, 6114 NULL, device_xname(sc->sc_dev), "Tx multicast / port"); 6115 evcnt_attach_dynamic(&isc->isc_tx_broadcast, EVCNT_TYPE_MISC, 6116 NULL, device_xname(sc->sc_dev), "Tx broadcast / port"); 6117 6118 evcnt_attach_dynamic(&isc->isc_vsi_tx_bytes, EVCNT_TYPE_MISC, 6119 NULL, device_xname(sc->sc_dev), "Tx bytes / vsi"); 6120 evcnt_attach_dynamic(&isc->isc_vsi_tx_errors, EVCNT_TYPE_MISC, 6121 NULL, device_xname(sc->sc_dev), "Tx errors / vsi"); 6122 evcnt_attach_dynamic(&isc->isc_vsi_tx_unicast, EVCNT_TYPE_MISC, 6123 NULL, device_xname(sc->sc_dev), "Tx unicast / vsi"); 6124 evcnt_attach_dynamic(&isc->isc_vsi_tx_multicast, EVCNT_TYPE_MISC, 6125 NULL, device_xname(sc->sc_dev), "Tx multicast / vsi"); 6126 evcnt_attach_dynamic(&isc->isc_vsi_tx_broadcast, EVCNT_TYPE_MISC, 6127 NULL, device_xname(sc->sc_dev), "Tx broadcast / vsi"); 6128 6129 sc->sc_stats_intval = ixl_param_stats_interval; 6130 callout_init(&sc->sc_stats_callout, CALLOUT_MPSAFE); 6131 callout_setfunc(&sc->sc_stats_callout, ixl_stats_callout, sc); 6132 ixl_work_set(&sc->sc_stats_task, ixl_stats_update, sc); 6133 6134 return 0; 6135 } 6136 6137 static void 6138 ixl_teardown_stats(struct ixl_softc *sc) 6139 { 6140 struct ixl_tx_ring *txr; 6141 struct ixl_rx_ring *rxr; 6142 struct ixl_stats_counters *isc; 6143 unsigned int i; 6144 6145 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6146 txr = sc->sc_qps[i].qp_txr; 6147 rxr = sc->sc_qps[i].qp_rxr; 6148 6149 evcnt_detach(&txr->txr_defragged); 6150 evcnt_detach(&txr->txr_defrag_failed); 6151 evcnt_detach(&txr->txr_pcqdrop); 6152 evcnt_detach(&txr->txr_transmitdef); 6153 evcnt_detach(&txr->txr_intr); 6154 evcnt_detach(&txr->txr_defer); 6155 6156 evcnt_detach(&rxr->rxr_mgethdr_failed); 6157 evcnt_detach(&rxr->rxr_mgetcl_failed); 6158 evcnt_detach(&rxr->rxr_mbuf_load_failed); 6159 evcnt_detach(&rxr->rxr_intr); 6160 evcnt_detach(&rxr->rxr_defer); 6161 } 6162 6163 isc = &sc->sc_stats_counters; 6164 evcnt_detach(&isc->isc_crc_errors); 6165 evcnt_detach(&isc->isc_illegal_bytes); 6166 evcnt_detach(&isc->isc_mac_local_faults); 6167 evcnt_detach(&isc->isc_mac_remote_faults); 6168 evcnt_detach(&isc->isc_link_xon_rx); 6169 evcnt_detach(&isc->isc_link_xon_tx); 6170 evcnt_detach(&isc->isc_link_xoff_rx); 6171 evcnt_detach(&isc->isc_link_xoff_tx); 6172 evcnt_detach(&isc->isc_rx_fragments); 6173 evcnt_detach(&isc->isc_rx_jabber); 6174 evcnt_detach(&isc->isc_rx_bytes); 6175 evcnt_detach(&isc->isc_rx_discards); 6176 evcnt_detach(&isc->isc_rx_unicast); 6177 evcnt_detach(&isc->isc_rx_multicast); 6178 evcnt_detach(&isc->isc_rx_broadcast); 6179 evcnt_detach(&isc->isc_rx_size_64); 6180 evcnt_detach(&isc->isc_rx_size_127); 6181 evcnt_detach(&isc->isc_rx_size_255); 6182 evcnt_detach(&isc->isc_rx_size_511); 6183 evcnt_detach(&isc->isc_rx_size_1023); 6184 evcnt_detach(&isc->isc_rx_size_1522); 6185 evcnt_detach(&isc->isc_rx_size_big); 6186 evcnt_detach(&isc->isc_rx_undersize); 6187 evcnt_detach(&isc->isc_rx_oversize); 6188 evcnt_detach(&isc->isc_tx_bytes); 6189 evcnt_detach(&isc->isc_tx_dropped_link_down); 6190 evcnt_detach(&isc->isc_tx_unicast); 6191 evcnt_detach(&isc->isc_tx_multicast); 6192 evcnt_detach(&isc->isc_tx_broadcast); 6193 evcnt_detach(&isc->isc_tx_size_64); 6194 evcnt_detach(&isc->isc_tx_size_127); 6195 evcnt_detach(&isc->isc_tx_size_255); 6196 evcnt_detach(&isc->isc_tx_size_511); 6197 evcnt_detach(&isc->isc_tx_size_1023); 6198 evcnt_detach(&isc->isc_tx_size_1522); 6199 evcnt_detach(&isc->isc_tx_size_big); 6200 evcnt_detach(&isc->isc_vsi_rx_discards); 6201 evcnt_detach(&isc->isc_vsi_rx_bytes); 6202 evcnt_detach(&isc->isc_vsi_rx_unicast); 6203 evcnt_detach(&isc->isc_vsi_rx_multicast); 6204 evcnt_detach(&isc->isc_vsi_rx_broadcast); 6205 evcnt_detach(&isc->isc_vsi_tx_errors); 6206 evcnt_detach(&isc->isc_vsi_tx_bytes); 6207 evcnt_detach(&isc->isc_vsi_tx_unicast); 6208 evcnt_detach(&isc->isc_vsi_tx_multicast); 6209 evcnt_detach(&isc->isc_vsi_tx_broadcast); 6210 6211 evcnt_detach(&sc->sc_event_atq); 6212 evcnt_detach(&sc->sc_event_link); 6213 evcnt_detach(&sc->sc_event_ecc_err); 6214 evcnt_detach(&sc->sc_event_pci_exception); 6215 evcnt_detach(&sc->sc_event_crit_err); 6216 6217 callout_destroy(&sc->sc_stats_callout); 6218 } 6219 6220 static void 6221 ixl_stats_callout(void *xsc) 6222 { 6223 struct ixl_softc *sc = xsc; 6224 6225 ixl_work_add(sc->sc_workq, &sc->sc_stats_task); 6226 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 6227 } 6228 6229 static uint64_t 6230 ixl_stat_delta(struct ixl_softc *sc, uint32_t reg_hi, uint32_t reg_lo, 6231 uint64_t *offset, bool has_offset) 6232 { 6233 uint64_t value, delta; 6234 int bitwidth; 6235 6236 bitwidth = reg_hi == 0 ? 32 : 48; 6237 6238 value = ixl_rd(sc, reg_lo); 6239 6240 if (bitwidth > 32) { 6241 value |= ((uint64_t)ixl_rd(sc, reg_hi) << 32); 6242 } 6243 6244 if (__predict_true(has_offset)) { 6245 delta = value; 6246 if (value < *offset) 6247 delta += ((uint64_t)1 << bitwidth); 6248 delta -= *offset; 6249 } else { 6250 delta = 0; 6251 } 6252 atomic_swap_64(offset, value); 6253 6254 return delta; 6255 } 6256 6257 static void 6258 ixl_stats_update(void *xsc) 6259 { 6260 struct ixl_softc *sc = xsc; 6261 struct ixl_stats_counters *isc; 6262 uint64_t delta; 6263 6264 isc = &sc->sc_stats_counters; 6265 6266 /* errors */ 6267 delta = ixl_stat_delta(sc, 6268 0, I40E_GLPRT_CRCERRS(sc->sc_port), 6269 &isc->isc_crc_errors_offset, isc->isc_has_offset); 6270 atomic_add_64(&isc->isc_crc_errors.ev_count, delta); 6271 6272 delta = ixl_stat_delta(sc, 6273 0, I40E_GLPRT_ILLERRC(sc->sc_port), 6274 &isc->isc_illegal_bytes_offset, isc->isc_has_offset); 6275 atomic_add_64(&isc->isc_illegal_bytes.ev_count, delta); 6276 6277 /* rx */ 6278 delta = ixl_stat_delta(sc, 6279 I40E_GLPRT_GORCH(sc->sc_port), I40E_GLPRT_GORCL(sc->sc_port), 6280 &isc->isc_rx_bytes_offset, isc->isc_has_offset); 6281 atomic_add_64(&isc->isc_rx_bytes.ev_count, delta); 6282 6283 delta = ixl_stat_delta(sc, 6284 0, I40E_GLPRT_RDPC(sc->sc_port), 6285 &isc->isc_rx_discards_offset, isc->isc_has_offset); 6286 atomic_add_64(&isc->isc_rx_discards.ev_count, delta); 6287 6288 delta = ixl_stat_delta(sc, 6289 I40E_GLPRT_UPRCH(sc->sc_port), I40E_GLPRT_UPRCL(sc->sc_port), 6290 &isc->isc_rx_unicast_offset, isc->isc_has_offset); 6291 atomic_add_64(&isc->isc_rx_unicast.ev_count, delta); 6292 6293 delta = ixl_stat_delta(sc, 6294 I40E_GLPRT_MPRCH(sc->sc_port), I40E_GLPRT_MPRCL(sc->sc_port), 6295 &isc->isc_rx_multicast_offset, isc->isc_has_offset); 6296 atomic_add_64(&isc->isc_rx_multicast.ev_count, delta); 6297 6298 delta = ixl_stat_delta(sc, 6299 I40E_GLPRT_BPRCH(sc->sc_port), I40E_GLPRT_BPRCL(sc->sc_port), 6300 &isc->isc_rx_broadcast_offset, isc->isc_has_offset); 6301 atomic_add_64(&isc->isc_rx_broadcast.ev_count, delta); 6302 6303 /* Packet size stats rx */ 6304 delta = ixl_stat_delta(sc, 6305 I40E_GLPRT_PRC64H(sc->sc_port), I40E_GLPRT_PRC64L(sc->sc_port), 6306 &isc->isc_rx_size_64_offset, isc->isc_has_offset); 6307 atomic_add_64(&isc->isc_rx_size_64.ev_count, delta); 6308 6309 delta = ixl_stat_delta(sc, 6310 I40E_GLPRT_PRC127H(sc->sc_port), I40E_GLPRT_PRC127L(sc->sc_port), 6311 &isc->isc_rx_size_127_offset, isc->isc_has_offset); 6312 atomic_add_64(&isc->isc_rx_size_127.ev_count, delta); 6313 6314 delta = ixl_stat_delta(sc, 6315 I40E_GLPRT_PRC255H(sc->sc_port), I40E_GLPRT_PRC255L(sc->sc_port), 6316 &isc->isc_rx_size_255_offset, isc->isc_has_offset); 6317 atomic_add_64(&isc->isc_rx_size_255.ev_count, delta); 6318 6319 delta = ixl_stat_delta(sc, 6320 I40E_GLPRT_PRC511H(sc->sc_port), I40E_GLPRT_PRC511L(sc->sc_port), 6321 &isc->isc_rx_size_511_offset, isc->isc_has_offset); 6322 atomic_add_64(&isc->isc_rx_size_511.ev_count, delta); 6323 6324 delta = ixl_stat_delta(sc, 6325 I40E_GLPRT_PRC1023H(sc->sc_port), I40E_GLPRT_PRC1023L(sc->sc_port), 6326 &isc->isc_rx_size_1023_offset, isc->isc_has_offset); 6327 atomic_add_64(&isc->isc_rx_size_1023.ev_count, delta); 6328 6329 delta = ixl_stat_delta(sc, 6330 I40E_GLPRT_PRC1522H(sc->sc_port), I40E_GLPRT_PRC1522L(sc->sc_port), 6331 &isc->isc_rx_size_1522_offset, isc->isc_has_offset); 6332 atomic_add_64(&isc->isc_rx_size_1522.ev_count, delta); 6333 6334 delta = ixl_stat_delta(sc, 6335 I40E_GLPRT_PRC9522H(sc->sc_port), I40E_GLPRT_PRC9522L(sc->sc_port), 6336 &isc->isc_rx_size_big_offset, isc->isc_has_offset); 6337 atomic_add_64(&isc->isc_rx_size_big.ev_count, delta); 6338 6339 delta = ixl_stat_delta(sc, 6340 0, I40E_GLPRT_RUC(sc->sc_port), 6341 &isc->isc_rx_undersize_offset, isc->isc_has_offset); 6342 atomic_add_64(&isc->isc_rx_undersize.ev_count, delta); 6343 6344 delta = ixl_stat_delta(sc, 6345 0, I40E_GLPRT_ROC(sc->sc_port), 6346 &isc->isc_rx_oversize_offset, isc->isc_has_offset); 6347 atomic_add_64(&isc->isc_rx_oversize.ev_count, delta); 6348 6349 /* tx */ 6350 delta = ixl_stat_delta(sc, 6351 I40E_GLPRT_GOTCH(sc->sc_port), I40E_GLPRT_GOTCL(sc->sc_port), 6352 &isc->isc_tx_bytes_offset, isc->isc_has_offset); 6353 atomic_add_64(&isc->isc_tx_bytes.ev_count, delta); 6354 6355 delta = ixl_stat_delta(sc, 6356 0, I40E_GLPRT_TDOLD(sc->sc_port), 6357 &isc->isc_tx_dropped_link_down_offset, isc->isc_has_offset); 6358 atomic_add_64(&isc->isc_tx_dropped_link_down.ev_count, delta); 6359 6360 delta = ixl_stat_delta(sc, 6361 I40E_GLPRT_UPTCH(sc->sc_port), I40E_GLPRT_UPTCL(sc->sc_port), 6362 &isc->isc_tx_unicast_offset, isc->isc_has_offset); 6363 atomic_add_64(&isc->isc_tx_unicast.ev_count, delta); 6364 6365 delta = ixl_stat_delta(sc, 6366 I40E_GLPRT_MPTCH(sc->sc_port), I40E_GLPRT_MPTCL(sc->sc_port), 6367 &isc->isc_tx_multicast_offset, isc->isc_has_offset); 6368 atomic_add_64(&isc->isc_tx_multicast.ev_count, delta); 6369 6370 delta = ixl_stat_delta(sc, 6371 I40E_GLPRT_BPTCH(sc->sc_port), I40E_GLPRT_BPTCL(sc->sc_port), 6372 &isc->isc_tx_broadcast_offset, isc->isc_has_offset); 6373 atomic_add_64(&isc->isc_tx_broadcast.ev_count, delta); 6374 6375 /* Packet size stats tx */ 6376 delta = ixl_stat_delta(sc, 6377 I40E_GLPRT_PTC64L(sc->sc_port), I40E_GLPRT_PTC64L(sc->sc_port), 6378 &isc->isc_tx_size_64_offset, isc->isc_has_offset); 6379 atomic_add_64(&isc->isc_tx_size_64.ev_count, delta); 6380 6381 delta = ixl_stat_delta(sc, 6382 I40E_GLPRT_PTC127H(sc->sc_port), I40E_GLPRT_PTC127L(sc->sc_port), 6383 &isc->isc_tx_size_127_offset, isc->isc_has_offset); 6384 atomic_add_64(&isc->isc_tx_size_127.ev_count, delta); 6385 6386 delta = ixl_stat_delta(sc, 6387 I40E_GLPRT_PTC255H(sc->sc_port), I40E_GLPRT_PTC255L(sc->sc_port), 6388 &isc->isc_tx_size_255_offset, isc->isc_has_offset); 6389 atomic_add_64(&isc->isc_tx_size_255.ev_count, delta); 6390 6391 delta = ixl_stat_delta(sc, 6392 I40E_GLPRT_PTC511H(sc->sc_port), I40E_GLPRT_PTC511L(sc->sc_port), 6393 &isc->isc_tx_size_511_offset, isc->isc_has_offset); 6394 atomic_add_64(&isc->isc_tx_size_511.ev_count, delta); 6395 6396 delta = ixl_stat_delta(sc, 6397 I40E_GLPRT_PTC1023H(sc->sc_port), I40E_GLPRT_PTC1023L(sc->sc_port), 6398 &isc->isc_tx_size_1023_offset, isc->isc_has_offset); 6399 atomic_add_64(&isc->isc_tx_size_1023.ev_count, delta); 6400 6401 delta = ixl_stat_delta(sc, 6402 I40E_GLPRT_PTC1522H(sc->sc_port), I40E_GLPRT_PTC1522L(sc->sc_port), 6403 &isc->isc_tx_size_1522_offset, isc->isc_has_offset); 6404 atomic_add_64(&isc->isc_tx_size_1522.ev_count, delta); 6405 6406 delta = ixl_stat_delta(sc, 6407 I40E_GLPRT_PTC9522H(sc->sc_port), I40E_GLPRT_PTC9522L(sc->sc_port), 6408 &isc->isc_tx_size_big_offset, isc->isc_has_offset); 6409 atomic_add_64(&isc->isc_tx_size_big.ev_count, delta); 6410 6411 /* mac faults */ 6412 delta = ixl_stat_delta(sc, 6413 0, I40E_GLPRT_MLFC(sc->sc_port), 6414 &isc->isc_mac_local_faults_offset, isc->isc_has_offset); 6415 atomic_add_64(&isc->isc_mac_local_faults.ev_count, delta); 6416 6417 delta = ixl_stat_delta(sc, 6418 0, I40E_GLPRT_MRFC(sc->sc_port), 6419 &isc->isc_mac_remote_faults_offset, isc->isc_has_offset); 6420 atomic_add_64(&isc->isc_mac_remote_faults.ev_count, delta); 6421 6422 /* Flow control (LFC) stats */ 6423 delta = ixl_stat_delta(sc, 6424 0, I40E_GLPRT_LXONRXC(sc->sc_port), 6425 &isc->isc_link_xon_rx_offset, isc->isc_has_offset); 6426 atomic_add_64(&isc->isc_link_xon_rx.ev_count, delta); 6427 6428 delta = ixl_stat_delta(sc, 6429 0, I40E_GLPRT_LXONTXC(sc->sc_port), 6430 &isc->isc_link_xon_tx_offset, isc->isc_has_offset); 6431 atomic_add_64(&isc->isc_link_xon_tx.ev_count, delta); 6432 6433 delta = ixl_stat_delta(sc, 6434 0, I40E_GLPRT_LXOFFRXC(sc->sc_port), 6435 &isc->isc_link_xoff_rx_offset, isc->isc_has_offset); 6436 atomic_add_64(&isc->isc_link_xoff_rx.ev_count, delta); 6437 6438 delta = ixl_stat_delta(sc, 6439 0, I40E_GLPRT_LXOFFTXC(sc->sc_port), 6440 &isc->isc_link_xoff_tx_offset, isc->isc_has_offset); 6441 atomic_add_64(&isc->isc_link_xoff_tx.ev_count, delta); 6442 6443 /* fragments */ 6444 delta = ixl_stat_delta(sc, 6445 0, I40E_GLPRT_RFC(sc->sc_port), 6446 &isc->isc_rx_fragments_offset, isc->isc_has_offset); 6447 atomic_add_64(&isc->isc_rx_fragments.ev_count, delta); 6448 6449 delta = ixl_stat_delta(sc, 6450 0, I40E_GLPRT_RJC(sc->sc_port), 6451 &isc->isc_rx_jabber_offset, isc->isc_has_offset); 6452 atomic_add_64(&isc->isc_rx_jabber.ev_count, delta); 6453 6454 /* VSI rx counters */ 6455 delta = ixl_stat_delta(sc, 6456 0, I40E_GLV_RDPC(sc->sc_vsi_stat_counter_idx), 6457 &isc->isc_vsi_rx_discards_offset, isc->isc_has_offset); 6458 atomic_add_64(&isc->isc_vsi_rx_discards.ev_count, delta); 6459 6460 delta = ixl_stat_delta(sc, 6461 I40E_GLV_GORCH(sc->sc_vsi_stat_counter_idx), 6462 I40E_GLV_GORCL(sc->sc_vsi_stat_counter_idx), 6463 &isc->isc_vsi_rx_bytes_offset, isc->isc_has_offset); 6464 atomic_add_64(&isc->isc_vsi_rx_bytes.ev_count, delta); 6465 6466 delta = ixl_stat_delta(sc, 6467 I40E_GLV_UPRCH(sc->sc_vsi_stat_counter_idx), 6468 I40E_GLV_UPRCL(sc->sc_vsi_stat_counter_idx), 6469 &isc->isc_vsi_rx_unicast_offset, isc->isc_has_offset); 6470 atomic_add_64(&isc->isc_vsi_rx_unicast.ev_count, delta); 6471 6472 delta = ixl_stat_delta(sc, 6473 I40E_GLV_MPRCH(sc->sc_vsi_stat_counter_idx), 6474 I40E_GLV_MPRCL(sc->sc_vsi_stat_counter_idx), 6475 &isc->isc_vsi_rx_multicast_offset, isc->isc_has_offset); 6476 atomic_add_64(&isc->isc_vsi_rx_multicast.ev_count, delta); 6477 6478 delta = ixl_stat_delta(sc, 6479 I40E_GLV_BPRCH(sc->sc_vsi_stat_counter_idx), 6480 I40E_GLV_BPRCL(sc->sc_vsi_stat_counter_idx), 6481 &isc->isc_vsi_rx_broadcast_offset, isc->isc_has_offset); 6482 atomic_add_64(&isc->isc_vsi_rx_broadcast.ev_count, delta); 6483 6484 /* VSI tx counters */ 6485 delta = ixl_stat_delta(sc, 6486 0, I40E_GLV_TEPC(sc->sc_vsi_stat_counter_idx), 6487 &isc->isc_vsi_tx_errors_offset, isc->isc_has_offset); 6488 atomic_add_64(&isc->isc_vsi_tx_errors.ev_count, delta); 6489 6490 delta = ixl_stat_delta(sc, 6491 I40E_GLV_GOTCH(sc->sc_vsi_stat_counter_idx), 6492 I40E_GLV_GOTCL(sc->sc_vsi_stat_counter_idx), 6493 &isc->isc_vsi_tx_bytes_offset, isc->isc_has_offset); 6494 atomic_add_64(&isc->isc_vsi_tx_bytes.ev_count, delta); 6495 6496 delta = ixl_stat_delta(sc, 6497 I40E_GLV_UPTCH(sc->sc_vsi_stat_counter_idx), 6498 I40E_GLV_UPTCL(sc->sc_vsi_stat_counter_idx), 6499 &isc->isc_vsi_tx_unicast_offset, isc->isc_has_offset); 6500 atomic_add_64(&isc->isc_vsi_tx_unicast.ev_count, delta); 6501 6502 delta = ixl_stat_delta(sc, 6503 I40E_GLV_MPTCH(sc->sc_vsi_stat_counter_idx), 6504 I40E_GLV_MPTCL(sc->sc_vsi_stat_counter_idx), 6505 &isc->isc_vsi_tx_multicast_offset, isc->isc_has_offset); 6506 atomic_add_64(&isc->isc_vsi_tx_multicast.ev_count, delta); 6507 6508 delta = ixl_stat_delta(sc, 6509 I40E_GLV_BPTCH(sc->sc_vsi_stat_counter_idx), 6510 I40E_GLV_BPTCL(sc->sc_vsi_stat_counter_idx), 6511 &isc->isc_vsi_tx_broadcast_offset, isc->isc_has_offset); 6512 atomic_add_64(&isc->isc_vsi_tx_broadcast.ev_count, delta); 6513 } 6514 6515 static int 6516 ixl_setup_sysctls(struct ixl_softc *sc) 6517 { 6518 const char *devname; 6519 struct sysctllog **log; 6520 const struct sysctlnode *rnode, *rxnode, *txnode; 6521 int error; 6522 6523 log = &sc->sc_sysctllog; 6524 devname = device_xname(sc->sc_dev); 6525 6526 error = sysctl_createv(log, 0, NULL, &rnode, 6527 0, CTLTYPE_NODE, devname, 6528 SYSCTL_DESCR("ixl information and settings"), 6529 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 6530 if (error) 6531 goto out; 6532 6533 error = sysctl_createv(log, 0, &rnode, NULL, 6534 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 6535 SYSCTL_DESCR("Use workqueue for packet processing"), 6536 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 6537 if (error) 6538 goto out; 6539 6540 error = sysctl_createv(log, 0, &rnode, NULL, 6541 CTLFLAG_READONLY, CTLTYPE_INT, "stats_interval", 6542 SYSCTL_DESCR("Statistics collection interval in milliseconds"), 6543 NULL, 0, &sc->sc_stats_intval, 0, CTL_CREATE, CTL_EOL); 6544 6545 error = sysctl_createv(log, 0, &rnode, &rxnode, 6546 0, CTLTYPE_NODE, "rx", 6547 SYSCTL_DESCR("ixl information and settings for Rx"), 6548 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6549 if (error) 6550 goto out; 6551 6552 error = sysctl_createv(log, 0, &rxnode, NULL, 6553 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6554 SYSCTL_DESCR("max number of Rx packets" 6555 " to process for interrupt processing"), 6556 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6557 if (error) 6558 goto out; 6559 6560 error = sysctl_createv(log, 0, &rxnode, NULL, 6561 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6562 SYSCTL_DESCR("max number of Rx packets" 6563 " to process for deferred processing"), 6564 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 6565 if (error) 6566 goto out; 6567 6568 error = sysctl_createv(log, 0, &rnode, &txnode, 6569 0, CTLTYPE_NODE, "tx", 6570 SYSCTL_DESCR("ixl information and settings for Tx"), 6571 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6572 if (error) 6573 goto out; 6574 6575 error = sysctl_createv(log, 0, &txnode, NULL, 6576 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6577 SYSCTL_DESCR("max number of Tx packets" 6578 " to process for interrupt processing"), 6579 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6580 if (error) 6581 goto out; 6582 6583 error = sysctl_createv(log, 0, &txnode, NULL, 6584 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6585 SYSCTL_DESCR("max number of Tx packets" 6586 " to process for deferred processing"), 6587 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 6588 if (error) 6589 goto out; 6590 6591 out: 6592 if (error) { 6593 aprint_error_dev(sc->sc_dev, 6594 "unable to create sysctl node\n"); 6595 sysctl_teardown(log); 6596 } 6597 6598 return error; 6599 } 6600 6601 static void 6602 ixl_teardown_sysctls(struct ixl_softc *sc) 6603 { 6604 6605 sysctl_teardown(&sc->sc_sysctllog); 6606 } 6607 6608 static struct workqueue * 6609 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 6610 { 6611 struct workqueue *wq; 6612 int error; 6613 6614 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 6615 prio, ipl, flags); 6616 6617 if (error) 6618 return NULL; 6619 6620 return wq; 6621 } 6622 6623 static void 6624 ixl_workq_destroy(struct workqueue *wq) 6625 { 6626 6627 workqueue_destroy(wq); 6628 } 6629 6630 static void 6631 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 6632 { 6633 6634 memset(work, 0, sizeof(*work)); 6635 work->ixw_func = func; 6636 work->ixw_arg = arg; 6637 } 6638 6639 static void 6640 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 6641 { 6642 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 6643 return; 6644 6645 kpreempt_disable(); 6646 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 6647 kpreempt_enable(); 6648 } 6649 6650 static void 6651 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 6652 { 6653 6654 workqueue_wait(wq, &work->ixw_cookie); 6655 } 6656 6657 static void 6658 ixl_workq_work(struct work *wk, void *context) 6659 { 6660 struct ixl_work *work; 6661 6662 work = container_of(wk, struct ixl_work, ixw_cookie); 6663 6664 atomic_swap_uint(&work->ixw_added, 0); 6665 work->ixw_func(work->ixw_arg); 6666 } 6667 6668 static int 6669 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 6670 { 6671 struct ixl_aq_desc iaq; 6672 6673 memset(&iaq, 0, sizeof(iaq)); 6674 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 6675 iaq.iaq_param[1] = htole32(reg); 6676 6677 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6678 return ETIMEDOUT; 6679 6680 switch (htole16(iaq.iaq_retval)) { 6681 case IXL_AQ_RC_OK: 6682 /* success */ 6683 break; 6684 case IXL_AQ_RC_EACCES: 6685 return EPERM; 6686 case IXL_AQ_RC_EAGAIN: 6687 return EAGAIN; 6688 default: 6689 return EIO; 6690 } 6691 6692 *rv = htole32(iaq.iaq_param[3]); 6693 return 0; 6694 } 6695 6696 static uint32_t 6697 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 6698 { 6699 uint32_t val; 6700 int rv, retry, retry_limit; 6701 6702 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6703 retry_limit = 5; 6704 } else { 6705 retry_limit = 0; 6706 } 6707 6708 for (retry = 0; retry < retry_limit; retry++) { 6709 rv = ixl_rx_ctl_read(sc, reg, &val); 6710 if (rv == 0) 6711 return val; 6712 else if (rv == EAGAIN) 6713 delaymsec(1); 6714 else 6715 break; 6716 } 6717 6718 val = ixl_rd(sc, reg); 6719 6720 return val; 6721 } 6722 6723 static int 6724 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6725 { 6726 struct ixl_aq_desc iaq; 6727 6728 memset(&iaq, 0, sizeof(iaq)); 6729 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 6730 iaq.iaq_param[1] = htole32(reg); 6731 iaq.iaq_param[3] = htole32(value); 6732 6733 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6734 return ETIMEDOUT; 6735 6736 switch (htole16(iaq.iaq_retval)) { 6737 case IXL_AQ_RC_OK: 6738 /* success */ 6739 break; 6740 case IXL_AQ_RC_EACCES: 6741 return EPERM; 6742 case IXL_AQ_RC_EAGAIN: 6743 return EAGAIN; 6744 default: 6745 return EIO; 6746 } 6747 6748 return 0; 6749 } 6750 6751 static void 6752 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6753 { 6754 int rv, retry, retry_limit; 6755 6756 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6757 retry_limit = 5; 6758 } else { 6759 retry_limit = 0; 6760 } 6761 6762 for (retry = 0; retry < retry_limit; retry++) { 6763 rv = ixl_rx_ctl_write(sc, reg, value); 6764 if (rv == 0) 6765 return; 6766 else if (rv == EAGAIN) 6767 delaymsec(1); 6768 else 6769 break; 6770 } 6771 6772 ixl_wr(sc, reg, value); 6773 } 6774 6775 static int 6776 ixl_nvm_lock(struct ixl_softc *sc, char rw) 6777 { 6778 struct ixl_aq_desc iaq; 6779 struct ixl_aq_req_resource_param *param; 6780 int rv; 6781 6782 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6783 return 0; 6784 6785 memset(&iaq, 0, sizeof(iaq)); 6786 iaq.iaq_opcode = htole16(IXL_AQ_OP_REQUEST_RESOURCE); 6787 6788 param = (struct ixl_aq_req_resource_param *)&iaq.iaq_param; 6789 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6790 if (rw == 'R') { 6791 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_READ); 6792 } else { 6793 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_WRITE); 6794 } 6795 6796 rv = ixl_atq_poll(sc, &iaq, 250); 6797 6798 if (rv != 0) 6799 return ETIMEDOUT; 6800 6801 switch (le16toh(iaq.iaq_retval)) { 6802 case IXL_AQ_RC_OK: 6803 break; 6804 case IXL_AQ_RC_EACCES: 6805 return EACCES; 6806 case IXL_AQ_RC_EBUSY: 6807 return EBUSY; 6808 case IXL_AQ_RC_EPERM: 6809 return EPERM; 6810 } 6811 6812 return 0; 6813 } 6814 6815 static int 6816 ixl_nvm_unlock(struct ixl_softc *sc) 6817 { 6818 struct ixl_aq_desc iaq; 6819 struct ixl_aq_rel_resource_param *param; 6820 int rv; 6821 6822 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6823 return 0; 6824 6825 memset(&iaq, 0, sizeof(iaq)); 6826 iaq.iaq_opcode = htole16(IXL_AQ_OP_RELEASE_RESOURCE); 6827 6828 param = (struct ixl_aq_rel_resource_param *)&iaq.iaq_param; 6829 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6830 6831 rv = ixl_atq_poll(sc, &iaq, 250); 6832 6833 if (rv != 0) 6834 return ETIMEDOUT; 6835 6836 switch (le16toh(iaq.iaq_retval)) { 6837 case IXL_AQ_RC_OK: 6838 break; 6839 default: 6840 return EIO; 6841 } 6842 return 0; 6843 } 6844 6845 static int 6846 ixl_srdone_poll(struct ixl_softc *sc) 6847 { 6848 int wait_count; 6849 uint32_t reg; 6850 6851 for (wait_count = 0; wait_count < IXL_SRRD_SRCTL_ATTEMPTS; 6852 wait_count++) { 6853 reg = ixl_rd(sc, I40E_GLNVM_SRCTL); 6854 if (ISSET(reg, I40E_GLNVM_SRCTL_DONE_MASK)) 6855 break; 6856 6857 delaymsec(5); 6858 } 6859 6860 if (wait_count == IXL_SRRD_SRCTL_ATTEMPTS) 6861 return -1; 6862 6863 return 0; 6864 } 6865 6866 static int 6867 ixl_nvm_read_srctl(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 6868 { 6869 uint32_t reg; 6870 6871 if (ixl_srdone_poll(sc) != 0) 6872 return ETIMEDOUT; 6873 6874 reg = ((uint32_t)offset << I40E_GLNVM_SRCTL_ADDR_SHIFT) | 6875 __BIT(I40E_GLNVM_SRCTL_START_SHIFT); 6876 ixl_wr(sc, I40E_GLNVM_SRCTL, reg); 6877 6878 if (ixl_srdone_poll(sc) != 0) { 6879 aprint_debug("NVM read error: couldn't access " 6880 "Shadow RAM address: 0x%x\n", offset); 6881 return ETIMEDOUT; 6882 } 6883 6884 reg = ixl_rd(sc, I40E_GLNVM_SRDATA); 6885 *data = (uint16_t)__SHIFTOUT(reg, I40E_GLNVM_SRDATA_RDDATA_MASK); 6886 6887 return 0; 6888 } 6889 6890 static int 6891 ixl_nvm_read_aq(struct ixl_softc *sc, uint16_t offset_word, 6892 void *data, size_t len) 6893 { 6894 struct ixl_dmamem *idm; 6895 struct ixl_aq_desc iaq; 6896 struct ixl_aq_nvm_param *param; 6897 uint32_t offset_bytes; 6898 int rv; 6899 6900 idm = &sc->sc_aqbuf; 6901 if (len > IXL_DMA_LEN(idm)) 6902 return ENOMEM; 6903 6904 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 6905 memset(&iaq, 0, sizeof(iaq)); 6906 iaq.iaq_opcode = htole16(IXL_AQ_OP_NVM_READ); 6907 iaq.iaq_flags = htole16(IXL_AQ_BUF | 6908 ((len > I40E_AQ_LARGE_BUF) ? IXL_AQ_LB : 0)); 6909 iaq.iaq_datalen = htole16(len); 6910 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 6911 6912 param = (struct ixl_aq_nvm_param *)iaq.iaq_param; 6913 param->command_flags = IXL_AQ_NVM_LAST_CMD; 6914 param->module_pointer = 0; 6915 param->length = htole16(len); 6916 offset_bytes = (uint32_t)offset_word * 2; 6917 offset_bytes &= 0x00FFFFFF; 6918 param->offset = htole32(offset_bytes); 6919 6920 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 6921 BUS_DMASYNC_PREREAD); 6922 6923 rv = ixl_atq_poll(sc, &iaq, 250); 6924 6925 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 6926 BUS_DMASYNC_POSTREAD); 6927 6928 if (rv != 0) { 6929 return ETIMEDOUT; 6930 } 6931 6932 switch (le16toh(iaq.iaq_retval)) { 6933 case IXL_AQ_RC_OK: 6934 break; 6935 case IXL_AQ_RC_EPERM: 6936 return EPERM; 6937 case IXL_AQ_RC_EINVAL: 6938 return EINVAL; 6939 case IXL_AQ_RC_EBUSY: 6940 return EBUSY; 6941 case IXL_AQ_RC_EIO: 6942 default: 6943 return EIO; 6944 } 6945 6946 memcpy(data, IXL_DMA_KVA(idm), len); 6947 6948 return 0; 6949 } 6950 6951 static int 6952 ixl_rd16_nvm(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 6953 { 6954 int error; 6955 uint16_t buf; 6956 6957 error = ixl_nvm_lock(sc, 'R'); 6958 if (error) 6959 return error; 6960 6961 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMREAD)) { 6962 error = ixl_nvm_read_aq(sc, offset, 6963 &buf, sizeof(buf)); 6964 if (error == 0) 6965 *data = le16toh(buf); 6966 } else { 6967 error = ixl_nvm_read_srctl(sc, offset, &buf); 6968 if (error == 0) 6969 *data = buf; 6970 } 6971 6972 ixl_nvm_unlock(sc); 6973 6974 return error; 6975 } 6976 6977 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 6978 6979 #ifdef _MODULE 6980 #include "ioconf.c" 6981 #endif 6982 6983 #ifdef _MODULE 6984 static void 6985 ixl_parse_modprop(prop_dictionary_t dict) 6986 { 6987 prop_object_t obj; 6988 int64_t val; 6989 uint64_t uval; 6990 6991 if (dict == NULL) 6992 return; 6993 6994 obj = prop_dictionary_get(dict, "nomsix"); 6995 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_BOOL) { 6996 ixl_param_nomsix = prop_bool_true((prop_bool_t)obj); 6997 } 6998 6999 obj = prop_dictionary_get(dict, "stats_interval"); 7000 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7001 val = prop_number_integer_value((prop_number_t)obj); 7002 7003 /* the range has no reason */ 7004 if (100 < val && val < 180000) { 7005 ixl_param_stats_interval = val; 7006 } 7007 } 7008 7009 obj = prop_dictionary_get(dict, "nqps_limit"); 7010 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7011 val = prop_number_integer_value((prop_number_t)obj); 7012 7013 if (val <= INT32_MAX) 7014 ixl_param_nqps_limit = val; 7015 } 7016 7017 obj = prop_dictionary_get(dict, "rx_ndescs"); 7018 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7019 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7020 7021 if (uval > 8) 7022 ixl_param_rx_ndescs = uval; 7023 } 7024 7025 obj = prop_dictionary_get(dict, "tx_ndescs"); 7026 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7027 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7028 7029 if (uval > IXL_TX_PKT_DESCS) 7030 ixl_param_tx_ndescs = uval; 7031 } 7032 7033 } 7034 #endif 7035 7036 static int 7037 if_ixl_modcmd(modcmd_t cmd, void *opaque) 7038 { 7039 int error = 0; 7040 7041 #ifdef _MODULE 7042 switch (cmd) { 7043 case MODULE_CMD_INIT: 7044 ixl_parse_modprop((prop_dictionary_t)opaque); 7045 error = config_init_component(cfdriver_ioconf_if_ixl, 7046 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7047 break; 7048 case MODULE_CMD_FINI: 7049 error = config_fini_component(cfdriver_ioconf_if_ixl, 7050 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7051 break; 7052 default: 7053 error = ENOTTY; 7054 break; 7055 } 7056 #endif 7057 7058 return error; 7059 } 7060
Indexes created Sat Sep 20 22:09:52 GMT 2025