if_ixl.c revision 1.79
1 /* $NetBSD: if_ixl.c,v 1.79 2022/03/24 08:54:16 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 __KERNEL_RCSID(0, "$NetBSD: if_ixl.c,v 1.79 2022/03/24 08:54:16 yamaguchi Exp $"); 78 79 #ifdef _KERNEL_OPT 80 #include "opt_net_mpsafe.h" 81 #include "opt_if_ixl.h" 82 #endif 83 84 #include <sys/param.h> 85 #include <sys/types.h> 86 87 #include <sys/bitops.h> 88 #include <sys/cpu.h> 89 #include <sys/device.h> 90 #include <sys/evcnt.h> 91 #include <sys/interrupt.h> 92 #include <sys/kmem.h> 93 #include <sys/module.h> 94 #include <sys/mutex.h> 95 #include <sys/pcq.h> 96 #include <sys/syslog.h> 97 #include <sys/workqueue.h> 98 99 #include <sys/bus.h> 100 101 #include <net/bpf.h> 102 #include <net/if.h> 103 #include <net/if_dl.h> 104 #include <net/if_media.h> 105 #include <net/if_ether.h> 106 #include <net/rss_config.h> 107 108 #include <netinet/tcp.h> /* for struct tcphdr */ 109 #include <netinet/udp.h> /* for struct udphdr */ 110 111 #include <dev/pci/pcivar.h> 112 #include <dev/pci/pcidevs.h> 113 114 #include <dev/pci/if_ixlreg.h> 115 #include <dev/pci/if_ixlvar.h> 116 117 #include <prop/proplib.h> 118 119 struct ixl_softc; /* defined */ 120 121 #define I40E_PF_RESET_WAIT_COUNT 200 122 #define I40E_AQ_LARGE_BUF 512 123 124 /* bitfields for Tx queue mapping in QTX_CTL */ 125 #define I40E_QTX_CTL_VF_QUEUE 0x0 126 #define I40E_QTX_CTL_VM_QUEUE 0x1 127 #define I40E_QTX_CTL_PF_QUEUE 0x2 128 129 #define I40E_QUEUE_TYPE_EOL 0x7ff 130 #define I40E_INTR_NOTX_QUEUE 0 131 132 #define I40E_QUEUE_TYPE_RX 0x0 133 #define I40E_QUEUE_TYPE_TX 0x1 134 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 135 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 136 137 #define I40E_ITR_INDEX_RX 0x0 138 #define I40E_ITR_INDEX_TX 0x1 139 #define I40E_ITR_INDEX_OTHER 0x2 140 #define I40E_ITR_INDEX_NONE 0x3 141 #define IXL_ITR_RX 0x7a /* 4K intrs/sec */ 142 #define IXL_ITR_TX 0x7a /* 4K intrs/sec */ 143 144 #define I40E_INTR_NOTX_QUEUE 0 145 #define I40E_INTR_NOTX_INTR 0 146 #define I40E_INTR_NOTX_RX_QUEUE 0 147 #define I40E_INTR_NOTX_TX_QUEUE 1 148 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 149 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 150 151 #define I40E_HASH_LUT_SIZE_128 0 152 153 #define IXL_ICR0_CRIT_ERR_MASK \ 154 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 155 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 156 I40E_PFINT_ICR0_PE_CRITERR_MASK) 157 158 #define IXL_QUEUE_MAX_XL710 64 159 #define IXL_QUEUE_MAX_X722 128 160 161 #define IXL_TX_PKT_DESCS 8 162 #define IXL_TX_PKT_MAXSIZE (MCLBYTES * IXL_TX_PKT_DESCS) 163 #define IXL_TX_QUEUE_ALIGN 128 164 #define IXL_RX_QUEUE_ALIGN 128 165 166 #define IXL_MCLBYTES (MCLBYTES - ETHER_ALIGN) 167 #define IXL_MTU_ETHERLEN ETHER_HDR_LEN \ 168 + ETHER_CRC_LEN 169 #if 0 170 #define IXL_MAX_MTU (9728 - IXL_MTU_ETHERLEN) 171 #else 172 /* (dbuff * 5) - ETHER_HDR_LEN - ETHER_CRC_LEN */ 173 #define IXL_MAX_MTU (9600 - IXL_MTU_ETHERLEN) 174 #endif 175 #define IXL_MIN_MTU (ETHER_MIN_LEN - ETHER_CRC_LEN) 176 177 #define IXL_PCIREG PCI_MAPREG_START 178 179 #define IXL_ITR0 0x0 180 #define IXL_ITR1 0x1 181 #define IXL_ITR2 0x2 182 #define IXL_NOITR 0x3 183 184 #define IXL_AQ_NUM 256 185 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 186 #define IXL_AQ_ALIGN 64 /* lol */ 187 #define IXL_AQ_BUFLEN 4096 188 189 #define IXL_HMC_ROUNDUP 512 190 #define IXL_HMC_PGSIZE 4096 191 #define IXL_HMC_DVASZ sizeof(uint64_t) 192 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 193 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 194 #define IXL_HMC_PDVALID 1ULL 195 196 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 197 198 #define IXL_SRRD_SRCTL_ATTEMPTS 100000 199 200 struct ixl_aq_regs { 201 bus_size_t atq_tail; 202 bus_size_t atq_head; 203 bus_size_t atq_len; 204 bus_size_t atq_bal; 205 bus_size_t atq_bah; 206 207 bus_size_t arq_tail; 208 bus_size_t arq_head; 209 bus_size_t arq_len; 210 bus_size_t arq_bal; 211 bus_size_t arq_bah; 212 213 uint32_t atq_len_enable; 214 uint32_t atq_tail_mask; 215 uint32_t atq_head_mask; 216 217 uint32_t arq_len_enable; 218 uint32_t arq_tail_mask; 219 uint32_t arq_head_mask; 220 }; 221 222 struct ixl_phy_type { 223 uint64_t phy_type; 224 uint64_t ifm_type; 225 }; 226 227 struct ixl_speed_type { 228 uint8_t dev_speed; 229 uint64_t net_speed; 230 }; 231 232 struct ixl_hmc_entry { 233 uint64_t hmc_base; 234 uint32_t hmc_count; 235 uint64_t hmc_size; 236 }; 237 238 enum ixl_hmc_types { 239 IXL_HMC_LAN_TX = 0, 240 IXL_HMC_LAN_RX, 241 IXL_HMC_FCOE_CTX, 242 IXL_HMC_FCOE_FILTER, 243 IXL_HMC_COUNT 244 }; 245 246 struct ixl_hmc_pack { 247 uint16_t offset; 248 uint16_t width; 249 uint16_t lsb; 250 }; 251 252 /* 253 * these hmc objects have weird sizes and alignments, so these are abstract 254 * representations of them that are nice for c to populate. 255 * 256 * the packing code relies on little-endian values being stored in the fields, 257 * no high bits in the fields being set, and the fields must be packed in the 258 * same order as they are in the ctx structure. 259 */ 260 261 struct ixl_hmc_rxq { 262 uint16_t head; 263 uint8_t cpuid; 264 uint64_t base; 265 #define IXL_HMC_RXQ_BASE_UNIT 128 266 uint16_t qlen; 267 uint16_t dbuff; 268 #define IXL_HMC_RXQ_DBUFF_UNIT 128 269 uint8_t hbuff; 270 #define IXL_HMC_RXQ_HBUFF_UNIT 64 271 uint8_t dtype; 272 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 273 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 274 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 275 uint8_t dsize; 276 #define IXL_HMC_RXQ_DSIZE_16 0 277 #define IXL_HMC_RXQ_DSIZE_32 1 278 uint8_t crcstrip; 279 uint8_t fc_ena; 280 uint8_t l2sel; 281 uint8_t hsplit_0; 282 uint8_t hsplit_1; 283 uint8_t showiv; 284 uint16_t rxmax; 285 uint8_t tphrdesc_ena; 286 uint8_t tphwdesc_ena; 287 uint8_t tphdata_ena; 288 uint8_t tphhead_ena; 289 uint8_t lrxqthresh; 290 uint8_t prefena; 291 }; 292 293 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 294 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 295 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 296 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 297 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 298 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 299 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 300 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 301 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 302 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 303 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 304 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 305 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 306 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 307 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 308 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 309 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 310 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 311 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 312 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 313 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 314 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 315 }; 316 317 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 318 319 struct ixl_hmc_txq { 320 uint16_t head; 321 uint8_t new_context; 322 uint64_t base; 323 #define IXL_HMC_TXQ_BASE_UNIT 128 324 uint8_t fc_ena; 325 uint8_t timesync_ena; 326 uint8_t fd_ena; 327 uint8_t alt_vlan_ena; 328 uint8_t cpuid; 329 uint16_t thead_wb; 330 uint8_t head_wb_ena; 331 #define IXL_HMC_TXQ_DESC_WB 0 332 #define IXL_HMC_TXQ_HEAD_WB 1 333 uint16_t qlen; 334 uint8_t tphrdesc_ena; 335 uint8_t tphrpacket_ena; 336 uint8_t tphwdesc_ena; 337 uint64_t head_wb_addr; 338 uint32_t crc; 339 uint16_t rdylist; 340 uint8_t rdylist_act; 341 }; 342 343 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 344 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 345 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 346 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 347 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 348 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 349 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 350 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 351 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 352 /* line 1 */ 353 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 354 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 355 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 356 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 357 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 358 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 359 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 360 /* line 7 */ 361 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 362 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 363 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 364 }; 365 366 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 367 368 struct ixl_work { 369 struct work ixw_cookie; 370 void (*ixw_func)(void *); 371 void *ixw_arg; 372 unsigned int ixw_added; 373 }; 374 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 375 376 struct ixl_tx_map { 377 struct mbuf *txm_m; 378 bus_dmamap_t txm_map; 379 unsigned int txm_eop; 380 }; 381 382 struct ixl_tx_ring { 383 kmutex_t txr_lock; 384 struct ixl_softc *txr_sc; 385 386 unsigned int txr_prod; 387 unsigned int txr_cons; 388 389 struct ixl_tx_map *txr_maps; 390 struct ixl_dmamem txr_mem; 391 392 bus_size_t txr_tail; 393 unsigned int txr_qid; 394 pcq_t *txr_intrq; 395 void *txr_si; 396 397 struct evcnt txr_defragged; 398 struct evcnt txr_defrag_failed; 399 struct evcnt txr_pcqdrop; 400 struct evcnt txr_transmitdef; 401 struct evcnt txr_intr; 402 struct evcnt txr_defer; 403 }; 404 405 struct ixl_rx_map { 406 struct mbuf *rxm_m; 407 bus_dmamap_t rxm_map; 408 }; 409 410 struct ixl_rx_ring { 411 kmutex_t rxr_lock; 412 413 unsigned int rxr_prod; 414 unsigned int rxr_cons; 415 416 struct ixl_rx_map *rxr_maps; 417 struct ixl_dmamem rxr_mem; 418 419 struct mbuf *rxr_m_head; 420 struct mbuf **rxr_m_tail; 421 422 bus_size_t rxr_tail; 423 unsigned int rxr_qid; 424 425 struct evcnt rxr_mgethdr_failed; 426 struct evcnt rxr_mgetcl_failed; 427 struct evcnt rxr_mbuf_load_failed; 428 struct evcnt rxr_intr; 429 struct evcnt rxr_defer; 430 }; 431 432 struct ixl_queue_pair { 433 struct ixl_softc *qp_sc; 434 struct ixl_tx_ring *qp_txr; 435 struct ixl_rx_ring *qp_rxr; 436 437 char qp_name[16]; 438 439 void *qp_si; 440 struct work qp_work; 441 bool qp_workqueue; 442 }; 443 444 struct ixl_atq { 445 struct ixl_aq_desc iatq_desc; 446 void (*iatq_fn)(struct ixl_softc *, 447 const struct ixl_aq_desc *); 448 }; 449 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 450 451 struct ixl_product { 452 unsigned int vendor_id; 453 unsigned int product_id; 454 }; 455 456 struct ixl_stats_counters { 457 bool isc_has_offset; 458 struct evcnt isc_crc_errors; 459 uint64_t isc_crc_errors_offset; 460 struct evcnt isc_illegal_bytes; 461 uint64_t isc_illegal_bytes_offset; 462 struct evcnt isc_rx_bytes; 463 uint64_t isc_rx_bytes_offset; 464 struct evcnt isc_rx_discards; 465 uint64_t isc_rx_discards_offset; 466 struct evcnt isc_rx_unicast; 467 uint64_t isc_rx_unicast_offset; 468 struct evcnt isc_rx_multicast; 469 uint64_t isc_rx_multicast_offset; 470 struct evcnt isc_rx_broadcast; 471 uint64_t isc_rx_broadcast_offset; 472 struct evcnt isc_rx_size_64; 473 uint64_t isc_rx_size_64_offset; 474 struct evcnt isc_rx_size_127; 475 uint64_t isc_rx_size_127_offset; 476 struct evcnt isc_rx_size_255; 477 uint64_t isc_rx_size_255_offset; 478 struct evcnt isc_rx_size_511; 479 uint64_t isc_rx_size_511_offset; 480 struct evcnt isc_rx_size_1023; 481 uint64_t isc_rx_size_1023_offset; 482 struct evcnt isc_rx_size_1522; 483 uint64_t isc_rx_size_1522_offset; 484 struct evcnt isc_rx_size_big; 485 uint64_t isc_rx_size_big_offset; 486 struct evcnt isc_rx_undersize; 487 uint64_t isc_rx_undersize_offset; 488 struct evcnt isc_rx_oversize; 489 uint64_t isc_rx_oversize_offset; 490 struct evcnt isc_rx_fragments; 491 uint64_t isc_rx_fragments_offset; 492 struct evcnt isc_rx_jabber; 493 uint64_t isc_rx_jabber_offset; 494 struct evcnt isc_tx_bytes; 495 uint64_t isc_tx_bytes_offset; 496 struct evcnt isc_tx_dropped_link_down; 497 uint64_t isc_tx_dropped_link_down_offset; 498 struct evcnt isc_tx_unicast; 499 uint64_t isc_tx_unicast_offset; 500 struct evcnt isc_tx_multicast; 501 uint64_t isc_tx_multicast_offset; 502 struct evcnt isc_tx_broadcast; 503 uint64_t isc_tx_broadcast_offset; 504 struct evcnt isc_tx_size_64; 505 uint64_t isc_tx_size_64_offset; 506 struct evcnt isc_tx_size_127; 507 uint64_t isc_tx_size_127_offset; 508 struct evcnt isc_tx_size_255; 509 uint64_t isc_tx_size_255_offset; 510 struct evcnt isc_tx_size_511; 511 uint64_t isc_tx_size_511_offset; 512 struct evcnt isc_tx_size_1023; 513 uint64_t isc_tx_size_1023_offset; 514 struct evcnt isc_tx_size_1522; 515 uint64_t isc_tx_size_1522_offset; 516 struct evcnt isc_tx_size_big; 517 uint64_t isc_tx_size_big_offset; 518 struct evcnt isc_mac_local_faults; 519 uint64_t isc_mac_local_faults_offset; 520 struct evcnt isc_mac_remote_faults; 521 uint64_t isc_mac_remote_faults_offset; 522 struct evcnt isc_link_xon_rx; 523 uint64_t isc_link_xon_rx_offset; 524 struct evcnt isc_link_xon_tx; 525 uint64_t isc_link_xon_tx_offset; 526 struct evcnt isc_link_xoff_rx; 527 uint64_t isc_link_xoff_rx_offset; 528 struct evcnt isc_link_xoff_tx; 529 uint64_t isc_link_xoff_tx_offset; 530 struct evcnt isc_vsi_rx_discards; 531 uint64_t isc_vsi_rx_discards_offset; 532 struct evcnt isc_vsi_rx_bytes; 533 uint64_t isc_vsi_rx_bytes_offset; 534 struct evcnt isc_vsi_rx_unicast; 535 uint64_t isc_vsi_rx_unicast_offset; 536 struct evcnt isc_vsi_rx_multicast; 537 uint64_t isc_vsi_rx_multicast_offset; 538 struct evcnt isc_vsi_rx_broadcast; 539 uint64_t isc_vsi_rx_broadcast_offset; 540 struct evcnt isc_vsi_tx_errors; 541 uint64_t isc_vsi_tx_errors_offset; 542 struct evcnt isc_vsi_tx_bytes; 543 uint64_t isc_vsi_tx_bytes_offset; 544 struct evcnt isc_vsi_tx_unicast; 545 uint64_t isc_vsi_tx_unicast_offset; 546 struct evcnt isc_vsi_tx_multicast; 547 uint64_t isc_vsi_tx_multicast_offset; 548 struct evcnt isc_vsi_tx_broadcast; 549 uint64_t isc_vsi_tx_broadcast_offset; 550 }; 551 552 /* 553 * Locking notes: 554 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 555 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 556 * - more than one lock of them cannot be held at once. 557 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 558 * (a spin mutex). 559 * - the lock cannot held with txr_lock or rxr_lock. 560 * + a field named sc_arq_* is not protected by any lock. 561 * - operations for sc_arq_* is done in one context related to 562 * sc_arq_task. 563 * + other fields in ixl_softc is protected by sc_cfg_lock 564 * (an adaptive mutex) 565 * - It must be held before another lock is held, and It can be 566 * released after the other lock is released. 567 * */ 568 569 struct ixl_softc { 570 device_t sc_dev; 571 struct ethercom sc_ec; 572 bool sc_attached; 573 bool sc_dead; 574 uint32_t sc_port; 575 struct sysctllog *sc_sysctllog; 576 struct workqueue *sc_workq; 577 struct workqueue *sc_workq_txrx; 578 int sc_stats_intval; 579 callout_t sc_stats_callout; 580 struct ixl_work sc_stats_task; 581 struct ixl_stats_counters 582 sc_stats_counters; 583 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 584 struct ifmedia sc_media; 585 uint64_t sc_media_status; 586 uint64_t sc_media_active; 587 uint64_t sc_phy_types; 588 uint8_t sc_phy_abilities; 589 uint8_t sc_phy_linkspeed; 590 uint8_t sc_phy_fec_cfg; 591 uint16_t sc_eee_cap; 592 uint32_t sc_eeer_val; 593 uint8_t sc_d3_lpan; 594 kmutex_t sc_cfg_lock; 595 enum i40e_mac_type sc_mac_type; 596 uint32_t sc_rss_table_size; 597 uint32_t sc_rss_table_entry_width; 598 bool sc_txrx_workqueue; 599 u_int sc_tx_process_limit; 600 u_int sc_rx_process_limit; 601 u_int sc_tx_intr_process_limit; 602 u_int sc_rx_intr_process_limit; 603 604 int sc_cur_ec_capenable; 605 606 struct pci_attach_args sc_pa; 607 pci_intr_handle_t *sc_ihp; 608 void **sc_ihs; 609 unsigned int sc_nintrs; 610 611 bus_dma_tag_t sc_dmat; 612 bus_space_tag_t sc_memt; 613 bus_space_handle_t sc_memh; 614 bus_size_t sc_mems; 615 616 uint8_t sc_pf_id; 617 uint16_t sc_uplink_seid; /* le */ 618 uint16_t sc_downlink_seid; /* le */ 619 uint16_t sc_vsi_number; 620 uint16_t sc_vsi_stat_counter_idx; 621 uint16_t sc_seid; 622 unsigned int sc_base_queue; 623 624 pci_intr_type_t sc_intrtype; 625 unsigned int sc_msix_vector_queue; 626 627 struct ixl_dmamem sc_scratch; 628 struct ixl_dmamem sc_aqbuf; 629 630 const struct ixl_aq_regs * 631 sc_aq_regs; 632 uint32_t sc_aq_flags; 633 #define IXL_SC_AQ_FLAG_RXCTL __BIT(0) 634 #define IXL_SC_AQ_FLAG_NVMLOCK __BIT(1) 635 #define IXL_SC_AQ_FLAG_NVMREAD __BIT(2) 636 #define IXL_SC_AQ_FLAG_RSS __BIT(3) 637 638 kmutex_t sc_atq_lock; 639 kcondvar_t sc_atq_cv; 640 struct ixl_dmamem sc_atq; 641 unsigned int sc_atq_prod; 642 unsigned int sc_atq_cons; 643 644 struct ixl_dmamem sc_arq; 645 struct ixl_work sc_arq_task; 646 struct ixl_aq_bufs sc_arq_idle; 647 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 648 unsigned int sc_arq_prod; 649 unsigned int sc_arq_cons; 650 651 struct ixl_work sc_link_state_task; 652 struct ixl_atq sc_link_state_atq; 653 654 struct ixl_dmamem sc_hmc_sd; 655 struct ixl_dmamem sc_hmc_pd; 656 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 657 658 struct if_percpuq *sc_ipq; 659 unsigned int sc_tx_ring_ndescs; 660 unsigned int sc_rx_ring_ndescs; 661 unsigned int sc_nqueue_pairs; 662 unsigned int sc_nqueue_pairs_max; 663 unsigned int sc_nqueue_pairs_device; 664 struct ixl_queue_pair *sc_qps; 665 uint32_t sc_itr_rx; 666 uint32_t sc_itr_tx; 667 668 struct evcnt sc_event_atq; 669 struct evcnt sc_event_link; 670 struct evcnt sc_event_ecc_err; 671 struct evcnt sc_event_pci_exception; 672 struct evcnt sc_event_crit_err; 673 }; 674 675 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 676 #define IXL_TX_PROCESS_LIMIT 256 677 #define IXL_RX_PROCESS_LIMIT 256 678 #define IXL_TX_INTR_PROCESS_LIMIT 256 679 #define IXL_RX_INTR_PROCESS_LIMIT 0U 680 681 #define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx | \ 682 IFCAP_CSUM_TCPv4_Rx | \ 683 IFCAP_CSUM_UDPv4_Rx | \ 684 IFCAP_CSUM_TCPv6_Rx | \ 685 IFCAP_CSUM_UDPv6_Rx) 686 #define IXL_IFCAP_TXCSUM (IFCAP_CSUM_IPv4_Tx | \ 687 IFCAP_CSUM_TCPv4_Tx | \ 688 IFCAP_CSUM_UDPv4_Tx | \ 689 IFCAP_CSUM_TCPv6_Tx | \ 690 IFCAP_CSUM_UDPv6_Tx) 691 #define IXL_CSUM_ALL_OFFLOAD (M_CSUM_IPv4 | \ 692 M_CSUM_TCPv4 | M_CSUM_TCPv6 | \ 693 M_CSUM_UDPv4 | M_CSUM_UDPv6) 694 695 #define delaymsec(_x) DELAY(1000 * (_x)) 696 #ifdef IXL_DEBUG 697 #define DDPRINTF(sc, fmt, args...) \ 698 do { \ 699 if ((sc) != NULL) { \ 700 device_printf( \ 701 ((struct ixl_softc *)(sc))->sc_dev, \ 702 ""); \ 703 } \ 704 printf("%s:\t" fmt, __func__, ##args); \ 705 } while (0) 706 #else 707 #define DDPRINTF(sc, fmt, args...) __nothing 708 #endif 709 #ifndef IXL_STATS_INTERVAL_MSEC 710 #define IXL_STATS_INTERVAL_MSEC 10000 711 #endif 712 #ifndef IXL_QUEUE_NUM 713 #define IXL_QUEUE_NUM 0 714 #endif 715 716 static bool ixl_param_nomsix = false; 717 static int ixl_param_stats_interval = IXL_STATS_INTERVAL_MSEC; 718 static int ixl_param_nqps_limit = IXL_QUEUE_NUM; 719 static unsigned int ixl_param_tx_ndescs = 512; 720 static unsigned int ixl_param_rx_ndescs = 256; 721 722 static enum i40e_mac_type 723 ixl_mactype(pci_product_id_t); 724 static void ixl_pci_csr_setup(pci_chipset_tag_t, pcitag_t); 725 static void ixl_clear_hw(struct ixl_softc *); 726 static int ixl_pf_reset(struct ixl_softc *); 727 728 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 729 bus_size_t, bus_size_t); 730 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 731 732 static int ixl_arq_fill(struct ixl_softc *); 733 static void ixl_arq_unfill(struct ixl_softc *); 734 735 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 736 unsigned int); 737 static void ixl_atq_set(struct ixl_atq *, 738 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 739 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 740 static void ixl_atq_done(struct ixl_softc *); 741 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 742 static int ixl_atq_exec_locked(struct ixl_softc *, struct ixl_atq *); 743 static int ixl_get_version(struct ixl_softc *); 744 static int ixl_get_nvm_version(struct ixl_softc *); 745 static int ixl_get_hw_capabilities(struct ixl_softc *); 746 static int ixl_pxe_clear(struct ixl_softc *); 747 static int ixl_lldp_shut(struct ixl_softc *); 748 static int ixl_get_mac(struct ixl_softc *); 749 static int ixl_get_switch_config(struct ixl_softc *); 750 static int ixl_phy_mask_ints(struct ixl_softc *); 751 static int ixl_get_phy_info(struct ixl_softc *); 752 static int ixl_set_phy_config(struct ixl_softc *, uint8_t, uint8_t, bool); 753 static int ixl_set_phy_autoselect(struct ixl_softc *); 754 static int ixl_restart_an(struct ixl_softc *); 755 static int ixl_hmc(struct ixl_softc *); 756 static void ixl_hmc_free(struct ixl_softc *); 757 static int ixl_get_vsi(struct ixl_softc *); 758 static int ixl_set_vsi(struct ixl_softc *); 759 static void ixl_set_filter_control(struct ixl_softc *); 760 static void ixl_get_link_status(void *); 761 static int ixl_get_link_status_poll(struct ixl_softc *, int *); 762 static void ixl_get_link_status_done(struct ixl_softc *, 763 const struct ixl_aq_desc *); 764 static int ixl_set_link_status_locked(struct ixl_softc *, 765 const struct ixl_aq_desc *); 766 static uint64_t ixl_search_link_speed(uint8_t); 767 static uint8_t ixl_search_baudrate(uint64_t); 768 static void ixl_config_rss(struct ixl_softc *); 769 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 770 uint16_t, uint16_t); 771 static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *, 772 uint16_t, uint16_t); 773 static void ixl_arq(void *); 774 static void ixl_hmc_pack(void *, const void *, 775 const struct ixl_hmc_pack *, unsigned int); 776 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 777 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 778 static int ixl_rd16_nvm(struct ixl_softc *, uint16_t, uint16_t *); 779 780 static int ixl_match(device_t, cfdata_t, void *); 781 static void ixl_attach(device_t, device_t, void *); 782 static int ixl_detach(device_t, int); 783 784 static void ixl_media_add(struct ixl_softc *); 785 static int ixl_media_change(struct ifnet *); 786 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 787 static void ixl_watchdog(struct ifnet *); 788 static int ixl_ioctl(struct ifnet *, u_long, void *); 789 static void ixl_start(struct ifnet *); 790 static int ixl_transmit(struct ifnet *, struct mbuf *); 791 static void ixl_deferred_transmit(void *); 792 static int ixl_intr(void *); 793 static int ixl_queue_intr(void *); 794 static int ixl_other_intr(void *); 795 static void ixl_handle_queue(void *); 796 static void ixl_handle_queue_wk(struct work *, void *); 797 static void ixl_sched_handle_queue(struct ixl_softc *, 798 struct ixl_queue_pair *); 799 static int ixl_init(struct ifnet *); 800 static int ixl_init_locked(struct ixl_softc *); 801 static void ixl_stop(struct ifnet *, int); 802 static void ixl_stop_locked(struct ixl_softc *); 803 static int ixl_iff(struct ixl_softc *); 804 static int ixl_ifflags_cb(struct ethercom *); 805 static int ixl_setup_interrupts(struct ixl_softc *); 806 static int ixl_establish_intx(struct ixl_softc *); 807 static int ixl_establish_msix(struct ixl_softc *); 808 static void ixl_enable_queue_intr(struct ixl_softc *, 809 struct ixl_queue_pair *); 810 static void ixl_disable_queue_intr(struct ixl_softc *, 811 struct ixl_queue_pair *); 812 static void ixl_enable_other_intr(struct ixl_softc *); 813 static void ixl_disable_other_intr(struct ixl_softc *); 814 static void ixl_config_queue_intr(struct ixl_softc *); 815 static void ixl_config_other_intr(struct ixl_softc *); 816 817 static struct ixl_tx_ring * 818 ixl_txr_alloc(struct ixl_softc *, unsigned int); 819 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 820 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 821 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 822 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 823 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 824 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 825 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 826 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 827 828 static struct ixl_rx_ring * 829 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 830 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 831 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 832 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 833 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 834 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 835 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 836 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 837 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 838 839 static struct workqueue * 840 ixl_workq_create(const char *, pri_t, int, int); 841 static void ixl_workq_destroy(struct workqueue *); 842 static int ixl_workqs_teardown(device_t); 843 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 844 static void ixl_work_add(struct workqueue *, struct ixl_work *); 845 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 846 static void ixl_workq_work(struct work *, void *); 847 static const struct ixl_product * 848 ixl_lookup(const struct pci_attach_args *pa); 849 static void ixl_link_state_update(struct ixl_softc *, 850 const struct ixl_aq_desc *); 851 static int ixl_vlan_cb(struct ethercom *, uint16_t, bool); 852 static int ixl_setup_vlan_hwfilter(struct ixl_softc *); 853 static void ixl_teardown_vlan_hwfilter(struct ixl_softc *); 854 static int ixl_update_macvlan(struct ixl_softc *); 855 static int ixl_setup_interrupts(struct ixl_softc *); 856 static void ixl_teardown_interrupts(struct ixl_softc *); 857 static int ixl_setup_stats(struct ixl_softc *); 858 static void ixl_teardown_stats(struct ixl_softc *); 859 static void ixl_stats_callout(void *); 860 static void ixl_stats_update(void *); 861 static int ixl_setup_sysctls(struct ixl_softc *); 862 static void ixl_teardown_sysctls(struct ixl_softc *); 863 static int ixl_sysctl_itr_handler(SYSCTLFN_PROTO); 864 static int ixl_queue_pairs_alloc(struct ixl_softc *); 865 static void ixl_queue_pairs_free(struct ixl_softc *); 866 867 static const struct ixl_phy_type ixl_phy_type_map[] = { 868 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 869 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 870 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 871 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 872 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 873 { 1ULL << IXL_PHY_TYPE_XAUI | 874 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 875 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 876 { 1ULL << IXL_PHY_TYPE_XLAUI | 877 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 878 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 879 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 880 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 881 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 882 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 883 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 884 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 885 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 886 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 887 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 888 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 889 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 890 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 891 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 892 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 893 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 894 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 895 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 896 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 897 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 898 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 899 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 900 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 901 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_ACC }, 902 { 1ULL << IXL_PHY_TYPE_2500BASE_T_1, IFM_2500_T }, 903 { 1ULL << IXL_PHY_TYPE_5000BASE_T_1, IFM_5000_T }, 904 { 1ULL << IXL_PHY_TYPE_2500BASE_T_2, IFM_2500_T }, 905 { 1ULL << IXL_PHY_TYPE_5000BASE_T_2, IFM_5000_T }, 906 }; 907 908 static const struct ixl_speed_type ixl_speed_type_map[] = { 909 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 910 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 911 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 912 { IXL_AQ_LINK_SPEED_5000MB, IF_Mbps(5000) }, 913 { IXL_AQ_LINK_SPEED_2500MB, IF_Mbps(2500) }, 914 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 915 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 916 }; 917 918 static const struct ixl_aq_regs ixl_pf_aq_regs = { 919 .atq_tail = I40E_PF_ATQT, 920 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 921 .atq_head = I40E_PF_ATQH, 922 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 923 .atq_len = I40E_PF_ATQLEN, 924 .atq_bal = I40E_PF_ATQBAL, 925 .atq_bah = I40E_PF_ATQBAH, 926 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 927 928 .arq_tail = I40E_PF_ARQT, 929 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 930 .arq_head = I40E_PF_ARQH, 931 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 932 .arq_len = I40E_PF_ARQLEN, 933 .arq_bal = I40E_PF_ARQBAL, 934 .arq_bah = I40E_PF_ARQBAH, 935 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 936 }; 937 938 #define ixl_rd(_s, _r) \ 939 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 940 #define ixl_wr(_s, _r, _v) \ 941 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 942 #define ixl_barrier(_s, _r, _l, _o) \ 943 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 944 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 945 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 946 947 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 948 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 949 DVF_DETACH_SHUTDOWN); 950 951 static const struct ixl_product ixl_products[] = { 952 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 953 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 954 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 955 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 956 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 957 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 958 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_1 }, 959 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_2 }, 960 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 961 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 962 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 963 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 964 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 965 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 966 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 967 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 968 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 969 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 970 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 971 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_SFP }, 972 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_BP }, 973 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_V710_5G_T}, 974 /* required last entry */ 975 {0, 0} 976 }; 977 978 static const struct ixl_product * 979 ixl_lookup(const struct pci_attach_args *pa) 980 { 981 const struct ixl_product *ixlp; 982 983 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 984 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 985 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 986 return ixlp; 987 } 988 989 return NULL; 990 } 991 992 static int 993 ixl_match(device_t parent, cfdata_t match, void *aux) 994 { 995 const struct pci_attach_args *pa = aux; 996 997 return (ixl_lookup(pa) != NULL) ? 1 : 0; 998 } 999 1000 static void 1001 ixl_attach(device_t parent, device_t self, void *aux) 1002 { 1003 struct ixl_softc *sc; 1004 struct pci_attach_args *pa = aux; 1005 struct ifnet *ifp; 1006 pcireg_t memtype; 1007 uint32_t firstq, port, ari, func; 1008 char xnamebuf[32]; 1009 int tries, rv, link; 1010 1011 sc = device_private(self); 1012 sc->sc_dev = self; 1013 ifp = &sc->sc_ec.ec_if; 1014 1015 sc->sc_pa = *pa; 1016 sc->sc_dmat = (pci_dma64_available(pa)) ? 1017 pa->pa_dmat64 : pa->pa_dmat; 1018 sc->sc_aq_regs = &ixl_pf_aq_regs; 1019 1020 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(pa->pa_id)); 1021 1022 ixl_pci_csr_setup(pa->pa_pc, pa->pa_tag); 1023 1024 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 1025 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 1026 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 1027 aprint_error(": unable to map registers\n"); 1028 return; 1029 } 1030 1031 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 1032 1033 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 1034 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 1035 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 1036 sc->sc_base_queue = firstq; 1037 1038 ixl_clear_hw(sc); 1039 if (ixl_pf_reset(sc) == -1) { 1040 /* error printed by ixl pf_reset */ 1041 goto unmap; 1042 } 1043 1044 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 1045 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 1046 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 1047 sc->sc_port = port; 1048 aprint_normal(": port %u", sc->sc_port); 1049 1050 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 1051 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 1052 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 1053 1054 func = ixl_rd(sc, I40E_PF_FUNC_RID); 1055 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 1056 1057 /* initialise the adminq */ 1058 1059 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 1060 1061 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 1062 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1063 aprint_error("\n" "%s: unable to allocate atq\n", 1064 device_xname(self)); 1065 goto unmap; 1066 } 1067 1068 SIMPLEQ_INIT(&sc->sc_arq_idle); 1069 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 1070 sc->sc_arq_cons = 0; 1071 sc->sc_arq_prod = 0; 1072 1073 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 1074 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1075 aprint_error("\n" "%s: unable to allocate arq\n", 1076 device_xname(self)); 1077 goto free_atq; 1078 } 1079 1080 if (!ixl_arq_fill(sc)) { 1081 aprint_error("\n" "%s: unable to fill arq descriptors\n", 1082 device_xname(self)); 1083 goto free_arq; 1084 } 1085 1086 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1087 0, IXL_DMA_LEN(&sc->sc_atq), 1088 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1089 1090 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1091 0, IXL_DMA_LEN(&sc->sc_arq), 1092 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1093 1094 for (tries = 0; tries < 10; tries++) { 1095 sc->sc_atq_cons = 0; 1096 sc->sc_atq_prod = 0; 1097 1098 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1099 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1100 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1101 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1102 1103 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1104 1105 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1106 ixl_dmamem_lo(&sc->sc_atq)); 1107 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1108 ixl_dmamem_hi(&sc->sc_atq)); 1109 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1110 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1111 1112 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1113 ixl_dmamem_lo(&sc->sc_arq)); 1114 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1115 ixl_dmamem_hi(&sc->sc_arq)); 1116 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1117 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1118 1119 rv = ixl_get_version(sc); 1120 if (rv == 0) 1121 break; 1122 if (rv != ETIMEDOUT) { 1123 aprint_error(", unable to get firmware version\n"); 1124 goto shutdown; 1125 } 1126 1127 delaymsec(100); 1128 } 1129 1130 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1131 1132 if (ixl_dmamem_alloc(sc, &sc->sc_aqbuf, IXL_AQ_BUFLEN, 0) != 0) { 1133 aprint_error_dev(self, ", unable to allocate nvm buffer\n"); 1134 goto shutdown; 1135 } 1136 1137 ixl_get_nvm_version(sc); 1138 1139 if (sc->sc_mac_type == I40E_MAC_X722) 1140 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_X722; 1141 else 1142 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_XL710; 1143 1144 rv = ixl_get_hw_capabilities(sc); 1145 if (rv != 0) { 1146 aprint_error(", GET HW CAPABILITIES %s\n", 1147 rv == ETIMEDOUT ? "timeout" : "error"); 1148 goto free_aqbuf; 1149 } 1150 1151 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1152 if (ixl_param_nqps_limit > 0) { 1153 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_max, 1154 ixl_param_nqps_limit); 1155 } 1156 1157 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1158 sc->sc_tx_ring_ndescs = ixl_param_tx_ndescs; 1159 sc->sc_rx_ring_ndescs = ixl_param_rx_ndescs; 1160 1161 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1162 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1163 KASSERT(sc->sc_rx_ring_ndescs == 1164 (1U << (fls32(sc->sc_rx_ring_ndescs) - 1))); 1165 KASSERT(sc->sc_tx_ring_ndescs == 1166 (1U << (fls32(sc->sc_tx_ring_ndescs) - 1))); 1167 1168 if (ixl_get_mac(sc) != 0) { 1169 /* error printed by ixl_get_mac */ 1170 goto free_aqbuf; 1171 } 1172 1173 aprint_normal("\n"); 1174 aprint_naive("\n"); 1175 1176 aprint_normal_dev(self, "Ethernet address %s\n", 1177 ether_sprintf(sc->sc_enaddr)); 1178 1179 rv = ixl_pxe_clear(sc); 1180 if (rv != 0) { 1181 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1182 rv == ETIMEDOUT ? "timeout" : "error"); 1183 } 1184 1185 ixl_set_filter_control(sc); 1186 1187 if (ixl_hmc(sc) != 0) { 1188 /* error printed by ixl_hmc */ 1189 goto free_aqbuf; 1190 } 1191 1192 if (ixl_lldp_shut(sc) != 0) { 1193 /* error printed by ixl_lldp_shut */ 1194 goto free_hmc; 1195 } 1196 1197 if (ixl_phy_mask_ints(sc) != 0) { 1198 /* error printed by ixl_phy_mask_ints */ 1199 goto free_hmc; 1200 } 1201 1202 if (ixl_restart_an(sc) != 0) { 1203 /* error printed by ixl_restart_an */ 1204 goto free_hmc; 1205 } 1206 1207 if (ixl_get_switch_config(sc) != 0) { 1208 /* error printed by ixl_get_switch_config */ 1209 goto free_hmc; 1210 } 1211 1212 rv = ixl_get_link_status_poll(sc, NULL); 1213 if (rv != 0) { 1214 aprint_error_dev(self, "GET LINK STATUS %s\n", 1215 rv == ETIMEDOUT ? "timeout" : "error"); 1216 goto free_hmc; 1217 } 1218 1219 /* 1220 * The FW often returns EIO in "Get PHY Abilities" command 1221 * if there is no delay 1222 */ 1223 DELAY(500); 1224 if (ixl_get_phy_info(sc) != 0) { 1225 /* error printed by ixl_get_phy_info */ 1226 goto free_hmc; 1227 } 1228 1229 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1230 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1231 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1232 goto free_hmc; 1233 } 1234 1235 rv = ixl_get_vsi(sc); 1236 if (rv != 0) { 1237 aprint_error_dev(self, "GET VSI %s %d\n", 1238 rv == ETIMEDOUT ? "timeout" : "error", rv); 1239 goto free_scratch; 1240 } 1241 1242 rv = ixl_set_vsi(sc); 1243 if (rv != 0) { 1244 aprint_error_dev(self, "UPDATE VSI error %s %d\n", 1245 rv == ETIMEDOUT ? "timeout" : "error", rv); 1246 goto free_scratch; 1247 } 1248 1249 if (ixl_queue_pairs_alloc(sc) != 0) { 1250 /* error printed by ixl_queue_pairs_alloc */ 1251 goto free_scratch; 1252 } 1253 1254 if (ixl_setup_interrupts(sc) != 0) { 1255 /* error printed by ixl_setup_interrupts */ 1256 goto free_queue_pairs; 1257 } 1258 1259 if (ixl_setup_stats(sc) != 0) { 1260 aprint_error_dev(self, "failed to setup event counters\n"); 1261 goto teardown_intrs; 1262 } 1263 1264 if (ixl_setup_sysctls(sc) != 0) { 1265 /* error printed by ixl_setup_sysctls */ 1266 goto teardown_stats; 1267 } 1268 1269 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1270 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1271 IPL_NET, WQ_MPSAFE); 1272 if (sc->sc_workq == NULL) 1273 goto teardown_sysctls; 1274 1275 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1276 rv = workqueue_create(&sc->sc_workq_txrx, xnamebuf, ixl_handle_queue_wk, 1277 sc, IXL_WORKQUEUE_PRI, IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1278 if (rv != 0) { 1279 sc->sc_workq_txrx = NULL; 1280 goto teardown_wqs; 1281 } 1282 1283 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1284 cv_init(&sc->sc_atq_cv, xnamebuf); 1285 1286 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1287 1288 ifp->if_softc = sc; 1289 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1290 ifp->if_extflags = IFEF_MPSAFE; 1291 ifp->if_ioctl = ixl_ioctl; 1292 ifp->if_start = ixl_start; 1293 ifp->if_transmit = ixl_transmit; 1294 ifp->if_watchdog = ixl_watchdog; 1295 ifp->if_init = ixl_init; 1296 ifp->if_stop = ixl_stop; 1297 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1298 IFQ_SET_READY(&ifp->if_snd); 1299 ifp->if_capabilities |= IXL_IFCAP_RXCSUM; 1300 ifp->if_capabilities |= IXL_IFCAP_TXCSUM; 1301 #if 0 1302 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1303 #endif 1304 ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb); 1305 sc->sc_ec.ec_capabilities |= ETHERCAP_JUMBO_MTU; 1306 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1307 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1308 1309 sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities; 1310 /* Disable VLAN_HWFILTER by default */ 1311 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1312 1313 sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable; 1314 1315 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1316 ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, ixl_media_change, 1317 ixl_media_status, &sc->sc_cfg_lock); 1318 1319 ixl_media_add(sc); 1320 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1321 if (ISSET(sc->sc_phy_abilities, 1322 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1323 ifmedia_add(&sc->sc_media, 1324 IFM_ETHER | IFM_AUTO | IFM_FLOW, 0, NULL); 1325 } 1326 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_NONE, 0, NULL); 1327 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1328 1329 if_initialize(ifp); 1330 1331 sc->sc_ipq = if_percpuq_create(ifp); 1332 if_deferred_start_init(ifp, NULL); 1333 ether_ifattach(ifp, sc->sc_enaddr); 1334 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1335 1336 rv = ixl_get_link_status_poll(sc, &link); 1337 if (rv != 0) 1338 link = LINK_STATE_UNKNOWN; 1339 if_link_state_change(ifp, link); 1340 1341 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 1342 ixl_work_set(&sc->sc_link_state_task, ixl_get_link_status, sc); 1343 1344 ixl_config_other_intr(sc); 1345 ixl_enable_other_intr(sc); 1346 1347 ixl_set_phy_autoselect(sc); 1348 1349 /* remove default mac filter and replace it so we can see vlans */ 1350 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 1351 if (rv != ENOENT) { 1352 aprint_debug_dev(self, 1353 "unable to remove macvlan %u\n", rv); 1354 } 1355 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1356 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1357 if (rv != ENOENT) { 1358 aprint_debug_dev(self, 1359 "unable to remove macvlan, ignore vlan %u\n", rv); 1360 } 1361 1362 if (ixl_update_macvlan(sc) != 0) { 1363 aprint_debug_dev(self, 1364 "couldn't enable vlan hardware filter\n"); 1365 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1366 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1367 } 1368 1369 sc->sc_txrx_workqueue = true; 1370 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1371 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1372 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1373 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1374 1375 ixl_stats_update(sc); 1376 sc->sc_stats_counters.isc_has_offset = true; 1377 1378 if (pmf_device_register(self, NULL, NULL) != true) 1379 aprint_debug_dev(self, "couldn't establish power handler\n"); 1380 sc->sc_itr_rx = IXL_ITR_RX; 1381 sc->sc_itr_tx = IXL_ITR_TX; 1382 sc->sc_attached = true; 1383 if_register(ifp); 1384 1385 return; 1386 1387 teardown_wqs: 1388 config_finalize_register(self, ixl_workqs_teardown); 1389 teardown_sysctls: 1390 ixl_teardown_sysctls(sc); 1391 teardown_stats: 1392 ixl_teardown_stats(sc); 1393 teardown_intrs: 1394 ixl_teardown_interrupts(sc); 1395 free_queue_pairs: 1396 ixl_queue_pairs_free(sc); 1397 free_scratch: 1398 ixl_dmamem_free(sc, &sc->sc_scratch); 1399 free_hmc: 1400 ixl_hmc_free(sc); 1401 free_aqbuf: 1402 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1403 shutdown: 1404 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1405 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1406 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1407 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1408 1409 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1410 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1411 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1412 1413 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1414 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1415 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1416 1417 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1418 0, IXL_DMA_LEN(&sc->sc_arq), 1419 BUS_DMASYNC_POSTREAD); 1420 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1421 0, IXL_DMA_LEN(&sc->sc_atq), 1422 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1423 1424 ixl_arq_unfill(sc); 1425 free_arq: 1426 ixl_dmamem_free(sc, &sc->sc_arq); 1427 free_atq: 1428 ixl_dmamem_free(sc, &sc->sc_atq); 1429 unmap: 1430 mutex_destroy(&sc->sc_atq_lock); 1431 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1432 mutex_destroy(&sc->sc_cfg_lock); 1433 sc->sc_mems = 0; 1434 1435 sc->sc_attached = false; 1436 } 1437 1438 static int 1439 ixl_detach(device_t self, int flags) 1440 { 1441 struct ixl_softc *sc = device_private(self); 1442 struct ifnet *ifp = &sc->sc_ec.ec_if; 1443 1444 if (!sc->sc_attached) 1445 return 0; 1446 1447 ixl_stop(ifp, 1); 1448 1449 ixl_disable_other_intr(sc); 1450 1451 callout_halt(&sc->sc_stats_callout, NULL); 1452 ixl_work_wait(sc->sc_workq, &sc->sc_stats_task); 1453 1454 /* wait for ATQ handler */ 1455 mutex_enter(&sc->sc_atq_lock); 1456 mutex_exit(&sc->sc_atq_lock); 1457 1458 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1459 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1460 1461 if (sc->sc_workq != NULL) { 1462 ixl_workq_destroy(sc->sc_workq); 1463 sc->sc_workq = NULL; 1464 } 1465 1466 if (sc->sc_workq_txrx != NULL) { 1467 workqueue_destroy(sc->sc_workq_txrx); 1468 sc->sc_workq_txrx = NULL; 1469 } 1470 1471 if_percpuq_destroy(sc->sc_ipq); 1472 ether_ifdetach(ifp); 1473 if_detach(ifp); 1474 ifmedia_fini(&sc->sc_media); 1475 1476 ixl_teardown_interrupts(sc); 1477 ixl_teardown_stats(sc); 1478 ixl_teardown_sysctls(sc); 1479 1480 ixl_queue_pairs_free(sc); 1481 1482 ixl_dmamem_free(sc, &sc->sc_scratch); 1483 ixl_hmc_free(sc); 1484 1485 /* shutdown */ 1486 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1487 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1488 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1489 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1490 1491 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1492 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1493 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1494 1495 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1496 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1497 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1498 1499 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1500 0, IXL_DMA_LEN(&sc->sc_arq), 1501 BUS_DMASYNC_POSTREAD); 1502 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1503 0, IXL_DMA_LEN(&sc->sc_atq), 1504 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1505 1506 ixl_arq_unfill(sc); 1507 1508 ixl_dmamem_free(sc, &sc->sc_arq); 1509 ixl_dmamem_free(sc, &sc->sc_atq); 1510 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1511 1512 cv_destroy(&sc->sc_atq_cv); 1513 mutex_destroy(&sc->sc_atq_lock); 1514 1515 if (sc->sc_mems != 0) { 1516 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1517 sc->sc_mems = 0; 1518 } 1519 1520 mutex_destroy(&sc->sc_cfg_lock); 1521 1522 return 0; 1523 } 1524 1525 static int 1526 ixl_workqs_teardown(device_t self) 1527 { 1528 struct ixl_softc *sc = device_private(self); 1529 1530 if (sc->sc_workq != NULL) { 1531 ixl_workq_destroy(sc->sc_workq); 1532 sc->sc_workq = NULL; 1533 } 1534 1535 if (sc->sc_workq_txrx != NULL) { 1536 workqueue_destroy(sc->sc_workq_txrx); 1537 sc->sc_workq_txrx = NULL; 1538 } 1539 1540 return 0; 1541 } 1542 1543 static int 1544 ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 1545 { 1546 struct ifnet *ifp = &ec->ec_if; 1547 struct ixl_softc *sc = ifp->if_softc; 1548 int rv; 1549 1550 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1551 return 0; 1552 } 1553 1554 if (set) { 1555 rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid, 1556 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1557 if (rv == 0) { 1558 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 1559 vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1560 } 1561 } else { 1562 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid, 1563 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1564 (void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid, 1565 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1566 } 1567 1568 return rv; 1569 } 1570 1571 static void 1572 ixl_media_add(struct ixl_softc *sc) 1573 { 1574 struct ifmedia *ifm = &sc->sc_media; 1575 const struct ixl_phy_type *itype; 1576 unsigned int i; 1577 bool flow; 1578 1579 if (ISSET(sc->sc_phy_abilities, 1580 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1581 flow = true; 1582 } else { 1583 flow = false; 1584 } 1585 1586 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1587 itype = &ixl_phy_type_map[i]; 1588 1589 if (ISSET(sc->sc_phy_types, itype->phy_type)) { 1590 ifmedia_add(ifm, 1591 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1592 1593 if (flow) { 1594 ifmedia_add(ifm, 1595 IFM_ETHER | IFM_FDX | IFM_FLOW | 1596 itype->ifm_type, 0, NULL); 1597 } 1598 1599 if (itype->ifm_type != IFM_100_TX) 1600 continue; 1601 1602 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1603 0, NULL); 1604 if (flow) { 1605 ifmedia_add(ifm, 1606 IFM_ETHER | IFM_FLOW | itype->ifm_type, 1607 0, NULL); 1608 } 1609 } 1610 } 1611 } 1612 1613 static void 1614 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1615 { 1616 struct ixl_softc *sc = ifp->if_softc; 1617 1618 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1619 1620 ifmr->ifm_status = sc->sc_media_status; 1621 ifmr->ifm_active = sc->sc_media_active; 1622 } 1623 1624 static int 1625 ixl_media_change(struct ifnet *ifp) 1626 { 1627 struct ixl_softc *sc = ifp->if_softc; 1628 struct ifmedia *ifm = &sc->sc_media; 1629 uint64_t ifm_active = sc->sc_media_active; 1630 uint8_t link_speed, abilities; 1631 1632 switch (IFM_SUBTYPE(ifm_active)) { 1633 case IFM_1000_SGMII: 1634 case IFM_1000_KX: 1635 case IFM_10G_KX4: 1636 case IFM_10G_KR: 1637 case IFM_40G_KR4: 1638 case IFM_20G_KR2: 1639 case IFM_25G_KR: 1640 /* backplanes */ 1641 return EINVAL; 1642 } 1643 1644 abilities = IXL_PHY_ABILITY_AUTONEGO | IXL_PHY_ABILITY_LINKUP; 1645 1646 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1647 case IFM_AUTO: 1648 link_speed = sc->sc_phy_linkspeed; 1649 break; 1650 case IFM_NONE: 1651 link_speed = 0; 1652 CLR(abilities, IXL_PHY_ABILITY_LINKUP); 1653 break; 1654 default: 1655 link_speed = ixl_search_baudrate( 1656 ifmedia_baudrate(ifm->ifm_media)); 1657 } 1658 1659 if (ISSET(abilities, IXL_PHY_ABILITY_LINKUP)) { 1660 if (ISSET(link_speed, sc->sc_phy_linkspeed) == 0) 1661 return EINVAL; 1662 } 1663 1664 if (ifm->ifm_media & IFM_FLOW) { 1665 abilities |= sc->sc_phy_abilities & 1666 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX); 1667 } 1668 1669 return ixl_set_phy_config(sc, link_speed, abilities, false); 1670 } 1671 1672 static void 1673 ixl_watchdog(struct ifnet *ifp) 1674 { 1675 1676 } 1677 1678 static void 1679 ixl_del_all_multiaddr(struct ixl_softc *sc) 1680 { 1681 struct ethercom *ec = &sc->sc_ec; 1682 struct ether_multi *enm; 1683 struct ether_multistep step; 1684 1685 ETHER_LOCK(ec); 1686 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1687 ETHER_NEXT_MULTI(step, enm)) { 1688 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1689 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1690 } 1691 ETHER_UNLOCK(ec); 1692 } 1693 1694 static int 1695 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1696 { 1697 struct ifnet *ifp = &sc->sc_ec.ec_if; 1698 int rv; 1699 1700 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1701 return 0; 1702 1703 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1704 ixl_del_all_multiaddr(sc); 1705 SET(ifp->if_flags, IFF_ALLMULTI); 1706 return ENETRESET; 1707 } 1708 1709 /* multicast address can not use VLAN HWFILTER */ 1710 rv = ixl_add_macvlan(sc, addrlo, 0, 1711 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1712 1713 if (rv == ENOSPC) { 1714 ixl_del_all_multiaddr(sc); 1715 SET(ifp->if_flags, IFF_ALLMULTI); 1716 return ENETRESET; 1717 } 1718 1719 return rv; 1720 } 1721 1722 static int 1723 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1724 { 1725 struct ifnet *ifp = &sc->sc_ec.ec_if; 1726 struct ethercom *ec = &sc->sc_ec; 1727 struct ether_multi *enm, *enm_last; 1728 struct ether_multistep step; 1729 int error, rv = 0; 1730 1731 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1732 ixl_remove_macvlan(sc, addrlo, 0, 1733 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1734 return 0; 1735 } 1736 1737 ETHER_LOCK(ec); 1738 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1739 ETHER_NEXT_MULTI(step, enm)) { 1740 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1741 ETHER_ADDR_LEN) != 0) { 1742 goto out; 1743 } 1744 } 1745 1746 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1747 ETHER_NEXT_MULTI(step, enm)) { 1748 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1749 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1750 if (error != 0) 1751 break; 1752 } 1753 1754 if (enm != NULL) { 1755 enm_last = enm; 1756 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1757 ETHER_NEXT_MULTI(step, enm)) { 1758 if (enm == enm_last) 1759 break; 1760 1761 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1762 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1763 } 1764 } else { 1765 CLR(ifp->if_flags, IFF_ALLMULTI); 1766 rv = ENETRESET; 1767 } 1768 1769 out: 1770 ETHER_UNLOCK(ec); 1771 return rv; 1772 } 1773 1774 static int 1775 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1776 { 1777 struct ifreq *ifr = (struct ifreq *)data; 1778 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1779 const struct sockaddr *sa; 1780 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1781 int s, error = 0; 1782 unsigned int nmtu; 1783 1784 switch (cmd) { 1785 case SIOCSIFMTU: 1786 nmtu = ifr->ifr_mtu; 1787 1788 if (nmtu < IXL_MIN_MTU || nmtu > IXL_MAX_MTU) { 1789 error = EINVAL; 1790 break; 1791 } 1792 if (ifp->if_mtu != nmtu) { 1793 s = splnet(); 1794 error = ether_ioctl(ifp, cmd, data); 1795 splx(s); 1796 if (error == ENETRESET) 1797 error = ixl_init(ifp); 1798 } 1799 break; 1800 case SIOCADDMULTI: 1801 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1802 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1803 error = ether_multiaddr(sa, addrlo, addrhi); 1804 if (error != 0) 1805 return error; 1806 1807 error = ixl_add_multi(sc, addrlo, addrhi); 1808 if (error != 0 && error != ENETRESET) { 1809 ether_delmulti(sa, &sc->sc_ec); 1810 error = EIO; 1811 } 1812 } 1813 break; 1814 1815 case SIOCDELMULTI: 1816 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1817 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1818 error = ether_multiaddr(sa, addrlo, addrhi); 1819 if (error != 0) 1820 return error; 1821 1822 error = ixl_del_multi(sc, addrlo, addrhi); 1823 } 1824 break; 1825 1826 default: 1827 s = splnet(); 1828 error = ether_ioctl(ifp, cmd, data); 1829 splx(s); 1830 } 1831 1832 if (error == ENETRESET) 1833 error = ixl_iff(sc); 1834 1835 return error; 1836 } 1837 1838 static enum i40e_mac_type 1839 ixl_mactype(pci_product_id_t id) 1840 { 1841 1842 switch (id) { 1843 case PCI_PRODUCT_INTEL_XL710_SFP: 1844 case PCI_PRODUCT_INTEL_XL710_KX_B: 1845 case PCI_PRODUCT_INTEL_XL710_KX_C: 1846 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1847 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1848 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1849 case PCI_PRODUCT_INTEL_X710_10G_T_1: 1850 case PCI_PRODUCT_INTEL_X710_10G_T_2: 1851 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1852 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1853 case PCI_PRODUCT_INTEL_X710_T4_10G: 1854 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1855 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1856 case PCI_PRODUCT_INTEL_X710_10G_SFP: 1857 case PCI_PRODUCT_INTEL_X710_10G_BP: 1858 return I40E_MAC_XL710; 1859 1860 case PCI_PRODUCT_INTEL_X722_KX: 1861 case PCI_PRODUCT_INTEL_X722_QSFP: 1862 case PCI_PRODUCT_INTEL_X722_SFP: 1863 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1864 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1865 case PCI_PRODUCT_INTEL_X722_I_SFP: 1866 return I40E_MAC_X722; 1867 } 1868 1869 return I40E_MAC_GENERIC; 1870 } 1871 1872 static void 1873 ixl_pci_csr_setup(pci_chipset_tag_t pc, pcitag_t tag) 1874 { 1875 pcireg_t csr; 1876 1877 csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 1878 csr |= (PCI_COMMAND_MASTER_ENABLE | 1879 PCI_COMMAND_MEM_ENABLE); 1880 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr); 1881 } 1882 1883 static inline void * 1884 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1885 { 1886 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1887 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1888 1889 if (i >= e->hmc_count) 1890 return NULL; 1891 1892 kva += e->hmc_base; 1893 kva += i * e->hmc_size; 1894 1895 return kva; 1896 } 1897 1898 static inline size_t 1899 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1900 { 1901 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1902 1903 return e->hmc_size; 1904 } 1905 1906 static void 1907 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1908 { 1909 struct ixl_rx_ring *rxr = qp->qp_rxr; 1910 1911 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1912 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1913 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1914 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1915 ixl_flush(sc); 1916 } 1917 1918 static void 1919 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1920 { 1921 struct ixl_rx_ring *rxr = qp->qp_rxr; 1922 1923 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1924 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1925 ixl_flush(sc); 1926 } 1927 1928 static void 1929 ixl_enable_other_intr(struct ixl_softc *sc) 1930 { 1931 1932 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1933 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1934 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1935 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1936 ixl_flush(sc); 1937 } 1938 1939 static void 1940 ixl_disable_other_intr(struct ixl_softc *sc) 1941 { 1942 1943 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1944 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1945 ixl_flush(sc); 1946 } 1947 1948 static int 1949 ixl_reinit(struct ixl_softc *sc) 1950 { 1951 struct ixl_rx_ring *rxr; 1952 struct ixl_tx_ring *txr; 1953 unsigned int i; 1954 uint32_t reg; 1955 1956 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1957 1958 if (ixl_get_vsi(sc) != 0) 1959 return EIO; 1960 1961 if (ixl_set_vsi(sc) != 0) 1962 return EIO; 1963 1964 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1965 txr = sc->sc_qps[i].qp_txr; 1966 rxr = sc->sc_qps[i].qp_rxr; 1967 1968 ixl_txr_config(sc, txr); 1969 ixl_rxr_config(sc, rxr); 1970 } 1971 1972 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1973 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 1974 1975 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1976 txr = sc->sc_qps[i].qp_txr; 1977 rxr = sc->sc_qps[i].qp_rxr; 1978 1979 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 1980 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 1981 ixl_flush(sc); 1982 1983 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 1984 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 1985 1986 /* ixl_rxfill() needs lock held */ 1987 mutex_enter(&rxr->rxr_lock); 1988 ixl_rxfill(sc, rxr); 1989 mutex_exit(&rxr->rxr_lock); 1990 1991 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 1992 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 1993 ixl_wr(sc, I40E_QRX_ENA(i), reg); 1994 if (ixl_rxr_enabled(sc, rxr) != 0) 1995 goto stop; 1996 1997 ixl_txr_qdis(sc, txr, 1); 1998 1999 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2000 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2001 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2002 2003 if (ixl_txr_enabled(sc, txr) != 0) 2004 goto stop; 2005 } 2006 2007 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2008 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2009 2010 return 0; 2011 2012 stop: 2013 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2014 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2015 2016 return ETIMEDOUT; 2017 } 2018 2019 static int 2020 ixl_init_locked(struct ixl_softc *sc) 2021 { 2022 struct ifnet *ifp = &sc->sc_ec.ec_if; 2023 unsigned int i; 2024 int error, eccap_change; 2025 2026 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2027 2028 if (ISSET(ifp->if_flags, IFF_RUNNING)) 2029 ixl_stop_locked(sc); 2030 2031 if (sc->sc_dead) { 2032 return ENXIO; 2033 } 2034 2035 eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable; 2036 if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING)) 2037 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 2038 2039 if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) { 2040 if (ixl_update_macvlan(sc) == 0) { 2041 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 2042 } else { 2043 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 2044 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 2045 } 2046 } 2047 2048 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 2049 sc->sc_nqueue_pairs = 1; 2050 else 2051 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 2052 2053 error = ixl_reinit(sc); 2054 if (error) { 2055 ixl_stop_locked(sc); 2056 return error; 2057 } 2058 2059 SET(ifp->if_flags, IFF_RUNNING); 2060 CLR(ifp->if_flags, IFF_OACTIVE); 2061 2062 ixl_config_rss(sc); 2063 ixl_config_queue_intr(sc); 2064 2065 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2066 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 2067 } 2068 2069 error = ixl_iff(sc); 2070 if (error) { 2071 ixl_stop_locked(sc); 2072 return error; 2073 } 2074 2075 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 2076 2077 return 0; 2078 } 2079 2080 static int 2081 ixl_init(struct ifnet *ifp) 2082 { 2083 struct ixl_softc *sc = ifp->if_softc; 2084 int error; 2085 2086 mutex_enter(&sc->sc_cfg_lock); 2087 error = ixl_init_locked(sc); 2088 mutex_exit(&sc->sc_cfg_lock); 2089 2090 if (error == 0) 2091 (void)ixl_get_link_status(sc); 2092 2093 return error; 2094 } 2095 2096 static int 2097 ixl_iff(struct ixl_softc *sc) 2098 { 2099 struct ifnet *ifp = &sc->sc_ec.ec_if; 2100 struct ixl_atq iatq; 2101 struct ixl_aq_desc *iaq; 2102 struct ixl_aq_vsi_promisc_param *param; 2103 uint16_t flag_add, flag_del; 2104 int error; 2105 2106 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2107 return 0; 2108 2109 memset(&iatq, 0, sizeof(iatq)); 2110 2111 iaq = &iatq.iatq_desc; 2112 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 2113 2114 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 2115 param->flags = htole16(0); 2116 2117 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER) 2118 || ISSET(ifp->if_flags, IFF_PROMISC)) { 2119 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2120 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2121 } 2122 2123 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 2124 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2125 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2126 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 2127 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2128 } 2129 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2130 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2131 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2132 param->seid = sc->sc_seid; 2133 2134 error = ixl_atq_exec(sc, &iatq); 2135 if (error) 2136 return error; 2137 2138 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 2139 return EIO; 2140 2141 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 2142 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 2143 flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH; 2144 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH; 2145 } else { 2146 flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN; 2147 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN; 2148 } 2149 2150 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del); 2151 2152 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2153 ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add); 2154 } 2155 return 0; 2156 } 2157 2158 static void 2159 ixl_stop_rendezvous(struct ixl_softc *sc) 2160 { 2161 struct ixl_tx_ring *txr; 2162 struct ixl_rx_ring *rxr; 2163 unsigned int i; 2164 2165 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2166 txr = sc->sc_qps[i].qp_txr; 2167 rxr = sc->sc_qps[i].qp_rxr; 2168 2169 mutex_enter(&txr->txr_lock); 2170 mutex_exit(&txr->txr_lock); 2171 2172 mutex_enter(&rxr->rxr_lock); 2173 mutex_exit(&rxr->rxr_lock); 2174 2175 sc->sc_qps[i].qp_workqueue = false; 2176 workqueue_wait(sc->sc_workq_txrx, 2177 &sc->sc_qps[i].qp_work); 2178 } 2179 } 2180 2181 static void 2182 ixl_stop_locked(struct ixl_softc *sc) 2183 { 2184 struct ifnet *ifp = &sc->sc_ec.ec_if; 2185 struct ixl_rx_ring *rxr; 2186 struct ixl_tx_ring *txr; 2187 unsigned int i; 2188 uint32_t reg; 2189 2190 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2191 2192 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 2193 callout_stop(&sc->sc_stats_callout); 2194 2195 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2196 txr = sc->sc_qps[i].qp_txr; 2197 rxr = sc->sc_qps[i].qp_rxr; 2198 2199 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 2200 2201 mutex_enter(&txr->txr_lock); 2202 ixl_txr_qdis(sc, txr, 0); 2203 mutex_exit(&txr->txr_lock); 2204 } 2205 2206 /* XXX wait at least 400 usec for all tx queues in one go */ 2207 ixl_flush(sc); 2208 DELAY(500); 2209 2210 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2211 txr = sc->sc_qps[i].qp_txr; 2212 rxr = sc->sc_qps[i].qp_rxr; 2213 2214 mutex_enter(&txr->txr_lock); 2215 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2216 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2217 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2218 mutex_exit(&txr->txr_lock); 2219 2220 mutex_enter(&rxr->rxr_lock); 2221 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2222 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2223 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2224 mutex_exit(&rxr->rxr_lock); 2225 } 2226 2227 /* XXX short wait for all queue disables to settle */ 2228 ixl_flush(sc); 2229 DELAY(50); 2230 2231 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2232 txr = sc->sc_qps[i].qp_txr; 2233 rxr = sc->sc_qps[i].qp_rxr; 2234 2235 mutex_enter(&txr->txr_lock); 2236 if (ixl_txr_disabled(sc, txr) != 0) { 2237 mutex_exit(&txr->txr_lock); 2238 goto die; 2239 } 2240 mutex_exit(&txr->txr_lock); 2241 2242 mutex_enter(&rxr->rxr_lock); 2243 if (ixl_rxr_disabled(sc, rxr) != 0) { 2244 mutex_exit(&rxr->rxr_lock); 2245 goto die; 2246 } 2247 mutex_exit(&rxr->rxr_lock); 2248 } 2249 2250 ixl_stop_rendezvous(sc); 2251 2252 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2253 txr = sc->sc_qps[i].qp_txr; 2254 rxr = sc->sc_qps[i].qp_rxr; 2255 2256 mutex_enter(&txr->txr_lock); 2257 ixl_txr_unconfig(sc, txr); 2258 mutex_exit(&txr->txr_lock); 2259 2260 mutex_enter(&rxr->rxr_lock); 2261 ixl_rxr_unconfig(sc, rxr); 2262 mutex_exit(&rxr->rxr_lock); 2263 2264 ixl_txr_clean(sc, txr); 2265 ixl_rxr_clean(sc, rxr); 2266 } 2267 2268 return; 2269 die: 2270 sc->sc_dead = true; 2271 log(LOG_CRIT, "%s: failed to shut down rings", 2272 device_xname(sc->sc_dev)); 2273 return; 2274 } 2275 2276 static void 2277 ixl_stop(struct ifnet *ifp, int disable) 2278 { 2279 struct ixl_softc *sc = ifp->if_softc; 2280 2281 mutex_enter(&sc->sc_cfg_lock); 2282 ixl_stop_locked(sc); 2283 mutex_exit(&sc->sc_cfg_lock); 2284 } 2285 2286 static int 2287 ixl_queue_pairs_alloc(struct ixl_softc *sc) 2288 { 2289 struct ixl_queue_pair *qp; 2290 unsigned int i; 2291 size_t sz; 2292 2293 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2294 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 2295 2296 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2297 qp = &sc->sc_qps[i]; 2298 2299 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2300 ixl_handle_queue, qp); 2301 if (qp->qp_si == NULL) 2302 goto free; 2303 2304 qp->qp_txr = ixl_txr_alloc(sc, i); 2305 if (qp->qp_txr == NULL) 2306 goto free; 2307 2308 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2309 if (qp->qp_rxr == NULL) 2310 goto free; 2311 2312 qp->qp_sc = sc; 2313 snprintf(qp->qp_name, sizeof(qp->qp_name), 2314 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2315 } 2316 2317 return 0; 2318 free: 2319 if (sc->sc_qps != NULL) { 2320 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2321 qp = &sc->sc_qps[i]; 2322 2323 if (qp->qp_txr != NULL) 2324 ixl_txr_free(sc, qp->qp_txr); 2325 if (qp->qp_rxr != NULL) 2326 ixl_rxr_free(sc, qp->qp_rxr); 2327 if (qp->qp_si != NULL) 2328 softint_disestablish(qp->qp_si); 2329 } 2330 2331 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2332 kmem_free(sc->sc_qps, sz); 2333 sc->sc_qps = NULL; 2334 } 2335 2336 return -1; 2337 } 2338 2339 static void 2340 ixl_queue_pairs_free(struct ixl_softc *sc) 2341 { 2342 struct ixl_queue_pair *qp; 2343 unsigned int i; 2344 size_t sz; 2345 2346 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2347 qp = &sc->sc_qps[i]; 2348 ixl_txr_free(sc, qp->qp_txr); 2349 ixl_rxr_free(sc, qp->qp_rxr); 2350 softint_disestablish(qp->qp_si); 2351 } 2352 2353 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2354 kmem_free(sc->sc_qps, sz); 2355 sc->sc_qps = NULL; 2356 } 2357 2358 static struct ixl_tx_ring * 2359 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2360 { 2361 struct ixl_tx_ring *txr = NULL; 2362 struct ixl_tx_map *maps = NULL, *txm; 2363 unsigned int i; 2364 2365 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2366 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2367 KM_SLEEP); 2368 2369 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2370 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2371 IXL_TX_QUEUE_ALIGN) != 0) 2372 goto free; 2373 2374 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2375 txm = &maps[i]; 2376 2377 if (bus_dmamap_create(sc->sc_dmat, IXL_TX_PKT_MAXSIZE, 2378 IXL_TX_PKT_DESCS, IXL_TX_PKT_MAXSIZE, 0, 2379 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2380 goto uncreate; 2381 2382 txm->txm_eop = -1; 2383 txm->txm_m = NULL; 2384 } 2385 2386 txr->txr_cons = txr->txr_prod = 0; 2387 txr->txr_maps = maps; 2388 2389 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2390 if (txr->txr_intrq == NULL) 2391 goto uncreate; 2392 2393 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2394 ixl_deferred_transmit, txr); 2395 if (txr->txr_si == NULL) 2396 goto destroy_pcq; 2397 2398 txr->txr_tail = I40E_QTX_TAIL(qid); 2399 txr->txr_qid = qid; 2400 txr->txr_sc = sc; 2401 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2402 2403 return txr; 2404 2405 destroy_pcq: 2406 pcq_destroy(txr->txr_intrq); 2407 uncreate: 2408 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2409 txm = &maps[i]; 2410 2411 if (txm->txm_map == NULL) 2412 continue; 2413 2414 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2415 } 2416 2417 ixl_dmamem_free(sc, &txr->txr_mem); 2418 free: 2419 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2420 kmem_free(txr, sizeof(*txr)); 2421 2422 return NULL; 2423 } 2424 2425 static void 2426 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2427 { 2428 unsigned int qid; 2429 bus_size_t reg; 2430 uint32_t r; 2431 2432 qid = txr->txr_qid + sc->sc_base_queue; 2433 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2434 qid %= 128; 2435 2436 r = ixl_rd(sc, reg); 2437 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2438 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2439 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2440 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2441 ixl_wr(sc, reg, r); 2442 } 2443 2444 static void 2445 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2446 { 2447 struct ixl_hmc_txq txq; 2448 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2449 void *hmc; 2450 2451 memset(&txq, 0, sizeof(txq)); 2452 txq.head = htole16(txr->txr_cons); 2453 txq.new_context = 1; 2454 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2455 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2456 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2457 txq.tphrdesc_ena = 0; 2458 txq.tphrpacket_ena = 0; 2459 txq.tphwdesc_ena = 0; 2460 txq.rdylist = data->qs_handle[0]; 2461 2462 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2463 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2464 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2465 __arraycount(ixl_hmc_pack_txq)); 2466 } 2467 2468 static void 2469 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2470 { 2471 void *hmc; 2472 2473 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2474 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2475 txr->txr_cons = txr->txr_prod = 0; 2476 } 2477 2478 static void 2479 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2480 { 2481 struct ixl_tx_map *maps, *txm; 2482 bus_dmamap_t map; 2483 unsigned int i; 2484 2485 maps = txr->txr_maps; 2486 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2487 txm = &maps[i]; 2488 2489 if (txm->txm_m == NULL) 2490 continue; 2491 2492 map = txm->txm_map; 2493 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2494 BUS_DMASYNC_POSTWRITE); 2495 bus_dmamap_unload(sc->sc_dmat, map); 2496 2497 m_freem(txm->txm_m); 2498 txm->txm_m = NULL; 2499 } 2500 } 2501 2502 static int 2503 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2504 { 2505 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2506 uint32_t reg; 2507 int i; 2508 2509 for (i = 0; i < 10; i++) { 2510 reg = ixl_rd(sc, ena); 2511 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2512 return 0; 2513 2514 delaymsec(10); 2515 } 2516 2517 return ETIMEDOUT; 2518 } 2519 2520 static int 2521 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2522 { 2523 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2524 uint32_t reg; 2525 int i; 2526 2527 KASSERT(mutex_owned(&txr->txr_lock)); 2528 2529 for (i = 0; i < 10; i++) { 2530 reg = ixl_rd(sc, ena); 2531 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2532 return 0; 2533 2534 delaymsec(10); 2535 } 2536 2537 return ETIMEDOUT; 2538 } 2539 2540 static void 2541 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2542 { 2543 struct ixl_tx_map *maps, *txm; 2544 struct mbuf *m; 2545 unsigned int i; 2546 2547 softint_disestablish(txr->txr_si); 2548 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2549 m_freem(m); 2550 pcq_destroy(txr->txr_intrq); 2551 2552 maps = txr->txr_maps; 2553 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2554 txm = &maps[i]; 2555 2556 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2557 } 2558 2559 ixl_dmamem_free(sc, &txr->txr_mem); 2560 mutex_destroy(&txr->txr_lock); 2561 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2562 kmem_free(txr, sizeof(*txr)); 2563 } 2564 2565 static inline int 2566 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2567 struct ixl_tx_ring *txr) 2568 { 2569 struct mbuf *m; 2570 int error; 2571 2572 KASSERT(mutex_owned(&txr->txr_lock)); 2573 2574 m = *m0; 2575 2576 error = bus_dmamap_load_mbuf(dmat, map, m, 2577 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2578 if (error != EFBIG) 2579 return error; 2580 2581 m = m_defrag(m, M_DONTWAIT); 2582 if (m != NULL) { 2583 *m0 = m; 2584 txr->txr_defragged.ev_count++; 2585 2586 error = bus_dmamap_load_mbuf(dmat, map, m, 2587 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2588 } else { 2589 txr->txr_defrag_failed.ev_count++; 2590 error = ENOBUFS; 2591 } 2592 2593 return error; 2594 } 2595 2596 static inline int 2597 ixl_tx_setup_offloads(struct mbuf *m, uint64_t *cmd_txd) 2598 { 2599 struct ether_header *eh; 2600 size_t len; 2601 uint64_t cmd; 2602 2603 cmd = 0; 2604 2605 eh = mtod(m, struct ether_header *); 2606 switch (htons(eh->ether_type)) { 2607 case ETHERTYPE_IP: 2608 case ETHERTYPE_IPV6: 2609 len = ETHER_HDR_LEN; 2610 break; 2611 case ETHERTYPE_VLAN: 2612 len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2613 break; 2614 default: 2615 len = 0; 2616 } 2617 cmd |= ((len >> 1) << IXL_TX_DESC_MACLEN_SHIFT); 2618 2619 if (m->m_pkthdr.csum_flags & 2620 (M_CSUM_TSOv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) { 2621 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4; 2622 } 2623 if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) { 2624 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4_CSUM; 2625 } 2626 2627 if (m->m_pkthdr.csum_flags & 2628 (M_CSUM_TSOv6 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) { 2629 cmd |= IXL_TX_DESC_CMD_IIPT_IPV6; 2630 } 2631 2632 switch (cmd & IXL_TX_DESC_CMD_IIPT_MASK) { 2633 case IXL_TX_DESC_CMD_IIPT_IPV4: 2634 case IXL_TX_DESC_CMD_IIPT_IPV4_CSUM: 2635 len = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); 2636 break; 2637 case IXL_TX_DESC_CMD_IIPT_IPV6: 2638 len = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); 2639 break; 2640 default: 2641 len = 0; 2642 } 2643 cmd |= ((len >> 2) << IXL_TX_DESC_IPLEN_SHIFT); 2644 2645 if (m->m_pkthdr.csum_flags & 2646 (M_CSUM_TSOv4 | M_CSUM_TSOv6 | M_CSUM_TCPv4 | M_CSUM_TCPv6)) { 2647 len = sizeof(struct tcphdr); 2648 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_TCP; 2649 } else if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) { 2650 len = sizeof(struct udphdr); 2651 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_UDP; 2652 } else { 2653 len = 0; 2654 } 2655 cmd |= ((len >> 2) << IXL_TX_DESC_L4LEN_SHIFT); 2656 2657 *cmd_txd |= cmd; 2658 return 0; 2659 } 2660 2661 static void 2662 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2663 bool is_transmit) 2664 { 2665 struct ixl_softc *sc = ifp->if_softc; 2666 struct ixl_tx_desc *ring, *txd; 2667 struct ixl_tx_map *txm; 2668 bus_dmamap_t map; 2669 struct mbuf *m; 2670 uint64_t cmd, cmd_txd; 2671 unsigned int prod, free, last, i; 2672 unsigned int mask; 2673 int post = 0; 2674 2675 KASSERT(mutex_owned(&txr->txr_lock)); 2676 2677 if (!ISSET(ifp->if_flags, IFF_RUNNING) 2678 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2679 if (!is_transmit) 2680 IFQ_PURGE(&ifp->if_snd); 2681 return; 2682 } 2683 2684 prod = txr->txr_prod; 2685 free = txr->txr_cons; 2686 if (free <= prod) 2687 free += sc->sc_tx_ring_ndescs; 2688 free -= prod; 2689 2690 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2691 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2692 2693 ring = IXL_DMA_KVA(&txr->txr_mem); 2694 mask = sc->sc_tx_ring_ndescs - 1; 2695 last = prod; 2696 cmd = 0; 2697 txd = NULL; 2698 2699 for (;;) { 2700 if (free <= IXL_TX_PKT_DESCS) { 2701 if (!is_transmit) 2702 SET(ifp->if_flags, IFF_OACTIVE); 2703 break; 2704 } 2705 2706 if (is_transmit) 2707 m = pcq_get(txr->txr_intrq); 2708 else 2709 IFQ_DEQUEUE(&ifp->if_snd, m); 2710 2711 if (m == NULL) 2712 break; 2713 2714 txm = &txr->txr_maps[prod]; 2715 map = txm->txm_map; 2716 2717 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2718 if_statinc(ifp, if_oerrors); 2719 m_freem(m); 2720 continue; 2721 } 2722 2723 cmd_txd = 0; 2724 if (m->m_pkthdr.csum_flags & IXL_CSUM_ALL_OFFLOAD) { 2725 ixl_tx_setup_offloads(m, &cmd_txd); 2726 } 2727 2728 if (vlan_has_tag(m)) { 2729 cmd_txd |= (uint64_t)vlan_get_tag(m) << 2730 IXL_TX_DESC_L2TAG1_SHIFT; 2731 cmd_txd |= IXL_TX_DESC_CMD_IL2TAG1; 2732 } 2733 2734 bus_dmamap_sync(sc->sc_dmat, map, 0, 2735 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2736 2737 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2738 txd = &ring[prod]; 2739 2740 cmd = (uint64_t)map->dm_segs[i].ds_len << 2741 IXL_TX_DESC_BSIZE_SHIFT; 2742 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2743 cmd |= cmd_txd; 2744 2745 txd->addr = htole64(map->dm_segs[i].ds_addr); 2746 txd->cmd = htole64(cmd); 2747 2748 last = prod; 2749 2750 prod++; 2751 prod &= mask; 2752 } 2753 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2754 txd->cmd = htole64(cmd); 2755 2756 txm->txm_m = m; 2757 txm->txm_eop = last; 2758 2759 bpf_mtap(ifp, m, BPF_D_OUT); 2760 2761 free -= i; 2762 post = 1; 2763 } 2764 2765 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2766 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2767 2768 if (post) { 2769 txr->txr_prod = prod; 2770 ixl_wr(sc, txr->txr_tail, prod); 2771 } 2772 } 2773 2774 static int 2775 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2776 { 2777 struct ifnet *ifp = &sc->sc_ec.ec_if; 2778 struct ixl_tx_desc *ring, *txd; 2779 struct ixl_tx_map *txm; 2780 struct mbuf *m; 2781 bus_dmamap_t map; 2782 unsigned int cons, prod, last; 2783 unsigned int mask; 2784 uint64_t dtype; 2785 int done = 0, more = 0; 2786 2787 KASSERT(mutex_owned(&txr->txr_lock)); 2788 2789 prod = txr->txr_prod; 2790 cons = txr->txr_cons; 2791 2792 if (cons == prod) 2793 return 0; 2794 2795 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2796 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2797 2798 ring = IXL_DMA_KVA(&txr->txr_mem); 2799 mask = sc->sc_tx_ring_ndescs - 1; 2800 2801 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2802 2803 do { 2804 if (txlimit-- <= 0) { 2805 more = 1; 2806 break; 2807 } 2808 2809 txm = &txr->txr_maps[cons]; 2810 last = txm->txm_eop; 2811 txd = &ring[last]; 2812 2813 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2814 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2815 break; 2816 2817 map = txm->txm_map; 2818 2819 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2820 BUS_DMASYNC_POSTWRITE); 2821 bus_dmamap_unload(sc->sc_dmat, map); 2822 2823 m = txm->txm_m; 2824 if (m != NULL) { 2825 if_statinc_ref(nsr, if_opackets); 2826 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 2827 if (ISSET(m->m_flags, M_MCAST)) 2828 if_statinc_ref(nsr, if_omcasts); 2829 m_freem(m); 2830 } 2831 2832 txm->txm_m = NULL; 2833 txm->txm_eop = -1; 2834 2835 cons = last + 1; 2836 cons &= mask; 2837 done = 1; 2838 } while (cons != prod); 2839 2840 IF_STAT_PUTREF(ifp); 2841 2842 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2843 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2844 2845 txr->txr_cons = cons; 2846 2847 if (done) { 2848 softint_schedule(txr->txr_si); 2849 if (txr->txr_qid == 0) { 2850 CLR(ifp->if_flags, IFF_OACTIVE); 2851 if_schedule_deferred_start(ifp); 2852 } 2853 } 2854 2855 return more; 2856 } 2857 2858 static void 2859 ixl_start(struct ifnet *ifp) 2860 { 2861 struct ixl_softc *sc; 2862 struct ixl_tx_ring *txr; 2863 2864 sc = ifp->if_softc; 2865 txr = sc->sc_qps[0].qp_txr; 2866 2867 mutex_enter(&txr->txr_lock); 2868 ixl_tx_common_locked(ifp, txr, false); 2869 mutex_exit(&txr->txr_lock); 2870 } 2871 2872 static inline unsigned int 2873 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2874 { 2875 u_int cpuid; 2876 2877 cpuid = cpu_index(curcpu()); 2878 2879 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2880 } 2881 2882 static int 2883 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2884 { 2885 struct ixl_softc *sc; 2886 struct ixl_tx_ring *txr; 2887 unsigned int qid; 2888 2889 sc = ifp->if_softc; 2890 qid = ixl_select_txqueue(sc, m); 2891 2892 txr = sc->sc_qps[qid].qp_txr; 2893 2894 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2895 mutex_enter(&txr->txr_lock); 2896 txr->txr_pcqdrop.ev_count++; 2897 mutex_exit(&txr->txr_lock); 2898 2899 m_freem(m); 2900 return ENOBUFS; 2901 } 2902 2903 if (mutex_tryenter(&txr->txr_lock)) { 2904 ixl_tx_common_locked(ifp, txr, true); 2905 mutex_exit(&txr->txr_lock); 2906 } else { 2907 kpreempt_disable(); 2908 softint_schedule(txr->txr_si); 2909 kpreempt_enable(); 2910 } 2911 2912 return 0; 2913 } 2914 2915 static void 2916 ixl_deferred_transmit(void *xtxr) 2917 { 2918 struct ixl_tx_ring *txr = xtxr; 2919 struct ixl_softc *sc = txr->txr_sc; 2920 struct ifnet *ifp = &sc->sc_ec.ec_if; 2921 2922 mutex_enter(&txr->txr_lock); 2923 txr->txr_transmitdef.ev_count++; 2924 if (pcq_peek(txr->txr_intrq) != NULL) 2925 ixl_tx_common_locked(ifp, txr, true); 2926 mutex_exit(&txr->txr_lock); 2927 } 2928 2929 static struct ixl_rx_ring * 2930 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2931 { 2932 struct ixl_rx_ring *rxr = NULL; 2933 struct ixl_rx_map *maps = NULL, *rxm; 2934 unsigned int i; 2935 2936 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2937 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2938 KM_SLEEP); 2939 2940 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2941 sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs, 2942 IXL_RX_QUEUE_ALIGN) != 0) 2943 goto free; 2944 2945 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2946 rxm = &maps[i]; 2947 2948 if (bus_dmamap_create(sc->sc_dmat, 2949 IXL_MCLBYTES, 1, IXL_MCLBYTES, 0, 2950 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2951 goto uncreate; 2952 2953 rxm->rxm_m = NULL; 2954 } 2955 2956 rxr->rxr_cons = rxr->rxr_prod = 0; 2957 rxr->rxr_m_head = NULL; 2958 rxr->rxr_m_tail = &rxr->rxr_m_head; 2959 rxr->rxr_maps = maps; 2960 2961 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2962 rxr->rxr_qid = qid; 2963 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2964 2965 return rxr; 2966 2967 uncreate: 2968 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2969 rxm = &maps[i]; 2970 2971 if (rxm->rxm_map == NULL) 2972 continue; 2973 2974 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2975 } 2976 2977 ixl_dmamem_free(sc, &rxr->rxr_mem); 2978 free: 2979 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2980 kmem_free(rxr, sizeof(*rxr)); 2981 2982 return NULL; 2983 } 2984 2985 static void 2986 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2987 { 2988 struct ixl_rx_map *maps, *rxm; 2989 bus_dmamap_t map; 2990 unsigned int i; 2991 2992 maps = rxr->rxr_maps; 2993 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2994 rxm = &maps[i]; 2995 2996 if (rxm->rxm_m == NULL) 2997 continue; 2998 2999 map = rxm->rxm_map; 3000 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3001 BUS_DMASYNC_POSTWRITE); 3002 bus_dmamap_unload(sc->sc_dmat, map); 3003 3004 m_freem(rxm->rxm_m); 3005 rxm->rxm_m = NULL; 3006 } 3007 3008 m_freem(rxr->rxr_m_head); 3009 rxr->rxr_m_head = NULL; 3010 rxr->rxr_m_tail = &rxr->rxr_m_head; 3011 3012 rxr->rxr_prod = rxr->rxr_cons = 0; 3013 } 3014 3015 static int 3016 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3017 { 3018 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3019 uint32_t reg; 3020 int i; 3021 3022 for (i = 0; i < 10; i++) { 3023 reg = ixl_rd(sc, ena); 3024 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 3025 return 0; 3026 3027 delaymsec(10); 3028 } 3029 3030 return ETIMEDOUT; 3031 } 3032 3033 static int 3034 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3035 { 3036 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3037 uint32_t reg; 3038 int i; 3039 3040 KASSERT(mutex_owned(&rxr->rxr_lock)); 3041 3042 for (i = 0; i < 10; i++) { 3043 reg = ixl_rd(sc, ena); 3044 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 3045 return 0; 3046 3047 delaymsec(10); 3048 } 3049 3050 return ETIMEDOUT; 3051 } 3052 3053 static void 3054 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3055 { 3056 struct ixl_hmc_rxq rxq; 3057 struct ifnet *ifp = &sc->sc_ec.ec_if; 3058 uint16_t rxmax; 3059 void *hmc; 3060 3061 memset(&rxq, 0, sizeof(rxq)); 3062 rxmax = ifp->if_mtu + IXL_MTU_ETHERLEN; 3063 3064 rxq.head = htole16(rxr->rxr_cons); 3065 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 3066 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 3067 rxq.dbuff = htole16(IXL_MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 3068 rxq.hbuff = 0; 3069 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 3070 rxq.dsize = IXL_HMC_RXQ_DSIZE_32; 3071 rxq.crcstrip = 1; 3072 rxq.l2sel = 1; 3073 rxq.showiv = 1; 3074 rxq.rxmax = htole16(rxmax); 3075 rxq.tphrdesc_ena = 0; 3076 rxq.tphwdesc_ena = 0; 3077 rxq.tphdata_ena = 0; 3078 rxq.tphhead_ena = 0; 3079 rxq.lrxqthresh = 0; 3080 rxq.prefena = 1; 3081 3082 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3083 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3084 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 3085 __arraycount(ixl_hmc_pack_rxq)); 3086 } 3087 3088 static void 3089 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3090 { 3091 void *hmc; 3092 3093 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3094 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3095 rxr->rxr_cons = rxr->rxr_prod = 0; 3096 } 3097 3098 static void 3099 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3100 { 3101 struct ixl_rx_map *maps, *rxm; 3102 unsigned int i; 3103 3104 maps = rxr->rxr_maps; 3105 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3106 rxm = &maps[i]; 3107 3108 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 3109 } 3110 3111 ixl_dmamem_free(sc, &rxr->rxr_mem); 3112 mutex_destroy(&rxr->rxr_lock); 3113 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3114 kmem_free(rxr, sizeof(*rxr)); 3115 } 3116 3117 static inline void 3118 ixl_rx_csum(struct mbuf *m, uint64_t qword) 3119 { 3120 int flags_mask; 3121 3122 if (!ISSET(qword, IXL_RX_DESC_L3L4P)) { 3123 /* No L3 or L4 checksum was calculated */ 3124 return; 3125 } 3126 3127 switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) { 3128 case IXL_RX_DESC_PTYPE_IPV4FRAG: 3129 case IXL_RX_DESC_PTYPE_IPV4: 3130 case IXL_RX_DESC_PTYPE_SCTPV4: 3131 case IXL_RX_DESC_PTYPE_ICMPV4: 3132 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3133 break; 3134 case IXL_RX_DESC_PTYPE_TCPV4: 3135 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3136 flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD; 3137 break; 3138 case IXL_RX_DESC_PTYPE_UDPV4: 3139 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3140 flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD; 3141 break; 3142 case IXL_RX_DESC_PTYPE_TCPV6: 3143 flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD; 3144 break; 3145 case IXL_RX_DESC_PTYPE_UDPV6: 3146 flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD; 3147 break; 3148 default: 3149 flags_mask = 0; 3150 } 3151 3152 m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 | 3153 M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6)); 3154 3155 if (ISSET(qword, IXL_RX_DESC_IPE)) { 3156 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD); 3157 } 3158 3159 if (ISSET(qword, IXL_RX_DESC_L4E)) { 3160 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD); 3161 } 3162 } 3163 3164 static int 3165 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 3166 { 3167 struct ifnet *ifp = &sc->sc_ec.ec_if; 3168 struct ixl_rx_wb_desc_32 *ring, *rxd; 3169 struct ixl_rx_map *rxm; 3170 bus_dmamap_t map; 3171 unsigned int cons, prod; 3172 struct mbuf *m; 3173 uint64_t word, word0; 3174 unsigned int len; 3175 unsigned int mask; 3176 int done = 0, more = 0; 3177 3178 KASSERT(mutex_owned(&rxr->rxr_lock)); 3179 3180 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3181 return 0; 3182 3183 prod = rxr->rxr_prod; 3184 cons = rxr->rxr_cons; 3185 3186 if (cons == prod) 3187 return 0; 3188 3189 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3190 0, IXL_DMA_LEN(&rxr->rxr_mem), 3191 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3192 3193 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3194 mask = sc->sc_rx_ring_ndescs - 1; 3195 3196 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 3197 3198 do { 3199 if (rxlimit-- <= 0) { 3200 more = 1; 3201 break; 3202 } 3203 3204 rxd = &ring[cons]; 3205 3206 word = le64toh(rxd->qword1); 3207 3208 if (!ISSET(word, IXL_RX_DESC_DD)) 3209 break; 3210 3211 rxm = &rxr->rxr_maps[cons]; 3212 3213 map = rxm->rxm_map; 3214 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3215 BUS_DMASYNC_POSTREAD); 3216 bus_dmamap_unload(sc->sc_dmat, map); 3217 3218 m = rxm->rxm_m; 3219 rxm->rxm_m = NULL; 3220 3221 KASSERT(m != NULL); 3222 3223 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 3224 m->m_len = len; 3225 m->m_pkthdr.len = 0; 3226 3227 m->m_next = NULL; 3228 *rxr->rxr_m_tail = m; 3229 rxr->rxr_m_tail = &m->m_next; 3230 3231 m = rxr->rxr_m_head; 3232 m->m_pkthdr.len += len; 3233 3234 if (ISSET(word, IXL_RX_DESC_EOP)) { 3235 word0 = le64toh(rxd->qword0); 3236 3237 if (ISSET(word, IXL_RX_DESC_L2TAG1P)) { 3238 vlan_set_tag(m, 3239 __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK)); 3240 } 3241 3242 if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0) 3243 ixl_rx_csum(m, word); 3244 3245 if (!ISSET(word, 3246 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 3247 m_set_rcvif(m, ifp); 3248 if_statinc_ref(nsr, if_ipackets); 3249 if_statadd_ref(nsr, if_ibytes, 3250 m->m_pkthdr.len); 3251 if_percpuq_enqueue(sc->sc_ipq, m); 3252 } else { 3253 if_statinc_ref(nsr, if_ierrors); 3254 m_freem(m); 3255 } 3256 3257 rxr->rxr_m_head = NULL; 3258 rxr->rxr_m_tail = &rxr->rxr_m_head; 3259 } 3260 3261 cons++; 3262 cons &= mask; 3263 3264 done = 1; 3265 } while (cons != prod); 3266 3267 if (done) { 3268 rxr->rxr_cons = cons; 3269 if (ixl_rxfill(sc, rxr) == -1) 3270 if_statinc_ref(nsr, if_iqdrops); 3271 } 3272 3273 IF_STAT_PUTREF(ifp); 3274 3275 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3276 0, IXL_DMA_LEN(&rxr->rxr_mem), 3277 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3278 3279 return more; 3280 } 3281 3282 static int 3283 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3284 { 3285 struct ixl_rx_rd_desc_32 *ring, *rxd; 3286 struct ixl_rx_map *rxm; 3287 bus_dmamap_t map; 3288 struct mbuf *m; 3289 unsigned int prod; 3290 unsigned int slots; 3291 unsigned int mask; 3292 int post = 0, error = 0; 3293 3294 KASSERT(mutex_owned(&rxr->rxr_lock)); 3295 3296 prod = rxr->rxr_prod; 3297 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 3298 sc->sc_rx_ring_ndescs); 3299 3300 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3301 mask = sc->sc_rx_ring_ndescs - 1; 3302 3303 if (__predict_false(slots <= 0)) 3304 return -1; 3305 3306 do { 3307 rxm = &rxr->rxr_maps[prod]; 3308 3309 MGETHDR(m, M_DONTWAIT, MT_DATA); 3310 if (m == NULL) { 3311 rxr->rxr_mgethdr_failed.ev_count++; 3312 error = -1; 3313 break; 3314 } 3315 3316 MCLGET(m, M_DONTWAIT); 3317 if (!ISSET(m->m_flags, M_EXT)) { 3318 rxr->rxr_mgetcl_failed.ev_count++; 3319 error = -1; 3320 m_freem(m); 3321 break; 3322 } 3323 3324 m->m_len = m->m_pkthdr.len = MCLBYTES; 3325 m_adj(m, ETHER_ALIGN); 3326 3327 map = rxm->rxm_map; 3328 3329 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 3330 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 3331 rxr->rxr_mbuf_load_failed.ev_count++; 3332 error = -1; 3333 m_freem(m); 3334 break; 3335 } 3336 3337 rxm->rxm_m = m; 3338 3339 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3340 BUS_DMASYNC_PREREAD); 3341 3342 rxd = &ring[prod]; 3343 3344 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 3345 rxd->haddr = htole64(0); 3346 3347 prod++; 3348 prod &= mask; 3349 3350 post = 1; 3351 3352 } while (--slots); 3353 3354 if (post) { 3355 rxr->rxr_prod = prod; 3356 ixl_wr(sc, rxr->rxr_tail, prod); 3357 } 3358 3359 return error; 3360 } 3361 3362 static inline int 3363 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 3364 u_int txlimit, struct evcnt *txevcnt, 3365 u_int rxlimit, struct evcnt *rxevcnt) 3366 { 3367 struct ixl_tx_ring *txr = qp->qp_txr; 3368 struct ixl_rx_ring *rxr = qp->qp_rxr; 3369 int txmore, rxmore; 3370 int rv; 3371 3372 mutex_enter(&txr->txr_lock); 3373 txevcnt->ev_count++; 3374 txmore = ixl_txeof(sc, txr, txlimit); 3375 mutex_exit(&txr->txr_lock); 3376 3377 mutex_enter(&rxr->rxr_lock); 3378 rxevcnt->ev_count++; 3379 rxmore = ixl_rxeof(sc, rxr, rxlimit); 3380 mutex_exit(&rxr->rxr_lock); 3381 3382 rv = txmore | (rxmore << 1); 3383 3384 return rv; 3385 } 3386 3387 static void 3388 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 3389 { 3390 3391 if (qp->qp_workqueue) 3392 workqueue_enqueue(sc->sc_workq_txrx, &qp->qp_work, NULL); 3393 else 3394 softint_schedule(qp->qp_si); 3395 } 3396 3397 static int 3398 ixl_intr(void *xsc) 3399 { 3400 struct ixl_softc *sc = xsc; 3401 struct ixl_tx_ring *txr; 3402 struct ixl_rx_ring *rxr; 3403 uint32_t icr, rxintr, txintr; 3404 int rv = 0; 3405 unsigned int i; 3406 3407 KASSERT(sc != NULL); 3408 3409 ixl_enable_other_intr(sc); 3410 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3411 3412 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3413 atomic_inc_64(&sc->sc_event_atq.ev_count); 3414 ixl_atq_done(sc); 3415 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3416 rv = 1; 3417 } 3418 3419 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3420 atomic_inc_64(&sc->sc_event_link.ev_count); 3421 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3422 rv = 1; 3423 } 3424 3425 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 3426 txintr = icr & I40E_INTR_NOTX_TX_MASK; 3427 3428 if (txintr || rxintr) { 3429 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 3430 txr = sc->sc_qps[i].qp_txr; 3431 rxr = sc->sc_qps[i].qp_rxr; 3432 3433 ixl_handle_queue_common(sc, &sc->sc_qps[i], 3434 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 3435 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 3436 } 3437 rv = 1; 3438 } 3439 3440 return rv; 3441 } 3442 3443 static int 3444 ixl_queue_intr(void *xqp) 3445 { 3446 struct ixl_queue_pair *qp = xqp; 3447 struct ixl_tx_ring *txr = qp->qp_txr; 3448 struct ixl_rx_ring *rxr = qp->qp_rxr; 3449 struct ixl_softc *sc = qp->qp_sc; 3450 u_int txlimit, rxlimit; 3451 int more; 3452 3453 txlimit = sc->sc_tx_intr_process_limit; 3454 rxlimit = sc->sc_rx_intr_process_limit; 3455 qp->qp_workqueue = sc->sc_txrx_workqueue; 3456 3457 more = ixl_handle_queue_common(sc, qp, 3458 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3459 3460 if (more != 0) { 3461 ixl_sched_handle_queue(sc, qp); 3462 } else { 3463 /* for ALTQ */ 3464 if (txr->txr_qid == 0) 3465 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3466 softint_schedule(txr->txr_si); 3467 3468 ixl_enable_queue_intr(sc, qp); 3469 } 3470 3471 return 1; 3472 } 3473 3474 static void 3475 ixl_handle_queue_wk(struct work *wk, void *xsc) 3476 { 3477 struct ixl_queue_pair *qp; 3478 3479 qp = container_of(wk, struct ixl_queue_pair, qp_work); 3480 ixl_handle_queue(qp); 3481 } 3482 3483 static void 3484 ixl_handle_queue(void *xqp) 3485 { 3486 struct ixl_queue_pair *qp = xqp; 3487 struct ixl_softc *sc = qp->qp_sc; 3488 struct ixl_tx_ring *txr = qp->qp_txr; 3489 struct ixl_rx_ring *rxr = qp->qp_rxr; 3490 u_int txlimit, rxlimit; 3491 int more; 3492 3493 txlimit = sc->sc_tx_process_limit; 3494 rxlimit = sc->sc_rx_process_limit; 3495 3496 more = ixl_handle_queue_common(sc, qp, 3497 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3498 3499 if (more != 0) 3500 ixl_sched_handle_queue(sc, qp); 3501 else 3502 ixl_enable_queue_intr(sc, qp); 3503 } 3504 3505 static inline void 3506 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3507 { 3508 uint32_t hmc_idx, hmc_isvf; 3509 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3510 3511 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3512 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3513 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3514 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3515 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3516 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3517 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3518 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3519 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3520 3521 device_printf(sc->sc_dev, 3522 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3523 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3524 } 3525 3526 static int 3527 ixl_other_intr(void *xsc) 3528 { 3529 struct ixl_softc *sc = xsc; 3530 uint32_t icr, mask, reg; 3531 int rv; 3532 3533 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3534 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3535 3536 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3537 atomic_inc_64(&sc->sc_event_atq.ev_count); 3538 ixl_atq_done(sc); 3539 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3540 rv = 1; 3541 } 3542 3543 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3544 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3545 device_printf(sc->sc_dev, "link stat changed\n"); 3546 3547 atomic_inc_64(&sc->sc_event_link.ev_count); 3548 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3549 rv = 1; 3550 } 3551 3552 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3553 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3554 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3555 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3556 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3557 3558 device_printf(sc->sc_dev, "GRST: %s\n", 3559 reg == I40E_RESET_CORER ? "CORER" : 3560 reg == I40E_RESET_GLOBR ? "GLOBR" : 3561 reg == I40E_RESET_EMPR ? "EMPR" : 3562 "POR"); 3563 } 3564 3565 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3566 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3567 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3568 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3569 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3570 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3571 3572 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3573 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3574 device_printf(sc->sc_dev, "critical error\n"); 3575 } 3576 3577 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3578 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3579 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3580 ixl_print_hmc_error(sc, reg); 3581 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3582 } 3583 3584 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3585 ixl_flush(sc); 3586 ixl_enable_other_intr(sc); 3587 return rv; 3588 } 3589 3590 static void 3591 ixl_get_link_status_done(struct ixl_softc *sc, 3592 const struct ixl_aq_desc *iaq) 3593 { 3594 struct ixl_aq_desc iaq_buf; 3595 3596 memcpy(&iaq_buf, iaq, sizeof(iaq_buf)); 3597 3598 /* 3599 * The lock can be released here 3600 * because there is no post processing about ATQ 3601 */ 3602 mutex_exit(&sc->sc_atq_lock); 3603 ixl_link_state_update(sc, &iaq_buf); 3604 mutex_enter(&sc->sc_atq_lock); 3605 } 3606 3607 static void 3608 ixl_get_link_status(void *xsc) 3609 { 3610 struct ixl_softc *sc = xsc; 3611 struct ixl_aq_desc *iaq; 3612 struct ixl_aq_link_param *param; 3613 int error; 3614 3615 mutex_enter(&sc->sc_atq_lock); 3616 3617 iaq = &sc->sc_link_state_atq.iatq_desc; 3618 memset(iaq, 0, sizeof(*iaq)); 3619 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3620 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3621 param->notify = IXL_AQ_LINK_NOTIFY; 3622 3623 error = ixl_atq_exec_locked(sc, &sc->sc_link_state_atq); 3624 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 3625 3626 if (error == 0) { 3627 ixl_get_link_status_done(sc, iaq); 3628 } 3629 3630 mutex_exit(&sc->sc_atq_lock); 3631 } 3632 3633 static void 3634 ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3635 { 3636 struct ifnet *ifp = &sc->sc_ec.ec_if; 3637 int link_state; 3638 3639 mutex_enter(&sc->sc_cfg_lock); 3640 link_state = ixl_set_link_status_locked(sc, iaq); 3641 mutex_exit(&sc->sc_cfg_lock); 3642 3643 if (ifp->if_link_state != link_state) 3644 if_link_state_change(ifp, link_state); 3645 3646 if (link_state != LINK_STATE_DOWN) { 3647 kpreempt_disable(); 3648 if_schedule_deferred_start(ifp); 3649 kpreempt_enable(); 3650 } 3651 } 3652 3653 static void 3654 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3655 const char *msg) 3656 { 3657 char buf[512]; 3658 size_t len; 3659 3660 len = sizeof(buf); 3661 buf[--len] = '\0'; 3662 3663 device_printf(sc->sc_dev, "%s\n", msg); 3664 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3665 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3666 buf, le16toh(iaq->iaq_opcode)); 3667 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3668 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3669 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3670 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3671 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3672 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3673 } 3674 3675 static void 3676 ixl_arq(void *xsc) 3677 { 3678 struct ixl_softc *sc = xsc; 3679 struct ixl_aq_desc *arq, *iaq; 3680 struct ixl_aq_buf *aqb; 3681 unsigned int cons = sc->sc_arq_cons; 3682 unsigned int prod; 3683 int done = 0; 3684 3685 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3686 sc->sc_aq_regs->arq_head_mask; 3687 3688 if (cons == prod) 3689 goto done; 3690 3691 arq = IXL_DMA_KVA(&sc->sc_arq); 3692 3693 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3694 0, IXL_DMA_LEN(&sc->sc_arq), 3695 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3696 3697 do { 3698 iaq = &arq[cons]; 3699 aqb = sc->sc_arq_live[cons]; 3700 3701 KASSERT(aqb != NULL); 3702 3703 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3704 BUS_DMASYNC_POSTREAD); 3705 3706 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3707 ixl_aq_dump(sc, iaq, "arq event"); 3708 3709 switch (iaq->iaq_opcode) { 3710 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3711 ixl_link_state_update(sc, iaq); 3712 break; 3713 } 3714 3715 memset(iaq, 0, sizeof(*iaq)); 3716 sc->sc_arq_live[cons] = NULL; 3717 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3718 3719 cons++; 3720 cons &= IXL_AQ_MASK; 3721 3722 done = 1; 3723 } while (cons != prod); 3724 3725 if (done) { 3726 sc->sc_arq_cons = cons; 3727 ixl_arq_fill(sc); 3728 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3729 0, IXL_DMA_LEN(&sc->sc_arq), 3730 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3731 } 3732 3733 done: 3734 ixl_enable_other_intr(sc); 3735 } 3736 3737 static void 3738 ixl_atq_set(struct ixl_atq *iatq, 3739 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3740 { 3741 3742 iatq->iatq_fn = fn; 3743 } 3744 3745 static int 3746 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3747 { 3748 struct ixl_aq_desc *atq, *slot; 3749 unsigned int prod, cons, prod_next; 3750 3751 /* assert locked */ 3752 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3753 3754 atq = IXL_DMA_KVA(&sc->sc_atq); 3755 prod = sc->sc_atq_prod; 3756 cons = sc->sc_atq_cons; 3757 prod_next = (prod +1) & IXL_AQ_MASK; 3758 3759 if (cons == prod_next) 3760 return ENOMEM; 3761 3762 slot = &atq[prod]; 3763 3764 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3765 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3766 3767 KASSERT(iatq->iatq_fn != NULL); 3768 *slot = iatq->iatq_desc; 3769 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3770 3771 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3772 ixl_aq_dump(sc, slot, "atq command"); 3773 3774 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3775 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3776 3777 sc->sc_atq_prod = prod_next; 3778 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3779 3780 return 0; 3781 } 3782 3783 static void 3784 ixl_atq_done_locked(struct ixl_softc *sc) 3785 { 3786 struct ixl_aq_desc *atq, *slot; 3787 struct ixl_atq *iatq; 3788 unsigned int cons; 3789 unsigned int prod; 3790 3791 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3792 3793 prod = sc->sc_atq_prod; 3794 cons = sc->sc_atq_cons; 3795 3796 if (prod == cons) 3797 return; 3798 3799 atq = IXL_DMA_KVA(&sc->sc_atq); 3800 3801 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3802 0, IXL_DMA_LEN(&sc->sc_atq), 3803 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3804 3805 do { 3806 slot = &atq[cons]; 3807 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3808 break; 3809 3810 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3811 iatq->iatq_desc = *slot; 3812 3813 memset(slot, 0, sizeof(*slot)); 3814 3815 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3816 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3817 3818 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3819 3820 cons++; 3821 cons &= IXL_AQ_MASK; 3822 } while (cons != prod); 3823 3824 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3825 0, IXL_DMA_LEN(&sc->sc_atq), 3826 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3827 3828 sc->sc_atq_cons = cons; 3829 } 3830 3831 static void 3832 ixl_atq_done(struct ixl_softc *sc) 3833 { 3834 3835 mutex_enter(&sc->sc_atq_lock); 3836 ixl_atq_done_locked(sc); 3837 mutex_exit(&sc->sc_atq_lock); 3838 } 3839 3840 static void 3841 ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3842 { 3843 3844 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3845 3846 cv_signal(&sc->sc_atq_cv); 3847 } 3848 3849 static int 3850 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3851 { 3852 int error; 3853 3854 mutex_enter(&sc->sc_atq_lock); 3855 error = ixl_atq_exec_locked(sc, iatq); 3856 mutex_exit(&sc->sc_atq_lock); 3857 3858 return error; 3859 } 3860 3861 static int 3862 ixl_atq_exec_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3863 { 3864 int error; 3865 3866 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3867 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3868 3869 ixl_atq_set(iatq, ixl_wakeup); 3870 3871 error = ixl_atq_post_locked(sc, iatq); 3872 if (error) 3873 return error; 3874 3875 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3876 IXL_ATQ_EXEC_TIMEOUT); 3877 3878 return error; 3879 } 3880 3881 static int 3882 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3883 { 3884 struct ixl_aq_desc *atq, *slot; 3885 unsigned int prod; 3886 unsigned int t = 0; 3887 3888 mutex_enter(&sc->sc_atq_lock); 3889 3890 atq = IXL_DMA_KVA(&sc->sc_atq); 3891 prod = sc->sc_atq_prod; 3892 slot = atq + prod; 3893 3894 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3895 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3896 3897 *slot = *iaq; 3898 slot->iaq_flags |= htole16(IXL_AQ_SI); 3899 3900 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3901 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3902 3903 prod++; 3904 prod &= IXL_AQ_MASK; 3905 sc->sc_atq_prod = prod; 3906 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3907 3908 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3909 delaymsec(1); 3910 3911 if (t++ > tm) { 3912 mutex_exit(&sc->sc_atq_lock); 3913 return ETIMEDOUT; 3914 } 3915 } 3916 3917 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3918 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3919 *iaq = *slot; 3920 memset(slot, 0, sizeof(*slot)); 3921 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3922 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3923 3924 sc->sc_atq_cons = prod; 3925 3926 mutex_exit(&sc->sc_atq_lock); 3927 3928 return 0; 3929 } 3930 3931 static int 3932 ixl_get_version(struct ixl_softc *sc) 3933 { 3934 struct ixl_aq_desc iaq; 3935 uint32_t fwbuild, fwver, apiver; 3936 uint16_t api_maj_ver, api_min_ver; 3937 3938 memset(&iaq, 0, sizeof(iaq)); 3939 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3940 3941 iaq.iaq_retval = le16toh(23); 3942 3943 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3944 return ETIMEDOUT; 3945 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3946 return EIO; 3947 3948 fwbuild = le32toh(iaq.iaq_param[1]); 3949 fwver = le32toh(iaq.iaq_param[2]); 3950 apiver = le32toh(iaq.iaq_param[3]); 3951 3952 api_maj_ver = (uint16_t)apiver; 3953 api_min_ver = (uint16_t)(apiver >> 16); 3954 3955 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 3956 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 3957 3958 if (sc->sc_mac_type == I40E_MAC_X722) { 3959 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK | 3960 IXL_SC_AQ_FLAG_NVMREAD); 3961 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 3962 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS); 3963 } 3964 3965 #define IXL_API_VER(maj, min) (((uint32_t)(maj) << 16) | (min)) 3966 if (IXL_API_VER(api_maj_ver, api_min_ver) >= IXL_API_VER(1, 5)) { 3967 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 3968 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK); 3969 } 3970 #undef IXL_API_VER 3971 3972 return 0; 3973 } 3974 3975 static int 3976 ixl_get_nvm_version(struct ixl_softc *sc) 3977 { 3978 uint16_t nvmver, cfg_ptr, eetrack_hi, eetrack_lo, oem_hi, oem_lo; 3979 uint32_t eetrack, oem; 3980 uint16_t nvm_maj_ver, nvm_min_ver, oem_build; 3981 uint8_t oem_ver, oem_patch; 3982 3983 nvmver = cfg_ptr = eetrack_hi = eetrack_lo = oem_hi = oem_lo = 0; 3984 ixl_rd16_nvm(sc, I40E_SR_NVM_DEV_STARTER_VERSION, &nvmver); 3985 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_HI, &eetrack_hi); 3986 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_LO, &eetrack_lo); 3987 ixl_rd16_nvm(sc, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr); 3988 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF, &oem_hi); 3989 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF + 1, &oem_lo); 3990 3991 nvm_maj_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_HI_MASK); 3992 nvm_min_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_LO_MASK); 3993 eetrack = ((uint32_t)eetrack_hi << 16) | eetrack_lo; 3994 oem = ((uint32_t)oem_hi << 16) | oem_lo; 3995 oem_ver = __SHIFTOUT(oem, IXL_NVM_OEMVERSION_MASK); 3996 oem_build = __SHIFTOUT(oem, IXL_NVM_OEMBUILD_MASK); 3997 oem_patch = __SHIFTOUT(oem, IXL_NVM_OEMPATCH_MASK); 3998 3999 aprint_normal(" nvm %x.%02x etid %08x oem %d.%d.%d", 4000 nvm_maj_ver, nvm_min_ver, eetrack, 4001 oem_ver, oem_build, oem_patch); 4002 4003 return 0; 4004 } 4005 4006 static int 4007 ixl_pxe_clear(struct ixl_softc *sc) 4008 { 4009 struct ixl_aq_desc iaq; 4010 int rv; 4011 4012 memset(&iaq, 0, sizeof(iaq)); 4013 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 4014 iaq.iaq_param[0] = htole32(0x2); 4015 4016 rv = ixl_atq_poll(sc, &iaq, 250); 4017 4018 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 4019 4020 if (rv != 0) 4021 return ETIMEDOUT; 4022 4023 switch (iaq.iaq_retval) { 4024 case htole16(IXL_AQ_RC_OK): 4025 case htole16(IXL_AQ_RC_EEXIST): 4026 break; 4027 default: 4028 return EIO; 4029 } 4030 4031 return 0; 4032 } 4033 4034 static int 4035 ixl_lldp_shut(struct ixl_softc *sc) 4036 { 4037 struct ixl_aq_desc iaq; 4038 4039 memset(&iaq, 0, sizeof(iaq)); 4040 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 4041 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 4042 4043 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4044 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 4045 return -1; 4046 } 4047 4048 switch (iaq.iaq_retval) { 4049 case htole16(IXL_AQ_RC_EMODE): 4050 case htole16(IXL_AQ_RC_EPERM): 4051 /* ignore silently */ 4052 default: 4053 break; 4054 } 4055 4056 return 0; 4057 } 4058 4059 static void 4060 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 4061 { 4062 uint16_t id; 4063 uint32_t number, logical_id; 4064 4065 id = le16toh(cap->cap_id); 4066 number = le32toh(cap->number); 4067 logical_id = le32toh(cap->logical_id); 4068 4069 switch (id) { 4070 case IXL_AQ_CAP_RSS: 4071 sc->sc_rss_table_size = number; 4072 sc->sc_rss_table_entry_width = logical_id; 4073 break; 4074 case IXL_AQ_CAP_RXQ: 4075 case IXL_AQ_CAP_TXQ: 4076 sc->sc_nqueue_pairs_device = MIN(number, 4077 sc->sc_nqueue_pairs_device); 4078 break; 4079 } 4080 } 4081 4082 static int 4083 ixl_get_hw_capabilities(struct ixl_softc *sc) 4084 { 4085 struct ixl_dmamem idm; 4086 struct ixl_aq_desc iaq; 4087 struct ixl_aq_capability *caps; 4088 size_t i, ncaps; 4089 bus_size_t caps_size; 4090 uint16_t status; 4091 int rv; 4092 4093 caps_size = sizeof(caps[0]) * 40; 4094 memset(&iaq, 0, sizeof(iaq)); 4095 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 4096 4097 do { 4098 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 4099 return -1; 4100 } 4101 4102 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4103 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4104 iaq.iaq_datalen = htole16(caps_size); 4105 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4106 4107 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4108 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 4109 4110 rv = ixl_atq_poll(sc, &iaq, 250); 4111 4112 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4113 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 4114 4115 if (rv != 0) { 4116 aprint_error(", HW capabilities timeout\n"); 4117 goto done; 4118 } 4119 4120 status = le16toh(iaq.iaq_retval); 4121 4122 if (status == IXL_AQ_RC_ENOMEM) { 4123 caps_size = le16toh(iaq.iaq_datalen); 4124 ixl_dmamem_free(sc, &idm); 4125 } 4126 } while (status == IXL_AQ_RC_ENOMEM); 4127 4128 if (status != IXL_AQ_RC_OK) { 4129 aprint_error(", HW capabilities error\n"); 4130 goto done; 4131 } 4132 4133 caps = IXL_DMA_KVA(&idm); 4134 ncaps = le16toh(iaq.iaq_param[1]); 4135 4136 for (i = 0; i < ncaps; i++) { 4137 ixl_parse_hw_capability(sc, &caps[i]); 4138 } 4139 4140 done: 4141 ixl_dmamem_free(sc, &idm); 4142 return rv; 4143 } 4144 4145 static int 4146 ixl_get_mac(struct ixl_softc *sc) 4147 { 4148 struct ixl_dmamem idm; 4149 struct ixl_aq_desc iaq; 4150 struct ixl_aq_mac_addresses *addrs; 4151 int rv; 4152 4153 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 4154 aprint_error(", unable to allocate mac addresses\n"); 4155 return -1; 4156 } 4157 4158 memset(&iaq, 0, sizeof(iaq)); 4159 iaq.iaq_flags = htole16(IXL_AQ_BUF); 4160 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 4161 iaq.iaq_datalen = htole16(sizeof(*addrs)); 4162 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4163 4164 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4165 BUS_DMASYNC_PREREAD); 4166 4167 rv = ixl_atq_poll(sc, &iaq, 250); 4168 4169 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4170 BUS_DMASYNC_POSTREAD); 4171 4172 if (rv != 0) { 4173 aprint_error(", MAC ADDRESS READ timeout\n"); 4174 rv = -1; 4175 goto done; 4176 } 4177 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4178 aprint_error(", MAC ADDRESS READ error\n"); 4179 rv = -1; 4180 goto done; 4181 } 4182 4183 addrs = IXL_DMA_KVA(&idm); 4184 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 4185 printf(", port address is not valid\n"); 4186 goto done; 4187 } 4188 4189 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 4190 rv = 0; 4191 4192 done: 4193 ixl_dmamem_free(sc, &idm); 4194 return rv; 4195 } 4196 4197 static int 4198 ixl_get_switch_config(struct ixl_softc *sc) 4199 { 4200 struct ixl_dmamem idm; 4201 struct ixl_aq_desc iaq; 4202 struct ixl_aq_switch_config *hdr; 4203 struct ixl_aq_switch_config_element *elms, *elm; 4204 unsigned int nelm, i; 4205 int rv; 4206 4207 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4208 aprint_error_dev(sc->sc_dev, 4209 "unable to allocate switch config buffer\n"); 4210 return -1; 4211 } 4212 4213 memset(&iaq, 0, sizeof(iaq)); 4214 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4215 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4216 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 4217 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 4218 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4219 4220 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4221 BUS_DMASYNC_PREREAD); 4222 4223 rv = ixl_atq_poll(sc, &iaq, 250); 4224 4225 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4226 BUS_DMASYNC_POSTREAD); 4227 4228 if (rv != 0) { 4229 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 4230 rv = -1; 4231 goto done; 4232 } 4233 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4234 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 4235 rv = -1; 4236 goto done; 4237 } 4238 4239 hdr = IXL_DMA_KVA(&idm); 4240 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 4241 4242 nelm = le16toh(hdr->num_reported); 4243 if (nelm < 1) { 4244 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 4245 rv = -1; 4246 goto done; 4247 } 4248 4249 for (i = 0; i < nelm; i++) { 4250 elm = &elms[i]; 4251 4252 aprint_debug_dev(sc->sc_dev, 4253 "type %x revision %u seid %04x\n", 4254 elm->type, elm->revision, le16toh(elm->seid)); 4255 aprint_debug_dev(sc->sc_dev, 4256 "uplink %04x downlink %04x\n", 4257 le16toh(elm->uplink_seid), 4258 le16toh(elm->downlink_seid)); 4259 aprint_debug_dev(sc->sc_dev, 4260 "conntype %x scheduler %04x extra %04x\n", 4261 elm->connection_type, 4262 le16toh(elm->scheduler_id), 4263 le16toh(elm->element_info)); 4264 } 4265 4266 elm = &elms[0]; 4267 4268 sc->sc_uplink_seid = elm->uplink_seid; 4269 sc->sc_downlink_seid = elm->downlink_seid; 4270 sc->sc_seid = elm->seid; 4271 4272 if ((sc->sc_uplink_seid == htole16(0)) != 4273 (sc->sc_downlink_seid == htole16(0))) { 4274 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 4275 rv = -1; 4276 goto done; 4277 } 4278 4279 done: 4280 ixl_dmamem_free(sc, &idm); 4281 return rv; 4282 } 4283 4284 static int 4285 ixl_phy_mask_ints(struct ixl_softc *sc) 4286 { 4287 struct ixl_aq_desc iaq; 4288 4289 memset(&iaq, 0, sizeof(iaq)); 4290 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 4291 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 4292 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 4293 IXL_AQ_PHY_EV_MEDIA_NA)); 4294 4295 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4296 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 4297 return -1; 4298 } 4299 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4300 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 4301 return -1; 4302 } 4303 4304 return 0; 4305 } 4306 4307 static int 4308 ixl_get_phy_abilities(struct ixl_softc *sc, struct ixl_dmamem *idm) 4309 { 4310 struct ixl_aq_desc iaq; 4311 int rv; 4312 4313 memset(&iaq, 0, sizeof(iaq)); 4314 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4315 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4316 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 4317 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 4318 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 4319 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 4320 4321 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4322 BUS_DMASYNC_PREREAD); 4323 4324 rv = ixl_atq_poll(sc, &iaq, 250); 4325 4326 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4327 BUS_DMASYNC_POSTREAD); 4328 4329 if (rv != 0) 4330 return -1; 4331 4332 return le16toh(iaq.iaq_retval); 4333 } 4334 4335 static int 4336 ixl_get_phy_info(struct ixl_softc *sc) 4337 { 4338 struct ixl_dmamem idm; 4339 struct ixl_aq_phy_abilities *phy; 4340 int rv; 4341 4342 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4343 aprint_error_dev(sc->sc_dev, 4344 "unable to allocate phy abilities buffer\n"); 4345 return -1; 4346 } 4347 4348 rv = ixl_get_phy_abilities(sc, &idm); 4349 switch (rv) { 4350 case -1: 4351 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 4352 goto done; 4353 case IXL_AQ_RC_OK: 4354 break; 4355 case IXL_AQ_RC_EIO: 4356 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 4357 goto done; 4358 default: 4359 aprint_error_dev(sc->sc_dev, 4360 "GET PHY ABILITIIES error %u\n", rv); 4361 goto done; 4362 } 4363 4364 phy = IXL_DMA_KVA(&idm); 4365 4366 sc->sc_phy_types = le32toh(phy->phy_type); 4367 sc->sc_phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 4368 4369 sc->sc_phy_abilities = phy->abilities; 4370 sc->sc_phy_linkspeed = phy->link_speed; 4371 sc->sc_phy_fec_cfg = phy->fec_cfg_curr_mod_ext_info & 4372 (IXL_AQ_ENABLE_FEC_KR | IXL_AQ_ENABLE_FEC_RS | 4373 IXL_AQ_REQUEST_FEC_KR | IXL_AQ_REQUEST_FEC_RS); 4374 sc->sc_eee_cap = phy->eee_capability; 4375 sc->sc_eeer_val = phy->eeer_val; 4376 sc->sc_d3_lpan = phy->d3_lpan; 4377 4378 rv = 0; 4379 4380 done: 4381 ixl_dmamem_free(sc, &idm); 4382 return rv; 4383 } 4384 4385 static int 4386 ixl_set_phy_config(struct ixl_softc *sc, 4387 uint8_t link_speed, uint8_t abilities, bool polling) 4388 { 4389 struct ixl_aq_phy_param *param; 4390 struct ixl_atq iatq; 4391 struct ixl_aq_desc *iaq; 4392 int error; 4393 4394 memset(&iatq, 0, sizeof(iatq)); 4395 4396 iaq = &iatq.iatq_desc; 4397 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_CONFIG); 4398 param = (struct ixl_aq_phy_param *)&iaq->iaq_param; 4399 param->phy_types = htole32((uint32_t)sc->sc_phy_types); 4400 param->phy_type_ext = (uint8_t)(sc->sc_phy_types >> 32); 4401 param->link_speed = link_speed; 4402 param->abilities = abilities | IXL_AQ_PHY_ABILITY_AUTO_LINK; 4403 param->fec_cfg = sc->sc_phy_fec_cfg; 4404 param->eee_capability = sc->sc_eee_cap; 4405 param->eeer_val = sc->sc_eeer_val; 4406 param->d3_lpan = sc->sc_d3_lpan; 4407 4408 if (polling) 4409 error = ixl_atq_poll(sc, iaq, 250); 4410 else 4411 error = ixl_atq_exec(sc, &iatq); 4412 4413 if (error != 0) 4414 return error; 4415 4416 switch (le16toh(iaq->iaq_retval)) { 4417 case IXL_AQ_RC_OK: 4418 break; 4419 case IXL_AQ_RC_EPERM: 4420 return EPERM; 4421 default: 4422 return EIO; 4423 } 4424 4425 return 0; 4426 } 4427 4428 static int 4429 ixl_set_phy_autoselect(struct ixl_softc *sc) 4430 { 4431 uint8_t link_speed, abilities; 4432 4433 link_speed = sc->sc_phy_linkspeed; 4434 abilities = IXL_PHY_ABILITY_LINKUP | IXL_PHY_ABILITY_AUTONEGO; 4435 4436 return ixl_set_phy_config(sc, link_speed, abilities, true); 4437 } 4438 4439 static int 4440 ixl_get_link_status_poll(struct ixl_softc *sc, int *l) 4441 { 4442 struct ixl_aq_desc iaq; 4443 struct ixl_aq_link_param *param; 4444 int link; 4445 4446 memset(&iaq, 0, sizeof(iaq)); 4447 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 4448 param = (struct ixl_aq_link_param *)iaq.iaq_param; 4449 param->notify = IXL_AQ_LINK_NOTIFY; 4450 4451 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4452 return ETIMEDOUT; 4453 } 4454 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4455 return EIO; 4456 } 4457 4458 /* It is unneccessary to hold lock */ 4459 link = ixl_set_link_status_locked(sc, &iaq); 4460 4461 if (l != NULL) 4462 *l = link; 4463 4464 return 0; 4465 } 4466 4467 static int 4468 ixl_get_vsi(struct ixl_softc *sc) 4469 { 4470 struct ixl_dmamem *vsi = &sc->sc_scratch; 4471 struct ixl_aq_desc iaq; 4472 struct ixl_aq_vsi_param *param; 4473 struct ixl_aq_vsi_reply *reply; 4474 struct ixl_aq_vsi_data *data; 4475 int rv; 4476 4477 /* grumble, vsi info isn't "known" at compile time */ 4478 4479 memset(&iaq, 0, sizeof(iaq)); 4480 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4481 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4482 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 4483 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4484 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4485 4486 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4487 param->uplink_seid = sc->sc_seid; 4488 4489 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4490 BUS_DMASYNC_PREREAD); 4491 4492 rv = ixl_atq_poll(sc, &iaq, 250); 4493 4494 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4495 BUS_DMASYNC_POSTREAD); 4496 4497 if (rv != 0) { 4498 return ETIMEDOUT; 4499 } 4500 4501 switch (le16toh(iaq.iaq_retval)) { 4502 case IXL_AQ_RC_OK: 4503 break; 4504 case IXL_AQ_RC_ENOENT: 4505 return ENOENT; 4506 case IXL_AQ_RC_EACCES: 4507 return EACCES; 4508 default: 4509 return EIO; 4510 } 4511 4512 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 4513 sc->sc_vsi_number = le16toh(reply->vsi_number); 4514 data = IXL_DMA_KVA(vsi); 4515 sc->sc_vsi_stat_counter_idx = le16toh(data->stat_counter_idx); 4516 4517 return 0; 4518 } 4519 4520 static int 4521 ixl_set_vsi(struct ixl_softc *sc) 4522 { 4523 struct ixl_dmamem *vsi = &sc->sc_scratch; 4524 struct ixl_aq_desc iaq; 4525 struct ixl_aq_vsi_param *param; 4526 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 4527 unsigned int qnum; 4528 uint16_t val; 4529 int rv; 4530 4531 qnum = sc->sc_nqueue_pairs - 1; 4532 4533 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 4534 IXL_AQ_VSI_VALID_VLAN); 4535 4536 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 4537 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 4538 data->queue_mapping[0] = htole16(0); 4539 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 4540 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 4541 4542 val = le16toh(data->port_vlan_flags); 4543 CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK); 4544 SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL); 4545 4546 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { 4547 SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH); 4548 } else { 4549 SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING); 4550 } 4551 4552 data->port_vlan_flags = htole16(val); 4553 4554 /* grumble, vsi info isn't "known" at compile time */ 4555 4556 memset(&iaq, 0, sizeof(iaq)); 4557 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4558 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4559 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 4560 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4561 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4562 4563 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4564 param->uplink_seid = sc->sc_seid; 4565 4566 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4567 BUS_DMASYNC_PREWRITE); 4568 4569 rv = ixl_atq_poll(sc, &iaq, 250); 4570 4571 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4572 BUS_DMASYNC_POSTWRITE); 4573 4574 if (rv != 0) { 4575 return ETIMEDOUT; 4576 } 4577 4578 switch (le16toh(iaq.iaq_retval)) { 4579 case IXL_AQ_RC_OK: 4580 break; 4581 case IXL_AQ_RC_ENOENT: 4582 return ENOENT; 4583 case IXL_AQ_RC_EACCES: 4584 return EACCES; 4585 default: 4586 return EIO; 4587 } 4588 4589 return 0; 4590 } 4591 4592 static void 4593 ixl_set_filter_control(struct ixl_softc *sc) 4594 { 4595 uint32_t reg; 4596 4597 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4598 4599 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4600 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4601 4602 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4603 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4604 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4605 4606 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4607 } 4608 4609 static inline void 4610 ixl_get_default_rss_key(uint32_t *buf, size_t len) 4611 { 4612 size_t cplen; 4613 uint8_t rss_seed[RSS_KEYSIZE]; 4614 4615 rss_getkey(rss_seed); 4616 memset(buf, 0, len); 4617 4618 cplen = MIN(len, sizeof(rss_seed)); 4619 memcpy(buf, rss_seed, cplen); 4620 } 4621 4622 static int 4623 ixl_set_rss_key(struct ixl_softc *sc, uint8_t *key, size_t keylen) 4624 { 4625 struct ixl_dmamem *idm; 4626 struct ixl_atq iatq; 4627 struct ixl_aq_desc *iaq; 4628 struct ixl_aq_rss_key_param *param; 4629 struct ixl_aq_rss_key_data *data; 4630 size_t len, datalen, stdlen, extlen; 4631 uint16_t vsi_id; 4632 int rv; 4633 4634 memset(&iatq, 0, sizeof(iatq)); 4635 iaq = &iatq.iatq_desc; 4636 idm = &sc->sc_aqbuf; 4637 4638 datalen = sizeof(*data); 4639 4640 /*XXX The buf size has to be less than the size of the register */ 4641 datalen = MIN(IXL_RSS_KEY_SIZE_REG * sizeof(uint32_t), datalen); 4642 4643 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4644 (datalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4645 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_KEY); 4646 iaq->iaq_datalen = htole16(datalen); 4647 4648 param = (struct ixl_aq_rss_key_param *)iaq->iaq_param; 4649 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSKEY_VSI_ID_SHIFT) | 4650 IXL_AQ_RSSKEY_VSI_VALID; 4651 param->vsi_id = htole16(vsi_id); 4652 4653 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4654 data = IXL_DMA_KVA(idm); 4655 4656 len = MIN(keylen, datalen); 4657 stdlen = MIN(sizeof(data->standard_rss_key), len); 4658 memcpy(data->standard_rss_key, key, stdlen); 4659 len = (len > stdlen) ? (len - stdlen) : 0; 4660 4661 extlen = MIN(sizeof(data->extended_hash_key), len); 4662 extlen = (stdlen < keylen) ? 0 : keylen - stdlen; 4663 memcpy(data->extended_hash_key, key + stdlen, extlen); 4664 4665 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4666 4667 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4668 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4669 4670 rv = ixl_atq_exec(sc, &iatq); 4671 4672 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4673 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4674 4675 if (rv != 0) { 4676 return ETIMEDOUT; 4677 } 4678 4679 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4680 return EIO; 4681 } 4682 4683 return 0; 4684 } 4685 4686 static int 4687 ixl_set_rss_lut(struct ixl_softc *sc, uint8_t *lut, size_t lutlen) 4688 { 4689 struct ixl_dmamem *idm; 4690 struct ixl_atq iatq; 4691 struct ixl_aq_desc *iaq; 4692 struct ixl_aq_rss_lut_param *param; 4693 uint16_t vsi_id; 4694 uint8_t *data; 4695 size_t dmalen; 4696 int rv; 4697 4698 memset(&iatq, 0, sizeof(iatq)); 4699 iaq = &iatq.iatq_desc; 4700 idm = &sc->sc_aqbuf; 4701 4702 dmalen = MIN(lutlen, IXL_DMA_LEN(idm)); 4703 4704 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4705 (dmalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4706 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_LUT); 4707 iaq->iaq_datalen = htole16(dmalen); 4708 4709 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4710 data = IXL_DMA_KVA(idm); 4711 memcpy(data, lut, dmalen); 4712 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4713 4714 param = (struct ixl_aq_rss_lut_param *)iaq->iaq_param; 4715 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSLUT_VSI_ID_SHIFT) | 4716 IXL_AQ_RSSLUT_VSI_VALID; 4717 param->vsi_id = htole16(vsi_id); 4718 param->flags = htole16(IXL_AQ_RSSLUT_TABLE_TYPE_PF << 4719 IXL_AQ_RSSLUT_TABLE_TYPE_SHIFT); 4720 4721 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4722 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4723 4724 rv = ixl_atq_exec(sc, &iatq); 4725 4726 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4727 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4728 4729 if (rv != 0) { 4730 return ETIMEDOUT; 4731 } 4732 4733 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4734 return EIO; 4735 } 4736 4737 return 0; 4738 } 4739 4740 static int 4741 ixl_register_rss_key(struct ixl_softc *sc) 4742 { 4743 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4744 int rv; 4745 size_t i; 4746 4747 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4748 4749 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4750 rv = ixl_set_rss_key(sc, (uint8_t*)rss_seed, 4751 sizeof(rss_seed)); 4752 } else { 4753 rv = 0; 4754 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4755 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4756 } 4757 } 4758 4759 return rv; 4760 } 4761 4762 static void 4763 ixl_register_rss_pctype(struct ixl_softc *sc) 4764 { 4765 uint64_t set_hena = 0; 4766 uint32_t hena0, hena1; 4767 4768 /* 4769 * We use TCP/UDP with IPv4/IPv6 by default. 4770 * Note: the device can not use just IP header in each 4771 * TCP/UDP packets for the RSS hash calculation. 4772 */ 4773 if (sc->sc_mac_type == I40E_MAC_X722) 4774 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4775 else 4776 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4777 4778 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4779 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4780 4781 SET(hena0, set_hena); 4782 SET(hena1, set_hena >> 32); 4783 4784 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4785 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4786 } 4787 4788 static int 4789 ixl_register_rss_hlut(struct ixl_softc *sc) 4790 { 4791 unsigned int qid; 4792 uint8_t hlut_buf[512], lut_mask; 4793 uint32_t *hluts; 4794 size_t i, hluts_num; 4795 int rv; 4796 4797 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4798 4799 for (i = 0; i < sc->sc_rss_table_size; i++) { 4800 qid = i % sc->sc_nqueue_pairs; 4801 hlut_buf[i] = qid & lut_mask; 4802 } 4803 4804 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4805 rv = ixl_set_rss_lut(sc, hlut_buf, sizeof(hlut_buf)); 4806 } else { 4807 rv = 0; 4808 hluts = (uint32_t *)hlut_buf; 4809 hluts_num = sc->sc_rss_table_size >> 2; 4810 for (i = 0; i < hluts_num; i++) { 4811 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4812 } 4813 ixl_flush(sc); 4814 } 4815 4816 return rv; 4817 } 4818 4819 static void 4820 ixl_config_rss(struct ixl_softc *sc) 4821 { 4822 4823 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4824 4825 ixl_register_rss_key(sc); 4826 ixl_register_rss_pctype(sc); 4827 ixl_register_rss_hlut(sc); 4828 } 4829 4830 static const struct ixl_phy_type * 4831 ixl_search_phy_type(uint8_t phy_type) 4832 { 4833 const struct ixl_phy_type *itype; 4834 uint64_t mask; 4835 unsigned int i; 4836 4837 if (phy_type >= 64) 4838 return NULL; 4839 4840 mask = 1ULL << phy_type; 4841 4842 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4843 itype = &ixl_phy_type_map[i]; 4844 4845 if (ISSET(itype->phy_type, mask)) 4846 return itype; 4847 } 4848 4849 return NULL; 4850 } 4851 4852 static uint64_t 4853 ixl_search_link_speed(uint8_t link_speed) 4854 { 4855 const struct ixl_speed_type *type; 4856 unsigned int i; 4857 4858 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4859 type = &ixl_speed_type_map[i]; 4860 4861 if (ISSET(type->dev_speed, link_speed)) 4862 return type->net_speed; 4863 } 4864 4865 return 0; 4866 } 4867 4868 static uint8_t 4869 ixl_search_baudrate(uint64_t baudrate) 4870 { 4871 const struct ixl_speed_type *type; 4872 unsigned int i; 4873 4874 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4875 type = &ixl_speed_type_map[i]; 4876 4877 if (type->net_speed == baudrate) { 4878 return type->dev_speed; 4879 } 4880 } 4881 4882 return 0; 4883 } 4884 4885 static int 4886 ixl_restart_an(struct ixl_softc *sc) 4887 { 4888 struct ixl_aq_desc iaq; 4889 4890 memset(&iaq, 0, sizeof(iaq)); 4891 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4892 iaq.iaq_param[0] = 4893 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4894 4895 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4896 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4897 return -1; 4898 } 4899 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4900 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4901 return -1; 4902 } 4903 4904 return 0; 4905 } 4906 4907 static int 4908 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4909 uint16_t vlan, uint16_t flags) 4910 { 4911 struct ixl_aq_desc iaq; 4912 struct ixl_aq_add_macvlan *param; 4913 struct ixl_aq_add_macvlan_elem *elem; 4914 4915 memset(&iaq, 0, sizeof(iaq)); 4916 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4917 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4918 iaq.iaq_datalen = htole16(sizeof(*elem)); 4919 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4920 4921 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4922 param->num_addrs = htole16(1); 4923 param->seid0 = htole16(0x8000) | sc->sc_seid; 4924 param->seid1 = 0; 4925 param->seid2 = 0; 4926 4927 elem = IXL_DMA_KVA(&sc->sc_scratch); 4928 memset(elem, 0, sizeof(*elem)); 4929 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4930 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4931 elem->vlan = htole16(vlan); 4932 4933 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4934 return IXL_AQ_RC_EINVAL; 4935 } 4936 4937 switch (le16toh(iaq.iaq_retval)) { 4938 case IXL_AQ_RC_OK: 4939 break; 4940 case IXL_AQ_RC_ENOSPC: 4941 return ENOSPC; 4942 case IXL_AQ_RC_ENOENT: 4943 return ENOENT; 4944 case IXL_AQ_RC_EACCES: 4945 return EACCES; 4946 case IXL_AQ_RC_EEXIST: 4947 return EEXIST; 4948 case IXL_AQ_RC_EINVAL: 4949 return EINVAL; 4950 default: 4951 return EIO; 4952 } 4953 4954 return 0; 4955 } 4956 4957 static int 4958 ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4959 uint16_t vlan, uint16_t flags) 4960 { 4961 struct ixl_aq_desc iaq; 4962 struct ixl_aq_remove_macvlan *param; 4963 struct ixl_aq_remove_macvlan_elem *elem; 4964 4965 memset(&iaq, 0, sizeof(iaq)); 4966 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4967 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 4968 iaq.iaq_datalen = htole16(sizeof(*elem)); 4969 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4970 4971 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 4972 param->num_addrs = htole16(1); 4973 param->seid0 = htole16(0x8000) | sc->sc_seid; 4974 param->seid1 = 0; 4975 param->seid2 = 0; 4976 4977 elem = IXL_DMA_KVA(&sc->sc_scratch); 4978 memset(elem, 0, sizeof(*elem)); 4979 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4980 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 4981 elem->vlan = htole16(vlan); 4982 4983 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4984 return EINVAL; 4985 } 4986 4987 switch (le16toh(iaq.iaq_retval)) { 4988 case IXL_AQ_RC_OK: 4989 break; 4990 case IXL_AQ_RC_ENOENT: 4991 return ENOENT; 4992 case IXL_AQ_RC_EACCES: 4993 return EACCES; 4994 case IXL_AQ_RC_EINVAL: 4995 return EINVAL; 4996 default: 4997 return EIO; 4998 } 4999 5000 return 0; 5001 } 5002 5003 static int 5004 ixl_hmc(struct ixl_softc *sc) 5005 { 5006 struct { 5007 uint32_t count; 5008 uint32_t minsize; 5009 bus_size_t objsiz; 5010 bus_size_t setoff; 5011 bus_size_t setcnt; 5012 } regs[] = { 5013 { 5014 0, 5015 IXL_HMC_TXQ_MINSIZE, 5016 I40E_GLHMC_LANTXOBJSZ, 5017 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 5018 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 5019 }, 5020 { 5021 0, 5022 IXL_HMC_RXQ_MINSIZE, 5023 I40E_GLHMC_LANRXOBJSZ, 5024 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 5025 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 5026 }, 5027 { 5028 0, 5029 0, 5030 I40E_GLHMC_FCOEDDPOBJSZ, 5031 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 5032 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 5033 }, 5034 { 5035 0, 5036 0, 5037 I40E_GLHMC_FCOEFOBJSZ, 5038 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 5039 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 5040 }, 5041 }; 5042 struct ixl_hmc_entry *e; 5043 uint64_t size, dva; 5044 uint8_t *kva; 5045 uint64_t *sdpage; 5046 unsigned int i; 5047 int npages, tables; 5048 uint32_t reg; 5049 5050 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 5051 5052 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 5053 ixl_rd(sc, I40E_GLHMC_LANQMAX); 5054 5055 size = 0; 5056 for (i = 0; i < __arraycount(regs); i++) { 5057 e = &sc->sc_hmc_entries[i]; 5058 5059 e->hmc_count = regs[i].count; 5060 reg = ixl_rd(sc, regs[i].objsiz); 5061 e->hmc_size = IXL_BIT_ULL(0x3F & reg); 5062 e->hmc_base = size; 5063 5064 if ((e->hmc_size * 8) < regs[i].minsize) { 5065 aprint_error_dev(sc->sc_dev, 5066 "kernel hmc entry is too big\n"); 5067 return -1; 5068 } 5069 5070 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 5071 } 5072 size = roundup(size, IXL_HMC_PGSIZE); 5073 npages = size / IXL_HMC_PGSIZE; 5074 5075 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 5076 5077 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 5078 aprint_error_dev(sc->sc_dev, 5079 "unable to allocate hmc pd memory\n"); 5080 return -1; 5081 } 5082 5083 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 5084 IXL_HMC_PGSIZE) != 0) { 5085 aprint_error_dev(sc->sc_dev, 5086 "unable to allocate hmc sd memory\n"); 5087 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5088 return -1; 5089 } 5090 5091 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 5092 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 5093 5094 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 5095 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 5096 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5097 5098 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 5099 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 5100 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 5101 5102 for (i = 0; (int)i < npages; i++) { 5103 *sdpage = htole64(dva | IXL_HMC_PDVALID); 5104 sdpage++; 5105 5106 dva += IXL_HMC_PGSIZE; 5107 } 5108 5109 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 5110 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 5111 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5112 5113 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 5114 for (i = 0; (int)i < tables; i++) { 5115 uint32_t count; 5116 5117 KASSERT(npages >= 0); 5118 5119 count = ((unsigned int)npages > IXL_HMC_PGS) ? 5120 IXL_HMC_PGS : (unsigned int)npages; 5121 5122 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 5123 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 5124 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 5125 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 5126 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 5127 ixl_wr(sc, I40E_PFHMC_SDCMD, 5128 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 5129 5130 npages -= IXL_HMC_PGS; 5131 dva += IXL_HMC_PGSIZE; 5132 } 5133 5134 for (i = 0; i < __arraycount(regs); i++) { 5135 e = &sc->sc_hmc_entries[i]; 5136 5137 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 5138 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 5139 } 5140 5141 return 0; 5142 } 5143 5144 static void 5145 ixl_hmc_free(struct ixl_softc *sc) 5146 { 5147 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 5148 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5149 } 5150 5151 static void 5152 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 5153 unsigned int npacking) 5154 { 5155 uint8_t *dst = d; 5156 const uint8_t *src = s; 5157 unsigned int i; 5158 5159 for (i = 0; i < npacking; i++) { 5160 const struct ixl_hmc_pack *pack = &packing[i]; 5161 unsigned int offset = pack->lsb / 8; 5162 unsigned int align = pack->lsb % 8; 5163 const uint8_t *in = src + pack->offset; 5164 uint8_t *out = dst + offset; 5165 int width = pack->width; 5166 unsigned int inbits = 0; 5167 5168 if (align) { 5169 inbits = (*in++) << align; 5170 *out++ |= (inbits & 0xff); 5171 inbits >>= 8; 5172 5173 width -= 8 - align; 5174 } 5175 5176 while (width >= 8) { 5177 inbits |= (*in++) << align; 5178 *out++ = (inbits & 0xff); 5179 inbits >>= 8; 5180 5181 width -= 8; 5182 } 5183 5184 if (width > 0) { 5185 inbits |= (*in) << align; 5186 *out |= (inbits & ((1 << width) - 1)); 5187 } 5188 } 5189 } 5190 5191 static struct ixl_aq_buf * 5192 ixl_aqb_alloc(struct ixl_softc *sc) 5193 { 5194 struct ixl_aq_buf *aqb; 5195 5196 aqb = kmem_alloc(sizeof(*aqb), KM_SLEEP); 5197 5198 aqb->aqb_size = IXL_AQ_BUFLEN; 5199 5200 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 5201 aqb->aqb_size, 0, 5202 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 5203 goto free; 5204 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 5205 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 5206 BUS_DMA_WAITOK) != 0) 5207 goto destroy; 5208 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 5209 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 5210 goto dma_free; 5211 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 5212 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 5213 goto unmap; 5214 5215 return aqb; 5216 unmap: 5217 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5218 dma_free: 5219 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5220 destroy: 5221 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5222 free: 5223 kmem_free(aqb, sizeof(*aqb)); 5224 5225 return NULL; 5226 } 5227 5228 static void 5229 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 5230 { 5231 5232 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 5233 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5234 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5235 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5236 kmem_free(aqb, sizeof(*aqb)); 5237 } 5238 5239 static int 5240 ixl_arq_fill(struct ixl_softc *sc) 5241 { 5242 struct ixl_aq_buf *aqb; 5243 struct ixl_aq_desc *arq, *iaq; 5244 unsigned int prod = sc->sc_arq_prod; 5245 unsigned int n; 5246 int post = 0; 5247 5248 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 5249 IXL_AQ_NUM); 5250 arq = IXL_DMA_KVA(&sc->sc_arq); 5251 5252 if (__predict_false(n <= 0)) 5253 return 0; 5254 5255 do { 5256 aqb = sc->sc_arq_live[prod]; 5257 iaq = &arq[prod]; 5258 5259 if (aqb == NULL) { 5260 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 5261 if (aqb != NULL) { 5262 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5263 ixl_aq_buf, aqb_entry); 5264 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 5265 break; 5266 } 5267 5268 sc->sc_arq_live[prod] = aqb; 5269 memset(aqb->aqb_data, 0, aqb->aqb_size); 5270 5271 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 5272 aqb->aqb_size, BUS_DMASYNC_PREREAD); 5273 5274 iaq->iaq_flags = htole16(IXL_AQ_BUF | 5275 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 5276 IXL_AQ_LB : 0)); 5277 iaq->iaq_opcode = 0; 5278 iaq->iaq_datalen = htole16(aqb->aqb_size); 5279 iaq->iaq_retval = 0; 5280 iaq->iaq_cookie = 0; 5281 iaq->iaq_param[0] = 0; 5282 iaq->iaq_param[1] = 0; 5283 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 5284 } 5285 5286 prod++; 5287 prod &= IXL_AQ_MASK; 5288 5289 post = 1; 5290 5291 } while (--n); 5292 5293 if (post) { 5294 sc->sc_arq_prod = prod; 5295 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 5296 } 5297 5298 return post; 5299 } 5300 5301 static void 5302 ixl_arq_unfill(struct ixl_softc *sc) 5303 { 5304 struct ixl_aq_buf *aqb; 5305 unsigned int i; 5306 5307 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 5308 aqb = sc->sc_arq_live[i]; 5309 if (aqb == NULL) 5310 continue; 5311 5312 sc->sc_arq_live[i] = NULL; 5313 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 5314 BUS_DMASYNC_POSTREAD); 5315 ixl_aqb_free(sc, aqb); 5316 } 5317 5318 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 5319 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5320 ixl_aq_buf, aqb_entry); 5321 ixl_aqb_free(sc, aqb); 5322 } 5323 } 5324 5325 static void 5326 ixl_clear_hw(struct ixl_softc *sc) 5327 { 5328 uint32_t num_queues, base_queue; 5329 uint32_t num_pf_int; 5330 uint32_t num_vf_int; 5331 uint32_t num_vfs; 5332 uint32_t i, j; 5333 uint32_t val; 5334 uint32_t eol = 0x7ff; 5335 5336 /* get number of interrupts, queues, and vfs */ 5337 val = ixl_rd(sc, I40E_GLPCI_CNF2); 5338 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 5339 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 5340 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 5341 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 5342 5343 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 5344 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 5345 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 5346 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 5347 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 5348 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 5349 num_queues = (j - base_queue) + 1; 5350 else 5351 num_queues = 0; 5352 5353 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 5354 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 5355 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 5356 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 5357 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 5358 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 5359 num_vfs = (j - i) + 1; 5360 else 5361 num_vfs = 0; 5362 5363 /* stop all the interrupts */ 5364 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5365 ixl_flush(sc); 5366 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 5367 for (i = 0; i < num_pf_int - 2; i++) 5368 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 5369 ixl_flush(sc); 5370 5371 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 5372 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5373 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 5374 for (i = 0; i < num_pf_int - 2; i++) 5375 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 5376 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5377 for (i = 0; i < num_vfs; i++) 5378 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 5379 for (i = 0; i < num_vf_int - 2; i++) 5380 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 5381 5382 /* warn the HW of the coming Tx disables */ 5383 for (i = 0; i < num_queues; i++) { 5384 uint32_t abs_queue_idx = base_queue + i; 5385 uint32_t reg_block = 0; 5386 5387 if (abs_queue_idx >= 128) { 5388 reg_block = abs_queue_idx / 128; 5389 abs_queue_idx %= 128; 5390 } 5391 5392 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 5393 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 5394 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 5395 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 5396 5397 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 5398 } 5399 delaymsec(400); 5400 5401 /* stop all the queues */ 5402 for (i = 0; i < num_queues; i++) { 5403 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 5404 ixl_wr(sc, I40E_QTX_ENA(i), 0); 5405 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 5406 ixl_wr(sc, I40E_QRX_ENA(i), 0); 5407 } 5408 5409 /* short wait for all queue disables to settle */ 5410 delaymsec(50); 5411 } 5412 5413 static int 5414 ixl_pf_reset(struct ixl_softc *sc) 5415 { 5416 uint32_t cnt = 0; 5417 uint32_t cnt1 = 0; 5418 uint32_t reg = 0, reg0 = 0; 5419 uint32_t grst_del; 5420 5421 /* 5422 * Poll for Global Reset steady state in case of recent GRST. 5423 * The grst delay value is in 100ms units, and we'll wait a 5424 * couple counts longer to be sure we don't just miss the end. 5425 */ 5426 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 5427 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 5428 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 5429 5430 grst_del = grst_del * 20; 5431 5432 for (cnt = 0; cnt < grst_del; cnt++) { 5433 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 5434 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 5435 break; 5436 delaymsec(100); 5437 } 5438 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5439 aprint_error(", Global reset polling failed to complete\n"); 5440 return -1; 5441 } 5442 5443 /* Now Wait for the FW to be ready */ 5444 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 5445 reg = ixl_rd(sc, I40E_GLNVM_ULD); 5446 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5447 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 5448 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5449 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 5450 break; 5451 5452 delaymsec(10); 5453 } 5454 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5455 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 5456 aprint_error(", wait for FW Reset complete timed out " 5457 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 5458 return -1; 5459 } 5460 5461 /* 5462 * If there was a Global Reset in progress when we got here, 5463 * we don't need to do the PF Reset 5464 */ 5465 if (cnt == 0) { 5466 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5467 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 5468 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 5469 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5470 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 5471 break; 5472 delaymsec(1); 5473 5474 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 5475 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5476 aprint_error(", Core reset upcoming." 5477 " Skipping PF reset reset request\n"); 5478 return -1; 5479 } 5480 } 5481 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 5482 aprint_error(", PF reset polling failed to complete" 5483 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 5484 return -1; 5485 } 5486 } 5487 5488 return 0; 5489 } 5490 5491 static int 5492 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 5493 bus_size_t size, bus_size_t align) 5494 { 5495 ixm->ixm_size = size; 5496 5497 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 5498 ixm->ixm_size, 0, 5499 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 5500 &ixm->ixm_map) != 0) 5501 return 1; 5502 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 5503 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 5504 BUS_DMA_WAITOK) != 0) 5505 goto destroy; 5506 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 5507 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 5508 goto free; 5509 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 5510 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 5511 goto unmap; 5512 5513 memset(ixm->ixm_kva, 0, ixm->ixm_size); 5514 5515 return 0; 5516 unmap: 5517 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5518 free: 5519 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5520 destroy: 5521 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5522 return 1; 5523 } 5524 5525 static void 5526 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 5527 { 5528 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 5529 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5530 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5531 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5532 } 5533 5534 static int 5535 ixl_setup_vlan_hwfilter(struct ixl_softc *sc) 5536 { 5537 struct ethercom *ec = &sc->sc_ec; 5538 struct vlanid_list *vlanidp; 5539 int rv; 5540 5541 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5542 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5543 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5544 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5545 5546 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5547 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5548 if (rv != 0) 5549 return rv; 5550 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5551 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5552 if (rv != 0) 5553 return rv; 5554 5555 ETHER_LOCK(ec); 5556 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5557 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 5558 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5559 if (rv != 0) 5560 break; 5561 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 5562 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5563 if (rv != 0) 5564 break; 5565 } 5566 ETHER_UNLOCK(ec); 5567 5568 return rv; 5569 } 5570 5571 static void 5572 ixl_teardown_vlan_hwfilter(struct ixl_softc *sc) 5573 { 5574 struct vlanid_list *vlanidp; 5575 struct ethercom *ec = &sc->sc_ec; 5576 5577 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5578 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5579 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5580 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5581 5582 ETHER_LOCK(ec); 5583 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5584 ixl_remove_macvlan(sc, sc->sc_enaddr, 5585 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5586 ixl_remove_macvlan(sc, etherbroadcastaddr, 5587 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5588 } 5589 ETHER_UNLOCK(ec); 5590 5591 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5592 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5593 ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5594 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5595 } 5596 5597 static int 5598 ixl_update_macvlan(struct ixl_softc *sc) 5599 { 5600 int rv = 0; 5601 int next_ec_capenable = sc->sc_ec.ec_capenable; 5602 5603 if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 5604 rv = ixl_setup_vlan_hwfilter(sc); 5605 if (rv != 0) 5606 ixl_teardown_vlan_hwfilter(sc); 5607 } else { 5608 ixl_teardown_vlan_hwfilter(sc); 5609 } 5610 5611 return rv; 5612 } 5613 5614 static int 5615 ixl_ifflags_cb(struct ethercom *ec) 5616 { 5617 struct ifnet *ifp = &ec->ec_if; 5618 struct ixl_softc *sc = ifp->if_softc; 5619 int rv, change; 5620 5621 mutex_enter(&sc->sc_cfg_lock); 5622 5623 change = ec->ec_capenable ^ sc->sc_cur_ec_capenable; 5624 5625 if (ISSET(change, ETHERCAP_VLAN_HWTAGGING)) { 5626 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 5627 rv = ENETRESET; 5628 goto out; 5629 } 5630 5631 if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) { 5632 rv = ixl_update_macvlan(sc); 5633 if (rv == 0) { 5634 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 5635 } else { 5636 CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER); 5637 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 5638 } 5639 } 5640 5641 rv = ixl_iff(sc); 5642 out: 5643 mutex_exit(&sc->sc_cfg_lock); 5644 5645 return rv; 5646 } 5647 5648 static int 5649 ixl_set_link_status_locked(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 5650 { 5651 const struct ixl_aq_link_status *status; 5652 const struct ixl_phy_type *itype; 5653 5654 uint64_t ifm_active = IFM_ETHER; 5655 uint64_t ifm_status = IFM_AVALID; 5656 int link_state = LINK_STATE_DOWN; 5657 uint64_t baudrate = 0; 5658 5659 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 5660 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) { 5661 ifm_active |= IFM_NONE; 5662 goto done; 5663 } 5664 5665 ifm_active |= IFM_FDX; 5666 ifm_status |= IFM_ACTIVE; 5667 link_state = LINK_STATE_UP; 5668 5669 itype = ixl_search_phy_type(status->phy_type); 5670 if (itype != NULL) 5671 ifm_active |= itype->ifm_type; 5672 5673 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 5674 ifm_active |= IFM_ETH_TXPAUSE; 5675 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 5676 ifm_active |= IFM_ETH_RXPAUSE; 5677 5678 baudrate = ixl_search_link_speed(status->link_speed); 5679 5680 done: 5681 /* sc->sc_cfg_lock held expect during attach */ 5682 sc->sc_media_active = ifm_active; 5683 sc->sc_media_status = ifm_status; 5684 5685 sc->sc_ec.ec_if.if_baudrate = baudrate; 5686 5687 return link_state; 5688 } 5689 5690 static int 5691 ixl_establish_intx(struct ixl_softc *sc) 5692 { 5693 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5694 pci_intr_handle_t *intr; 5695 char xnamebuf[32]; 5696 char intrbuf[PCI_INTRSTR_LEN]; 5697 char const *intrstr; 5698 5699 KASSERT(sc->sc_nintrs == 1); 5700 5701 intr = &sc->sc_ihp[0]; 5702 5703 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 5704 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 5705 device_xname(sc->sc_dev)); 5706 5707 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 5708 sc, xnamebuf); 5709 5710 if (sc->sc_ihs[0] == NULL) { 5711 aprint_error_dev(sc->sc_dev, 5712 "unable to establish interrupt at %s\n", intrstr); 5713 return -1; 5714 } 5715 5716 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 5717 return 0; 5718 } 5719 5720 static int 5721 ixl_establish_msix(struct ixl_softc *sc) 5722 { 5723 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5724 kcpuset_t *affinity; 5725 unsigned int vector = 0; 5726 unsigned int i; 5727 int affinity_to, r; 5728 char xnamebuf[32]; 5729 char intrbuf[PCI_INTRSTR_LEN]; 5730 char const *intrstr; 5731 5732 kcpuset_create(&affinity, false); 5733 5734 /* the "other" intr is mapped to vector 0 */ 5735 vector = 0; 5736 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5737 intrbuf, sizeof(intrbuf)); 5738 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 5739 device_xname(sc->sc_dev)); 5740 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5741 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 5742 sc, xnamebuf); 5743 if (sc->sc_ihs[vector] == NULL) { 5744 aprint_error_dev(sc->sc_dev, 5745 "unable to establish interrupt at %s\n", intrstr); 5746 goto fail; 5747 } 5748 5749 aprint_normal_dev(sc->sc_dev, "other interrupt at %s", intrstr); 5750 5751 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5752 affinity_to = (affinity_to + sc->sc_nqueue_pairs_max) % ncpu; 5753 5754 kcpuset_zero(affinity); 5755 kcpuset_set(affinity, affinity_to); 5756 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5757 if (r == 0) { 5758 aprint_normal(", affinity to %u", affinity_to); 5759 } 5760 aprint_normal("\n"); 5761 vector++; 5762 5763 sc->sc_msix_vector_queue = vector; 5764 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5765 5766 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5767 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5768 intrbuf, sizeof(intrbuf)); 5769 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 5770 device_xname(sc->sc_dev), i); 5771 5772 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5773 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 5774 (void *)&sc->sc_qps[i], xnamebuf); 5775 5776 if (sc->sc_ihs[vector] == NULL) { 5777 aprint_error_dev(sc->sc_dev, 5778 "unable to establish interrupt at %s\n", intrstr); 5779 goto fail; 5780 } 5781 5782 aprint_normal_dev(sc->sc_dev, 5783 "for TXRX%d interrupt at %s", i, intrstr); 5784 5785 kcpuset_zero(affinity); 5786 kcpuset_set(affinity, affinity_to); 5787 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5788 if (r == 0) { 5789 aprint_normal(", affinity to %u", affinity_to); 5790 affinity_to = (affinity_to + 1) % ncpu; 5791 } 5792 aprint_normal("\n"); 5793 vector++; 5794 } 5795 5796 kcpuset_destroy(affinity); 5797 5798 return 0; 5799 fail: 5800 for (i = 0; i < vector; i++) { 5801 pci_intr_disestablish(pc, sc->sc_ihs[i]); 5802 } 5803 5804 sc->sc_msix_vector_queue = 0; 5805 sc->sc_msix_vector_queue = 0; 5806 kcpuset_destroy(affinity); 5807 5808 return -1; 5809 } 5810 5811 static void 5812 ixl_config_queue_intr(struct ixl_softc *sc) 5813 { 5814 unsigned int i, vector; 5815 5816 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5817 vector = sc->sc_msix_vector_queue; 5818 } else { 5819 vector = I40E_INTR_NOTX_INTR; 5820 5821 ixl_wr(sc, I40E_PFINT_LNKLST0, 5822 (I40E_INTR_NOTX_QUEUE << 5823 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5824 (I40E_QUEUE_TYPE_RX << 5825 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5826 } 5827 5828 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5829 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5830 ixl_flush(sc); 5831 5832 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5833 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5834 (I40E_QUEUE_TYPE_RX << 5835 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5836 5837 ixl_wr(sc, I40E_QINT_RQCTL(i), 5838 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5839 (I40E_ITR_INDEX_RX << 5840 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5841 (I40E_INTR_NOTX_RX_QUEUE << 5842 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5843 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5844 (I40E_QUEUE_TYPE_TX << 5845 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5846 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5847 5848 ixl_wr(sc, I40E_QINT_TQCTL(i), 5849 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5850 (I40E_ITR_INDEX_TX << 5851 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5852 (I40E_INTR_NOTX_TX_QUEUE << 5853 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5854 (I40E_QUEUE_TYPE_EOL << 5855 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5856 (I40E_QUEUE_TYPE_RX << 5857 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5858 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5859 5860 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5861 ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_RX, i), 5862 sc->sc_itr_rx); 5863 ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_TX, i), 5864 sc->sc_itr_tx); 5865 vector++; 5866 } 5867 } 5868 ixl_flush(sc); 5869 5870 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), sc->sc_itr_rx); 5871 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), sc->sc_itr_tx); 5872 ixl_flush(sc); 5873 } 5874 5875 static void 5876 ixl_config_other_intr(struct ixl_softc *sc) 5877 { 5878 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5879 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5880 5881 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5882 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5883 I40E_PFINT_ICR0_ENA_GRST_MASK | 5884 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5885 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5886 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5887 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5888 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5889 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5890 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5891 5892 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5893 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5894 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5895 (I40E_ITR_INDEX_OTHER << 5896 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5897 ixl_flush(sc); 5898 } 5899 5900 static int 5901 ixl_setup_interrupts(struct ixl_softc *sc) 5902 { 5903 struct pci_attach_args *pa = &sc->sc_pa; 5904 pci_intr_type_t max_type, intr_type; 5905 int counts[PCI_INTR_TYPE_SIZE]; 5906 int error; 5907 unsigned int i; 5908 bool retry; 5909 5910 memset(counts, 0, sizeof(counts)); 5911 max_type = PCI_INTR_TYPE_MSIX; 5912 /* QPs + other interrupt */ 5913 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5914 counts[PCI_INTR_TYPE_INTX] = 1; 5915 5916 if (ixl_param_nomsix) 5917 counts[PCI_INTR_TYPE_MSIX] = 0; 5918 5919 do { 5920 retry = false; 5921 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5922 if (error != 0) { 5923 aprint_error_dev(sc->sc_dev, 5924 "couldn't map interrupt\n"); 5925 break; 5926 } 5927 5928 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5929 sc->sc_nintrs = counts[intr_type]; 5930 KASSERT(sc->sc_nintrs > 0); 5931 5932 for (i = 0; i < sc->sc_nintrs; i++) { 5933 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5934 PCI_INTR_MPSAFE, true); 5935 } 5936 5937 sc->sc_ihs = kmem_zalloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5938 KM_SLEEP); 5939 5940 if (intr_type == PCI_INTR_TYPE_MSIX) { 5941 error = ixl_establish_msix(sc); 5942 if (error) { 5943 counts[PCI_INTR_TYPE_MSIX] = 0; 5944 retry = true; 5945 } 5946 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5947 error = ixl_establish_intx(sc); 5948 } else { 5949 error = -1; 5950 } 5951 5952 if (error) { 5953 kmem_free(sc->sc_ihs, 5954 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5955 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5956 } else { 5957 sc->sc_intrtype = intr_type; 5958 } 5959 } while (retry); 5960 5961 return error; 5962 } 5963 5964 static void 5965 ixl_teardown_interrupts(struct ixl_softc *sc) 5966 { 5967 struct pci_attach_args *pa = &sc->sc_pa; 5968 unsigned int i; 5969 5970 for (i = 0; i < sc->sc_nintrs; i++) { 5971 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 5972 } 5973 5974 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5975 5976 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5977 sc->sc_ihs = NULL; 5978 sc->sc_nintrs = 0; 5979 } 5980 5981 static int 5982 ixl_setup_stats(struct ixl_softc *sc) 5983 { 5984 struct ixl_queue_pair *qp; 5985 struct ixl_tx_ring *txr; 5986 struct ixl_rx_ring *rxr; 5987 struct ixl_stats_counters *isc; 5988 unsigned int i; 5989 5990 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5991 qp = &sc->sc_qps[i]; 5992 txr = qp->qp_txr; 5993 rxr = qp->qp_rxr; 5994 5995 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 5996 NULL, qp->qp_name, "m_defrag successed"); 5997 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 5998 NULL, qp->qp_name, "m_defrag_failed"); 5999 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 6000 NULL, qp->qp_name, "Dropped in pcq"); 6001 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 6002 NULL, qp->qp_name, "Deferred transmit"); 6003 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 6004 NULL, qp->qp_name, "Interrupt on queue"); 6005 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 6006 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6007 6008 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 6009 NULL, qp->qp_name, "MGETHDR failed"); 6010 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 6011 NULL, qp->qp_name, "MCLGET failed"); 6012 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 6013 EVCNT_TYPE_MISC, NULL, qp->qp_name, 6014 "bus_dmamap_load_mbuf failed"); 6015 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 6016 NULL, qp->qp_name, "Interrupt on queue"); 6017 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 6018 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6019 } 6020 6021 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 6022 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 6023 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 6024 NULL, device_xname(sc->sc_dev), "Link status event"); 6025 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 6026 NULL, device_xname(sc->sc_dev), "ECC error"); 6027 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 6028 NULL, device_xname(sc->sc_dev), "PCI exception"); 6029 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 6030 NULL, device_xname(sc->sc_dev), "Critical error"); 6031 6032 isc = &sc->sc_stats_counters; 6033 evcnt_attach_dynamic(&isc->isc_crc_errors, EVCNT_TYPE_MISC, 6034 NULL, device_xname(sc->sc_dev), "CRC errors"); 6035 evcnt_attach_dynamic(&isc->isc_illegal_bytes, EVCNT_TYPE_MISC, 6036 NULL, device_xname(sc->sc_dev), "Illegal bytes"); 6037 evcnt_attach_dynamic(&isc->isc_mac_local_faults, EVCNT_TYPE_MISC, 6038 NULL, device_xname(sc->sc_dev), "Mac local faults"); 6039 evcnt_attach_dynamic(&isc->isc_mac_remote_faults, EVCNT_TYPE_MISC, 6040 NULL, device_xname(sc->sc_dev), "Mac remote faults"); 6041 evcnt_attach_dynamic(&isc->isc_link_xon_rx, EVCNT_TYPE_MISC, 6042 NULL, device_xname(sc->sc_dev), "Rx xon"); 6043 evcnt_attach_dynamic(&isc->isc_link_xon_tx, EVCNT_TYPE_MISC, 6044 NULL, device_xname(sc->sc_dev), "Tx xon"); 6045 evcnt_attach_dynamic(&isc->isc_link_xoff_rx, EVCNT_TYPE_MISC, 6046 NULL, device_xname(sc->sc_dev), "Rx xoff"); 6047 evcnt_attach_dynamic(&isc->isc_link_xoff_tx, EVCNT_TYPE_MISC, 6048 NULL, device_xname(sc->sc_dev), "Tx xoff"); 6049 evcnt_attach_dynamic(&isc->isc_rx_fragments, EVCNT_TYPE_MISC, 6050 NULL, device_xname(sc->sc_dev), "Rx fragments"); 6051 evcnt_attach_dynamic(&isc->isc_rx_jabber, EVCNT_TYPE_MISC, 6052 NULL, device_xname(sc->sc_dev), "Rx jabber"); 6053 6054 evcnt_attach_dynamic(&isc->isc_rx_size_64, EVCNT_TYPE_MISC, 6055 NULL, device_xname(sc->sc_dev), "Rx size 64"); 6056 evcnt_attach_dynamic(&isc->isc_rx_size_127, EVCNT_TYPE_MISC, 6057 NULL, device_xname(sc->sc_dev), "Rx size 127"); 6058 evcnt_attach_dynamic(&isc->isc_rx_size_255, EVCNT_TYPE_MISC, 6059 NULL, device_xname(sc->sc_dev), "Rx size 255"); 6060 evcnt_attach_dynamic(&isc->isc_rx_size_511, EVCNT_TYPE_MISC, 6061 NULL, device_xname(sc->sc_dev), "Rx size 511"); 6062 evcnt_attach_dynamic(&isc->isc_rx_size_1023, EVCNT_TYPE_MISC, 6063 NULL, device_xname(sc->sc_dev), "Rx size 1023"); 6064 evcnt_attach_dynamic(&isc->isc_rx_size_1522, EVCNT_TYPE_MISC, 6065 NULL, device_xname(sc->sc_dev), "Rx size 1522"); 6066 evcnt_attach_dynamic(&isc->isc_rx_size_big, EVCNT_TYPE_MISC, 6067 NULL, device_xname(sc->sc_dev), "Rx jumbo packets"); 6068 evcnt_attach_dynamic(&isc->isc_rx_undersize, EVCNT_TYPE_MISC, 6069 NULL, device_xname(sc->sc_dev), "Rx under size"); 6070 evcnt_attach_dynamic(&isc->isc_rx_oversize, EVCNT_TYPE_MISC, 6071 NULL, device_xname(sc->sc_dev), "Rx over size"); 6072 6073 evcnt_attach_dynamic(&isc->isc_rx_bytes, EVCNT_TYPE_MISC, 6074 NULL, device_xname(sc->sc_dev), "Rx bytes / port"); 6075 evcnt_attach_dynamic(&isc->isc_rx_discards, EVCNT_TYPE_MISC, 6076 NULL, device_xname(sc->sc_dev), "Rx discards / port"); 6077 evcnt_attach_dynamic(&isc->isc_rx_unicast, EVCNT_TYPE_MISC, 6078 NULL, device_xname(sc->sc_dev), "Rx unicast / port"); 6079 evcnt_attach_dynamic(&isc->isc_rx_multicast, EVCNT_TYPE_MISC, 6080 NULL, device_xname(sc->sc_dev), "Rx multicast / port"); 6081 evcnt_attach_dynamic(&isc->isc_rx_broadcast, EVCNT_TYPE_MISC, 6082 NULL, device_xname(sc->sc_dev), "Rx broadcast / port"); 6083 6084 evcnt_attach_dynamic(&isc->isc_vsi_rx_bytes, EVCNT_TYPE_MISC, 6085 NULL, device_xname(sc->sc_dev), "Rx bytes / vsi"); 6086 evcnt_attach_dynamic(&isc->isc_vsi_rx_discards, EVCNT_TYPE_MISC, 6087 NULL, device_xname(sc->sc_dev), "Rx discard / vsi"); 6088 evcnt_attach_dynamic(&isc->isc_vsi_rx_unicast, EVCNT_TYPE_MISC, 6089 NULL, device_xname(sc->sc_dev), "Rx unicast / vsi"); 6090 evcnt_attach_dynamic(&isc->isc_vsi_rx_multicast, EVCNT_TYPE_MISC, 6091 NULL, device_xname(sc->sc_dev), "Rx multicast / vsi"); 6092 evcnt_attach_dynamic(&isc->isc_vsi_rx_broadcast, EVCNT_TYPE_MISC, 6093 NULL, device_xname(sc->sc_dev), "Rx broadcast / vsi"); 6094 6095 evcnt_attach_dynamic(&isc->isc_tx_size_64, EVCNT_TYPE_MISC, 6096 NULL, device_xname(sc->sc_dev), "Tx size 64"); 6097 evcnt_attach_dynamic(&isc->isc_tx_size_127, EVCNT_TYPE_MISC, 6098 NULL, device_xname(sc->sc_dev), "Tx size 127"); 6099 evcnt_attach_dynamic(&isc->isc_tx_size_255, EVCNT_TYPE_MISC, 6100 NULL, device_xname(sc->sc_dev), "Tx size 255"); 6101 evcnt_attach_dynamic(&isc->isc_tx_size_511, EVCNT_TYPE_MISC, 6102 NULL, device_xname(sc->sc_dev), "Tx size 511"); 6103 evcnt_attach_dynamic(&isc->isc_tx_size_1023, EVCNT_TYPE_MISC, 6104 NULL, device_xname(sc->sc_dev), "Tx size 1023"); 6105 evcnt_attach_dynamic(&isc->isc_tx_size_1522, EVCNT_TYPE_MISC, 6106 NULL, device_xname(sc->sc_dev), "Tx size 1522"); 6107 evcnt_attach_dynamic(&isc->isc_tx_size_big, EVCNT_TYPE_MISC, 6108 NULL, device_xname(sc->sc_dev), "Tx jumbo packets"); 6109 6110 evcnt_attach_dynamic(&isc->isc_tx_bytes, EVCNT_TYPE_MISC, 6111 NULL, device_xname(sc->sc_dev), "Tx bytes / port"); 6112 evcnt_attach_dynamic(&isc->isc_tx_dropped_link_down, EVCNT_TYPE_MISC, 6113 NULL, device_xname(sc->sc_dev), 6114 "Tx dropped due to link down / port"); 6115 evcnt_attach_dynamic(&isc->isc_tx_unicast, EVCNT_TYPE_MISC, 6116 NULL, device_xname(sc->sc_dev), "Tx unicast / port"); 6117 evcnt_attach_dynamic(&isc->isc_tx_multicast, EVCNT_TYPE_MISC, 6118 NULL, device_xname(sc->sc_dev), "Tx multicast / port"); 6119 evcnt_attach_dynamic(&isc->isc_tx_broadcast, EVCNT_TYPE_MISC, 6120 NULL, device_xname(sc->sc_dev), "Tx broadcast / port"); 6121 6122 evcnt_attach_dynamic(&isc->isc_vsi_tx_bytes, EVCNT_TYPE_MISC, 6123 NULL, device_xname(sc->sc_dev), "Tx bytes / vsi"); 6124 evcnt_attach_dynamic(&isc->isc_vsi_tx_errors, EVCNT_TYPE_MISC, 6125 NULL, device_xname(sc->sc_dev), "Tx errors / vsi"); 6126 evcnt_attach_dynamic(&isc->isc_vsi_tx_unicast, EVCNT_TYPE_MISC, 6127 NULL, device_xname(sc->sc_dev), "Tx unicast / vsi"); 6128 evcnt_attach_dynamic(&isc->isc_vsi_tx_multicast, EVCNT_TYPE_MISC, 6129 NULL, device_xname(sc->sc_dev), "Tx multicast / vsi"); 6130 evcnt_attach_dynamic(&isc->isc_vsi_tx_broadcast, EVCNT_TYPE_MISC, 6131 NULL, device_xname(sc->sc_dev), "Tx broadcast / vsi"); 6132 6133 sc->sc_stats_intval = ixl_param_stats_interval; 6134 callout_init(&sc->sc_stats_callout, CALLOUT_MPSAFE); 6135 callout_setfunc(&sc->sc_stats_callout, ixl_stats_callout, sc); 6136 ixl_work_set(&sc->sc_stats_task, ixl_stats_update, sc); 6137 6138 return 0; 6139 } 6140 6141 static void 6142 ixl_teardown_stats(struct ixl_softc *sc) 6143 { 6144 struct ixl_tx_ring *txr; 6145 struct ixl_rx_ring *rxr; 6146 struct ixl_stats_counters *isc; 6147 unsigned int i; 6148 6149 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6150 txr = sc->sc_qps[i].qp_txr; 6151 rxr = sc->sc_qps[i].qp_rxr; 6152 6153 evcnt_detach(&txr->txr_defragged); 6154 evcnt_detach(&txr->txr_defrag_failed); 6155 evcnt_detach(&txr->txr_pcqdrop); 6156 evcnt_detach(&txr->txr_transmitdef); 6157 evcnt_detach(&txr->txr_intr); 6158 evcnt_detach(&txr->txr_defer); 6159 6160 evcnt_detach(&rxr->rxr_mgethdr_failed); 6161 evcnt_detach(&rxr->rxr_mgetcl_failed); 6162 evcnt_detach(&rxr->rxr_mbuf_load_failed); 6163 evcnt_detach(&rxr->rxr_intr); 6164 evcnt_detach(&rxr->rxr_defer); 6165 } 6166 6167 isc = &sc->sc_stats_counters; 6168 evcnt_detach(&isc->isc_crc_errors); 6169 evcnt_detach(&isc->isc_illegal_bytes); 6170 evcnt_detach(&isc->isc_mac_local_faults); 6171 evcnt_detach(&isc->isc_mac_remote_faults); 6172 evcnt_detach(&isc->isc_link_xon_rx); 6173 evcnt_detach(&isc->isc_link_xon_tx); 6174 evcnt_detach(&isc->isc_link_xoff_rx); 6175 evcnt_detach(&isc->isc_link_xoff_tx); 6176 evcnt_detach(&isc->isc_rx_fragments); 6177 evcnt_detach(&isc->isc_rx_jabber); 6178 evcnt_detach(&isc->isc_rx_bytes); 6179 evcnt_detach(&isc->isc_rx_discards); 6180 evcnt_detach(&isc->isc_rx_unicast); 6181 evcnt_detach(&isc->isc_rx_multicast); 6182 evcnt_detach(&isc->isc_rx_broadcast); 6183 evcnt_detach(&isc->isc_rx_size_64); 6184 evcnt_detach(&isc->isc_rx_size_127); 6185 evcnt_detach(&isc->isc_rx_size_255); 6186 evcnt_detach(&isc->isc_rx_size_511); 6187 evcnt_detach(&isc->isc_rx_size_1023); 6188 evcnt_detach(&isc->isc_rx_size_1522); 6189 evcnt_detach(&isc->isc_rx_size_big); 6190 evcnt_detach(&isc->isc_rx_undersize); 6191 evcnt_detach(&isc->isc_rx_oversize); 6192 evcnt_detach(&isc->isc_tx_bytes); 6193 evcnt_detach(&isc->isc_tx_dropped_link_down); 6194 evcnt_detach(&isc->isc_tx_unicast); 6195 evcnt_detach(&isc->isc_tx_multicast); 6196 evcnt_detach(&isc->isc_tx_broadcast); 6197 evcnt_detach(&isc->isc_tx_size_64); 6198 evcnt_detach(&isc->isc_tx_size_127); 6199 evcnt_detach(&isc->isc_tx_size_255); 6200 evcnt_detach(&isc->isc_tx_size_511); 6201 evcnt_detach(&isc->isc_tx_size_1023); 6202 evcnt_detach(&isc->isc_tx_size_1522); 6203 evcnt_detach(&isc->isc_tx_size_big); 6204 evcnt_detach(&isc->isc_vsi_rx_discards); 6205 evcnt_detach(&isc->isc_vsi_rx_bytes); 6206 evcnt_detach(&isc->isc_vsi_rx_unicast); 6207 evcnt_detach(&isc->isc_vsi_rx_multicast); 6208 evcnt_detach(&isc->isc_vsi_rx_broadcast); 6209 evcnt_detach(&isc->isc_vsi_tx_errors); 6210 evcnt_detach(&isc->isc_vsi_tx_bytes); 6211 evcnt_detach(&isc->isc_vsi_tx_unicast); 6212 evcnt_detach(&isc->isc_vsi_tx_multicast); 6213 evcnt_detach(&isc->isc_vsi_tx_broadcast); 6214 6215 evcnt_detach(&sc->sc_event_atq); 6216 evcnt_detach(&sc->sc_event_link); 6217 evcnt_detach(&sc->sc_event_ecc_err); 6218 evcnt_detach(&sc->sc_event_pci_exception); 6219 evcnt_detach(&sc->sc_event_crit_err); 6220 6221 callout_destroy(&sc->sc_stats_callout); 6222 } 6223 6224 static void 6225 ixl_stats_callout(void *xsc) 6226 { 6227 struct ixl_softc *sc = xsc; 6228 6229 ixl_work_add(sc->sc_workq, &sc->sc_stats_task); 6230 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 6231 } 6232 6233 static uint64_t 6234 ixl_stat_delta(struct ixl_softc *sc, uint32_t reg_hi, uint32_t reg_lo, 6235 uint64_t *offset, bool has_offset) 6236 { 6237 uint64_t value, delta; 6238 int bitwidth; 6239 6240 bitwidth = reg_hi == 0 ? 32 : 48; 6241 6242 value = ixl_rd(sc, reg_lo); 6243 6244 if (bitwidth > 32) { 6245 value |= ((uint64_t)ixl_rd(sc, reg_hi) << 32); 6246 } 6247 6248 if (__predict_true(has_offset)) { 6249 delta = value; 6250 if (value < *offset) 6251 delta += ((uint64_t)1 << bitwidth); 6252 delta -= *offset; 6253 } else { 6254 delta = 0; 6255 } 6256 atomic_swap_64(offset, value); 6257 6258 return delta; 6259 } 6260 6261 static void 6262 ixl_stats_update(void *xsc) 6263 { 6264 struct ixl_softc *sc = xsc; 6265 struct ixl_stats_counters *isc; 6266 uint64_t delta; 6267 6268 isc = &sc->sc_stats_counters; 6269 6270 /* errors */ 6271 delta = ixl_stat_delta(sc, 6272 0, I40E_GLPRT_CRCERRS(sc->sc_port), 6273 &isc->isc_crc_errors_offset, isc->isc_has_offset); 6274 atomic_add_64(&isc->isc_crc_errors.ev_count, delta); 6275 6276 delta = ixl_stat_delta(sc, 6277 0, I40E_GLPRT_ILLERRC(sc->sc_port), 6278 &isc->isc_illegal_bytes_offset, isc->isc_has_offset); 6279 atomic_add_64(&isc->isc_illegal_bytes.ev_count, delta); 6280 6281 /* rx */ 6282 delta = ixl_stat_delta(sc, 6283 I40E_GLPRT_GORCH(sc->sc_port), I40E_GLPRT_GORCL(sc->sc_port), 6284 &isc->isc_rx_bytes_offset, isc->isc_has_offset); 6285 atomic_add_64(&isc->isc_rx_bytes.ev_count, delta); 6286 6287 delta = ixl_stat_delta(sc, 6288 0, I40E_GLPRT_RDPC(sc->sc_port), 6289 &isc->isc_rx_discards_offset, isc->isc_has_offset); 6290 atomic_add_64(&isc->isc_rx_discards.ev_count, delta); 6291 6292 delta = ixl_stat_delta(sc, 6293 I40E_GLPRT_UPRCH(sc->sc_port), I40E_GLPRT_UPRCL(sc->sc_port), 6294 &isc->isc_rx_unicast_offset, isc->isc_has_offset); 6295 atomic_add_64(&isc->isc_rx_unicast.ev_count, delta); 6296 6297 delta = ixl_stat_delta(sc, 6298 I40E_GLPRT_MPRCH(sc->sc_port), I40E_GLPRT_MPRCL(sc->sc_port), 6299 &isc->isc_rx_multicast_offset, isc->isc_has_offset); 6300 atomic_add_64(&isc->isc_rx_multicast.ev_count, delta); 6301 6302 delta = ixl_stat_delta(sc, 6303 I40E_GLPRT_BPRCH(sc->sc_port), I40E_GLPRT_BPRCL(sc->sc_port), 6304 &isc->isc_rx_broadcast_offset, isc->isc_has_offset); 6305 atomic_add_64(&isc->isc_rx_broadcast.ev_count, delta); 6306 6307 /* Packet size stats rx */ 6308 delta = ixl_stat_delta(sc, 6309 I40E_GLPRT_PRC64H(sc->sc_port), I40E_GLPRT_PRC64L(sc->sc_port), 6310 &isc->isc_rx_size_64_offset, isc->isc_has_offset); 6311 atomic_add_64(&isc->isc_rx_size_64.ev_count, delta); 6312 6313 delta = ixl_stat_delta(sc, 6314 I40E_GLPRT_PRC127H(sc->sc_port), I40E_GLPRT_PRC127L(sc->sc_port), 6315 &isc->isc_rx_size_127_offset, isc->isc_has_offset); 6316 atomic_add_64(&isc->isc_rx_size_127.ev_count, delta); 6317 6318 delta = ixl_stat_delta(sc, 6319 I40E_GLPRT_PRC255H(sc->sc_port), I40E_GLPRT_PRC255L(sc->sc_port), 6320 &isc->isc_rx_size_255_offset, isc->isc_has_offset); 6321 atomic_add_64(&isc->isc_rx_size_255.ev_count, delta); 6322 6323 delta = ixl_stat_delta(sc, 6324 I40E_GLPRT_PRC511H(sc->sc_port), I40E_GLPRT_PRC511L(sc->sc_port), 6325 &isc->isc_rx_size_511_offset, isc->isc_has_offset); 6326 atomic_add_64(&isc->isc_rx_size_511.ev_count, delta); 6327 6328 delta = ixl_stat_delta(sc, 6329 I40E_GLPRT_PRC1023H(sc->sc_port), I40E_GLPRT_PRC1023L(sc->sc_port), 6330 &isc->isc_rx_size_1023_offset, isc->isc_has_offset); 6331 atomic_add_64(&isc->isc_rx_size_1023.ev_count, delta); 6332 6333 delta = ixl_stat_delta(sc, 6334 I40E_GLPRT_PRC1522H(sc->sc_port), I40E_GLPRT_PRC1522L(sc->sc_port), 6335 &isc->isc_rx_size_1522_offset, isc->isc_has_offset); 6336 atomic_add_64(&isc->isc_rx_size_1522.ev_count, delta); 6337 6338 delta = ixl_stat_delta(sc, 6339 I40E_GLPRT_PRC9522H(sc->sc_port), I40E_GLPRT_PRC9522L(sc->sc_port), 6340 &isc->isc_rx_size_big_offset, isc->isc_has_offset); 6341 atomic_add_64(&isc->isc_rx_size_big.ev_count, delta); 6342 6343 delta = ixl_stat_delta(sc, 6344 0, I40E_GLPRT_RUC(sc->sc_port), 6345 &isc->isc_rx_undersize_offset, isc->isc_has_offset); 6346 atomic_add_64(&isc->isc_rx_undersize.ev_count, delta); 6347 6348 delta = ixl_stat_delta(sc, 6349 0, I40E_GLPRT_ROC(sc->sc_port), 6350 &isc->isc_rx_oversize_offset, isc->isc_has_offset); 6351 atomic_add_64(&isc->isc_rx_oversize.ev_count, delta); 6352 6353 /* tx */ 6354 delta = ixl_stat_delta(sc, 6355 I40E_GLPRT_GOTCH(sc->sc_port), I40E_GLPRT_GOTCL(sc->sc_port), 6356 &isc->isc_tx_bytes_offset, isc->isc_has_offset); 6357 atomic_add_64(&isc->isc_tx_bytes.ev_count, delta); 6358 6359 delta = ixl_stat_delta(sc, 6360 0, I40E_GLPRT_TDOLD(sc->sc_port), 6361 &isc->isc_tx_dropped_link_down_offset, isc->isc_has_offset); 6362 atomic_add_64(&isc->isc_tx_dropped_link_down.ev_count, delta); 6363 6364 delta = ixl_stat_delta(sc, 6365 I40E_GLPRT_UPTCH(sc->sc_port), I40E_GLPRT_UPTCL(sc->sc_port), 6366 &isc->isc_tx_unicast_offset, isc->isc_has_offset); 6367 atomic_add_64(&isc->isc_tx_unicast.ev_count, delta); 6368 6369 delta = ixl_stat_delta(sc, 6370 I40E_GLPRT_MPTCH(sc->sc_port), I40E_GLPRT_MPTCL(sc->sc_port), 6371 &isc->isc_tx_multicast_offset, isc->isc_has_offset); 6372 atomic_add_64(&isc->isc_tx_multicast.ev_count, delta); 6373 6374 delta = ixl_stat_delta(sc, 6375 I40E_GLPRT_BPTCH(sc->sc_port), I40E_GLPRT_BPTCL(sc->sc_port), 6376 &isc->isc_tx_broadcast_offset, isc->isc_has_offset); 6377 atomic_add_64(&isc->isc_tx_broadcast.ev_count, delta); 6378 6379 /* Packet size stats tx */ 6380 delta = ixl_stat_delta(sc, 6381 I40E_GLPRT_PTC64L(sc->sc_port), I40E_GLPRT_PTC64L(sc->sc_port), 6382 &isc->isc_tx_size_64_offset, isc->isc_has_offset); 6383 atomic_add_64(&isc->isc_tx_size_64.ev_count, delta); 6384 6385 delta = ixl_stat_delta(sc, 6386 I40E_GLPRT_PTC127H(sc->sc_port), I40E_GLPRT_PTC127L(sc->sc_port), 6387 &isc->isc_tx_size_127_offset, isc->isc_has_offset); 6388 atomic_add_64(&isc->isc_tx_size_127.ev_count, delta); 6389 6390 delta = ixl_stat_delta(sc, 6391 I40E_GLPRT_PTC255H(sc->sc_port), I40E_GLPRT_PTC255L(sc->sc_port), 6392 &isc->isc_tx_size_255_offset, isc->isc_has_offset); 6393 atomic_add_64(&isc->isc_tx_size_255.ev_count, delta); 6394 6395 delta = ixl_stat_delta(sc, 6396 I40E_GLPRT_PTC511H(sc->sc_port), I40E_GLPRT_PTC511L(sc->sc_port), 6397 &isc->isc_tx_size_511_offset, isc->isc_has_offset); 6398 atomic_add_64(&isc->isc_tx_size_511.ev_count, delta); 6399 6400 delta = ixl_stat_delta(sc, 6401 I40E_GLPRT_PTC1023H(sc->sc_port), I40E_GLPRT_PTC1023L(sc->sc_port), 6402 &isc->isc_tx_size_1023_offset, isc->isc_has_offset); 6403 atomic_add_64(&isc->isc_tx_size_1023.ev_count, delta); 6404 6405 delta = ixl_stat_delta(sc, 6406 I40E_GLPRT_PTC1522H(sc->sc_port), I40E_GLPRT_PTC1522L(sc->sc_port), 6407 &isc->isc_tx_size_1522_offset, isc->isc_has_offset); 6408 atomic_add_64(&isc->isc_tx_size_1522.ev_count, delta); 6409 6410 delta = ixl_stat_delta(sc, 6411 I40E_GLPRT_PTC9522H(sc->sc_port), I40E_GLPRT_PTC9522L(sc->sc_port), 6412 &isc->isc_tx_size_big_offset, isc->isc_has_offset); 6413 atomic_add_64(&isc->isc_tx_size_big.ev_count, delta); 6414 6415 /* mac faults */ 6416 delta = ixl_stat_delta(sc, 6417 0, I40E_GLPRT_MLFC(sc->sc_port), 6418 &isc->isc_mac_local_faults_offset, isc->isc_has_offset); 6419 atomic_add_64(&isc->isc_mac_local_faults.ev_count, delta); 6420 6421 delta = ixl_stat_delta(sc, 6422 0, I40E_GLPRT_MRFC(sc->sc_port), 6423 &isc->isc_mac_remote_faults_offset, isc->isc_has_offset); 6424 atomic_add_64(&isc->isc_mac_remote_faults.ev_count, delta); 6425 6426 /* Flow control (LFC) stats */ 6427 delta = ixl_stat_delta(sc, 6428 0, I40E_GLPRT_LXONRXC(sc->sc_port), 6429 &isc->isc_link_xon_rx_offset, isc->isc_has_offset); 6430 atomic_add_64(&isc->isc_link_xon_rx.ev_count, delta); 6431 6432 delta = ixl_stat_delta(sc, 6433 0, I40E_GLPRT_LXONTXC(sc->sc_port), 6434 &isc->isc_link_xon_tx_offset, isc->isc_has_offset); 6435 atomic_add_64(&isc->isc_link_xon_tx.ev_count, delta); 6436 6437 delta = ixl_stat_delta(sc, 6438 0, I40E_GLPRT_LXOFFRXC(sc->sc_port), 6439 &isc->isc_link_xoff_rx_offset, isc->isc_has_offset); 6440 atomic_add_64(&isc->isc_link_xoff_rx.ev_count, delta); 6441 6442 delta = ixl_stat_delta(sc, 6443 0, I40E_GLPRT_LXOFFTXC(sc->sc_port), 6444 &isc->isc_link_xoff_tx_offset, isc->isc_has_offset); 6445 atomic_add_64(&isc->isc_link_xoff_tx.ev_count, delta); 6446 6447 /* fragments */ 6448 delta = ixl_stat_delta(sc, 6449 0, I40E_GLPRT_RFC(sc->sc_port), 6450 &isc->isc_rx_fragments_offset, isc->isc_has_offset); 6451 atomic_add_64(&isc->isc_rx_fragments.ev_count, delta); 6452 6453 delta = ixl_stat_delta(sc, 6454 0, I40E_GLPRT_RJC(sc->sc_port), 6455 &isc->isc_rx_jabber_offset, isc->isc_has_offset); 6456 atomic_add_64(&isc->isc_rx_jabber.ev_count, delta); 6457 6458 /* VSI rx counters */ 6459 delta = ixl_stat_delta(sc, 6460 0, I40E_GLV_RDPC(sc->sc_vsi_stat_counter_idx), 6461 &isc->isc_vsi_rx_discards_offset, isc->isc_has_offset); 6462 atomic_add_64(&isc->isc_vsi_rx_discards.ev_count, delta); 6463 6464 delta = ixl_stat_delta(sc, 6465 I40E_GLV_GORCH(sc->sc_vsi_stat_counter_idx), 6466 I40E_GLV_GORCL(sc->sc_vsi_stat_counter_idx), 6467 &isc->isc_vsi_rx_bytes_offset, isc->isc_has_offset); 6468 atomic_add_64(&isc->isc_vsi_rx_bytes.ev_count, delta); 6469 6470 delta = ixl_stat_delta(sc, 6471 I40E_GLV_UPRCH(sc->sc_vsi_stat_counter_idx), 6472 I40E_GLV_UPRCL(sc->sc_vsi_stat_counter_idx), 6473 &isc->isc_vsi_rx_unicast_offset, isc->isc_has_offset); 6474 atomic_add_64(&isc->isc_vsi_rx_unicast.ev_count, delta); 6475 6476 delta = ixl_stat_delta(sc, 6477 I40E_GLV_MPRCH(sc->sc_vsi_stat_counter_idx), 6478 I40E_GLV_MPRCL(sc->sc_vsi_stat_counter_idx), 6479 &isc->isc_vsi_rx_multicast_offset, isc->isc_has_offset); 6480 atomic_add_64(&isc->isc_vsi_rx_multicast.ev_count, delta); 6481 6482 delta = ixl_stat_delta(sc, 6483 I40E_GLV_BPRCH(sc->sc_vsi_stat_counter_idx), 6484 I40E_GLV_BPRCL(sc->sc_vsi_stat_counter_idx), 6485 &isc->isc_vsi_rx_broadcast_offset, isc->isc_has_offset); 6486 atomic_add_64(&isc->isc_vsi_rx_broadcast.ev_count, delta); 6487 6488 /* VSI tx counters */ 6489 delta = ixl_stat_delta(sc, 6490 0, I40E_GLV_TEPC(sc->sc_vsi_stat_counter_idx), 6491 &isc->isc_vsi_tx_errors_offset, isc->isc_has_offset); 6492 atomic_add_64(&isc->isc_vsi_tx_errors.ev_count, delta); 6493 6494 delta = ixl_stat_delta(sc, 6495 I40E_GLV_GOTCH(sc->sc_vsi_stat_counter_idx), 6496 I40E_GLV_GOTCL(sc->sc_vsi_stat_counter_idx), 6497 &isc->isc_vsi_tx_bytes_offset, isc->isc_has_offset); 6498 atomic_add_64(&isc->isc_vsi_tx_bytes.ev_count, delta); 6499 6500 delta = ixl_stat_delta(sc, 6501 I40E_GLV_UPTCH(sc->sc_vsi_stat_counter_idx), 6502 I40E_GLV_UPTCL(sc->sc_vsi_stat_counter_idx), 6503 &isc->isc_vsi_tx_unicast_offset, isc->isc_has_offset); 6504 atomic_add_64(&isc->isc_vsi_tx_unicast.ev_count, delta); 6505 6506 delta = ixl_stat_delta(sc, 6507 I40E_GLV_MPTCH(sc->sc_vsi_stat_counter_idx), 6508 I40E_GLV_MPTCL(sc->sc_vsi_stat_counter_idx), 6509 &isc->isc_vsi_tx_multicast_offset, isc->isc_has_offset); 6510 atomic_add_64(&isc->isc_vsi_tx_multicast.ev_count, delta); 6511 6512 delta = ixl_stat_delta(sc, 6513 I40E_GLV_BPTCH(sc->sc_vsi_stat_counter_idx), 6514 I40E_GLV_BPTCL(sc->sc_vsi_stat_counter_idx), 6515 &isc->isc_vsi_tx_broadcast_offset, isc->isc_has_offset); 6516 atomic_add_64(&isc->isc_vsi_tx_broadcast.ev_count, delta); 6517 } 6518 6519 static int 6520 ixl_setup_sysctls(struct ixl_softc *sc) 6521 { 6522 const char *devname; 6523 struct sysctllog **log; 6524 const struct sysctlnode *rnode, *rxnode, *txnode; 6525 int error; 6526 6527 log = &sc->sc_sysctllog; 6528 devname = device_xname(sc->sc_dev); 6529 6530 error = sysctl_createv(log, 0, NULL, &rnode, 6531 0, CTLTYPE_NODE, devname, 6532 SYSCTL_DESCR("ixl information and settings"), 6533 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 6534 if (error) 6535 goto out; 6536 6537 error = sysctl_createv(log, 0, &rnode, NULL, 6538 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 6539 SYSCTL_DESCR("Use workqueue for packet processing"), 6540 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 6541 if (error) 6542 goto out; 6543 6544 error = sysctl_createv(log, 0, &rnode, NULL, 6545 CTLFLAG_READONLY, CTLTYPE_INT, "stats_interval", 6546 SYSCTL_DESCR("Statistics collection interval in milliseconds"), 6547 NULL, 0, &sc->sc_stats_intval, 0, CTL_CREATE, CTL_EOL); 6548 6549 error = sysctl_createv(log, 0, &rnode, &rxnode, 6550 0, CTLTYPE_NODE, "rx", 6551 SYSCTL_DESCR("ixl information and settings for Rx"), 6552 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6553 if (error) 6554 goto out; 6555 6556 error = sysctl_createv(log, 0, &rxnode, NULL, 6557 CTLFLAG_READWRITE, CTLTYPE_INT, "itr", 6558 SYSCTL_DESCR("Interrupt Throttling"), 6559 ixl_sysctl_itr_handler, 0, 6560 (void *)sc, 0, CTL_CREATE, CTL_EOL); 6561 if (error) 6562 goto out; 6563 6564 error = sysctl_createv(log, 0, &rxnode, NULL, 6565 CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num", 6566 SYSCTL_DESCR("the number of rx descriptors"), 6567 NULL, 0, &sc->sc_rx_ring_ndescs, 0, CTL_CREATE, CTL_EOL); 6568 if (error) 6569 goto out; 6570 6571 error = sysctl_createv(log, 0, &rxnode, NULL, 6572 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6573 SYSCTL_DESCR("max number of Rx packets" 6574 " to process for interrupt processing"), 6575 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6576 if (error) 6577 goto out; 6578 6579 error = sysctl_createv(log, 0, &rxnode, NULL, 6580 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6581 SYSCTL_DESCR("max number of Rx packets" 6582 " to process for deferred processing"), 6583 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 6584 if (error) 6585 goto out; 6586 6587 error = sysctl_createv(log, 0, &rnode, &txnode, 6588 0, CTLTYPE_NODE, "tx", 6589 SYSCTL_DESCR("ixl information and settings for Tx"), 6590 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6591 if (error) 6592 goto out; 6593 6594 error = sysctl_createv(log, 0, &txnode, NULL, 6595 CTLFLAG_READWRITE, CTLTYPE_INT, "itr", 6596 SYSCTL_DESCR("Interrupt Throttling"), 6597 ixl_sysctl_itr_handler, 0, 6598 (void *)sc, 0, CTL_CREATE, CTL_EOL); 6599 if (error) 6600 goto out; 6601 6602 error = sysctl_createv(log, 0, &txnode, NULL, 6603 CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num", 6604 SYSCTL_DESCR("the number of tx descriptors"), 6605 NULL, 0, &sc->sc_tx_ring_ndescs, 0, CTL_CREATE, CTL_EOL); 6606 if (error) 6607 goto out; 6608 6609 error = sysctl_createv(log, 0, &txnode, NULL, 6610 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6611 SYSCTL_DESCR("max number of Tx packets" 6612 " to process for interrupt processing"), 6613 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6614 if (error) 6615 goto out; 6616 6617 error = sysctl_createv(log, 0, &txnode, NULL, 6618 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6619 SYSCTL_DESCR("max number of Tx packets" 6620 " to process for deferred processing"), 6621 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 6622 if (error) 6623 goto out; 6624 6625 out: 6626 if (error) { 6627 aprint_error_dev(sc->sc_dev, 6628 "unable to create sysctl node\n"); 6629 sysctl_teardown(log); 6630 } 6631 6632 return error; 6633 } 6634 6635 static void 6636 ixl_teardown_sysctls(struct ixl_softc *sc) 6637 { 6638 6639 sysctl_teardown(&sc->sc_sysctllog); 6640 } 6641 6642 static bool 6643 ixl_sysctlnode_is_rx(struct sysctlnode *node) 6644 { 6645 6646 if (strstr(node->sysctl_parent->sysctl_name, "rx") != NULL) 6647 return true; 6648 6649 return false; 6650 } 6651 6652 static int 6653 ixl_sysctl_itr_handler(SYSCTLFN_ARGS) 6654 { 6655 struct sysctlnode node = *rnode; 6656 struct ixl_softc *sc = (struct ixl_softc *)node.sysctl_data; 6657 struct ifnet *ifp = &sc->sc_ec.ec_if; 6658 uint32_t newitr, *itrptr; 6659 int error; 6660 6661 if (ixl_sysctlnode_is_rx(&node)) { 6662 itrptr = &sc->sc_itr_rx; 6663 } else { 6664 itrptr = &sc->sc_itr_tx; 6665 } 6666 6667 newitr = *itrptr; 6668 node.sysctl_data = &newitr; 6669 node.sysctl_size = sizeof(newitr); 6670 6671 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6672 6673 if (error || newp == NULL) 6674 return error; 6675 6676 /* ITRs are applied in ixl_init() for simple implementaion */ 6677 if (ISSET(ifp->if_flags, IFF_RUNNING)) 6678 return EBUSY; 6679 6680 if (newitr > 0x07ff) 6681 return EINVAL; 6682 6683 *itrptr = newitr; 6684 6685 return 0; 6686 } 6687 6688 static struct workqueue * 6689 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 6690 { 6691 struct workqueue *wq; 6692 int error; 6693 6694 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 6695 prio, ipl, flags); 6696 6697 if (error) 6698 return NULL; 6699 6700 return wq; 6701 } 6702 6703 static void 6704 ixl_workq_destroy(struct workqueue *wq) 6705 { 6706 6707 workqueue_destroy(wq); 6708 } 6709 6710 static void 6711 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 6712 { 6713 6714 memset(work, 0, sizeof(*work)); 6715 work->ixw_func = func; 6716 work->ixw_arg = arg; 6717 } 6718 6719 static void 6720 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 6721 { 6722 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 6723 return; 6724 6725 kpreempt_disable(); 6726 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 6727 kpreempt_enable(); 6728 } 6729 6730 static void 6731 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 6732 { 6733 6734 workqueue_wait(wq, &work->ixw_cookie); 6735 } 6736 6737 static void 6738 ixl_workq_work(struct work *wk, void *context) 6739 { 6740 struct ixl_work *work; 6741 6742 work = container_of(wk, struct ixl_work, ixw_cookie); 6743 6744 atomic_swap_uint(&work->ixw_added, 0); 6745 work->ixw_func(work->ixw_arg); 6746 } 6747 6748 static int 6749 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 6750 { 6751 struct ixl_aq_desc iaq; 6752 6753 memset(&iaq, 0, sizeof(iaq)); 6754 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 6755 iaq.iaq_param[1] = htole32(reg); 6756 6757 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6758 return ETIMEDOUT; 6759 6760 switch (htole16(iaq.iaq_retval)) { 6761 case IXL_AQ_RC_OK: 6762 /* success */ 6763 break; 6764 case IXL_AQ_RC_EACCES: 6765 return EPERM; 6766 case IXL_AQ_RC_EAGAIN: 6767 return EAGAIN; 6768 default: 6769 return EIO; 6770 } 6771 6772 *rv = htole32(iaq.iaq_param[3]); 6773 return 0; 6774 } 6775 6776 static uint32_t 6777 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 6778 { 6779 uint32_t val; 6780 int rv, retry, retry_limit; 6781 6782 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6783 retry_limit = 5; 6784 } else { 6785 retry_limit = 0; 6786 } 6787 6788 for (retry = 0; retry < retry_limit; retry++) { 6789 rv = ixl_rx_ctl_read(sc, reg, &val); 6790 if (rv == 0) 6791 return val; 6792 else if (rv == EAGAIN) 6793 delaymsec(1); 6794 else 6795 break; 6796 } 6797 6798 val = ixl_rd(sc, reg); 6799 6800 return val; 6801 } 6802 6803 static int 6804 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6805 { 6806 struct ixl_aq_desc iaq; 6807 6808 memset(&iaq, 0, sizeof(iaq)); 6809 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 6810 iaq.iaq_param[1] = htole32(reg); 6811 iaq.iaq_param[3] = htole32(value); 6812 6813 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6814 return ETIMEDOUT; 6815 6816 switch (htole16(iaq.iaq_retval)) { 6817 case IXL_AQ_RC_OK: 6818 /* success */ 6819 break; 6820 case IXL_AQ_RC_EACCES: 6821 return EPERM; 6822 case IXL_AQ_RC_EAGAIN: 6823 return EAGAIN; 6824 default: 6825 return EIO; 6826 } 6827 6828 return 0; 6829 } 6830 6831 static void 6832 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6833 { 6834 int rv, retry, retry_limit; 6835 6836 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6837 retry_limit = 5; 6838 } else { 6839 retry_limit = 0; 6840 } 6841 6842 for (retry = 0; retry < retry_limit; retry++) { 6843 rv = ixl_rx_ctl_write(sc, reg, value); 6844 if (rv == 0) 6845 return; 6846 else if (rv == EAGAIN) 6847 delaymsec(1); 6848 else 6849 break; 6850 } 6851 6852 ixl_wr(sc, reg, value); 6853 } 6854 6855 static int 6856 ixl_nvm_lock(struct ixl_softc *sc, char rw) 6857 { 6858 struct ixl_aq_desc iaq; 6859 struct ixl_aq_req_resource_param *param; 6860 int rv; 6861 6862 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6863 return 0; 6864 6865 memset(&iaq, 0, sizeof(iaq)); 6866 iaq.iaq_opcode = htole16(IXL_AQ_OP_REQUEST_RESOURCE); 6867 6868 param = (struct ixl_aq_req_resource_param *)&iaq.iaq_param; 6869 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6870 if (rw == 'R') { 6871 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_READ); 6872 } else { 6873 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_WRITE); 6874 } 6875 6876 rv = ixl_atq_poll(sc, &iaq, 250); 6877 6878 if (rv != 0) 6879 return ETIMEDOUT; 6880 6881 switch (le16toh(iaq.iaq_retval)) { 6882 case IXL_AQ_RC_OK: 6883 break; 6884 case IXL_AQ_RC_EACCES: 6885 return EACCES; 6886 case IXL_AQ_RC_EBUSY: 6887 return EBUSY; 6888 case IXL_AQ_RC_EPERM: 6889 return EPERM; 6890 } 6891 6892 return 0; 6893 } 6894 6895 static int 6896 ixl_nvm_unlock(struct ixl_softc *sc) 6897 { 6898 struct ixl_aq_desc iaq; 6899 struct ixl_aq_rel_resource_param *param; 6900 int rv; 6901 6902 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6903 return 0; 6904 6905 memset(&iaq, 0, sizeof(iaq)); 6906 iaq.iaq_opcode = htole16(IXL_AQ_OP_RELEASE_RESOURCE); 6907 6908 param = (struct ixl_aq_rel_resource_param *)&iaq.iaq_param; 6909 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6910 6911 rv = ixl_atq_poll(sc, &iaq, 250); 6912 6913 if (rv != 0) 6914 return ETIMEDOUT; 6915 6916 switch (le16toh(iaq.iaq_retval)) { 6917 case IXL_AQ_RC_OK: 6918 break; 6919 default: 6920 return EIO; 6921 } 6922 return 0; 6923 } 6924 6925 static int 6926 ixl_srdone_poll(struct ixl_softc *sc) 6927 { 6928 int wait_count; 6929 uint32_t reg; 6930 6931 for (wait_count = 0; wait_count < IXL_SRRD_SRCTL_ATTEMPTS; 6932 wait_count++) { 6933 reg = ixl_rd(sc, I40E_GLNVM_SRCTL); 6934 if (ISSET(reg, I40E_GLNVM_SRCTL_DONE_MASK)) 6935 break; 6936 6937 delaymsec(5); 6938 } 6939 6940 if (wait_count == IXL_SRRD_SRCTL_ATTEMPTS) 6941 return -1; 6942 6943 return 0; 6944 } 6945 6946 static int 6947 ixl_nvm_read_srctl(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 6948 { 6949 uint32_t reg; 6950 6951 if (ixl_srdone_poll(sc) != 0) 6952 return ETIMEDOUT; 6953 6954 reg = ((uint32_t)offset << I40E_GLNVM_SRCTL_ADDR_SHIFT) | 6955 __BIT(I40E_GLNVM_SRCTL_START_SHIFT); 6956 ixl_wr(sc, I40E_GLNVM_SRCTL, reg); 6957 6958 if (ixl_srdone_poll(sc) != 0) { 6959 aprint_debug("NVM read error: couldn't access " 6960 "Shadow RAM address: 0x%x\n", offset); 6961 return ETIMEDOUT; 6962 } 6963 6964 reg = ixl_rd(sc, I40E_GLNVM_SRDATA); 6965 *data = (uint16_t)__SHIFTOUT(reg, I40E_GLNVM_SRDATA_RDDATA_MASK); 6966 6967 return 0; 6968 } 6969 6970 static int 6971 ixl_nvm_read_aq(struct ixl_softc *sc, uint16_t offset_word, 6972 void *data, size_t len) 6973 { 6974 struct ixl_dmamem *idm; 6975 struct ixl_aq_desc iaq; 6976 struct ixl_aq_nvm_param *param; 6977 uint32_t offset_bytes; 6978 int rv; 6979 6980 idm = &sc->sc_aqbuf; 6981 if (len > IXL_DMA_LEN(idm)) 6982 return ENOMEM; 6983 6984 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 6985 memset(&iaq, 0, sizeof(iaq)); 6986 iaq.iaq_opcode = htole16(IXL_AQ_OP_NVM_READ); 6987 iaq.iaq_flags = htole16(IXL_AQ_BUF | 6988 ((len > I40E_AQ_LARGE_BUF) ? IXL_AQ_LB : 0)); 6989 iaq.iaq_datalen = htole16(len); 6990 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 6991 6992 param = (struct ixl_aq_nvm_param *)iaq.iaq_param; 6993 param->command_flags = IXL_AQ_NVM_LAST_CMD; 6994 param->module_pointer = 0; 6995 param->length = htole16(len); 6996 offset_bytes = (uint32_t)offset_word * 2; 6997 offset_bytes &= 0x00FFFFFF; 6998 param->offset = htole32(offset_bytes); 6999 7000 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 7001 BUS_DMASYNC_PREREAD); 7002 7003 rv = ixl_atq_poll(sc, &iaq, 250); 7004 7005 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 7006 BUS_DMASYNC_POSTREAD); 7007 7008 if (rv != 0) { 7009 return ETIMEDOUT; 7010 } 7011 7012 switch (le16toh(iaq.iaq_retval)) { 7013 case IXL_AQ_RC_OK: 7014 break; 7015 case IXL_AQ_RC_EPERM: 7016 return EPERM; 7017 case IXL_AQ_RC_EINVAL: 7018 return EINVAL; 7019 case IXL_AQ_RC_EBUSY: 7020 return EBUSY; 7021 case IXL_AQ_RC_EIO: 7022 default: 7023 return EIO; 7024 } 7025 7026 memcpy(data, IXL_DMA_KVA(idm), len); 7027 7028 return 0; 7029 } 7030 7031 static int 7032 ixl_rd16_nvm(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 7033 { 7034 int error; 7035 uint16_t buf; 7036 7037 error = ixl_nvm_lock(sc, 'R'); 7038 if (error) 7039 return error; 7040 7041 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMREAD)) { 7042 error = ixl_nvm_read_aq(sc, offset, 7043 &buf, sizeof(buf)); 7044 if (error == 0) 7045 *data = le16toh(buf); 7046 } else { 7047 error = ixl_nvm_read_srctl(sc, offset, &buf); 7048 if (error == 0) 7049 *data = buf; 7050 } 7051 7052 ixl_nvm_unlock(sc); 7053 7054 return error; 7055 } 7056 7057 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 7058 7059 #ifdef _MODULE 7060 #include "ioconf.c" 7061 #endif 7062 7063 #ifdef _MODULE 7064 static void 7065 ixl_parse_modprop(prop_dictionary_t dict) 7066 { 7067 prop_object_t obj; 7068 int64_t val; 7069 uint64_t uval; 7070 7071 if (dict == NULL) 7072 return; 7073 7074 obj = prop_dictionary_get(dict, "nomsix"); 7075 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_BOOL) { 7076 ixl_param_nomsix = prop_bool_true((prop_bool_t)obj); 7077 } 7078 7079 obj = prop_dictionary_get(dict, "stats_interval"); 7080 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7081 val = prop_number_signed_value((prop_number_t)obj); 7082 7083 /* the range has no reason */ 7084 if (100 < val && val < 180000) { 7085 ixl_param_stats_interval = val; 7086 } 7087 } 7088 7089 obj = prop_dictionary_get(dict, "nqps_limit"); 7090 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7091 val = prop_number_signed_value((prop_number_t)obj); 7092 7093 if (val <= INT32_MAX) 7094 ixl_param_nqps_limit = val; 7095 } 7096 7097 obj = prop_dictionary_get(dict, "rx_ndescs"); 7098 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7099 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7100 7101 if (uval > 8) 7102 ixl_param_rx_ndescs = uval; 7103 } 7104 7105 obj = prop_dictionary_get(dict, "tx_ndescs"); 7106 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7107 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7108 7109 if (uval > IXL_TX_PKT_DESCS) 7110 ixl_param_tx_ndescs = uval; 7111 } 7112 7113 } 7114 #endif 7115 7116 static int 7117 if_ixl_modcmd(modcmd_t cmd, void *opaque) 7118 { 7119 int error = 0; 7120 7121 #ifdef _MODULE 7122 switch (cmd) { 7123 case MODULE_CMD_INIT: 7124 ixl_parse_modprop((prop_dictionary_t)opaque); 7125 error = config_init_component(cfdriver_ioconf_if_ixl, 7126 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7127 break; 7128 case MODULE_CMD_FINI: 7129 error = config_fini_component(cfdriver_ioconf_if_ixl, 7130 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7131 break; 7132 default: 7133 error = ENOTTY; 7134 break; 7135 } 7136 #endif 7137 7138 return error; 7139 } 7140
Indexes created Sat Sep 20 22:09:52 GMT 2025