if_ixl.c revision 1.78
1 /* $NetBSD: if_ixl.c,v 1.78 2022/03/16 05:26:37 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 __KERNEL_RCSID(0, "$NetBSD: if_ixl.c,v 1.78 2022/03/16 05:26:37 yamaguchi Exp $"); 78 79 #ifdef _KERNEL_OPT 80 #include "opt_net_mpsafe.h" 81 #include "opt_if_ixl.h" 82 #endif 83 84 #include <sys/param.h> 85 #include <sys/types.h> 86 87 #include <sys/bitops.h> 88 #include <sys/cpu.h> 89 #include <sys/device.h> 90 #include <sys/evcnt.h> 91 #include <sys/interrupt.h> 92 #include <sys/kmem.h> 93 #include <sys/module.h> 94 #include <sys/mutex.h> 95 #include <sys/pcq.h> 96 #include <sys/syslog.h> 97 #include <sys/workqueue.h> 98 99 #include <sys/bus.h> 100 101 #include <net/bpf.h> 102 #include <net/if.h> 103 #include <net/if_dl.h> 104 #include <net/if_media.h> 105 #include <net/if_ether.h> 106 #include <net/rss_config.h> 107 108 #include <netinet/tcp.h> /* for struct tcphdr */ 109 #include <netinet/udp.h> /* for struct udphdr */ 110 111 #include <dev/pci/pcivar.h> 112 #include <dev/pci/pcidevs.h> 113 114 #include <dev/pci/if_ixlreg.h> 115 #include <dev/pci/if_ixlvar.h> 116 117 #include <prop/proplib.h> 118 119 struct ixl_softc; /* defined */ 120 121 #define I40E_PF_RESET_WAIT_COUNT 200 122 #define I40E_AQ_LARGE_BUF 512 123 124 /* bitfields for Tx queue mapping in QTX_CTL */ 125 #define I40E_QTX_CTL_VF_QUEUE 0x0 126 #define I40E_QTX_CTL_VM_QUEUE 0x1 127 #define I40E_QTX_CTL_PF_QUEUE 0x2 128 129 #define I40E_QUEUE_TYPE_EOL 0x7ff 130 #define I40E_INTR_NOTX_QUEUE 0 131 132 #define I40E_QUEUE_TYPE_RX 0x0 133 #define I40E_QUEUE_TYPE_TX 0x1 134 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 135 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 136 137 #define I40E_ITR_INDEX_RX 0x0 138 #define I40E_ITR_INDEX_TX 0x1 139 #define I40E_ITR_INDEX_OTHER 0x2 140 #define I40E_ITR_INDEX_NONE 0x3 141 #define IXL_ITR_RX 0x7a /* 4K intrs/sec */ 142 #define IXL_ITR_TX 0x7a /* 4K intrs/sec */ 143 144 #define I40E_INTR_NOTX_QUEUE 0 145 #define I40E_INTR_NOTX_INTR 0 146 #define I40E_INTR_NOTX_RX_QUEUE 0 147 #define I40E_INTR_NOTX_TX_QUEUE 1 148 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 149 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 150 151 #define I40E_HASH_LUT_SIZE_128 0 152 153 #define IXL_ICR0_CRIT_ERR_MASK \ 154 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 155 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 156 I40E_PFINT_ICR0_PE_CRITERR_MASK) 157 158 #define IXL_QUEUE_MAX_XL710 64 159 #define IXL_QUEUE_MAX_X722 128 160 161 #define IXL_TX_PKT_DESCS 8 162 #define IXL_TX_PKT_MAXSIZE (MCLBYTES * IXL_TX_PKT_DESCS) 163 #define IXL_TX_QUEUE_ALIGN 128 164 #define IXL_RX_QUEUE_ALIGN 128 165 166 #define IXL_MCLBYTES (MCLBYTES - ETHER_ALIGN) 167 #define IXL_MTU_ETHERLEN ETHER_HDR_LEN \ 168 + ETHER_CRC_LEN 169 #if 0 170 #define IXL_MAX_MTU (9728 - IXL_MTU_ETHERLEN) 171 #else 172 /* (dbuff * 5) - ETHER_HDR_LEN - ETHER_CRC_LEN */ 173 #define IXL_MAX_MTU (9600 - IXL_MTU_ETHERLEN) 174 #endif 175 #define IXL_MIN_MTU (ETHER_MIN_LEN - ETHER_CRC_LEN) 176 177 #define IXL_PCIREG PCI_MAPREG_START 178 179 #define IXL_ITR0 0x0 180 #define IXL_ITR1 0x1 181 #define IXL_ITR2 0x2 182 #define IXL_NOITR 0x3 183 184 #define IXL_AQ_NUM 256 185 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 186 #define IXL_AQ_ALIGN 64 /* lol */ 187 #define IXL_AQ_BUFLEN 4096 188 189 #define IXL_HMC_ROUNDUP 512 190 #define IXL_HMC_PGSIZE 4096 191 #define IXL_HMC_DVASZ sizeof(uint64_t) 192 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 193 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 194 #define IXL_HMC_PDVALID 1ULL 195 196 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 197 198 #define IXL_SRRD_SRCTL_ATTEMPTS 100000 199 200 struct ixl_aq_regs { 201 bus_size_t atq_tail; 202 bus_size_t atq_head; 203 bus_size_t atq_len; 204 bus_size_t atq_bal; 205 bus_size_t atq_bah; 206 207 bus_size_t arq_tail; 208 bus_size_t arq_head; 209 bus_size_t arq_len; 210 bus_size_t arq_bal; 211 bus_size_t arq_bah; 212 213 uint32_t atq_len_enable; 214 uint32_t atq_tail_mask; 215 uint32_t atq_head_mask; 216 217 uint32_t arq_len_enable; 218 uint32_t arq_tail_mask; 219 uint32_t arq_head_mask; 220 }; 221 222 struct ixl_phy_type { 223 uint64_t phy_type; 224 uint64_t ifm_type; 225 }; 226 227 struct ixl_speed_type { 228 uint8_t dev_speed; 229 uint64_t net_speed; 230 }; 231 232 struct ixl_hmc_entry { 233 uint64_t hmc_base; 234 uint32_t hmc_count; 235 uint64_t hmc_size; 236 }; 237 238 enum ixl_hmc_types { 239 IXL_HMC_LAN_TX = 0, 240 IXL_HMC_LAN_RX, 241 IXL_HMC_FCOE_CTX, 242 IXL_HMC_FCOE_FILTER, 243 IXL_HMC_COUNT 244 }; 245 246 struct ixl_hmc_pack { 247 uint16_t offset; 248 uint16_t width; 249 uint16_t lsb; 250 }; 251 252 /* 253 * these hmc objects have weird sizes and alignments, so these are abstract 254 * representations of them that are nice for c to populate. 255 * 256 * the packing code relies on little-endian values being stored in the fields, 257 * no high bits in the fields being set, and the fields must be packed in the 258 * same order as they are in the ctx structure. 259 */ 260 261 struct ixl_hmc_rxq { 262 uint16_t head; 263 uint8_t cpuid; 264 uint64_t base; 265 #define IXL_HMC_RXQ_BASE_UNIT 128 266 uint16_t qlen; 267 uint16_t dbuff; 268 #define IXL_HMC_RXQ_DBUFF_UNIT 128 269 uint8_t hbuff; 270 #define IXL_HMC_RXQ_HBUFF_UNIT 64 271 uint8_t dtype; 272 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 273 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 274 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 275 uint8_t dsize; 276 #define IXL_HMC_RXQ_DSIZE_16 0 277 #define IXL_HMC_RXQ_DSIZE_32 1 278 uint8_t crcstrip; 279 uint8_t fc_ena; 280 uint8_t l2sel; 281 uint8_t hsplit_0; 282 uint8_t hsplit_1; 283 uint8_t showiv; 284 uint16_t rxmax; 285 uint8_t tphrdesc_ena; 286 uint8_t tphwdesc_ena; 287 uint8_t tphdata_ena; 288 uint8_t tphhead_ena; 289 uint8_t lrxqthresh; 290 uint8_t prefena; 291 }; 292 293 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 294 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 295 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 296 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 297 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 298 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 299 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 300 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 301 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 302 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 303 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 304 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 305 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 306 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 307 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 308 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 309 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 310 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 311 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 312 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 313 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 314 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 315 }; 316 317 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 318 319 struct ixl_hmc_txq { 320 uint16_t head; 321 uint8_t new_context; 322 uint64_t base; 323 #define IXL_HMC_TXQ_BASE_UNIT 128 324 uint8_t fc_ena; 325 uint8_t timesync_ena; 326 uint8_t fd_ena; 327 uint8_t alt_vlan_ena; 328 uint8_t cpuid; 329 uint16_t thead_wb; 330 uint8_t head_wb_ena; 331 #define IXL_HMC_TXQ_DESC_WB 0 332 #define IXL_HMC_TXQ_HEAD_WB 1 333 uint16_t qlen; 334 uint8_t tphrdesc_ena; 335 uint8_t tphrpacket_ena; 336 uint8_t tphwdesc_ena; 337 uint64_t head_wb_addr; 338 uint32_t crc; 339 uint16_t rdylist; 340 uint8_t rdylist_act; 341 }; 342 343 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 344 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 345 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 346 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 347 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 348 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 349 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 350 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 351 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 352 /* line 1 */ 353 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 354 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 355 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 356 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 357 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 358 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 359 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 360 /* line 7 */ 361 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 362 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 363 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 364 }; 365 366 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 367 368 struct ixl_work { 369 struct work ixw_cookie; 370 void (*ixw_func)(void *); 371 void *ixw_arg; 372 unsigned int ixw_added; 373 }; 374 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 375 376 struct ixl_tx_map { 377 struct mbuf *txm_m; 378 bus_dmamap_t txm_map; 379 unsigned int txm_eop; 380 }; 381 382 struct ixl_tx_ring { 383 kmutex_t txr_lock; 384 struct ixl_softc *txr_sc; 385 386 unsigned int txr_prod; 387 unsigned int txr_cons; 388 389 struct ixl_tx_map *txr_maps; 390 struct ixl_dmamem txr_mem; 391 392 bus_size_t txr_tail; 393 unsigned int txr_qid; 394 pcq_t *txr_intrq; 395 void *txr_si; 396 397 struct evcnt txr_defragged; 398 struct evcnt txr_defrag_failed; 399 struct evcnt txr_pcqdrop; 400 struct evcnt txr_transmitdef; 401 struct evcnt txr_intr; 402 struct evcnt txr_defer; 403 }; 404 405 struct ixl_rx_map { 406 struct mbuf *rxm_m; 407 bus_dmamap_t rxm_map; 408 }; 409 410 struct ixl_rx_ring { 411 kmutex_t rxr_lock; 412 413 unsigned int rxr_prod; 414 unsigned int rxr_cons; 415 416 struct ixl_rx_map *rxr_maps; 417 struct ixl_dmamem rxr_mem; 418 419 struct mbuf *rxr_m_head; 420 struct mbuf **rxr_m_tail; 421 422 bus_size_t rxr_tail; 423 unsigned int rxr_qid; 424 425 struct evcnt rxr_mgethdr_failed; 426 struct evcnt rxr_mgetcl_failed; 427 struct evcnt rxr_mbuf_load_failed; 428 struct evcnt rxr_intr; 429 struct evcnt rxr_defer; 430 }; 431 432 struct ixl_queue_pair { 433 struct ixl_softc *qp_sc; 434 struct ixl_tx_ring *qp_txr; 435 struct ixl_rx_ring *qp_rxr; 436 437 char qp_name[16]; 438 439 void *qp_si; 440 struct work qp_work; 441 bool qp_workqueue; 442 }; 443 444 struct ixl_atq { 445 struct ixl_aq_desc iatq_desc; 446 void (*iatq_fn)(struct ixl_softc *, 447 const struct ixl_aq_desc *); 448 }; 449 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 450 451 struct ixl_product { 452 unsigned int vendor_id; 453 unsigned int product_id; 454 }; 455 456 struct ixl_stats_counters { 457 bool isc_has_offset; 458 struct evcnt isc_crc_errors; 459 uint64_t isc_crc_errors_offset; 460 struct evcnt isc_illegal_bytes; 461 uint64_t isc_illegal_bytes_offset; 462 struct evcnt isc_rx_bytes; 463 uint64_t isc_rx_bytes_offset; 464 struct evcnt isc_rx_discards; 465 uint64_t isc_rx_discards_offset; 466 struct evcnt isc_rx_unicast; 467 uint64_t isc_rx_unicast_offset; 468 struct evcnt isc_rx_multicast; 469 uint64_t isc_rx_multicast_offset; 470 struct evcnt isc_rx_broadcast; 471 uint64_t isc_rx_broadcast_offset; 472 struct evcnt isc_rx_size_64; 473 uint64_t isc_rx_size_64_offset; 474 struct evcnt isc_rx_size_127; 475 uint64_t isc_rx_size_127_offset; 476 struct evcnt isc_rx_size_255; 477 uint64_t isc_rx_size_255_offset; 478 struct evcnt isc_rx_size_511; 479 uint64_t isc_rx_size_511_offset; 480 struct evcnt isc_rx_size_1023; 481 uint64_t isc_rx_size_1023_offset; 482 struct evcnt isc_rx_size_1522; 483 uint64_t isc_rx_size_1522_offset; 484 struct evcnt isc_rx_size_big; 485 uint64_t isc_rx_size_big_offset; 486 struct evcnt isc_rx_undersize; 487 uint64_t isc_rx_undersize_offset; 488 struct evcnt isc_rx_oversize; 489 uint64_t isc_rx_oversize_offset; 490 struct evcnt isc_rx_fragments; 491 uint64_t isc_rx_fragments_offset; 492 struct evcnt isc_rx_jabber; 493 uint64_t isc_rx_jabber_offset; 494 struct evcnt isc_tx_bytes; 495 uint64_t isc_tx_bytes_offset; 496 struct evcnt isc_tx_dropped_link_down; 497 uint64_t isc_tx_dropped_link_down_offset; 498 struct evcnt isc_tx_unicast; 499 uint64_t isc_tx_unicast_offset; 500 struct evcnt isc_tx_multicast; 501 uint64_t isc_tx_multicast_offset; 502 struct evcnt isc_tx_broadcast; 503 uint64_t isc_tx_broadcast_offset; 504 struct evcnt isc_tx_size_64; 505 uint64_t isc_tx_size_64_offset; 506 struct evcnt isc_tx_size_127; 507 uint64_t isc_tx_size_127_offset; 508 struct evcnt isc_tx_size_255; 509 uint64_t isc_tx_size_255_offset; 510 struct evcnt isc_tx_size_511; 511 uint64_t isc_tx_size_511_offset; 512 struct evcnt isc_tx_size_1023; 513 uint64_t isc_tx_size_1023_offset; 514 struct evcnt isc_tx_size_1522; 515 uint64_t isc_tx_size_1522_offset; 516 struct evcnt isc_tx_size_big; 517 uint64_t isc_tx_size_big_offset; 518 struct evcnt isc_mac_local_faults; 519 uint64_t isc_mac_local_faults_offset; 520 struct evcnt isc_mac_remote_faults; 521 uint64_t isc_mac_remote_faults_offset; 522 struct evcnt isc_link_xon_rx; 523 uint64_t isc_link_xon_rx_offset; 524 struct evcnt isc_link_xon_tx; 525 uint64_t isc_link_xon_tx_offset; 526 struct evcnt isc_link_xoff_rx; 527 uint64_t isc_link_xoff_rx_offset; 528 struct evcnt isc_link_xoff_tx; 529 uint64_t isc_link_xoff_tx_offset; 530 struct evcnt isc_vsi_rx_discards; 531 uint64_t isc_vsi_rx_discards_offset; 532 struct evcnt isc_vsi_rx_bytes; 533 uint64_t isc_vsi_rx_bytes_offset; 534 struct evcnt isc_vsi_rx_unicast; 535 uint64_t isc_vsi_rx_unicast_offset; 536 struct evcnt isc_vsi_rx_multicast; 537 uint64_t isc_vsi_rx_multicast_offset; 538 struct evcnt isc_vsi_rx_broadcast; 539 uint64_t isc_vsi_rx_broadcast_offset; 540 struct evcnt isc_vsi_tx_errors; 541 uint64_t isc_vsi_tx_errors_offset; 542 struct evcnt isc_vsi_tx_bytes; 543 uint64_t isc_vsi_tx_bytes_offset; 544 struct evcnt isc_vsi_tx_unicast; 545 uint64_t isc_vsi_tx_unicast_offset; 546 struct evcnt isc_vsi_tx_multicast; 547 uint64_t isc_vsi_tx_multicast_offset; 548 struct evcnt isc_vsi_tx_broadcast; 549 uint64_t isc_vsi_tx_broadcast_offset; 550 }; 551 552 /* 553 * Locking notes: 554 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 555 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 556 * - more than one lock of them cannot be held at once. 557 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 558 * (a spin mutex). 559 * - the lock cannot held with txr_lock or rxr_lock. 560 * + a field named sc_arq_* is not protected by any lock. 561 * - operations for sc_arq_* is done in one context related to 562 * sc_arq_task. 563 * + other fields in ixl_softc is protected by sc_cfg_lock 564 * (an adaptive mutex) 565 * - It must be held before another lock is held, and It can be 566 * released after the other lock is released. 567 * */ 568 569 struct ixl_softc { 570 device_t sc_dev; 571 struct ethercom sc_ec; 572 bool sc_attached; 573 bool sc_dead; 574 uint32_t sc_port; 575 struct sysctllog *sc_sysctllog; 576 struct workqueue *sc_workq; 577 struct workqueue *sc_workq_txrx; 578 int sc_stats_intval; 579 callout_t sc_stats_callout; 580 struct ixl_work sc_stats_task; 581 struct ixl_stats_counters 582 sc_stats_counters; 583 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 584 struct ifmedia sc_media; 585 uint64_t sc_media_status; 586 uint64_t sc_media_active; 587 uint64_t sc_phy_types; 588 uint8_t sc_phy_abilities; 589 uint8_t sc_phy_linkspeed; 590 uint8_t sc_phy_fec_cfg; 591 uint16_t sc_eee_cap; 592 uint32_t sc_eeer_val; 593 uint8_t sc_d3_lpan; 594 kmutex_t sc_cfg_lock; 595 enum i40e_mac_type sc_mac_type; 596 uint32_t sc_rss_table_size; 597 uint32_t sc_rss_table_entry_width; 598 bool sc_txrx_workqueue; 599 u_int sc_tx_process_limit; 600 u_int sc_rx_process_limit; 601 u_int sc_tx_intr_process_limit; 602 u_int sc_rx_intr_process_limit; 603 604 int sc_cur_ec_capenable; 605 606 struct pci_attach_args sc_pa; 607 pci_intr_handle_t *sc_ihp; 608 void **sc_ihs; 609 unsigned int sc_nintrs; 610 611 bus_dma_tag_t sc_dmat; 612 bus_space_tag_t sc_memt; 613 bus_space_handle_t sc_memh; 614 bus_size_t sc_mems; 615 616 uint8_t sc_pf_id; 617 uint16_t sc_uplink_seid; /* le */ 618 uint16_t sc_downlink_seid; /* le */ 619 uint16_t sc_vsi_number; 620 uint16_t sc_vsi_stat_counter_idx; 621 uint16_t sc_seid; 622 unsigned int sc_base_queue; 623 624 pci_intr_type_t sc_intrtype; 625 unsigned int sc_msix_vector_queue; 626 627 struct ixl_dmamem sc_scratch; 628 struct ixl_dmamem sc_aqbuf; 629 630 const struct ixl_aq_regs * 631 sc_aq_regs; 632 uint32_t sc_aq_flags; 633 #define IXL_SC_AQ_FLAG_RXCTL __BIT(0) 634 #define IXL_SC_AQ_FLAG_NVMLOCK __BIT(1) 635 #define IXL_SC_AQ_FLAG_NVMREAD __BIT(2) 636 #define IXL_SC_AQ_FLAG_RSS __BIT(3) 637 638 kmutex_t sc_atq_lock; 639 kcondvar_t sc_atq_cv; 640 struct ixl_dmamem sc_atq; 641 unsigned int sc_atq_prod; 642 unsigned int sc_atq_cons; 643 644 struct ixl_dmamem sc_arq; 645 struct ixl_work sc_arq_task; 646 struct ixl_aq_bufs sc_arq_idle; 647 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 648 unsigned int sc_arq_prod; 649 unsigned int sc_arq_cons; 650 651 struct ixl_work sc_link_state_task; 652 struct ixl_atq sc_link_state_atq; 653 654 struct ixl_dmamem sc_hmc_sd; 655 struct ixl_dmamem sc_hmc_pd; 656 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 657 658 struct if_percpuq *sc_ipq; 659 unsigned int sc_tx_ring_ndescs; 660 unsigned int sc_rx_ring_ndescs; 661 unsigned int sc_nqueue_pairs; 662 unsigned int sc_nqueue_pairs_max; 663 unsigned int sc_nqueue_pairs_device; 664 struct ixl_queue_pair *sc_qps; 665 uint32_t sc_itr_rx; 666 uint32_t sc_itr_tx; 667 668 struct evcnt sc_event_atq; 669 struct evcnt sc_event_link; 670 struct evcnt sc_event_ecc_err; 671 struct evcnt sc_event_pci_exception; 672 struct evcnt sc_event_crit_err; 673 }; 674 675 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 676 #define IXL_TX_PROCESS_LIMIT 256 677 #define IXL_RX_PROCESS_LIMIT 256 678 #define IXL_TX_INTR_PROCESS_LIMIT 256 679 #define IXL_RX_INTR_PROCESS_LIMIT 0U 680 681 #define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx | \ 682 IFCAP_CSUM_TCPv4_Rx | \ 683 IFCAP_CSUM_UDPv4_Rx | \ 684 IFCAP_CSUM_TCPv6_Rx | \ 685 IFCAP_CSUM_UDPv6_Rx) 686 #define IXL_IFCAP_TXCSUM (IFCAP_CSUM_IPv4_Tx | \ 687 IFCAP_CSUM_TCPv4_Tx | \ 688 IFCAP_CSUM_UDPv4_Tx | \ 689 IFCAP_CSUM_TCPv6_Tx | \ 690 IFCAP_CSUM_UDPv6_Tx) 691 #define IXL_CSUM_ALL_OFFLOAD (M_CSUM_IPv4 | \ 692 M_CSUM_TCPv4 | M_CSUM_TCPv6 | \ 693 M_CSUM_UDPv4 | M_CSUM_UDPv6) 694 695 #define delaymsec(_x) DELAY(1000 * (_x)) 696 #ifdef IXL_DEBUG 697 #define DDPRINTF(sc, fmt, args...) \ 698 do { \ 699 if ((sc) != NULL) { \ 700 device_printf( \ 701 ((struct ixl_softc *)(sc))->sc_dev, \ 702 ""); \ 703 } \ 704 printf("%s:\t" fmt, __func__, ##args); \ 705 } while (0) 706 #else 707 #define DDPRINTF(sc, fmt, args...) __nothing 708 #endif 709 #ifndef IXL_STATS_INTERVAL_MSEC 710 #define IXL_STATS_INTERVAL_MSEC 10000 711 #endif 712 #ifndef IXL_QUEUE_NUM 713 #define IXL_QUEUE_NUM 0 714 #endif 715 716 static bool ixl_param_nomsix = false; 717 static int ixl_param_stats_interval = IXL_STATS_INTERVAL_MSEC; 718 static int ixl_param_nqps_limit = IXL_QUEUE_NUM; 719 static unsigned int ixl_param_tx_ndescs = 512; 720 static unsigned int ixl_param_rx_ndescs = 256; 721 722 static enum i40e_mac_type 723 ixl_mactype(pci_product_id_t); 724 static void ixl_pci_csr_setup(pci_chipset_tag_t, pcitag_t); 725 static void ixl_clear_hw(struct ixl_softc *); 726 static int ixl_pf_reset(struct ixl_softc *); 727 728 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 729 bus_size_t, bus_size_t); 730 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 731 732 static int ixl_arq_fill(struct ixl_softc *); 733 static void ixl_arq_unfill(struct ixl_softc *); 734 735 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 736 unsigned int); 737 static void ixl_atq_set(struct ixl_atq *, 738 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 739 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 740 static void ixl_atq_done(struct ixl_softc *); 741 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 742 static int ixl_atq_exec_locked(struct ixl_softc *, struct ixl_atq *); 743 static int ixl_get_version(struct ixl_softc *); 744 static int ixl_get_nvm_version(struct ixl_softc *); 745 static int ixl_get_hw_capabilities(struct ixl_softc *); 746 static int ixl_pxe_clear(struct ixl_softc *); 747 static int ixl_lldp_shut(struct ixl_softc *); 748 static int ixl_get_mac(struct ixl_softc *); 749 static int ixl_get_switch_config(struct ixl_softc *); 750 static int ixl_phy_mask_ints(struct ixl_softc *); 751 static int ixl_get_phy_info(struct ixl_softc *); 752 static int ixl_set_phy_config(struct ixl_softc *, uint8_t, uint8_t, bool); 753 static int ixl_set_phy_autoselect(struct ixl_softc *); 754 static int ixl_restart_an(struct ixl_softc *); 755 static int ixl_hmc(struct ixl_softc *); 756 static void ixl_hmc_free(struct ixl_softc *); 757 static int ixl_get_vsi(struct ixl_softc *); 758 static int ixl_set_vsi(struct ixl_softc *); 759 static void ixl_set_filter_control(struct ixl_softc *); 760 static void ixl_get_link_status(void *); 761 static int ixl_get_link_status_poll(struct ixl_softc *, int *); 762 static void ixl_get_link_status_done(struct ixl_softc *, 763 const struct ixl_aq_desc *); 764 static int ixl_set_link_status_locked(struct ixl_softc *, 765 const struct ixl_aq_desc *); 766 static uint64_t ixl_search_link_speed(uint8_t); 767 static uint8_t ixl_search_baudrate(uint64_t); 768 static void ixl_config_rss(struct ixl_softc *); 769 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 770 uint16_t, uint16_t); 771 static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *, 772 uint16_t, uint16_t); 773 static void ixl_arq(void *); 774 static void ixl_hmc_pack(void *, const void *, 775 const struct ixl_hmc_pack *, unsigned int); 776 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 777 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 778 static int ixl_rd16_nvm(struct ixl_softc *, uint16_t, uint16_t *); 779 780 static int ixl_match(device_t, cfdata_t, void *); 781 static void ixl_attach(device_t, device_t, void *); 782 static int ixl_detach(device_t, int); 783 784 static void ixl_media_add(struct ixl_softc *); 785 static int ixl_media_change(struct ifnet *); 786 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 787 static void ixl_watchdog(struct ifnet *); 788 static int ixl_ioctl(struct ifnet *, u_long, void *); 789 static void ixl_start(struct ifnet *); 790 static int ixl_transmit(struct ifnet *, struct mbuf *); 791 static void ixl_deferred_transmit(void *); 792 static int ixl_intr(void *); 793 static int ixl_queue_intr(void *); 794 static int ixl_other_intr(void *); 795 static void ixl_handle_queue(void *); 796 static void ixl_handle_queue_wk(struct work *, void *); 797 static void ixl_sched_handle_queue(struct ixl_softc *, 798 struct ixl_queue_pair *); 799 static int ixl_init(struct ifnet *); 800 static int ixl_init_locked(struct ixl_softc *); 801 static void ixl_stop(struct ifnet *, int); 802 static void ixl_stop_locked(struct ixl_softc *); 803 static int ixl_iff(struct ixl_softc *); 804 static int ixl_ifflags_cb(struct ethercom *); 805 static int ixl_setup_interrupts(struct ixl_softc *); 806 static int ixl_establish_intx(struct ixl_softc *); 807 static int ixl_establish_msix(struct ixl_softc *); 808 static void ixl_enable_queue_intr(struct ixl_softc *, 809 struct ixl_queue_pair *); 810 static void ixl_disable_queue_intr(struct ixl_softc *, 811 struct ixl_queue_pair *); 812 static void ixl_enable_other_intr(struct ixl_softc *); 813 static void ixl_disable_other_intr(struct ixl_softc *); 814 static void ixl_config_queue_intr(struct ixl_softc *); 815 static void ixl_config_other_intr(struct ixl_softc *); 816 817 static struct ixl_tx_ring * 818 ixl_txr_alloc(struct ixl_softc *, unsigned int); 819 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 820 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 821 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 822 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 823 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 824 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 825 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 826 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 827 828 static struct ixl_rx_ring * 829 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 830 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 831 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 832 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 833 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 834 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 835 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 836 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 837 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 838 839 static struct workqueue * 840 ixl_workq_create(const char *, pri_t, int, int); 841 static void ixl_workq_destroy(struct workqueue *); 842 static int ixl_workqs_teardown(device_t); 843 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 844 static void ixl_work_add(struct workqueue *, struct ixl_work *); 845 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 846 static void ixl_workq_work(struct work *, void *); 847 static const struct ixl_product * 848 ixl_lookup(const struct pci_attach_args *pa); 849 static void ixl_link_state_update(struct ixl_softc *, 850 const struct ixl_aq_desc *); 851 static int ixl_vlan_cb(struct ethercom *, uint16_t, bool); 852 static int ixl_setup_vlan_hwfilter(struct ixl_softc *); 853 static void ixl_teardown_vlan_hwfilter(struct ixl_softc *); 854 static int ixl_update_macvlan(struct ixl_softc *); 855 static int ixl_setup_interrupts(struct ixl_softc *); 856 static void ixl_teardown_interrupts(struct ixl_softc *); 857 static int ixl_setup_stats(struct ixl_softc *); 858 static void ixl_teardown_stats(struct ixl_softc *); 859 static void ixl_stats_callout(void *); 860 static void ixl_stats_update(void *); 861 static int ixl_setup_sysctls(struct ixl_softc *); 862 static void ixl_teardown_sysctls(struct ixl_softc *); 863 static int ixl_sysctl_itr_handler(SYSCTLFN_PROTO); 864 static int ixl_queue_pairs_alloc(struct ixl_softc *); 865 static void ixl_queue_pairs_free(struct ixl_softc *); 866 867 static const struct ixl_phy_type ixl_phy_type_map[] = { 868 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 869 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 870 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 871 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 872 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 873 { 1ULL << IXL_PHY_TYPE_XAUI | 874 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 875 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 876 { 1ULL << IXL_PHY_TYPE_XLAUI | 877 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 878 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 879 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 880 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 881 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 882 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 883 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 884 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 885 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 886 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 887 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 888 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 889 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 890 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 891 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 892 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 893 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 894 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 895 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 896 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 897 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 898 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 899 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 900 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 901 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_ACC }, 902 { 1ULL << IXL_PHY_TYPE_2500BASE_T_1, IFM_2500_T }, 903 { 1ULL << IXL_PHY_TYPE_5000BASE_T_1, IFM_5000_T }, 904 { 1ULL << IXL_PHY_TYPE_2500BASE_T_2, IFM_2500_T }, 905 { 1ULL << IXL_PHY_TYPE_5000BASE_T_2, IFM_5000_T }, 906 }; 907 908 static const struct ixl_speed_type ixl_speed_type_map[] = { 909 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 910 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 911 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 912 { IXL_AQ_LINK_SPEED_5000MB, IF_Mbps(5000) }, 913 { IXL_AQ_LINK_SPEED_2500MB, IF_Mbps(2500) }, 914 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 915 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 916 }; 917 918 static const struct ixl_aq_regs ixl_pf_aq_regs = { 919 .atq_tail = I40E_PF_ATQT, 920 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 921 .atq_head = I40E_PF_ATQH, 922 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 923 .atq_len = I40E_PF_ATQLEN, 924 .atq_bal = I40E_PF_ATQBAL, 925 .atq_bah = I40E_PF_ATQBAH, 926 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 927 928 .arq_tail = I40E_PF_ARQT, 929 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 930 .arq_head = I40E_PF_ARQH, 931 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 932 .arq_len = I40E_PF_ARQLEN, 933 .arq_bal = I40E_PF_ARQBAL, 934 .arq_bah = I40E_PF_ARQBAH, 935 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 936 }; 937 938 #define ixl_rd(_s, _r) \ 939 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 940 #define ixl_wr(_s, _r, _v) \ 941 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 942 #define ixl_barrier(_s, _r, _l, _o) \ 943 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 944 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 945 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 946 947 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 948 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 949 DVF_DETACH_SHUTDOWN); 950 951 static const struct ixl_product ixl_products[] = { 952 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 953 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 954 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 955 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 956 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 957 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 958 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_1 }, 959 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_2 }, 960 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 961 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 962 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 963 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 964 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 965 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 966 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 967 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 968 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 969 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 970 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 971 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_SFP }, 972 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_BP }, 973 /* required last entry */ 974 {0, 0} 975 }; 976 977 static const struct ixl_product * 978 ixl_lookup(const struct pci_attach_args *pa) 979 { 980 const struct ixl_product *ixlp; 981 982 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 983 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 984 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 985 return ixlp; 986 } 987 988 return NULL; 989 } 990 991 static int 992 ixl_match(device_t parent, cfdata_t match, void *aux) 993 { 994 const struct pci_attach_args *pa = aux; 995 996 return (ixl_lookup(pa) != NULL) ? 1 : 0; 997 } 998 999 static void 1000 ixl_attach(device_t parent, device_t self, void *aux) 1001 { 1002 struct ixl_softc *sc; 1003 struct pci_attach_args *pa = aux; 1004 struct ifnet *ifp; 1005 pcireg_t memtype; 1006 uint32_t firstq, port, ari, func; 1007 char xnamebuf[32]; 1008 int tries, rv, link; 1009 1010 sc = device_private(self); 1011 sc->sc_dev = self; 1012 ifp = &sc->sc_ec.ec_if; 1013 1014 sc->sc_pa = *pa; 1015 sc->sc_dmat = (pci_dma64_available(pa)) ? 1016 pa->pa_dmat64 : pa->pa_dmat; 1017 sc->sc_aq_regs = &ixl_pf_aq_regs; 1018 1019 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(pa->pa_id)); 1020 1021 ixl_pci_csr_setup(pa->pa_pc, pa->pa_tag); 1022 1023 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 1024 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 1025 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 1026 aprint_error(": unable to map registers\n"); 1027 return; 1028 } 1029 1030 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 1031 1032 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 1033 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 1034 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 1035 sc->sc_base_queue = firstq; 1036 1037 ixl_clear_hw(sc); 1038 if (ixl_pf_reset(sc) == -1) { 1039 /* error printed by ixl pf_reset */ 1040 goto unmap; 1041 } 1042 1043 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 1044 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 1045 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 1046 sc->sc_port = port; 1047 aprint_normal(": port %u", sc->sc_port); 1048 1049 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 1050 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 1051 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 1052 1053 func = ixl_rd(sc, I40E_PF_FUNC_RID); 1054 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 1055 1056 /* initialise the adminq */ 1057 1058 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 1059 1060 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 1061 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1062 aprint_error("\n" "%s: unable to allocate atq\n", 1063 device_xname(self)); 1064 goto unmap; 1065 } 1066 1067 SIMPLEQ_INIT(&sc->sc_arq_idle); 1068 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 1069 sc->sc_arq_cons = 0; 1070 sc->sc_arq_prod = 0; 1071 1072 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 1073 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1074 aprint_error("\n" "%s: unable to allocate arq\n", 1075 device_xname(self)); 1076 goto free_atq; 1077 } 1078 1079 if (!ixl_arq_fill(sc)) { 1080 aprint_error("\n" "%s: unable to fill arq descriptors\n", 1081 device_xname(self)); 1082 goto free_arq; 1083 } 1084 1085 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1086 0, IXL_DMA_LEN(&sc->sc_atq), 1087 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1088 1089 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1090 0, IXL_DMA_LEN(&sc->sc_arq), 1091 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1092 1093 for (tries = 0; tries < 10; tries++) { 1094 sc->sc_atq_cons = 0; 1095 sc->sc_atq_prod = 0; 1096 1097 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1098 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1099 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1100 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1101 1102 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1103 1104 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1105 ixl_dmamem_lo(&sc->sc_atq)); 1106 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1107 ixl_dmamem_hi(&sc->sc_atq)); 1108 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1109 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1110 1111 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1112 ixl_dmamem_lo(&sc->sc_arq)); 1113 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1114 ixl_dmamem_hi(&sc->sc_arq)); 1115 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1116 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1117 1118 rv = ixl_get_version(sc); 1119 if (rv == 0) 1120 break; 1121 if (rv != ETIMEDOUT) { 1122 aprint_error(", unable to get firmware version\n"); 1123 goto shutdown; 1124 } 1125 1126 delaymsec(100); 1127 } 1128 1129 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1130 1131 if (ixl_dmamem_alloc(sc, &sc->sc_aqbuf, IXL_AQ_BUFLEN, 0) != 0) { 1132 aprint_error_dev(self, ", unable to allocate nvm buffer\n"); 1133 goto shutdown; 1134 } 1135 1136 ixl_get_nvm_version(sc); 1137 1138 if (sc->sc_mac_type == I40E_MAC_X722) 1139 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_X722; 1140 else 1141 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_XL710; 1142 1143 rv = ixl_get_hw_capabilities(sc); 1144 if (rv != 0) { 1145 aprint_error(", GET HW CAPABILITIES %s\n", 1146 rv == ETIMEDOUT ? "timeout" : "error"); 1147 goto free_aqbuf; 1148 } 1149 1150 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1151 if (ixl_param_nqps_limit > 0) { 1152 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_max, 1153 ixl_param_nqps_limit); 1154 } 1155 1156 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1157 sc->sc_tx_ring_ndescs = ixl_param_tx_ndescs; 1158 sc->sc_rx_ring_ndescs = ixl_param_rx_ndescs; 1159 1160 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1161 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1162 KASSERT(sc->sc_rx_ring_ndescs == 1163 (1U << (fls32(sc->sc_rx_ring_ndescs) - 1))); 1164 KASSERT(sc->sc_tx_ring_ndescs == 1165 (1U << (fls32(sc->sc_tx_ring_ndescs) - 1))); 1166 1167 if (ixl_get_mac(sc) != 0) { 1168 /* error printed by ixl_get_mac */ 1169 goto free_aqbuf; 1170 } 1171 1172 aprint_normal("\n"); 1173 aprint_naive("\n"); 1174 1175 aprint_normal_dev(self, "Ethernet address %s\n", 1176 ether_sprintf(sc->sc_enaddr)); 1177 1178 rv = ixl_pxe_clear(sc); 1179 if (rv != 0) { 1180 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1181 rv == ETIMEDOUT ? "timeout" : "error"); 1182 } 1183 1184 ixl_set_filter_control(sc); 1185 1186 if (ixl_hmc(sc) != 0) { 1187 /* error printed by ixl_hmc */ 1188 goto free_aqbuf; 1189 } 1190 1191 if (ixl_lldp_shut(sc) != 0) { 1192 /* error printed by ixl_lldp_shut */ 1193 goto free_hmc; 1194 } 1195 1196 if (ixl_phy_mask_ints(sc) != 0) { 1197 /* error printed by ixl_phy_mask_ints */ 1198 goto free_hmc; 1199 } 1200 1201 if (ixl_restart_an(sc) != 0) { 1202 /* error printed by ixl_restart_an */ 1203 goto free_hmc; 1204 } 1205 1206 if (ixl_get_switch_config(sc) != 0) { 1207 /* error printed by ixl_get_switch_config */ 1208 goto free_hmc; 1209 } 1210 1211 rv = ixl_get_link_status_poll(sc, NULL); 1212 if (rv != 0) { 1213 aprint_error_dev(self, "GET LINK STATUS %s\n", 1214 rv == ETIMEDOUT ? "timeout" : "error"); 1215 goto free_hmc; 1216 } 1217 1218 /* 1219 * The FW often returns EIO in "Get PHY Abilities" command 1220 * if there is no delay 1221 */ 1222 DELAY(500); 1223 if (ixl_get_phy_info(sc) != 0) { 1224 /* error printed by ixl_get_phy_info */ 1225 goto free_hmc; 1226 } 1227 1228 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1229 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1230 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1231 goto free_hmc; 1232 } 1233 1234 rv = ixl_get_vsi(sc); 1235 if (rv != 0) { 1236 aprint_error_dev(self, "GET VSI %s %d\n", 1237 rv == ETIMEDOUT ? "timeout" : "error", rv); 1238 goto free_scratch; 1239 } 1240 1241 rv = ixl_set_vsi(sc); 1242 if (rv != 0) { 1243 aprint_error_dev(self, "UPDATE VSI error %s %d\n", 1244 rv == ETIMEDOUT ? "timeout" : "error", rv); 1245 goto free_scratch; 1246 } 1247 1248 if (ixl_queue_pairs_alloc(sc) != 0) { 1249 /* error printed by ixl_queue_pairs_alloc */ 1250 goto free_scratch; 1251 } 1252 1253 if (ixl_setup_interrupts(sc) != 0) { 1254 /* error printed by ixl_setup_interrupts */ 1255 goto free_queue_pairs; 1256 } 1257 1258 if (ixl_setup_stats(sc) != 0) { 1259 aprint_error_dev(self, "failed to setup event counters\n"); 1260 goto teardown_intrs; 1261 } 1262 1263 if (ixl_setup_sysctls(sc) != 0) { 1264 /* error printed by ixl_setup_sysctls */ 1265 goto teardown_stats; 1266 } 1267 1268 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1269 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1270 IPL_NET, WQ_MPSAFE); 1271 if (sc->sc_workq == NULL) 1272 goto teardown_sysctls; 1273 1274 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1275 rv = workqueue_create(&sc->sc_workq_txrx, xnamebuf, ixl_handle_queue_wk, 1276 sc, IXL_WORKQUEUE_PRI, IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1277 if (rv != 0) { 1278 sc->sc_workq_txrx = NULL; 1279 goto teardown_wqs; 1280 } 1281 1282 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1283 cv_init(&sc->sc_atq_cv, xnamebuf); 1284 1285 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1286 1287 ifp->if_softc = sc; 1288 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1289 ifp->if_extflags = IFEF_MPSAFE; 1290 ifp->if_ioctl = ixl_ioctl; 1291 ifp->if_start = ixl_start; 1292 ifp->if_transmit = ixl_transmit; 1293 ifp->if_watchdog = ixl_watchdog; 1294 ifp->if_init = ixl_init; 1295 ifp->if_stop = ixl_stop; 1296 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1297 IFQ_SET_READY(&ifp->if_snd); 1298 ifp->if_capabilities |= IXL_IFCAP_RXCSUM; 1299 ifp->if_capabilities |= IXL_IFCAP_TXCSUM; 1300 #if 0 1301 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1302 #endif 1303 ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb); 1304 sc->sc_ec.ec_capabilities |= ETHERCAP_JUMBO_MTU; 1305 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1306 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1307 1308 sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities; 1309 /* Disable VLAN_HWFILTER by default */ 1310 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1311 1312 sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable; 1313 1314 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1315 ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, ixl_media_change, 1316 ixl_media_status, &sc->sc_cfg_lock); 1317 1318 ixl_media_add(sc); 1319 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1320 if (ISSET(sc->sc_phy_abilities, 1321 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1322 ifmedia_add(&sc->sc_media, 1323 IFM_ETHER | IFM_AUTO | IFM_FLOW, 0, NULL); 1324 } 1325 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_NONE, 0, NULL); 1326 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1327 1328 if_initialize(ifp); 1329 1330 sc->sc_ipq = if_percpuq_create(ifp); 1331 if_deferred_start_init(ifp, NULL); 1332 ether_ifattach(ifp, sc->sc_enaddr); 1333 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1334 1335 rv = ixl_get_link_status_poll(sc, &link); 1336 if (rv != 0) 1337 link = LINK_STATE_UNKNOWN; 1338 if_link_state_change(ifp, link); 1339 1340 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 1341 ixl_work_set(&sc->sc_link_state_task, ixl_get_link_status, sc); 1342 1343 ixl_config_other_intr(sc); 1344 ixl_enable_other_intr(sc); 1345 1346 ixl_set_phy_autoselect(sc); 1347 1348 /* remove default mac filter and replace it so we can see vlans */ 1349 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 1350 if (rv != ENOENT) { 1351 aprint_debug_dev(self, 1352 "unable to remove macvlan %u\n", rv); 1353 } 1354 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1355 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1356 if (rv != ENOENT) { 1357 aprint_debug_dev(self, 1358 "unable to remove macvlan, ignore vlan %u\n", rv); 1359 } 1360 1361 if (ixl_update_macvlan(sc) != 0) { 1362 aprint_debug_dev(self, 1363 "couldn't enable vlan hardware filter\n"); 1364 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1365 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1366 } 1367 1368 sc->sc_txrx_workqueue = true; 1369 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1370 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1371 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1372 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1373 1374 ixl_stats_update(sc); 1375 sc->sc_stats_counters.isc_has_offset = true; 1376 1377 if (pmf_device_register(self, NULL, NULL) != true) 1378 aprint_debug_dev(self, "couldn't establish power handler\n"); 1379 sc->sc_itr_rx = IXL_ITR_RX; 1380 sc->sc_itr_tx = IXL_ITR_TX; 1381 sc->sc_attached = true; 1382 if_register(ifp); 1383 1384 return; 1385 1386 teardown_wqs: 1387 config_finalize_register(self, ixl_workqs_teardown); 1388 teardown_sysctls: 1389 ixl_teardown_sysctls(sc); 1390 teardown_stats: 1391 ixl_teardown_stats(sc); 1392 teardown_intrs: 1393 ixl_teardown_interrupts(sc); 1394 free_queue_pairs: 1395 ixl_queue_pairs_free(sc); 1396 free_scratch: 1397 ixl_dmamem_free(sc, &sc->sc_scratch); 1398 free_hmc: 1399 ixl_hmc_free(sc); 1400 free_aqbuf: 1401 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1402 shutdown: 1403 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1404 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1405 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1406 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1407 1408 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1409 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1410 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1411 1412 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1413 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1414 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1415 1416 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1417 0, IXL_DMA_LEN(&sc->sc_arq), 1418 BUS_DMASYNC_POSTREAD); 1419 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1420 0, IXL_DMA_LEN(&sc->sc_atq), 1421 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1422 1423 ixl_arq_unfill(sc); 1424 free_arq: 1425 ixl_dmamem_free(sc, &sc->sc_arq); 1426 free_atq: 1427 ixl_dmamem_free(sc, &sc->sc_atq); 1428 unmap: 1429 mutex_destroy(&sc->sc_atq_lock); 1430 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1431 mutex_destroy(&sc->sc_cfg_lock); 1432 sc->sc_mems = 0; 1433 1434 sc->sc_attached = false; 1435 } 1436 1437 static int 1438 ixl_detach(device_t self, int flags) 1439 { 1440 struct ixl_softc *sc = device_private(self); 1441 struct ifnet *ifp = &sc->sc_ec.ec_if; 1442 1443 if (!sc->sc_attached) 1444 return 0; 1445 1446 ixl_stop(ifp, 1); 1447 1448 ixl_disable_other_intr(sc); 1449 1450 callout_halt(&sc->sc_stats_callout, NULL); 1451 ixl_work_wait(sc->sc_workq, &sc->sc_stats_task); 1452 1453 /* wait for ATQ handler */ 1454 mutex_enter(&sc->sc_atq_lock); 1455 mutex_exit(&sc->sc_atq_lock); 1456 1457 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1458 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1459 1460 if (sc->sc_workq != NULL) { 1461 ixl_workq_destroy(sc->sc_workq); 1462 sc->sc_workq = NULL; 1463 } 1464 1465 if (sc->sc_workq_txrx != NULL) { 1466 workqueue_destroy(sc->sc_workq_txrx); 1467 sc->sc_workq_txrx = NULL; 1468 } 1469 1470 if_percpuq_destroy(sc->sc_ipq); 1471 ether_ifdetach(ifp); 1472 if_detach(ifp); 1473 ifmedia_fini(&sc->sc_media); 1474 1475 ixl_teardown_interrupts(sc); 1476 ixl_teardown_stats(sc); 1477 ixl_teardown_sysctls(sc); 1478 1479 ixl_queue_pairs_free(sc); 1480 1481 ixl_dmamem_free(sc, &sc->sc_scratch); 1482 ixl_hmc_free(sc); 1483 1484 /* shutdown */ 1485 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1486 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1487 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1488 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1489 1490 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1491 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1492 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1493 1494 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1495 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1496 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1497 1498 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1499 0, IXL_DMA_LEN(&sc->sc_arq), 1500 BUS_DMASYNC_POSTREAD); 1501 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1502 0, IXL_DMA_LEN(&sc->sc_atq), 1503 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1504 1505 ixl_arq_unfill(sc); 1506 1507 ixl_dmamem_free(sc, &sc->sc_arq); 1508 ixl_dmamem_free(sc, &sc->sc_atq); 1509 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1510 1511 cv_destroy(&sc->sc_atq_cv); 1512 mutex_destroy(&sc->sc_atq_lock); 1513 1514 if (sc->sc_mems != 0) { 1515 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1516 sc->sc_mems = 0; 1517 } 1518 1519 mutex_destroy(&sc->sc_cfg_lock); 1520 1521 return 0; 1522 } 1523 1524 static int 1525 ixl_workqs_teardown(device_t self) 1526 { 1527 struct ixl_softc *sc = device_private(self); 1528 1529 if (sc->sc_workq != NULL) { 1530 ixl_workq_destroy(sc->sc_workq); 1531 sc->sc_workq = NULL; 1532 } 1533 1534 if (sc->sc_workq_txrx != NULL) { 1535 workqueue_destroy(sc->sc_workq_txrx); 1536 sc->sc_workq_txrx = NULL; 1537 } 1538 1539 return 0; 1540 } 1541 1542 static int 1543 ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 1544 { 1545 struct ifnet *ifp = &ec->ec_if; 1546 struct ixl_softc *sc = ifp->if_softc; 1547 int rv; 1548 1549 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1550 return 0; 1551 } 1552 1553 if (set) { 1554 rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid, 1555 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1556 if (rv == 0) { 1557 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 1558 vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1559 } 1560 } else { 1561 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid, 1562 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1563 (void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid, 1564 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1565 } 1566 1567 return rv; 1568 } 1569 1570 static void 1571 ixl_media_add(struct ixl_softc *sc) 1572 { 1573 struct ifmedia *ifm = &sc->sc_media; 1574 const struct ixl_phy_type *itype; 1575 unsigned int i; 1576 bool flow; 1577 1578 if (ISSET(sc->sc_phy_abilities, 1579 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1580 flow = true; 1581 } else { 1582 flow = false; 1583 } 1584 1585 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1586 itype = &ixl_phy_type_map[i]; 1587 1588 if (ISSET(sc->sc_phy_types, itype->phy_type)) { 1589 ifmedia_add(ifm, 1590 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1591 1592 if (flow) { 1593 ifmedia_add(ifm, 1594 IFM_ETHER | IFM_FDX | IFM_FLOW | 1595 itype->ifm_type, 0, NULL); 1596 } 1597 1598 if (itype->ifm_type != IFM_100_TX) 1599 continue; 1600 1601 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1602 0, NULL); 1603 if (flow) { 1604 ifmedia_add(ifm, 1605 IFM_ETHER | IFM_FLOW | itype->ifm_type, 1606 0, NULL); 1607 } 1608 } 1609 } 1610 } 1611 1612 static void 1613 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1614 { 1615 struct ixl_softc *sc = ifp->if_softc; 1616 1617 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1618 1619 ifmr->ifm_status = sc->sc_media_status; 1620 ifmr->ifm_active = sc->sc_media_active; 1621 } 1622 1623 static int 1624 ixl_media_change(struct ifnet *ifp) 1625 { 1626 struct ixl_softc *sc = ifp->if_softc; 1627 struct ifmedia *ifm = &sc->sc_media; 1628 uint64_t ifm_active = sc->sc_media_active; 1629 uint8_t link_speed, abilities; 1630 1631 switch (IFM_SUBTYPE(ifm_active)) { 1632 case IFM_1000_SGMII: 1633 case IFM_1000_KX: 1634 case IFM_10G_KX4: 1635 case IFM_10G_KR: 1636 case IFM_40G_KR4: 1637 case IFM_20G_KR2: 1638 case IFM_25G_KR: 1639 /* backplanes */ 1640 return EINVAL; 1641 } 1642 1643 abilities = IXL_PHY_ABILITY_AUTONEGO | IXL_PHY_ABILITY_LINKUP; 1644 1645 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1646 case IFM_AUTO: 1647 link_speed = sc->sc_phy_linkspeed; 1648 break; 1649 case IFM_NONE: 1650 link_speed = 0; 1651 CLR(abilities, IXL_PHY_ABILITY_LINKUP); 1652 break; 1653 default: 1654 link_speed = ixl_search_baudrate( 1655 ifmedia_baudrate(ifm->ifm_media)); 1656 } 1657 1658 if (ISSET(abilities, IXL_PHY_ABILITY_LINKUP)) { 1659 if (ISSET(link_speed, sc->sc_phy_linkspeed) == 0) 1660 return EINVAL; 1661 } 1662 1663 if (ifm->ifm_media & IFM_FLOW) { 1664 abilities |= sc->sc_phy_abilities & 1665 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX); 1666 } 1667 1668 return ixl_set_phy_config(sc, link_speed, abilities, false); 1669 } 1670 1671 static void 1672 ixl_watchdog(struct ifnet *ifp) 1673 { 1674 1675 } 1676 1677 static void 1678 ixl_del_all_multiaddr(struct ixl_softc *sc) 1679 { 1680 struct ethercom *ec = &sc->sc_ec; 1681 struct ether_multi *enm; 1682 struct ether_multistep step; 1683 1684 ETHER_LOCK(ec); 1685 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1686 ETHER_NEXT_MULTI(step, enm)) { 1687 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1688 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1689 } 1690 ETHER_UNLOCK(ec); 1691 } 1692 1693 static int 1694 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1695 { 1696 struct ifnet *ifp = &sc->sc_ec.ec_if; 1697 int rv; 1698 1699 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1700 return 0; 1701 1702 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1703 ixl_del_all_multiaddr(sc); 1704 SET(ifp->if_flags, IFF_ALLMULTI); 1705 return ENETRESET; 1706 } 1707 1708 /* multicast address can not use VLAN HWFILTER */ 1709 rv = ixl_add_macvlan(sc, addrlo, 0, 1710 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1711 1712 if (rv == ENOSPC) { 1713 ixl_del_all_multiaddr(sc); 1714 SET(ifp->if_flags, IFF_ALLMULTI); 1715 return ENETRESET; 1716 } 1717 1718 return rv; 1719 } 1720 1721 static int 1722 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1723 { 1724 struct ifnet *ifp = &sc->sc_ec.ec_if; 1725 struct ethercom *ec = &sc->sc_ec; 1726 struct ether_multi *enm, *enm_last; 1727 struct ether_multistep step; 1728 int error, rv = 0; 1729 1730 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1731 ixl_remove_macvlan(sc, addrlo, 0, 1732 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1733 return 0; 1734 } 1735 1736 ETHER_LOCK(ec); 1737 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1738 ETHER_NEXT_MULTI(step, enm)) { 1739 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1740 ETHER_ADDR_LEN) != 0) { 1741 goto out; 1742 } 1743 } 1744 1745 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1746 ETHER_NEXT_MULTI(step, enm)) { 1747 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1748 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1749 if (error != 0) 1750 break; 1751 } 1752 1753 if (enm != NULL) { 1754 enm_last = enm; 1755 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1756 ETHER_NEXT_MULTI(step, enm)) { 1757 if (enm == enm_last) 1758 break; 1759 1760 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1761 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1762 } 1763 } else { 1764 CLR(ifp->if_flags, IFF_ALLMULTI); 1765 rv = ENETRESET; 1766 } 1767 1768 out: 1769 ETHER_UNLOCK(ec); 1770 return rv; 1771 } 1772 1773 static int 1774 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1775 { 1776 struct ifreq *ifr = (struct ifreq *)data; 1777 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1778 const struct sockaddr *sa; 1779 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1780 int s, error = 0; 1781 unsigned int nmtu; 1782 1783 switch (cmd) { 1784 case SIOCSIFMTU: 1785 nmtu = ifr->ifr_mtu; 1786 1787 if (nmtu < IXL_MIN_MTU || nmtu > IXL_MAX_MTU) { 1788 error = EINVAL; 1789 break; 1790 } 1791 if (ifp->if_mtu != nmtu) { 1792 s = splnet(); 1793 error = ether_ioctl(ifp, cmd, data); 1794 splx(s); 1795 if (error == ENETRESET) 1796 error = ixl_init(ifp); 1797 } 1798 break; 1799 case SIOCADDMULTI: 1800 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1801 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1802 error = ether_multiaddr(sa, addrlo, addrhi); 1803 if (error != 0) 1804 return error; 1805 1806 error = ixl_add_multi(sc, addrlo, addrhi); 1807 if (error != 0 && error != ENETRESET) { 1808 ether_delmulti(sa, &sc->sc_ec); 1809 error = EIO; 1810 } 1811 } 1812 break; 1813 1814 case SIOCDELMULTI: 1815 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1816 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1817 error = ether_multiaddr(sa, addrlo, addrhi); 1818 if (error != 0) 1819 return error; 1820 1821 error = ixl_del_multi(sc, addrlo, addrhi); 1822 } 1823 break; 1824 1825 default: 1826 s = splnet(); 1827 error = ether_ioctl(ifp, cmd, data); 1828 splx(s); 1829 } 1830 1831 if (error == ENETRESET) 1832 error = ixl_iff(sc); 1833 1834 return error; 1835 } 1836 1837 static enum i40e_mac_type 1838 ixl_mactype(pci_product_id_t id) 1839 { 1840 1841 switch (id) { 1842 case PCI_PRODUCT_INTEL_XL710_SFP: 1843 case PCI_PRODUCT_INTEL_XL710_KX_B: 1844 case PCI_PRODUCT_INTEL_XL710_KX_C: 1845 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1846 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1847 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1848 case PCI_PRODUCT_INTEL_X710_10G_T_1: 1849 case PCI_PRODUCT_INTEL_X710_10G_T_2: 1850 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1851 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1852 case PCI_PRODUCT_INTEL_X710_T4_10G: 1853 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1854 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1855 case PCI_PRODUCT_INTEL_X710_10G_SFP: 1856 case PCI_PRODUCT_INTEL_X710_10G_BP: 1857 return I40E_MAC_XL710; 1858 1859 case PCI_PRODUCT_INTEL_X722_KX: 1860 case PCI_PRODUCT_INTEL_X722_QSFP: 1861 case PCI_PRODUCT_INTEL_X722_SFP: 1862 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1863 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1864 case PCI_PRODUCT_INTEL_X722_I_SFP: 1865 return I40E_MAC_X722; 1866 } 1867 1868 return I40E_MAC_GENERIC; 1869 } 1870 1871 static void 1872 ixl_pci_csr_setup(pci_chipset_tag_t pc, pcitag_t tag) 1873 { 1874 pcireg_t csr; 1875 1876 csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 1877 csr |= (PCI_COMMAND_MASTER_ENABLE | 1878 PCI_COMMAND_MEM_ENABLE); 1879 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr); 1880 } 1881 1882 static inline void * 1883 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1884 { 1885 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1886 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1887 1888 if (i >= e->hmc_count) 1889 return NULL; 1890 1891 kva += e->hmc_base; 1892 kva += i * e->hmc_size; 1893 1894 return kva; 1895 } 1896 1897 static inline size_t 1898 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1899 { 1900 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1901 1902 return e->hmc_size; 1903 } 1904 1905 static void 1906 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1907 { 1908 struct ixl_rx_ring *rxr = qp->qp_rxr; 1909 1910 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1911 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1912 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1913 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1914 ixl_flush(sc); 1915 } 1916 1917 static void 1918 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1919 { 1920 struct ixl_rx_ring *rxr = qp->qp_rxr; 1921 1922 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1923 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1924 ixl_flush(sc); 1925 } 1926 1927 static void 1928 ixl_enable_other_intr(struct ixl_softc *sc) 1929 { 1930 1931 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1932 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1933 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1934 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1935 ixl_flush(sc); 1936 } 1937 1938 static void 1939 ixl_disable_other_intr(struct ixl_softc *sc) 1940 { 1941 1942 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1943 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1944 ixl_flush(sc); 1945 } 1946 1947 static int 1948 ixl_reinit(struct ixl_softc *sc) 1949 { 1950 struct ixl_rx_ring *rxr; 1951 struct ixl_tx_ring *txr; 1952 unsigned int i; 1953 uint32_t reg; 1954 1955 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1956 1957 if (ixl_get_vsi(sc) != 0) 1958 return EIO; 1959 1960 if (ixl_set_vsi(sc) != 0) 1961 return EIO; 1962 1963 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1964 txr = sc->sc_qps[i].qp_txr; 1965 rxr = sc->sc_qps[i].qp_rxr; 1966 1967 ixl_txr_config(sc, txr); 1968 ixl_rxr_config(sc, rxr); 1969 } 1970 1971 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1972 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 1973 1974 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1975 txr = sc->sc_qps[i].qp_txr; 1976 rxr = sc->sc_qps[i].qp_rxr; 1977 1978 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 1979 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 1980 ixl_flush(sc); 1981 1982 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 1983 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 1984 1985 /* ixl_rxfill() needs lock held */ 1986 mutex_enter(&rxr->rxr_lock); 1987 ixl_rxfill(sc, rxr); 1988 mutex_exit(&rxr->rxr_lock); 1989 1990 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 1991 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 1992 ixl_wr(sc, I40E_QRX_ENA(i), reg); 1993 if (ixl_rxr_enabled(sc, rxr) != 0) 1994 goto stop; 1995 1996 ixl_txr_qdis(sc, txr, 1); 1997 1998 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 1999 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2000 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2001 2002 if (ixl_txr_enabled(sc, txr) != 0) 2003 goto stop; 2004 } 2005 2006 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2007 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2008 2009 return 0; 2010 2011 stop: 2012 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2013 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2014 2015 return ETIMEDOUT; 2016 } 2017 2018 static int 2019 ixl_init_locked(struct ixl_softc *sc) 2020 { 2021 struct ifnet *ifp = &sc->sc_ec.ec_if; 2022 unsigned int i; 2023 int error, eccap_change; 2024 2025 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2026 2027 if (ISSET(ifp->if_flags, IFF_RUNNING)) 2028 ixl_stop_locked(sc); 2029 2030 if (sc->sc_dead) { 2031 return ENXIO; 2032 } 2033 2034 eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable; 2035 if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING)) 2036 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 2037 2038 if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) { 2039 if (ixl_update_macvlan(sc) == 0) { 2040 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 2041 } else { 2042 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 2043 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 2044 } 2045 } 2046 2047 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 2048 sc->sc_nqueue_pairs = 1; 2049 else 2050 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 2051 2052 error = ixl_reinit(sc); 2053 if (error) { 2054 ixl_stop_locked(sc); 2055 return error; 2056 } 2057 2058 SET(ifp->if_flags, IFF_RUNNING); 2059 CLR(ifp->if_flags, IFF_OACTIVE); 2060 2061 ixl_config_rss(sc); 2062 ixl_config_queue_intr(sc); 2063 2064 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2065 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 2066 } 2067 2068 error = ixl_iff(sc); 2069 if (error) { 2070 ixl_stop_locked(sc); 2071 return error; 2072 } 2073 2074 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 2075 2076 return 0; 2077 } 2078 2079 static int 2080 ixl_init(struct ifnet *ifp) 2081 { 2082 struct ixl_softc *sc = ifp->if_softc; 2083 int error; 2084 2085 mutex_enter(&sc->sc_cfg_lock); 2086 error = ixl_init_locked(sc); 2087 mutex_exit(&sc->sc_cfg_lock); 2088 2089 if (error == 0) 2090 (void)ixl_get_link_status(sc); 2091 2092 return error; 2093 } 2094 2095 static int 2096 ixl_iff(struct ixl_softc *sc) 2097 { 2098 struct ifnet *ifp = &sc->sc_ec.ec_if; 2099 struct ixl_atq iatq; 2100 struct ixl_aq_desc *iaq; 2101 struct ixl_aq_vsi_promisc_param *param; 2102 uint16_t flag_add, flag_del; 2103 int error; 2104 2105 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2106 return 0; 2107 2108 memset(&iatq, 0, sizeof(iatq)); 2109 2110 iaq = &iatq.iatq_desc; 2111 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 2112 2113 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 2114 param->flags = htole16(0); 2115 2116 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER) 2117 || ISSET(ifp->if_flags, IFF_PROMISC)) { 2118 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2119 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2120 } 2121 2122 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 2123 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2124 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2125 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 2126 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2127 } 2128 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2129 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2130 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2131 param->seid = sc->sc_seid; 2132 2133 error = ixl_atq_exec(sc, &iatq); 2134 if (error) 2135 return error; 2136 2137 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 2138 return EIO; 2139 2140 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 2141 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 2142 flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH; 2143 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH; 2144 } else { 2145 flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN; 2146 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN; 2147 } 2148 2149 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del); 2150 2151 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2152 ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add); 2153 } 2154 return 0; 2155 } 2156 2157 static void 2158 ixl_stop_rendezvous(struct ixl_softc *sc) 2159 { 2160 struct ixl_tx_ring *txr; 2161 struct ixl_rx_ring *rxr; 2162 unsigned int i; 2163 2164 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2165 txr = sc->sc_qps[i].qp_txr; 2166 rxr = sc->sc_qps[i].qp_rxr; 2167 2168 mutex_enter(&txr->txr_lock); 2169 mutex_exit(&txr->txr_lock); 2170 2171 mutex_enter(&rxr->rxr_lock); 2172 mutex_exit(&rxr->rxr_lock); 2173 2174 sc->sc_qps[i].qp_workqueue = false; 2175 workqueue_wait(sc->sc_workq_txrx, 2176 &sc->sc_qps[i].qp_work); 2177 } 2178 } 2179 2180 static void 2181 ixl_stop_locked(struct ixl_softc *sc) 2182 { 2183 struct ifnet *ifp = &sc->sc_ec.ec_if; 2184 struct ixl_rx_ring *rxr; 2185 struct ixl_tx_ring *txr; 2186 unsigned int i; 2187 uint32_t reg; 2188 2189 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2190 2191 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 2192 callout_stop(&sc->sc_stats_callout); 2193 2194 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2195 txr = sc->sc_qps[i].qp_txr; 2196 rxr = sc->sc_qps[i].qp_rxr; 2197 2198 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 2199 2200 mutex_enter(&txr->txr_lock); 2201 ixl_txr_qdis(sc, txr, 0); 2202 mutex_exit(&txr->txr_lock); 2203 } 2204 2205 /* XXX wait at least 400 usec for all tx queues in one go */ 2206 ixl_flush(sc); 2207 DELAY(500); 2208 2209 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2210 txr = sc->sc_qps[i].qp_txr; 2211 rxr = sc->sc_qps[i].qp_rxr; 2212 2213 mutex_enter(&txr->txr_lock); 2214 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2215 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2216 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2217 mutex_exit(&txr->txr_lock); 2218 2219 mutex_enter(&rxr->rxr_lock); 2220 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2221 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2222 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2223 mutex_exit(&rxr->rxr_lock); 2224 } 2225 2226 /* XXX short wait for all queue disables to settle */ 2227 ixl_flush(sc); 2228 DELAY(50); 2229 2230 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2231 txr = sc->sc_qps[i].qp_txr; 2232 rxr = sc->sc_qps[i].qp_rxr; 2233 2234 mutex_enter(&txr->txr_lock); 2235 if (ixl_txr_disabled(sc, txr) != 0) { 2236 mutex_exit(&txr->txr_lock); 2237 goto die; 2238 } 2239 mutex_exit(&txr->txr_lock); 2240 2241 mutex_enter(&rxr->rxr_lock); 2242 if (ixl_rxr_disabled(sc, rxr) != 0) { 2243 mutex_exit(&rxr->rxr_lock); 2244 goto die; 2245 } 2246 mutex_exit(&rxr->rxr_lock); 2247 } 2248 2249 ixl_stop_rendezvous(sc); 2250 2251 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2252 txr = sc->sc_qps[i].qp_txr; 2253 rxr = sc->sc_qps[i].qp_rxr; 2254 2255 mutex_enter(&txr->txr_lock); 2256 ixl_txr_unconfig(sc, txr); 2257 mutex_exit(&txr->txr_lock); 2258 2259 mutex_enter(&rxr->rxr_lock); 2260 ixl_rxr_unconfig(sc, rxr); 2261 mutex_exit(&rxr->rxr_lock); 2262 2263 ixl_txr_clean(sc, txr); 2264 ixl_rxr_clean(sc, rxr); 2265 } 2266 2267 return; 2268 die: 2269 sc->sc_dead = true; 2270 log(LOG_CRIT, "%s: failed to shut down rings", 2271 device_xname(sc->sc_dev)); 2272 return; 2273 } 2274 2275 static void 2276 ixl_stop(struct ifnet *ifp, int disable) 2277 { 2278 struct ixl_softc *sc = ifp->if_softc; 2279 2280 mutex_enter(&sc->sc_cfg_lock); 2281 ixl_stop_locked(sc); 2282 mutex_exit(&sc->sc_cfg_lock); 2283 } 2284 2285 static int 2286 ixl_queue_pairs_alloc(struct ixl_softc *sc) 2287 { 2288 struct ixl_queue_pair *qp; 2289 unsigned int i; 2290 size_t sz; 2291 2292 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2293 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 2294 2295 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2296 qp = &sc->sc_qps[i]; 2297 2298 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2299 ixl_handle_queue, qp); 2300 if (qp->qp_si == NULL) 2301 goto free; 2302 2303 qp->qp_txr = ixl_txr_alloc(sc, i); 2304 if (qp->qp_txr == NULL) 2305 goto free; 2306 2307 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2308 if (qp->qp_rxr == NULL) 2309 goto free; 2310 2311 qp->qp_sc = sc; 2312 snprintf(qp->qp_name, sizeof(qp->qp_name), 2313 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2314 } 2315 2316 return 0; 2317 free: 2318 if (sc->sc_qps != NULL) { 2319 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2320 qp = &sc->sc_qps[i]; 2321 2322 if (qp->qp_txr != NULL) 2323 ixl_txr_free(sc, qp->qp_txr); 2324 if (qp->qp_rxr != NULL) 2325 ixl_rxr_free(sc, qp->qp_rxr); 2326 if (qp->qp_si != NULL) 2327 softint_disestablish(qp->qp_si); 2328 } 2329 2330 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2331 kmem_free(sc->sc_qps, sz); 2332 sc->sc_qps = NULL; 2333 } 2334 2335 return -1; 2336 } 2337 2338 static void 2339 ixl_queue_pairs_free(struct ixl_softc *sc) 2340 { 2341 struct ixl_queue_pair *qp; 2342 unsigned int i; 2343 size_t sz; 2344 2345 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2346 qp = &sc->sc_qps[i]; 2347 ixl_txr_free(sc, qp->qp_txr); 2348 ixl_rxr_free(sc, qp->qp_rxr); 2349 softint_disestablish(qp->qp_si); 2350 } 2351 2352 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2353 kmem_free(sc->sc_qps, sz); 2354 sc->sc_qps = NULL; 2355 } 2356 2357 static struct ixl_tx_ring * 2358 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2359 { 2360 struct ixl_tx_ring *txr = NULL; 2361 struct ixl_tx_map *maps = NULL, *txm; 2362 unsigned int i; 2363 2364 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2365 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2366 KM_SLEEP); 2367 2368 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2369 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2370 IXL_TX_QUEUE_ALIGN) != 0) 2371 goto free; 2372 2373 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2374 txm = &maps[i]; 2375 2376 if (bus_dmamap_create(sc->sc_dmat, IXL_TX_PKT_MAXSIZE, 2377 IXL_TX_PKT_DESCS, IXL_TX_PKT_MAXSIZE, 0, 2378 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2379 goto uncreate; 2380 2381 txm->txm_eop = -1; 2382 txm->txm_m = NULL; 2383 } 2384 2385 txr->txr_cons = txr->txr_prod = 0; 2386 txr->txr_maps = maps; 2387 2388 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2389 if (txr->txr_intrq == NULL) 2390 goto uncreate; 2391 2392 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2393 ixl_deferred_transmit, txr); 2394 if (txr->txr_si == NULL) 2395 goto destroy_pcq; 2396 2397 txr->txr_tail = I40E_QTX_TAIL(qid); 2398 txr->txr_qid = qid; 2399 txr->txr_sc = sc; 2400 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2401 2402 return txr; 2403 2404 destroy_pcq: 2405 pcq_destroy(txr->txr_intrq); 2406 uncreate: 2407 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2408 txm = &maps[i]; 2409 2410 if (txm->txm_map == NULL) 2411 continue; 2412 2413 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2414 } 2415 2416 ixl_dmamem_free(sc, &txr->txr_mem); 2417 free: 2418 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2419 kmem_free(txr, sizeof(*txr)); 2420 2421 return NULL; 2422 } 2423 2424 static void 2425 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2426 { 2427 unsigned int qid; 2428 bus_size_t reg; 2429 uint32_t r; 2430 2431 qid = txr->txr_qid + sc->sc_base_queue; 2432 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2433 qid %= 128; 2434 2435 r = ixl_rd(sc, reg); 2436 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2437 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2438 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2439 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2440 ixl_wr(sc, reg, r); 2441 } 2442 2443 static void 2444 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2445 { 2446 struct ixl_hmc_txq txq; 2447 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2448 void *hmc; 2449 2450 memset(&txq, 0, sizeof(txq)); 2451 txq.head = htole16(txr->txr_cons); 2452 txq.new_context = 1; 2453 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2454 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2455 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2456 txq.tphrdesc_ena = 0; 2457 txq.tphrpacket_ena = 0; 2458 txq.tphwdesc_ena = 0; 2459 txq.rdylist = data->qs_handle[0]; 2460 2461 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2462 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2463 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2464 __arraycount(ixl_hmc_pack_txq)); 2465 } 2466 2467 static void 2468 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2469 { 2470 void *hmc; 2471 2472 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2473 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2474 txr->txr_cons = txr->txr_prod = 0; 2475 } 2476 2477 static void 2478 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2479 { 2480 struct ixl_tx_map *maps, *txm; 2481 bus_dmamap_t map; 2482 unsigned int i; 2483 2484 maps = txr->txr_maps; 2485 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2486 txm = &maps[i]; 2487 2488 if (txm->txm_m == NULL) 2489 continue; 2490 2491 map = txm->txm_map; 2492 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2493 BUS_DMASYNC_POSTWRITE); 2494 bus_dmamap_unload(sc->sc_dmat, map); 2495 2496 m_freem(txm->txm_m); 2497 txm->txm_m = NULL; 2498 } 2499 } 2500 2501 static int 2502 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2503 { 2504 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2505 uint32_t reg; 2506 int i; 2507 2508 for (i = 0; i < 10; i++) { 2509 reg = ixl_rd(sc, ena); 2510 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2511 return 0; 2512 2513 delaymsec(10); 2514 } 2515 2516 return ETIMEDOUT; 2517 } 2518 2519 static int 2520 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2521 { 2522 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2523 uint32_t reg; 2524 int i; 2525 2526 KASSERT(mutex_owned(&txr->txr_lock)); 2527 2528 for (i = 0; i < 10; i++) { 2529 reg = ixl_rd(sc, ena); 2530 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2531 return 0; 2532 2533 delaymsec(10); 2534 } 2535 2536 return ETIMEDOUT; 2537 } 2538 2539 static void 2540 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2541 { 2542 struct ixl_tx_map *maps, *txm; 2543 struct mbuf *m; 2544 unsigned int i; 2545 2546 softint_disestablish(txr->txr_si); 2547 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2548 m_freem(m); 2549 pcq_destroy(txr->txr_intrq); 2550 2551 maps = txr->txr_maps; 2552 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2553 txm = &maps[i]; 2554 2555 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2556 } 2557 2558 ixl_dmamem_free(sc, &txr->txr_mem); 2559 mutex_destroy(&txr->txr_lock); 2560 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2561 kmem_free(txr, sizeof(*txr)); 2562 } 2563 2564 static inline int 2565 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2566 struct ixl_tx_ring *txr) 2567 { 2568 struct mbuf *m; 2569 int error; 2570 2571 KASSERT(mutex_owned(&txr->txr_lock)); 2572 2573 m = *m0; 2574 2575 error = bus_dmamap_load_mbuf(dmat, map, m, 2576 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2577 if (error != EFBIG) 2578 return error; 2579 2580 m = m_defrag(m, M_DONTWAIT); 2581 if (m != NULL) { 2582 *m0 = m; 2583 txr->txr_defragged.ev_count++; 2584 2585 error = bus_dmamap_load_mbuf(dmat, map, m, 2586 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2587 } else { 2588 txr->txr_defrag_failed.ev_count++; 2589 error = ENOBUFS; 2590 } 2591 2592 return error; 2593 } 2594 2595 static inline int 2596 ixl_tx_setup_offloads(struct mbuf *m, uint64_t *cmd_txd) 2597 { 2598 struct ether_header *eh; 2599 size_t len; 2600 uint64_t cmd; 2601 2602 cmd = 0; 2603 2604 eh = mtod(m, struct ether_header *); 2605 switch (htons(eh->ether_type)) { 2606 case ETHERTYPE_IP: 2607 case ETHERTYPE_IPV6: 2608 len = ETHER_HDR_LEN; 2609 break; 2610 case ETHERTYPE_VLAN: 2611 len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2612 break; 2613 default: 2614 len = 0; 2615 } 2616 cmd |= ((len >> 1) << IXL_TX_DESC_MACLEN_SHIFT); 2617 2618 if (m->m_pkthdr.csum_flags & 2619 (M_CSUM_TSOv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) { 2620 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4; 2621 } 2622 if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) { 2623 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4_CSUM; 2624 } 2625 2626 if (m->m_pkthdr.csum_flags & 2627 (M_CSUM_TSOv6 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) { 2628 cmd |= IXL_TX_DESC_CMD_IIPT_IPV6; 2629 } 2630 2631 switch (cmd & IXL_TX_DESC_CMD_IIPT_MASK) { 2632 case IXL_TX_DESC_CMD_IIPT_IPV4: 2633 case IXL_TX_DESC_CMD_IIPT_IPV4_CSUM: 2634 len = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); 2635 break; 2636 case IXL_TX_DESC_CMD_IIPT_IPV6: 2637 len = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); 2638 break; 2639 default: 2640 len = 0; 2641 } 2642 cmd |= ((len >> 2) << IXL_TX_DESC_IPLEN_SHIFT); 2643 2644 if (m->m_pkthdr.csum_flags & 2645 (M_CSUM_TSOv4 | M_CSUM_TSOv6 | M_CSUM_TCPv4 | M_CSUM_TCPv6)) { 2646 len = sizeof(struct tcphdr); 2647 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_TCP; 2648 } else if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) { 2649 len = sizeof(struct udphdr); 2650 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_UDP; 2651 } else { 2652 len = 0; 2653 } 2654 cmd |= ((len >> 2) << IXL_TX_DESC_L4LEN_SHIFT); 2655 2656 *cmd_txd |= cmd; 2657 return 0; 2658 } 2659 2660 static void 2661 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2662 bool is_transmit) 2663 { 2664 struct ixl_softc *sc = ifp->if_softc; 2665 struct ixl_tx_desc *ring, *txd; 2666 struct ixl_tx_map *txm; 2667 bus_dmamap_t map; 2668 struct mbuf *m; 2669 uint64_t cmd, cmd_txd; 2670 unsigned int prod, free, last, i; 2671 unsigned int mask; 2672 int post = 0; 2673 2674 KASSERT(mutex_owned(&txr->txr_lock)); 2675 2676 if (!ISSET(ifp->if_flags, IFF_RUNNING) 2677 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2678 if (!is_transmit) 2679 IFQ_PURGE(&ifp->if_snd); 2680 return; 2681 } 2682 2683 prod = txr->txr_prod; 2684 free = txr->txr_cons; 2685 if (free <= prod) 2686 free += sc->sc_tx_ring_ndescs; 2687 free -= prod; 2688 2689 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2690 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2691 2692 ring = IXL_DMA_KVA(&txr->txr_mem); 2693 mask = sc->sc_tx_ring_ndescs - 1; 2694 last = prod; 2695 cmd = 0; 2696 txd = NULL; 2697 2698 for (;;) { 2699 if (free <= IXL_TX_PKT_DESCS) { 2700 if (!is_transmit) 2701 SET(ifp->if_flags, IFF_OACTIVE); 2702 break; 2703 } 2704 2705 if (is_transmit) 2706 m = pcq_get(txr->txr_intrq); 2707 else 2708 IFQ_DEQUEUE(&ifp->if_snd, m); 2709 2710 if (m == NULL) 2711 break; 2712 2713 txm = &txr->txr_maps[prod]; 2714 map = txm->txm_map; 2715 2716 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2717 if_statinc(ifp, if_oerrors); 2718 m_freem(m); 2719 continue; 2720 } 2721 2722 cmd_txd = 0; 2723 if (m->m_pkthdr.csum_flags & IXL_CSUM_ALL_OFFLOAD) { 2724 ixl_tx_setup_offloads(m, &cmd_txd); 2725 } 2726 2727 if (vlan_has_tag(m)) { 2728 cmd_txd |= (uint64_t)vlan_get_tag(m) << 2729 IXL_TX_DESC_L2TAG1_SHIFT; 2730 cmd_txd |= IXL_TX_DESC_CMD_IL2TAG1; 2731 } 2732 2733 bus_dmamap_sync(sc->sc_dmat, map, 0, 2734 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2735 2736 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2737 txd = &ring[prod]; 2738 2739 cmd = (uint64_t)map->dm_segs[i].ds_len << 2740 IXL_TX_DESC_BSIZE_SHIFT; 2741 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2742 cmd |= cmd_txd; 2743 2744 txd->addr = htole64(map->dm_segs[i].ds_addr); 2745 txd->cmd = htole64(cmd); 2746 2747 last = prod; 2748 2749 prod++; 2750 prod &= mask; 2751 } 2752 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2753 txd->cmd = htole64(cmd); 2754 2755 txm->txm_m = m; 2756 txm->txm_eop = last; 2757 2758 bpf_mtap(ifp, m, BPF_D_OUT); 2759 2760 free -= i; 2761 post = 1; 2762 } 2763 2764 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2765 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2766 2767 if (post) { 2768 txr->txr_prod = prod; 2769 ixl_wr(sc, txr->txr_tail, prod); 2770 } 2771 } 2772 2773 static int 2774 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2775 { 2776 struct ifnet *ifp = &sc->sc_ec.ec_if; 2777 struct ixl_tx_desc *ring, *txd; 2778 struct ixl_tx_map *txm; 2779 struct mbuf *m; 2780 bus_dmamap_t map; 2781 unsigned int cons, prod, last; 2782 unsigned int mask; 2783 uint64_t dtype; 2784 int done = 0, more = 0; 2785 2786 KASSERT(mutex_owned(&txr->txr_lock)); 2787 2788 prod = txr->txr_prod; 2789 cons = txr->txr_cons; 2790 2791 if (cons == prod) 2792 return 0; 2793 2794 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2795 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2796 2797 ring = IXL_DMA_KVA(&txr->txr_mem); 2798 mask = sc->sc_tx_ring_ndescs - 1; 2799 2800 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2801 2802 do { 2803 if (txlimit-- <= 0) { 2804 more = 1; 2805 break; 2806 } 2807 2808 txm = &txr->txr_maps[cons]; 2809 last = txm->txm_eop; 2810 txd = &ring[last]; 2811 2812 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2813 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2814 break; 2815 2816 map = txm->txm_map; 2817 2818 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2819 BUS_DMASYNC_POSTWRITE); 2820 bus_dmamap_unload(sc->sc_dmat, map); 2821 2822 m = txm->txm_m; 2823 if (m != NULL) { 2824 if_statinc_ref(nsr, if_opackets); 2825 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 2826 if (ISSET(m->m_flags, M_MCAST)) 2827 if_statinc_ref(nsr, if_omcasts); 2828 m_freem(m); 2829 } 2830 2831 txm->txm_m = NULL; 2832 txm->txm_eop = -1; 2833 2834 cons = last + 1; 2835 cons &= mask; 2836 done = 1; 2837 } while (cons != prod); 2838 2839 IF_STAT_PUTREF(ifp); 2840 2841 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2842 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2843 2844 txr->txr_cons = cons; 2845 2846 if (done) { 2847 softint_schedule(txr->txr_si); 2848 if (txr->txr_qid == 0) { 2849 CLR(ifp->if_flags, IFF_OACTIVE); 2850 if_schedule_deferred_start(ifp); 2851 } 2852 } 2853 2854 return more; 2855 } 2856 2857 static void 2858 ixl_start(struct ifnet *ifp) 2859 { 2860 struct ixl_softc *sc; 2861 struct ixl_tx_ring *txr; 2862 2863 sc = ifp->if_softc; 2864 txr = sc->sc_qps[0].qp_txr; 2865 2866 mutex_enter(&txr->txr_lock); 2867 ixl_tx_common_locked(ifp, txr, false); 2868 mutex_exit(&txr->txr_lock); 2869 } 2870 2871 static inline unsigned int 2872 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2873 { 2874 u_int cpuid; 2875 2876 cpuid = cpu_index(curcpu()); 2877 2878 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2879 } 2880 2881 static int 2882 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2883 { 2884 struct ixl_softc *sc; 2885 struct ixl_tx_ring *txr; 2886 unsigned int qid; 2887 2888 sc = ifp->if_softc; 2889 qid = ixl_select_txqueue(sc, m); 2890 2891 txr = sc->sc_qps[qid].qp_txr; 2892 2893 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2894 mutex_enter(&txr->txr_lock); 2895 txr->txr_pcqdrop.ev_count++; 2896 mutex_exit(&txr->txr_lock); 2897 2898 m_freem(m); 2899 return ENOBUFS; 2900 } 2901 2902 if (mutex_tryenter(&txr->txr_lock)) { 2903 ixl_tx_common_locked(ifp, txr, true); 2904 mutex_exit(&txr->txr_lock); 2905 } else { 2906 kpreempt_disable(); 2907 softint_schedule(txr->txr_si); 2908 kpreempt_enable(); 2909 } 2910 2911 return 0; 2912 } 2913 2914 static void 2915 ixl_deferred_transmit(void *xtxr) 2916 { 2917 struct ixl_tx_ring *txr = xtxr; 2918 struct ixl_softc *sc = txr->txr_sc; 2919 struct ifnet *ifp = &sc->sc_ec.ec_if; 2920 2921 mutex_enter(&txr->txr_lock); 2922 txr->txr_transmitdef.ev_count++; 2923 if (pcq_peek(txr->txr_intrq) != NULL) 2924 ixl_tx_common_locked(ifp, txr, true); 2925 mutex_exit(&txr->txr_lock); 2926 } 2927 2928 static struct ixl_rx_ring * 2929 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2930 { 2931 struct ixl_rx_ring *rxr = NULL; 2932 struct ixl_rx_map *maps = NULL, *rxm; 2933 unsigned int i; 2934 2935 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2936 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2937 KM_SLEEP); 2938 2939 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2940 sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs, 2941 IXL_RX_QUEUE_ALIGN) != 0) 2942 goto free; 2943 2944 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2945 rxm = &maps[i]; 2946 2947 if (bus_dmamap_create(sc->sc_dmat, 2948 IXL_MCLBYTES, 1, IXL_MCLBYTES, 0, 2949 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2950 goto uncreate; 2951 2952 rxm->rxm_m = NULL; 2953 } 2954 2955 rxr->rxr_cons = rxr->rxr_prod = 0; 2956 rxr->rxr_m_head = NULL; 2957 rxr->rxr_m_tail = &rxr->rxr_m_head; 2958 rxr->rxr_maps = maps; 2959 2960 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2961 rxr->rxr_qid = qid; 2962 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2963 2964 return rxr; 2965 2966 uncreate: 2967 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2968 rxm = &maps[i]; 2969 2970 if (rxm->rxm_map == NULL) 2971 continue; 2972 2973 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2974 } 2975 2976 ixl_dmamem_free(sc, &rxr->rxr_mem); 2977 free: 2978 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2979 kmem_free(rxr, sizeof(*rxr)); 2980 2981 return NULL; 2982 } 2983 2984 static void 2985 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2986 { 2987 struct ixl_rx_map *maps, *rxm; 2988 bus_dmamap_t map; 2989 unsigned int i; 2990 2991 maps = rxr->rxr_maps; 2992 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2993 rxm = &maps[i]; 2994 2995 if (rxm->rxm_m == NULL) 2996 continue; 2997 2998 map = rxm->rxm_map; 2999 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3000 BUS_DMASYNC_POSTWRITE); 3001 bus_dmamap_unload(sc->sc_dmat, map); 3002 3003 m_freem(rxm->rxm_m); 3004 rxm->rxm_m = NULL; 3005 } 3006 3007 m_freem(rxr->rxr_m_head); 3008 rxr->rxr_m_head = NULL; 3009 rxr->rxr_m_tail = &rxr->rxr_m_head; 3010 3011 rxr->rxr_prod = rxr->rxr_cons = 0; 3012 } 3013 3014 static int 3015 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3016 { 3017 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3018 uint32_t reg; 3019 int i; 3020 3021 for (i = 0; i < 10; i++) { 3022 reg = ixl_rd(sc, ena); 3023 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 3024 return 0; 3025 3026 delaymsec(10); 3027 } 3028 3029 return ETIMEDOUT; 3030 } 3031 3032 static int 3033 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3034 { 3035 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3036 uint32_t reg; 3037 int i; 3038 3039 KASSERT(mutex_owned(&rxr->rxr_lock)); 3040 3041 for (i = 0; i < 10; i++) { 3042 reg = ixl_rd(sc, ena); 3043 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 3044 return 0; 3045 3046 delaymsec(10); 3047 } 3048 3049 return ETIMEDOUT; 3050 } 3051 3052 static void 3053 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3054 { 3055 struct ixl_hmc_rxq rxq; 3056 struct ifnet *ifp = &sc->sc_ec.ec_if; 3057 uint16_t rxmax; 3058 void *hmc; 3059 3060 memset(&rxq, 0, sizeof(rxq)); 3061 rxmax = ifp->if_mtu + IXL_MTU_ETHERLEN; 3062 3063 rxq.head = htole16(rxr->rxr_cons); 3064 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 3065 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 3066 rxq.dbuff = htole16(IXL_MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 3067 rxq.hbuff = 0; 3068 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 3069 rxq.dsize = IXL_HMC_RXQ_DSIZE_32; 3070 rxq.crcstrip = 1; 3071 rxq.l2sel = 1; 3072 rxq.showiv = 1; 3073 rxq.rxmax = htole16(rxmax); 3074 rxq.tphrdesc_ena = 0; 3075 rxq.tphwdesc_ena = 0; 3076 rxq.tphdata_ena = 0; 3077 rxq.tphhead_ena = 0; 3078 rxq.lrxqthresh = 0; 3079 rxq.prefena = 1; 3080 3081 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3082 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3083 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 3084 __arraycount(ixl_hmc_pack_rxq)); 3085 } 3086 3087 static void 3088 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3089 { 3090 void *hmc; 3091 3092 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3093 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3094 rxr->rxr_cons = rxr->rxr_prod = 0; 3095 } 3096 3097 static void 3098 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3099 { 3100 struct ixl_rx_map *maps, *rxm; 3101 unsigned int i; 3102 3103 maps = rxr->rxr_maps; 3104 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3105 rxm = &maps[i]; 3106 3107 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 3108 } 3109 3110 ixl_dmamem_free(sc, &rxr->rxr_mem); 3111 mutex_destroy(&rxr->rxr_lock); 3112 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3113 kmem_free(rxr, sizeof(*rxr)); 3114 } 3115 3116 static inline void 3117 ixl_rx_csum(struct mbuf *m, uint64_t qword) 3118 { 3119 int flags_mask; 3120 3121 if (!ISSET(qword, IXL_RX_DESC_L3L4P)) { 3122 /* No L3 or L4 checksum was calculated */ 3123 return; 3124 } 3125 3126 switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) { 3127 case IXL_RX_DESC_PTYPE_IPV4FRAG: 3128 case IXL_RX_DESC_PTYPE_IPV4: 3129 case IXL_RX_DESC_PTYPE_SCTPV4: 3130 case IXL_RX_DESC_PTYPE_ICMPV4: 3131 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3132 break; 3133 case IXL_RX_DESC_PTYPE_TCPV4: 3134 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3135 flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD; 3136 break; 3137 case IXL_RX_DESC_PTYPE_UDPV4: 3138 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3139 flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD; 3140 break; 3141 case IXL_RX_DESC_PTYPE_TCPV6: 3142 flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD; 3143 break; 3144 case IXL_RX_DESC_PTYPE_UDPV6: 3145 flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD; 3146 break; 3147 default: 3148 flags_mask = 0; 3149 } 3150 3151 m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 | 3152 M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6)); 3153 3154 if (ISSET(qword, IXL_RX_DESC_IPE)) { 3155 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD); 3156 } 3157 3158 if (ISSET(qword, IXL_RX_DESC_L4E)) { 3159 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD); 3160 } 3161 } 3162 3163 static int 3164 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 3165 { 3166 struct ifnet *ifp = &sc->sc_ec.ec_if; 3167 struct ixl_rx_wb_desc_32 *ring, *rxd; 3168 struct ixl_rx_map *rxm; 3169 bus_dmamap_t map; 3170 unsigned int cons, prod; 3171 struct mbuf *m; 3172 uint64_t word, word0; 3173 unsigned int len; 3174 unsigned int mask; 3175 int done = 0, more = 0; 3176 3177 KASSERT(mutex_owned(&rxr->rxr_lock)); 3178 3179 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3180 return 0; 3181 3182 prod = rxr->rxr_prod; 3183 cons = rxr->rxr_cons; 3184 3185 if (cons == prod) 3186 return 0; 3187 3188 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3189 0, IXL_DMA_LEN(&rxr->rxr_mem), 3190 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3191 3192 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3193 mask = sc->sc_rx_ring_ndescs - 1; 3194 3195 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 3196 3197 do { 3198 if (rxlimit-- <= 0) { 3199 more = 1; 3200 break; 3201 } 3202 3203 rxd = &ring[cons]; 3204 3205 word = le64toh(rxd->qword1); 3206 3207 if (!ISSET(word, IXL_RX_DESC_DD)) 3208 break; 3209 3210 rxm = &rxr->rxr_maps[cons]; 3211 3212 map = rxm->rxm_map; 3213 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3214 BUS_DMASYNC_POSTREAD); 3215 bus_dmamap_unload(sc->sc_dmat, map); 3216 3217 m = rxm->rxm_m; 3218 rxm->rxm_m = NULL; 3219 3220 KASSERT(m != NULL); 3221 3222 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 3223 m->m_len = len; 3224 m->m_pkthdr.len = 0; 3225 3226 m->m_next = NULL; 3227 *rxr->rxr_m_tail = m; 3228 rxr->rxr_m_tail = &m->m_next; 3229 3230 m = rxr->rxr_m_head; 3231 m->m_pkthdr.len += len; 3232 3233 if (ISSET(word, IXL_RX_DESC_EOP)) { 3234 word0 = le64toh(rxd->qword0); 3235 3236 if (ISSET(word, IXL_RX_DESC_L2TAG1P)) { 3237 vlan_set_tag(m, 3238 __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK)); 3239 } 3240 3241 if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0) 3242 ixl_rx_csum(m, word); 3243 3244 if (!ISSET(word, 3245 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 3246 m_set_rcvif(m, ifp); 3247 if_statinc_ref(nsr, if_ipackets); 3248 if_statadd_ref(nsr, if_ibytes, 3249 m->m_pkthdr.len); 3250 if_percpuq_enqueue(sc->sc_ipq, m); 3251 } else { 3252 if_statinc_ref(nsr, if_ierrors); 3253 m_freem(m); 3254 } 3255 3256 rxr->rxr_m_head = NULL; 3257 rxr->rxr_m_tail = &rxr->rxr_m_head; 3258 } 3259 3260 cons++; 3261 cons &= mask; 3262 3263 done = 1; 3264 } while (cons != prod); 3265 3266 if (done) { 3267 rxr->rxr_cons = cons; 3268 if (ixl_rxfill(sc, rxr) == -1) 3269 if_statinc_ref(nsr, if_iqdrops); 3270 } 3271 3272 IF_STAT_PUTREF(ifp); 3273 3274 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3275 0, IXL_DMA_LEN(&rxr->rxr_mem), 3276 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3277 3278 return more; 3279 } 3280 3281 static int 3282 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3283 { 3284 struct ixl_rx_rd_desc_32 *ring, *rxd; 3285 struct ixl_rx_map *rxm; 3286 bus_dmamap_t map; 3287 struct mbuf *m; 3288 unsigned int prod; 3289 unsigned int slots; 3290 unsigned int mask; 3291 int post = 0, error = 0; 3292 3293 KASSERT(mutex_owned(&rxr->rxr_lock)); 3294 3295 prod = rxr->rxr_prod; 3296 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 3297 sc->sc_rx_ring_ndescs); 3298 3299 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3300 mask = sc->sc_rx_ring_ndescs - 1; 3301 3302 if (__predict_false(slots <= 0)) 3303 return -1; 3304 3305 do { 3306 rxm = &rxr->rxr_maps[prod]; 3307 3308 MGETHDR(m, M_DONTWAIT, MT_DATA); 3309 if (m == NULL) { 3310 rxr->rxr_mgethdr_failed.ev_count++; 3311 error = -1; 3312 break; 3313 } 3314 3315 MCLGET(m, M_DONTWAIT); 3316 if (!ISSET(m->m_flags, M_EXT)) { 3317 rxr->rxr_mgetcl_failed.ev_count++; 3318 error = -1; 3319 m_freem(m); 3320 break; 3321 } 3322 3323 m->m_len = m->m_pkthdr.len = MCLBYTES; 3324 m_adj(m, ETHER_ALIGN); 3325 3326 map = rxm->rxm_map; 3327 3328 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 3329 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 3330 rxr->rxr_mbuf_load_failed.ev_count++; 3331 error = -1; 3332 m_freem(m); 3333 break; 3334 } 3335 3336 rxm->rxm_m = m; 3337 3338 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3339 BUS_DMASYNC_PREREAD); 3340 3341 rxd = &ring[prod]; 3342 3343 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 3344 rxd->haddr = htole64(0); 3345 3346 prod++; 3347 prod &= mask; 3348 3349 post = 1; 3350 3351 } while (--slots); 3352 3353 if (post) { 3354 rxr->rxr_prod = prod; 3355 ixl_wr(sc, rxr->rxr_tail, prod); 3356 } 3357 3358 return error; 3359 } 3360 3361 static inline int 3362 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 3363 u_int txlimit, struct evcnt *txevcnt, 3364 u_int rxlimit, struct evcnt *rxevcnt) 3365 { 3366 struct ixl_tx_ring *txr = qp->qp_txr; 3367 struct ixl_rx_ring *rxr = qp->qp_rxr; 3368 int txmore, rxmore; 3369 int rv; 3370 3371 mutex_enter(&txr->txr_lock); 3372 txevcnt->ev_count++; 3373 txmore = ixl_txeof(sc, txr, txlimit); 3374 mutex_exit(&txr->txr_lock); 3375 3376 mutex_enter(&rxr->rxr_lock); 3377 rxevcnt->ev_count++; 3378 rxmore = ixl_rxeof(sc, rxr, rxlimit); 3379 mutex_exit(&rxr->rxr_lock); 3380 3381 rv = txmore | (rxmore << 1); 3382 3383 return rv; 3384 } 3385 3386 static void 3387 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 3388 { 3389 3390 if (qp->qp_workqueue) 3391 workqueue_enqueue(sc->sc_workq_txrx, &qp->qp_work, NULL); 3392 else 3393 softint_schedule(qp->qp_si); 3394 } 3395 3396 static int 3397 ixl_intr(void *xsc) 3398 { 3399 struct ixl_softc *sc = xsc; 3400 struct ixl_tx_ring *txr; 3401 struct ixl_rx_ring *rxr; 3402 uint32_t icr, rxintr, txintr; 3403 int rv = 0; 3404 unsigned int i; 3405 3406 KASSERT(sc != NULL); 3407 3408 ixl_enable_other_intr(sc); 3409 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3410 3411 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3412 atomic_inc_64(&sc->sc_event_atq.ev_count); 3413 ixl_atq_done(sc); 3414 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3415 rv = 1; 3416 } 3417 3418 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3419 atomic_inc_64(&sc->sc_event_link.ev_count); 3420 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3421 rv = 1; 3422 } 3423 3424 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 3425 txintr = icr & I40E_INTR_NOTX_TX_MASK; 3426 3427 if (txintr || rxintr) { 3428 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 3429 txr = sc->sc_qps[i].qp_txr; 3430 rxr = sc->sc_qps[i].qp_rxr; 3431 3432 ixl_handle_queue_common(sc, &sc->sc_qps[i], 3433 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 3434 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 3435 } 3436 rv = 1; 3437 } 3438 3439 return rv; 3440 } 3441 3442 static int 3443 ixl_queue_intr(void *xqp) 3444 { 3445 struct ixl_queue_pair *qp = xqp; 3446 struct ixl_tx_ring *txr = qp->qp_txr; 3447 struct ixl_rx_ring *rxr = qp->qp_rxr; 3448 struct ixl_softc *sc = qp->qp_sc; 3449 u_int txlimit, rxlimit; 3450 int more; 3451 3452 txlimit = sc->sc_tx_intr_process_limit; 3453 rxlimit = sc->sc_rx_intr_process_limit; 3454 qp->qp_workqueue = sc->sc_txrx_workqueue; 3455 3456 more = ixl_handle_queue_common(sc, qp, 3457 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3458 3459 if (more != 0) { 3460 ixl_sched_handle_queue(sc, qp); 3461 } else { 3462 /* for ALTQ */ 3463 if (txr->txr_qid == 0) 3464 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3465 softint_schedule(txr->txr_si); 3466 3467 ixl_enable_queue_intr(sc, qp); 3468 } 3469 3470 return 1; 3471 } 3472 3473 static void 3474 ixl_handle_queue_wk(struct work *wk, void *xsc) 3475 { 3476 struct ixl_queue_pair *qp; 3477 3478 qp = container_of(wk, struct ixl_queue_pair, qp_work); 3479 ixl_handle_queue(qp); 3480 } 3481 3482 static void 3483 ixl_handle_queue(void *xqp) 3484 { 3485 struct ixl_queue_pair *qp = xqp; 3486 struct ixl_softc *sc = qp->qp_sc; 3487 struct ixl_tx_ring *txr = qp->qp_txr; 3488 struct ixl_rx_ring *rxr = qp->qp_rxr; 3489 u_int txlimit, rxlimit; 3490 int more; 3491 3492 txlimit = sc->sc_tx_process_limit; 3493 rxlimit = sc->sc_rx_process_limit; 3494 3495 more = ixl_handle_queue_common(sc, qp, 3496 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3497 3498 if (more != 0) 3499 ixl_sched_handle_queue(sc, qp); 3500 else 3501 ixl_enable_queue_intr(sc, qp); 3502 } 3503 3504 static inline void 3505 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3506 { 3507 uint32_t hmc_idx, hmc_isvf; 3508 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3509 3510 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3511 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3512 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3513 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3514 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3515 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3516 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3517 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3518 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3519 3520 device_printf(sc->sc_dev, 3521 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3522 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3523 } 3524 3525 static int 3526 ixl_other_intr(void *xsc) 3527 { 3528 struct ixl_softc *sc = xsc; 3529 uint32_t icr, mask, reg; 3530 int rv; 3531 3532 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3533 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3534 3535 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3536 atomic_inc_64(&sc->sc_event_atq.ev_count); 3537 ixl_atq_done(sc); 3538 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3539 rv = 1; 3540 } 3541 3542 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3543 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3544 device_printf(sc->sc_dev, "link stat changed\n"); 3545 3546 atomic_inc_64(&sc->sc_event_link.ev_count); 3547 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3548 rv = 1; 3549 } 3550 3551 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3552 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3553 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3554 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3555 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3556 3557 device_printf(sc->sc_dev, "GRST: %s\n", 3558 reg == I40E_RESET_CORER ? "CORER" : 3559 reg == I40E_RESET_GLOBR ? "GLOBR" : 3560 reg == I40E_RESET_EMPR ? "EMPR" : 3561 "POR"); 3562 } 3563 3564 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3565 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3566 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3567 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3568 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3569 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3570 3571 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3572 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3573 device_printf(sc->sc_dev, "critical error\n"); 3574 } 3575 3576 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3577 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3578 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3579 ixl_print_hmc_error(sc, reg); 3580 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3581 } 3582 3583 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3584 ixl_flush(sc); 3585 ixl_enable_other_intr(sc); 3586 return rv; 3587 } 3588 3589 static void 3590 ixl_get_link_status_done(struct ixl_softc *sc, 3591 const struct ixl_aq_desc *iaq) 3592 { 3593 struct ixl_aq_desc iaq_buf; 3594 3595 memcpy(&iaq_buf, iaq, sizeof(iaq_buf)); 3596 3597 /* 3598 * The lock can be released here 3599 * because there is no post processing about ATQ 3600 */ 3601 mutex_exit(&sc->sc_atq_lock); 3602 ixl_link_state_update(sc, &iaq_buf); 3603 mutex_enter(&sc->sc_atq_lock); 3604 } 3605 3606 static void 3607 ixl_get_link_status(void *xsc) 3608 { 3609 struct ixl_softc *sc = xsc; 3610 struct ixl_aq_desc *iaq; 3611 struct ixl_aq_link_param *param; 3612 int error; 3613 3614 mutex_enter(&sc->sc_atq_lock); 3615 3616 iaq = &sc->sc_link_state_atq.iatq_desc; 3617 memset(iaq, 0, sizeof(*iaq)); 3618 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3619 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3620 param->notify = IXL_AQ_LINK_NOTIFY; 3621 3622 error = ixl_atq_exec_locked(sc, &sc->sc_link_state_atq); 3623 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 3624 3625 if (error == 0) { 3626 ixl_get_link_status_done(sc, iaq); 3627 } 3628 3629 mutex_exit(&sc->sc_atq_lock); 3630 } 3631 3632 static void 3633 ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3634 { 3635 struct ifnet *ifp = &sc->sc_ec.ec_if; 3636 int link_state; 3637 3638 mutex_enter(&sc->sc_cfg_lock); 3639 link_state = ixl_set_link_status_locked(sc, iaq); 3640 mutex_exit(&sc->sc_cfg_lock); 3641 3642 if (ifp->if_link_state != link_state) 3643 if_link_state_change(ifp, link_state); 3644 3645 if (link_state != LINK_STATE_DOWN) { 3646 kpreempt_disable(); 3647 if_schedule_deferred_start(ifp); 3648 kpreempt_enable(); 3649 } 3650 } 3651 3652 static void 3653 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3654 const char *msg) 3655 { 3656 char buf[512]; 3657 size_t len; 3658 3659 len = sizeof(buf); 3660 buf[--len] = '\0'; 3661 3662 device_printf(sc->sc_dev, "%s\n", msg); 3663 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3664 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3665 buf, le16toh(iaq->iaq_opcode)); 3666 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3667 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3668 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3669 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3670 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3671 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3672 } 3673 3674 static void 3675 ixl_arq(void *xsc) 3676 { 3677 struct ixl_softc *sc = xsc; 3678 struct ixl_aq_desc *arq, *iaq; 3679 struct ixl_aq_buf *aqb; 3680 unsigned int cons = sc->sc_arq_cons; 3681 unsigned int prod; 3682 int done = 0; 3683 3684 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3685 sc->sc_aq_regs->arq_head_mask; 3686 3687 if (cons == prod) 3688 goto done; 3689 3690 arq = IXL_DMA_KVA(&sc->sc_arq); 3691 3692 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3693 0, IXL_DMA_LEN(&sc->sc_arq), 3694 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3695 3696 do { 3697 iaq = &arq[cons]; 3698 aqb = sc->sc_arq_live[cons]; 3699 3700 KASSERT(aqb != NULL); 3701 3702 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3703 BUS_DMASYNC_POSTREAD); 3704 3705 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3706 ixl_aq_dump(sc, iaq, "arq event"); 3707 3708 switch (iaq->iaq_opcode) { 3709 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3710 ixl_link_state_update(sc, iaq); 3711 break; 3712 } 3713 3714 memset(iaq, 0, sizeof(*iaq)); 3715 sc->sc_arq_live[cons] = NULL; 3716 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3717 3718 cons++; 3719 cons &= IXL_AQ_MASK; 3720 3721 done = 1; 3722 } while (cons != prod); 3723 3724 if (done) { 3725 sc->sc_arq_cons = cons; 3726 ixl_arq_fill(sc); 3727 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3728 0, IXL_DMA_LEN(&sc->sc_arq), 3729 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3730 } 3731 3732 done: 3733 ixl_enable_other_intr(sc); 3734 } 3735 3736 static void 3737 ixl_atq_set(struct ixl_atq *iatq, 3738 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3739 { 3740 3741 iatq->iatq_fn = fn; 3742 } 3743 3744 static int 3745 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3746 { 3747 struct ixl_aq_desc *atq, *slot; 3748 unsigned int prod, cons, prod_next; 3749 3750 /* assert locked */ 3751 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3752 3753 atq = IXL_DMA_KVA(&sc->sc_atq); 3754 prod = sc->sc_atq_prod; 3755 cons = sc->sc_atq_cons; 3756 prod_next = (prod +1) & IXL_AQ_MASK; 3757 3758 if (cons == prod_next) 3759 return ENOMEM; 3760 3761 slot = &atq[prod]; 3762 3763 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3764 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3765 3766 KASSERT(iatq->iatq_fn != NULL); 3767 *slot = iatq->iatq_desc; 3768 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3769 3770 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3771 ixl_aq_dump(sc, slot, "atq command"); 3772 3773 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3774 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3775 3776 sc->sc_atq_prod = prod_next; 3777 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3778 3779 return 0; 3780 } 3781 3782 static void 3783 ixl_atq_done_locked(struct ixl_softc *sc) 3784 { 3785 struct ixl_aq_desc *atq, *slot; 3786 struct ixl_atq *iatq; 3787 unsigned int cons; 3788 unsigned int prod; 3789 3790 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3791 3792 prod = sc->sc_atq_prod; 3793 cons = sc->sc_atq_cons; 3794 3795 if (prod == cons) 3796 return; 3797 3798 atq = IXL_DMA_KVA(&sc->sc_atq); 3799 3800 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3801 0, IXL_DMA_LEN(&sc->sc_atq), 3802 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3803 3804 do { 3805 slot = &atq[cons]; 3806 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3807 break; 3808 3809 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3810 iatq->iatq_desc = *slot; 3811 3812 memset(slot, 0, sizeof(*slot)); 3813 3814 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3815 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3816 3817 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3818 3819 cons++; 3820 cons &= IXL_AQ_MASK; 3821 } while (cons != prod); 3822 3823 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3824 0, IXL_DMA_LEN(&sc->sc_atq), 3825 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3826 3827 sc->sc_atq_cons = cons; 3828 } 3829 3830 static void 3831 ixl_atq_done(struct ixl_softc *sc) 3832 { 3833 3834 mutex_enter(&sc->sc_atq_lock); 3835 ixl_atq_done_locked(sc); 3836 mutex_exit(&sc->sc_atq_lock); 3837 } 3838 3839 static void 3840 ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3841 { 3842 3843 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3844 3845 cv_signal(&sc->sc_atq_cv); 3846 } 3847 3848 static int 3849 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3850 { 3851 int error; 3852 3853 mutex_enter(&sc->sc_atq_lock); 3854 error = ixl_atq_exec_locked(sc, iatq); 3855 mutex_exit(&sc->sc_atq_lock); 3856 3857 return error; 3858 } 3859 3860 static int 3861 ixl_atq_exec_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3862 { 3863 int error; 3864 3865 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3866 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3867 3868 ixl_atq_set(iatq, ixl_wakeup); 3869 3870 error = ixl_atq_post_locked(sc, iatq); 3871 if (error) 3872 return error; 3873 3874 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3875 IXL_ATQ_EXEC_TIMEOUT); 3876 3877 return error; 3878 } 3879 3880 static int 3881 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3882 { 3883 struct ixl_aq_desc *atq, *slot; 3884 unsigned int prod; 3885 unsigned int t = 0; 3886 3887 mutex_enter(&sc->sc_atq_lock); 3888 3889 atq = IXL_DMA_KVA(&sc->sc_atq); 3890 prod = sc->sc_atq_prod; 3891 slot = atq + prod; 3892 3893 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3894 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3895 3896 *slot = *iaq; 3897 slot->iaq_flags |= htole16(IXL_AQ_SI); 3898 3899 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3900 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3901 3902 prod++; 3903 prod &= IXL_AQ_MASK; 3904 sc->sc_atq_prod = prod; 3905 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3906 3907 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3908 delaymsec(1); 3909 3910 if (t++ > tm) { 3911 mutex_exit(&sc->sc_atq_lock); 3912 return ETIMEDOUT; 3913 } 3914 } 3915 3916 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3917 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3918 *iaq = *slot; 3919 memset(slot, 0, sizeof(*slot)); 3920 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3921 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3922 3923 sc->sc_atq_cons = prod; 3924 3925 mutex_exit(&sc->sc_atq_lock); 3926 3927 return 0; 3928 } 3929 3930 static int 3931 ixl_get_version(struct ixl_softc *sc) 3932 { 3933 struct ixl_aq_desc iaq; 3934 uint32_t fwbuild, fwver, apiver; 3935 uint16_t api_maj_ver, api_min_ver; 3936 3937 memset(&iaq, 0, sizeof(iaq)); 3938 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3939 3940 iaq.iaq_retval = le16toh(23); 3941 3942 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3943 return ETIMEDOUT; 3944 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3945 return EIO; 3946 3947 fwbuild = le32toh(iaq.iaq_param[1]); 3948 fwver = le32toh(iaq.iaq_param[2]); 3949 apiver = le32toh(iaq.iaq_param[3]); 3950 3951 api_maj_ver = (uint16_t)apiver; 3952 api_min_ver = (uint16_t)(apiver >> 16); 3953 3954 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 3955 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 3956 3957 if (sc->sc_mac_type == I40E_MAC_X722) { 3958 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK | 3959 IXL_SC_AQ_FLAG_NVMREAD); 3960 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 3961 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS); 3962 } 3963 3964 #define IXL_API_VER(maj, min) (((uint32_t)(maj) << 16) | (min)) 3965 if (IXL_API_VER(api_maj_ver, api_min_ver) >= IXL_API_VER(1, 5)) { 3966 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 3967 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK); 3968 } 3969 #undef IXL_API_VER 3970 3971 return 0; 3972 } 3973 3974 static int 3975 ixl_get_nvm_version(struct ixl_softc *sc) 3976 { 3977 uint16_t nvmver, cfg_ptr, eetrack_hi, eetrack_lo, oem_hi, oem_lo; 3978 uint32_t eetrack, oem; 3979 uint16_t nvm_maj_ver, nvm_min_ver, oem_build; 3980 uint8_t oem_ver, oem_patch; 3981 3982 nvmver = cfg_ptr = eetrack_hi = eetrack_lo = oem_hi = oem_lo = 0; 3983 ixl_rd16_nvm(sc, I40E_SR_NVM_DEV_STARTER_VERSION, &nvmver); 3984 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_HI, &eetrack_hi); 3985 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_LO, &eetrack_lo); 3986 ixl_rd16_nvm(sc, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr); 3987 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF, &oem_hi); 3988 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF + 1, &oem_lo); 3989 3990 nvm_maj_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_HI_MASK); 3991 nvm_min_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_LO_MASK); 3992 eetrack = ((uint32_t)eetrack_hi << 16) | eetrack_lo; 3993 oem = ((uint32_t)oem_hi << 16) | oem_lo; 3994 oem_ver = __SHIFTOUT(oem, IXL_NVM_OEMVERSION_MASK); 3995 oem_build = __SHIFTOUT(oem, IXL_NVM_OEMBUILD_MASK); 3996 oem_patch = __SHIFTOUT(oem, IXL_NVM_OEMPATCH_MASK); 3997 3998 aprint_normal(" nvm %x.%02x etid %08x oem %d.%d.%d", 3999 nvm_maj_ver, nvm_min_ver, eetrack, 4000 oem_ver, oem_build, oem_patch); 4001 4002 return 0; 4003 } 4004 4005 static int 4006 ixl_pxe_clear(struct ixl_softc *sc) 4007 { 4008 struct ixl_aq_desc iaq; 4009 int rv; 4010 4011 memset(&iaq, 0, sizeof(iaq)); 4012 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 4013 iaq.iaq_param[0] = htole32(0x2); 4014 4015 rv = ixl_atq_poll(sc, &iaq, 250); 4016 4017 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 4018 4019 if (rv != 0) 4020 return ETIMEDOUT; 4021 4022 switch (iaq.iaq_retval) { 4023 case htole16(IXL_AQ_RC_OK): 4024 case htole16(IXL_AQ_RC_EEXIST): 4025 break; 4026 default: 4027 return EIO; 4028 } 4029 4030 return 0; 4031 } 4032 4033 static int 4034 ixl_lldp_shut(struct ixl_softc *sc) 4035 { 4036 struct ixl_aq_desc iaq; 4037 4038 memset(&iaq, 0, sizeof(iaq)); 4039 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 4040 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 4041 4042 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4043 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 4044 return -1; 4045 } 4046 4047 switch (iaq.iaq_retval) { 4048 case htole16(IXL_AQ_RC_EMODE): 4049 case htole16(IXL_AQ_RC_EPERM): 4050 /* ignore silently */ 4051 default: 4052 break; 4053 } 4054 4055 return 0; 4056 } 4057 4058 static void 4059 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 4060 { 4061 uint16_t id; 4062 uint32_t number, logical_id; 4063 4064 id = le16toh(cap->cap_id); 4065 number = le32toh(cap->number); 4066 logical_id = le32toh(cap->logical_id); 4067 4068 switch (id) { 4069 case IXL_AQ_CAP_RSS: 4070 sc->sc_rss_table_size = number; 4071 sc->sc_rss_table_entry_width = logical_id; 4072 break; 4073 case IXL_AQ_CAP_RXQ: 4074 case IXL_AQ_CAP_TXQ: 4075 sc->sc_nqueue_pairs_device = MIN(number, 4076 sc->sc_nqueue_pairs_device); 4077 break; 4078 } 4079 } 4080 4081 static int 4082 ixl_get_hw_capabilities(struct ixl_softc *sc) 4083 { 4084 struct ixl_dmamem idm; 4085 struct ixl_aq_desc iaq; 4086 struct ixl_aq_capability *caps; 4087 size_t i, ncaps; 4088 bus_size_t caps_size; 4089 uint16_t status; 4090 int rv; 4091 4092 caps_size = sizeof(caps[0]) * 40; 4093 memset(&iaq, 0, sizeof(iaq)); 4094 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 4095 4096 do { 4097 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 4098 return -1; 4099 } 4100 4101 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4102 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4103 iaq.iaq_datalen = htole16(caps_size); 4104 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4105 4106 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4107 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 4108 4109 rv = ixl_atq_poll(sc, &iaq, 250); 4110 4111 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4112 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 4113 4114 if (rv != 0) { 4115 aprint_error(", HW capabilities timeout\n"); 4116 goto done; 4117 } 4118 4119 status = le16toh(iaq.iaq_retval); 4120 4121 if (status == IXL_AQ_RC_ENOMEM) { 4122 caps_size = le16toh(iaq.iaq_datalen); 4123 ixl_dmamem_free(sc, &idm); 4124 } 4125 } while (status == IXL_AQ_RC_ENOMEM); 4126 4127 if (status != IXL_AQ_RC_OK) { 4128 aprint_error(", HW capabilities error\n"); 4129 goto done; 4130 } 4131 4132 caps = IXL_DMA_KVA(&idm); 4133 ncaps = le16toh(iaq.iaq_param[1]); 4134 4135 for (i = 0; i < ncaps; i++) { 4136 ixl_parse_hw_capability(sc, &caps[i]); 4137 } 4138 4139 done: 4140 ixl_dmamem_free(sc, &idm); 4141 return rv; 4142 } 4143 4144 static int 4145 ixl_get_mac(struct ixl_softc *sc) 4146 { 4147 struct ixl_dmamem idm; 4148 struct ixl_aq_desc iaq; 4149 struct ixl_aq_mac_addresses *addrs; 4150 int rv; 4151 4152 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 4153 aprint_error(", unable to allocate mac addresses\n"); 4154 return -1; 4155 } 4156 4157 memset(&iaq, 0, sizeof(iaq)); 4158 iaq.iaq_flags = htole16(IXL_AQ_BUF); 4159 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 4160 iaq.iaq_datalen = htole16(sizeof(*addrs)); 4161 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4162 4163 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4164 BUS_DMASYNC_PREREAD); 4165 4166 rv = ixl_atq_poll(sc, &iaq, 250); 4167 4168 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4169 BUS_DMASYNC_POSTREAD); 4170 4171 if (rv != 0) { 4172 aprint_error(", MAC ADDRESS READ timeout\n"); 4173 rv = -1; 4174 goto done; 4175 } 4176 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4177 aprint_error(", MAC ADDRESS READ error\n"); 4178 rv = -1; 4179 goto done; 4180 } 4181 4182 addrs = IXL_DMA_KVA(&idm); 4183 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 4184 printf(", port address is not valid\n"); 4185 goto done; 4186 } 4187 4188 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 4189 rv = 0; 4190 4191 done: 4192 ixl_dmamem_free(sc, &idm); 4193 return rv; 4194 } 4195 4196 static int 4197 ixl_get_switch_config(struct ixl_softc *sc) 4198 { 4199 struct ixl_dmamem idm; 4200 struct ixl_aq_desc iaq; 4201 struct ixl_aq_switch_config *hdr; 4202 struct ixl_aq_switch_config_element *elms, *elm; 4203 unsigned int nelm, i; 4204 int rv; 4205 4206 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4207 aprint_error_dev(sc->sc_dev, 4208 "unable to allocate switch config buffer\n"); 4209 return -1; 4210 } 4211 4212 memset(&iaq, 0, sizeof(iaq)); 4213 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4214 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4215 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 4216 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 4217 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4218 4219 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4220 BUS_DMASYNC_PREREAD); 4221 4222 rv = ixl_atq_poll(sc, &iaq, 250); 4223 4224 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4225 BUS_DMASYNC_POSTREAD); 4226 4227 if (rv != 0) { 4228 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 4229 rv = -1; 4230 goto done; 4231 } 4232 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4233 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 4234 rv = -1; 4235 goto done; 4236 } 4237 4238 hdr = IXL_DMA_KVA(&idm); 4239 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 4240 4241 nelm = le16toh(hdr->num_reported); 4242 if (nelm < 1) { 4243 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 4244 rv = -1; 4245 goto done; 4246 } 4247 4248 for (i = 0; i < nelm; i++) { 4249 elm = &elms[i]; 4250 4251 aprint_debug_dev(sc->sc_dev, 4252 "type %x revision %u seid %04x\n", 4253 elm->type, elm->revision, le16toh(elm->seid)); 4254 aprint_debug_dev(sc->sc_dev, 4255 "uplink %04x downlink %04x\n", 4256 le16toh(elm->uplink_seid), 4257 le16toh(elm->downlink_seid)); 4258 aprint_debug_dev(sc->sc_dev, 4259 "conntype %x scheduler %04x extra %04x\n", 4260 elm->connection_type, 4261 le16toh(elm->scheduler_id), 4262 le16toh(elm->element_info)); 4263 } 4264 4265 elm = &elms[0]; 4266 4267 sc->sc_uplink_seid = elm->uplink_seid; 4268 sc->sc_downlink_seid = elm->downlink_seid; 4269 sc->sc_seid = elm->seid; 4270 4271 if ((sc->sc_uplink_seid == htole16(0)) != 4272 (sc->sc_downlink_seid == htole16(0))) { 4273 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 4274 rv = -1; 4275 goto done; 4276 } 4277 4278 done: 4279 ixl_dmamem_free(sc, &idm); 4280 return rv; 4281 } 4282 4283 static int 4284 ixl_phy_mask_ints(struct ixl_softc *sc) 4285 { 4286 struct ixl_aq_desc iaq; 4287 4288 memset(&iaq, 0, sizeof(iaq)); 4289 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 4290 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 4291 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 4292 IXL_AQ_PHY_EV_MEDIA_NA)); 4293 4294 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4295 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 4296 return -1; 4297 } 4298 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4299 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 4300 return -1; 4301 } 4302 4303 return 0; 4304 } 4305 4306 static int 4307 ixl_get_phy_abilities(struct ixl_softc *sc, struct ixl_dmamem *idm) 4308 { 4309 struct ixl_aq_desc iaq; 4310 int rv; 4311 4312 memset(&iaq, 0, sizeof(iaq)); 4313 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4314 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4315 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 4316 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 4317 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 4318 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 4319 4320 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4321 BUS_DMASYNC_PREREAD); 4322 4323 rv = ixl_atq_poll(sc, &iaq, 250); 4324 4325 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4326 BUS_DMASYNC_POSTREAD); 4327 4328 if (rv != 0) 4329 return -1; 4330 4331 return le16toh(iaq.iaq_retval); 4332 } 4333 4334 static int 4335 ixl_get_phy_info(struct ixl_softc *sc) 4336 { 4337 struct ixl_dmamem idm; 4338 struct ixl_aq_phy_abilities *phy; 4339 int rv; 4340 4341 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4342 aprint_error_dev(sc->sc_dev, 4343 "unable to allocate phy abilities buffer\n"); 4344 return -1; 4345 } 4346 4347 rv = ixl_get_phy_abilities(sc, &idm); 4348 switch (rv) { 4349 case -1: 4350 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 4351 goto done; 4352 case IXL_AQ_RC_OK: 4353 break; 4354 case IXL_AQ_RC_EIO: 4355 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 4356 goto done; 4357 default: 4358 aprint_error_dev(sc->sc_dev, 4359 "GET PHY ABILITIIES error %u\n", rv); 4360 goto done; 4361 } 4362 4363 phy = IXL_DMA_KVA(&idm); 4364 4365 sc->sc_phy_types = le32toh(phy->phy_type); 4366 sc->sc_phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 4367 4368 sc->sc_phy_abilities = phy->abilities; 4369 sc->sc_phy_linkspeed = phy->link_speed; 4370 sc->sc_phy_fec_cfg = phy->fec_cfg_curr_mod_ext_info & 4371 (IXL_AQ_ENABLE_FEC_KR | IXL_AQ_ENABLE_FEC_RS | 4372 IXL_AQ_REQUEST_FEC_KR | IXL_AQ_REQUEST_FEC_RS); 4373 sc->sc_eee_cap = phy->eee_capability; 4374 sc->sc_eeer_val = phy->eeer_val; 4375 sc->sc_d3_lpan = phy->d3_lpan; 4376 4377 rv = 0; 4378 4379 done: 4380 ixl_dmamem_free(sc, &idm); 4381 return rv; 4382 } 4383 4384 static int 4385 ixl_set_phy_config(struct ixl_softc *sc, 4386 uint8_t link_speed, uint8_t abilities, bool polling) 4387 { 4388 struct ixl_aq_phy_param *param; 4389 struct ixl_atq iatq; 4390 struct ixl_aq_desc *iaq; 4391 int error; 4392 4393 memset(&iatq, 0, sizeof(iatq)); 4394 4395 iaq = &iatq.iatq_desc; 4396 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_CONFIG); 4397 param = (struct ixl_aq_phy_param *)&iaq->iaq_param; 4398 param->phy_types = htole32((uint32_t)sc->sc_phy_types); 4399 param->phy_type_ext = (uint8_t)(sc->sc_phy_types >> 32); 4400 param->link_speed = link_speed; 4401 param->abilities = abilities | IXL_AQ_PHY_ABILITY_AUTO_LINK; 4402 param->fec_cfg = sc->sc_phy_fec_cfg; 4403 param->eee_capability = sc->sc_eee_cap; 4404 param->eeer_val = sc->sc_eeer_val; 4405 param->d3_lpan = sc->sc_d3_lpan; 4406 4407 if (polling) 4408 error = ixl_atq_poll(sc, iaq, 250); 4409 else 4410 error = ixl_atq_exec(sc, &iatq); 4411 4412 if (error != 0) 4413 return error; 4414 4415 switch (le16toh(iaq->iaq_retval)) { 4416 case IXL_AQ_RC_OK: 4417 break; 4418 case IXL_AQ_RC_EPERM: 4419 return EPERM; 4420 default: 4421 return EIO; 4422 } 4423 4424 return 0; 4425 } 4426 4427 static int 4428 ixl_set_phy_autoselect(struct ixl_softc *sc) 4429 { 4430 uint8_t link_speed, abilities; 4431 4432 link_speed = sc->sc_phy_linkspeed; 4433 abilities = IXL_PHY_ABILITY_LINKUP | IXL_PHY_ABILITY_AUTONEGO; 4434 4435 return ixl_set_phy_config(sc, link_speed, abilities, true); 4436 } 4437 4438 static int 4439 ixl_get_link_status_poll(struct ixl_softc *sc, int *l) 4440 { 4441 struct ixl_aq_desc iaq; 4442 struct ixl_aq_link_param *param; 4443 int link; 4444 4445 memset(&iaq, 0, sizeof(iaq)); 4446 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 4447 param = (struct ixl_aq_link_param *)iaq.iaq_param; 4448 param->notify = IXL_AQ_LINK_NOTIFY; 4449 4450 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4451 return ETIMEDOUT; 4452 } 4453 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4454 return EIO; 4455 } 4456 4457 /* It is unneccessary to hold lock */ 4458 link = ixl_set_link_status_locked(sc, &iaq); 4459 4460 if (l != NULL) 4461 *l = link; 4462 4463 return 0; 4464 } 4465 4466 static int 4467 ixl_get_vsi(struct ixl_softc *sc) 4468 { 4469 struct ixl_dmamem *vsi = &sc->sc_scratch; 4470 struct ixl_aq_desc iaq; 4471 struct ixl_aq_vsi_param *param; 4472 struct ixl_aq_vsi_reply *reply; 4473 struct ixl_aq_vsi_data *data; 4474 int rv; 4475 4476 /* grumble, vsi info isn't "known" at compile time */ 4477 4478 memset(&iaq, 0, sizeof(iaq)); 4479 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4480 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4481 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 4482 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4483 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4484 4485 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4486 param->uplink_seid = sc->sc_seid; 4487 4488 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4489 BUS_DMASYNC_PREREAD); 4490 4491 rv = ixl_atq_poll(sc, &iaq, 250); 4492 4493 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4494 BUS_DMASYNC_POSTREAD); 4495 4496 if (rv != 0) { 4497 return ETIMEDOUT; 4498 } 4499 4500 switch (le16toh(iaq.iaq_retval)) { 4501 case IXL_AQ_RC_OK: 4502 break; 4503 case IXL_AQ_RC_ENOENT: 4504 return ENOENT; 4505 case IXL_AQ_RC_EACCES: 4506 return EACCES; 4507 default: 4508 return EIO; 4509 } 4510 4511 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 4512 sc->sc_vsi_number = le16toh(reply->vsi_number); 4513 data = IXL_DMA_KVA(vsi); 4514 sc->sc_vsi_stat_counter_idx = le16toh(data->stat_counter_idx); 4515 4516 return 0; 4517 } 4518 4519 static int 4520 ixl_set_vsi(struct ixl_softc *sc) 4521 { 4522 struct ixl_dmamem *vsi = &sc->sc_scratch; 4523 struct ixl_aq_desc iaq; 4524 struct ixl_aq_vsi_param *param; 4525 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 4526 unsigned int qnum; 4527 uint16_t val; 4528 int rv; 4529 4530 qnum = sc->sc_nqueue_pairs - 1; 4531 4532 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 4533 IXL_AQ_VSI_VALID_VLAN); 4534 4535 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 4536 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 4537 data->queue_mapping[0] = htole16(0); 4538 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 4539 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 4540 4541 val = le16toh(data->port_vlan_flags); 4542 CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK); 4543 SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL); 4544 4545 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { 4546 SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH); 4547 } else { 4548 SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING); 4549 } 4550 4551 data->port_vlan_flags = htole16(val); 4552 4553 /* grumble, vsi info isn't "known" at compile time */ 4554 4555 memset(&iaq, 0, sizeof(iaq)); 4556 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4557 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4558 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 4559 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4560 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4561 4562 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4563 param->uplink_seid = sc->sc_seid; 4564 4565 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4566 BUS_DMASYNC_PREWRITE); 4567 4568 rv = ixl_atq_poll(sc, &iaq, 250); 4569 4570 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4571 BUS_DMASYNC_POSTWRITE); 4572 4573 if (rv != 0) { 4574 return ETIMEDOUT; 4575 } 4576 4577 switch (le16toh(iaq.iaq_retval)) { 4578 case IXL_AQ_RC_OK: 4579 break; 4580 case IXL_AQ_RC_ENOENT: 4581 return ENOENT; 4582 case IXL_AQ_RC_EACCES: 4583 return EACCES; 4584 default: 4585 return EIO; 4586 } 4587 4588 return 0; 4589 } 4590 4591 static void 4592 ixl_set_filter_control(struct ixl_softc *sc) 4593 { 4594 uint32_t reg; 4595 4596 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4597 4598 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4599 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4600 4601 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4602 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4603 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4604 4605 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4606 } 4607 4608 static inline void 4609 ixl_get_default_rss_key(uint32_t *buf, size_t len) 4610 { 4611 size_t cplen; 4612 uint8_t rss_seed[RSS_KEYSIZE]; 4613 4614 rss_getkey(rss_seed); 4615 memset(buf, 0, len); 4616 4617 cplen = MIN(len, sizeof(rss_seed)); 4618 memcpy(buf, rss_seed, cplen); 4619 } 4620 4621 static int 4622 ixl_set_rss_key(struct ixl_softc *sc, uint8_t *key, size_t keylen) 4623 { 4624 struct ixl_dmamem *idm; 4625 struct ixl_atq iatq; 4626 struct ixl_aq_desc *iaq; 4627 struct ixl_aq_rss_key_param *param; 4628 struct ixl_aq_rss_key_data *data; 4629 size_t len, datalen, stdlen, extlen; 4630 uint16_t vsi_id; 4631 int rv; 4632 4633 memset(&iatq, 0, sizeof(iatq)); 4634 iaq = &iatq.iatq_desc; 4635 idm = &sc->sc_aqbuf; 4636 4637 datalen = sizeof(*data); 4638 4639 /*XXX The buf size has to be less than the size of the register */ 4640 datalen = MIN(IXL_RSS_KEY_SIZE_REG * sizeof(uint32_t), datalen); 4641 4642 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4643 (datalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4644 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_KEY); 4645 iaq->iaq_datalen = htole16(datalen); 4646 4647 param = (struct ixl_aq_rss_key_param *)iaq->iaq_param; 4648 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSKEY_VSI_ID_SHIFT) | 4649 IXL_AQ_RSSKEY_VSI_VALID; 4650 param->vsi_id = htole16(vsi_id); 4651 4652 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4653 data = IXL_DMA_KVA(idm); 4654 4655 len = MIN(keylen, datalen); 4656 stdlen = MIN(sizeof(data->standard_rss_key), len); 4657 memcpy(data->standard_rss_key, key, stdlen); 4658 len = (len > stdlen) ? (len - stdlen) : 0; 4659 4660 extlen = MIN(sizeof(data->extended_hash_key), len); 4661 extlen = (stdlen < keylen) ? 0 : keylen - stdlen; 4662 memcpy(data->extended_hash_key, key + stdlen, extlen); 4663 4664 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4665 4666 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4667 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4668 4669 rv = ixl_atq_exec(sc, &iatq); 4670 4671 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4672 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4673 4674 if (rv != 0) { 4675 return ETIMEDOUT; 4676 } 4677 4678 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4679 return EIO; 4680 } 4681 4682 return 0; 4683 } 4684 4685 static int 4686 ixl_set_rss_lut(struct ixl_softc *sc, uint8_t *lut, size_t lutlen) 4687 { 4688 struct ixl_dmamem *idm; 4689 struct ixl_atq iatq; 4690 struct ixl_aq_desc *iaq; 4691 struct ixl_aq_rss_lut_param *param; 4692 uint16_t vsi_id; 4693 uint8_t *data; 4694 size_t dmalen; 4695 int rv; 4696 4697 memset(&iatq, 0, sizeof(iatq)); 4698 iaq = &iatq.iatq_desc; 4699 idm = &sc->sc_aqbuf; 4700 4701 dmalen = MIN(lutlen, IXL_DMA_LEN(idm)); 4702 4703 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4704 (dmalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4705 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_LUT); 4706 iaq->iaq_datalen = htole16(dmalen); 4707 4708 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4709 data = IXL_DMA_KVA(idm); 4710 memcpy(data, lut, dmalen); 4711 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4712 4713 param = (struct ixl_aq_rss_lut_param *)iaq->iaq_param; 4714 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSLUT_VSI_ID_SHIFT) | 4715 IXL_AQ_RSSLUT_VSI_VALID; 4716 param->vsi_id = htole16(vsi_id); 4717 param->flags = htole16(IXL_AQ_RSSLUT_TABLE_TYPE_PF << 4718 IXL_AQ_RSSLUT_TABLE_TYPE_SHIFT); 4719 4720 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4721 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4722 4723 rv = ixl_atq_exec(sc, &iatq); 4724 4725 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4726 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4727 4728 if (rv != 0) { 4729 return ETIMEDOUT; 4730 } 4731 4732 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4733 return EIO; 4734 } 4735 4736 return 0; 4737 } 4738 4739 static int 4740 ixl_register_rss_key(struct ixl_softc *sc) 4741 { 4742 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4743 int rv; 4744 size_t i; 4745 4746 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4747 4748 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4749 rv = ixl_set_rss_key(sc, (uint8_t*)rss_seed, 4750 sizeof(rss_seed)); 4751 } else { 4752 rv = 0; 4753 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4754 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4755 } 4756 } 4757 4758 return rv; 4759 } 4760 4761 static void 4762 ixl_register_rss_pctype(struct ixl_softc *sc) 4763 { 4764 uint64_t set_hena = 0; 4765 uint32_t hena0, hena1; 4766 4767 /* 4768 * We use TCP/UDP with IPv4/IPv6 by default. 4769 * Note: the device can not use just IP header in each 4770 * TCP/UDP packets for the RSS hash calculation. 4771 */ 4772 if (sc->sc_mac_type == I40E_MAC_X722) 4773 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4774 else 4775 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4776 4777 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4778 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4779 4780 SET(hena0, set_hena); 4781 SET(hena1, set_hena >> 32); 4782 4783 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4784 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4785 } 4786 4787 static int 4788 ixl_register_rss_hlut(struct ixl_softc *sc) 4789 { 4790 unsigned int qid; 4791 uint8_t hlut_buf[512], lut_mask; 4792 uint32_t *hluts; 4793 size_t i, hluts_num; 4794 int rv; 4795 4796 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4797 4798 for (i = 0; i < sc->sc_rss_table_size; i++) { 4799 qid = i % sc->sc_nqueue_pairs; 4800 hlut_buf[i] = qid & lut_mask; 4801 } 4802 4803 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4804 rv = ixl_set_rss_lut(sc, hlut_buf, sizeof(hlut_buf)); 4805 } else { 4806 rv = 0; 4807 hluts = (uint32_t *)hlut_buf; 4808 hluts_num = sc->sc_rss_table_size >> 2; 4809 for (i = 0; i < hluts_num; i++) { 4810 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4811 } 4812 ixl_flush(sc); 4813 } 4814 4815 return rv; 4816 } 4817 4818 static void 4819 ixl_config_rss(struct ixl_softc *sc) 4820 { 4821 4822 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4823 4824 ixl_register_rss_key(sc); 4825 ixl_register_rss_pctype(sc); 4826 ixl_register_rss_hlut(sc); 4827 } 4828 4829 static const struct ixl_phy_type * 4830 ixl_search_phy_type(uint8_t phy_type) 4831 { 4832 const struct ixl_phy_type *itype; 4833 uint64_t mask; 4834 unsigned int i; 4835 4836 if (phy_type >= 64) 4837 return NULL; 4838 4839 mask = 1ULL << phy_type; 4840 4841 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4842 itype = &ixl_phy_type_map[i]; 4843 4844 if (ISSET(itype->phy_type, mask)) 4845 return itype; 4846 } 4847 4848 return NULL; 4849 } 4850 4851 static uint64_t 4852 ixl_search_link_speed(uint8_t link_speed) 4853 { 4854 const struct ixl_speed_type *type; 4855 unsigned int i; 4856 4857 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4858 type = &ixl_speed_type_map[i]; 4859 4860 if (ISSET(type->dev_speed, link_speed)) 4861 return type->net_speed; 4862 } 4863 4864 return 0; 4865 } 4866 4867 static uint8_t 4868 ixl_search_baudrate(uint64_t baudrate) 4869 { 4870 const struct ixl_speed_type *type; 4871 unsigned int i; 4872 4873 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4874 type = &ixl_speed_type_map[i]; 4875 4876 if (type->net_speed == baudrate) { 4877 return type->dev_speed; 4878 } 4879 } 4880 4881 return 0; 4882 } 4883 4884 static int 4885 ixl_restart_an(struct ixl_softc *sc) 4886 { 4887 struct ixl_aq_desc iaq; 4888 4889 memset(&iaq, 0, sizeof(iaq)); 4890 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4891 iaq.iaq_param[0] = 4892 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4893 4894 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4895 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4896 return -1; 4897 } 4898 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4899 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4900 return -1; 4901 } 4902 4903 return 0; 4904 } 4905 4906 static int 4907 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4908 uint16_t vlan, uint16_t flags) 4909 { 4910 struct ixl_aq_desc iaq; 4911 struct ixl_aq_add_macvlan *param; 4912 struct ixl_aq_add_macvlan_elem *elem; 4913 4914 memset(&iaq, 0, sizeof(iaq)); 4915 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4916 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4917 iaq.iaq_datalen = htole16(sizeof(*elem)); 4918 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4919 4920 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4921 param->num_addrs = htole16(1); 4922 param->seid0 = htole16(0x8000) | sc->sc_seid; 4923 param->seid1 = 0; 4924 param->seid2 = 0; 4925 4926 elem = IXL_DMA_KVA(&sc->sc_scratch); 4927 memset(elem, 0, sizeof(*elem)); 4928 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4929 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4930 elem->vlan = htole16(vlan); 4931 4932 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4933 return IXL_AQ_RC_EINVAL; 4934 } 4935 4936 switch (le16toh(iaq.iaq_retval)) { 4937 case IXL_AQ_RC_OK: 4938 break; 4939 case IXL_AQ_RC_ENOSPC: 4940 return ENOSPC; 4941 case IXL_AQ_RC_ENOENT: 4942 return ENOENT; 4943 case IXL_AQ_RC_EACCES: 4944 return EACCES; 4945 case IXL_AQ_RC_EEXIST: 4946 return EEXIST; 4947 case IXL_AQ_RC_EINVAL: 4948 return EINVAL; 4949 default: 4950 return EIO; 4951 } 4952 4953 return 0; 4954 } 4955 4956 static int 4957 ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4958 uint16_t vlan, uint16_t flags) 4959 { 4960 struct ixl_aq_desc iaq; 4961 struct ixl_aq_remove_macvlan *param; 4962 struct ixl_aq_remove_macvlan_elem *elem; 4963 4964 memset(&iaq, 0, sizeof(iaq)); 4965 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4966 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 4967 iaq.iaq_datalen = htole16(sizeof(*elem)); 4968 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4969 4970 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 4971 param->num_addrs = htole16(1); 4972 param->seid0 = htole16(0x8000) | sc->sc_seid; 4973 param->seid1 = 0; 4974 param->seid2 = 0; 4975 4976 elem = IXL_DMA_KVA(&sc->sc_scratch); 4977 memset(elem, 0, sizeof(*elem)); 4978 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4979 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 4980 elem->vlan = htole16(vlan); 4981 4982 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4983 return EINVAL; 4984 } 4985 4986 switch (le16toh(iaq.iaq_retval)) { 4987 case IXL_AQ_RC_OK: 4988 break; 4989 case IXL_AQ_RC_ENOENT: 4990 return ENOENT; 4991 case IXL_AQ_RC_EACCES: 4992 return EACCES; 4993 case IXL_AQ_RC_EINVAL: 4994 return EINVAL; 4995 default: 4996 return EIO; 4997 } 4998 4999 return 0; 5000 } 5001 5002 static int 5003 ixl_hmc(struct ixl_softc *sc) 5004 { 5005 struct { 5006 uint32_t count; 5007 uint32_t minsize; 5008 bus_size_t objsiz; 5009 bus_size_t setoff; 5010 bus_size_t setcnt; 5011 } regs[] = { 5012 { 5013 0, 5014 IXL_HMC_TXQ_MINSIZE, 5015 I40E_GLHMC_LANTXOBJSZ, 5016 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 5017 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 5018 }, 5019 { 5020 0, 5021 IXL_HMC_RXQ_MINSIZE, 5022 I40E_GLHMC_LANRXOBJSZ, 5023 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 5024 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 5025 }, 5026 { 5027 0, 5028 0, 5029 I40E_GLHMC_FCOEDDPOBJSZ, 5030 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 5031 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 5032 }, 5033 { 5034 0, 5035 0, 5036 I40E_GLHMC_FCOEFOBJSZ, 5037 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 5038 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 5039 }, 5040 }; 5041 struct ixl_hmc_entry *e; 5042 uint64_t size, dva; 5043 uint8_t *kva; 5044 uint64_t *sdpage; 5045 unsigned int i; 5046 int npages, tables; 5047 uint32_t reg; 5048 5049 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 5050 5051 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 5052 ixl_rd(sc, I40E_GLHMC_LANQMAX); 5053 5054 size = 0; 5055 for (i = 0; i < __arraycount(regs); i++) { 5056 e = &sc->sc_hmc_entries[i]; 5057 5058 e->hmc_count = regs[i].count; 5059 reg = ixl_rd(sc, regs[i].objsiz); 5060 e->hmc_size = IXL_BIT_ULL(0x3F & reg); 5061 e->hmc_base = size; 5062 5063 if ((e->hmc_size * 8) < regs[i].minsize) { 5064 aprint_error_dev(sc->sc_dev, 5065 "kernel hmc entry is too big\n"); 5066 return -1; 5067 } 5068 5069 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 5070 } 5071 size = roundup(size, IXL_HMC_PGSIZE); 5072 npages = size / IXL_HMC_PGSIZE; 5073 5074 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 5075 5076 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 5077 aprint_error_dev(sc->sc_dev, 5078 "unable to allocate hmc pd memory\n"); 5079 return -1; 5080 } 5081 5082 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 5083 IXL_HMC_PGSIZE) != 0) { 5084 aprint_error_dev(sc->sc_dev, 5085 "unable to allocate hmc sd memory\n"); 5086 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5087 return -1; 5088 } 5089 5090 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 5091 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 5092 5093 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 5094 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 5095 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5096 5097 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 5098 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 5099 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 5100 5101 for (i = 0; (int)i < npages; i++) { 5102 *sdpage = htole64(dva | IXL_HMC_PDVALID); 5103 sdpage++; 5104 5105 dva += IXL_HMC_PGSIZE; 5106 } 5107 5108 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 5109 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 5110 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5111 5112 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 5113 for (i = 0; (int)i < tables; i++) { 5114 uint32_t count; 5115 5116 KASSERT(npages >= 0); 5117 5118 count = ((unsigned int)npages > IXL_HMC_PGS) ? 5119 IXL_HMC_PGS : (unsigned int)npages; 5120 5121 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 5122 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 5123 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 5124 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 5125 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 5126 ixl_wr(sc, I40E_PFHMC_SDCMD, 5127 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 5128 5129 npages -= IXL_HMC_PGS; 5130 dva += IXL_HMC_PGSIZE; 5131 } 5132 5133 for (i = 0; i < __arraycount(regs); i++) { 5134 e = &sc->sc_hmc_entries[i]; 5135 5136 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 5137 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 5138 } 5139 5140 return 0; 5141 } 5142 5143 static void 5144 ixl_hmc_free(struct ixl_softc *sc) 5145 { 5146 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 5147 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5148 } 5149 5150 static void 5151 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 5152 unsigned int npacking) 5153 { 5154 uint8_t *dst = d; 5155 const uint8_t *src = s; 5156 unsigned int i; 5157 5158 for (i = 0; i < npacking; i++) { 5159 const struct ixl_hmc_pack *pack = &packing[i]; 5160 unsigned int offset = pack->lsb / 8; 5161 unsigned int align = pack->lsb % 8; 5162 const uint8_t *in = src + pack->offset; 5163 uint8_t *out = dst + offset; 5164 int width = pack->width; 5165 unsigned int inbits = 0; 5166 5167 if (align) { 5168 inbits = (*in++) << align; 5169 *out++ |= (inbits & 0xff); 5170 inbits >>= 8; 5171 5172 width -= 8 - align; 5173 } 5174 5175 while (width >= 8) { 5176 inbits |= (*in++) << align; 5177 *out++ = (inbits & 0xff); 5178 inbits >>= 8; 5179 5180 width -= 8; 5181 } 5182 5183 if (width > 0) { 5184 inbits |= (*in) << align; 5185 *out |= (inbits & ((1 << width) - 1)); 5186 } 5187 } 5188 } 5189 5190 static struct ixl_aq_buf * 5191 ixl_aqb_alloc(struct ixl_softc *sc) 5192 { 5193 struct ixl_aq_buf *aqb; 5194 5195 aqb = kmem_alloc(sizeof(*aqb), KM_SLEEP); 5196 5197 aqb->aqb_size = IXL_AQ_BUFLEN; 5198 5199 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 5200 aqb->aqb_size, 0, 5201 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 5202 goto free; 5203 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 5204 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 5205 BUS_DMA_WAITOK) != 0) 5206 goto destroy; 5207 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 5208 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 5209 goto dma_free; 5210 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 5211 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 5212 goto unmap; 5213 5214 return aqb; 5215 unmap: 5216 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5217 dma_free: 5218 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5219 destroy: 5220 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5221 free: 5222 kmem_free(aqb, sizeof(*aqb)); 5223 5224 return NULL; 5225 } 5226 5227 static void 5228 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 5229 { 5230 5231 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 5232 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5233 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5234 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5235 kmem_free(aqb, sizeof(*aqb)); 5236 } 5237 5238 static int 5239 ixl_arq_fill(struct ixl_softc *sc) 5240 { 5241 struct ixl_aq_buf *aqb; 5242 struct ixl_aq_desc *arq, *iaq; 5243 unsigned int prod = sc->sc_arq_prod; 5244 unsigned int n; 5245 int post = 0; 5246 5247 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 5248 IXL_AQ_NUM); 5249 arq = IXL_DMA_KVA(&sc->sc_arq); 5250 5251 if (__predict_false(n <= 0)) 5252 return 0; 5253 5254 do { 5255 aqb = sc->sc_arq_live[prod]; 5256 iaq = &arq[prod]; 5257 5258 if (aqb == NULL) { 5259 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 5260 if (aqb != NULL) { 5261 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5262 ixl_aq_buf, aqb_entry); 5263 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 5264 break; 5265 } 5266 5267 sc->sc_arq_live[prod] = aqb; 5268 memset(aqb->aqb_data, 0, aqb->aqb_size); 5269 5270 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 5271 aqb->aqb_size, BUS_DMASYNC_PREREAD); 5272 5273 iaq->iaq_flags = htole16(IXL_AQ_BUF | 5274 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 5275 IXL_AQ_LB : 0)); 5276 iaq->iaq_opcode = 0; 5277 iaq->iaq_datalen = htole16(aqb->aqb_size); 5278 iaq->iaq_retval = 0; 5279 iaq->iaq_cookie = 0; 5280 iaq->iaq_param[0] = 0; 5281 iaq->iaq_param[1] = 0; 5282 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 5283 } 5284 5285 prod++; 5286 prod &= IXL_AQ_MASK; 5287 5288 post = 1; 5289 5290 } while (--n); 5291 5292 if (post) { 5293 sc->sc_arq_prod = prod; 5294 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 5295 } 5296 5297 return post; 5298 } 5299 5300 static void 5301 ixl_arq_unfill(struct ixl_softc *sc) 5302 { 5303 struct ixl_aq_buf *aqb; 5304 unsigned int i; 5305 5306 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 5307 aqb = sc->sc_arq_live[i]; 5308 if (aqb == NULL) 5309 continue; 5310 5311 sc->sc_arq_live[i] = NULL; 5312 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 5313 BUS_DMASYNC_POSTREAD); 5314 ixl_aqb_free(sc, aqb); 5315 } 5316 5317 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 5318 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5319 ixl_aq_buf, aqb_entry); 5320 ixl_aqb_free(sc, aqb); 5321 } 5322 } 5323 5324 static void 5325 ixl_clear_hw(struct ixl_softc *sc) 5326 { 5327 uint32_t num_queues, base_queue; 5328 uint32_t num_pf_int; 5329 uint32_t num_vf_int; 5330 uint32_t num_vfs; 5331 uint32_t i, j; 5332 uint32_t val; 5333 uint32_t eol = 0x7ff; 5334 5335 /* get number of interrupts, queues, and vfs */ 5336 val = ixl_rd(sc, I40E_GLPCI_CNF2); 5337 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 5338 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 5339 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 5340 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 5341 5342 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 5343 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 5344 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 5345 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 5346 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 5347 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 5348 num_queues = (j - base_queue) + 1; 5349 else 5350 num_queues = 0; 5351 5352 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 5353 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 5354 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 5355 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 5356 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 5357 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 5358 num_vfs = (j - i) + 1; 5359 else 5360 num_vfs = 0; 5361 5362 /* stop all the interrupts */ 5363 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5364 ixl_flush(sc); 5365 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 5366 for (i = 0; i < num_pf_int - 2; i++) 5367 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 5368 ixl_flush(sc); 5369 5370 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 5371 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5372 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 5373 for (i = 0; i < num_pf_int - 2; i++) 5374 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 5375 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5376 for (i = 0; i < num_vfs; i++) 5377 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 5378 for (i = 0; i < num_vf_int - 2; i++) 5379 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 5380 5381 /* warn the HW of the coming Tx disables */ 5382 for (i = 0; i < num_queues; i++) { 5383 uint32_t abs_queue_idx = base_queue + i; 5384 uint32_t reg_block = 0; 5385 5386 if (abs_queue_idx >= 128) { 5387 reg_block = abs_queue_idx / 128; 5388 abs_queue_idx %= 128; 5389 } 5390 5391 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 5392 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 5393 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 5394 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 5395 5396 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 5397 } 5398 delaymsec(400); 5399 5400 /* stop all the queues */ 5401 for (i = 0; i < num_queues; i++) { 5402 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 5403 ixl_wr(sc, I40E_QTX_ENA(i), 0); 5404 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 5405 ixl_wr(sc, I40E_QRX_ENA(i), 0); 5406 } 5407 5408 /* short wait for all queue disables to settle */ 5409 delaymsec(50); 5410 } 5411 5412 static int 5413 ixl_pf_reset(struct ixl_softc *sc) 5414 { 5415 uint32_t cnt = 0; 5416 uint32_t cnt1 = 0; 5417 uint32_t reg = 0, reg0 = 0; 5418 uint32_t grst_del; 5419 5420 /* 5421 * Poll for Global Reset steady state in case of recent GRST. 5422 * The grst delay value is in 100ms units, and we'll wait a 5423 * couple counts longer to be sure we don't just miss the end. 5424 */ 5425 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 5426 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 5427 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 5428 5429 grst_del = grst_del * 20; 5430 5431 for (cnt = 0; cnt < grst_del; cnt++) { 5432 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 5433 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 5434 break; 5435 delaymsec(100); 5436 } 5437 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5438 aprint_error(", Global reset polling failed to complete\n"); 5439 return -1; 5440 } 5441 5442 /* Now Wait for the FW to be ready */ 5443 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 5444 reg = ixl_rd(sc, I40E_GLNVM_ULD); 5445 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5446 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 5447 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5448 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 5449 break; 5450 5451 delaymsec(10); 5452 } 5453 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5454 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 5455 aprint_error(", wait for FW Reset complete timed out " 5456 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 5457 return -1; 5458 } 5459 5460 /* 5461 * If there was a Global Reset in progress when we got here, 5462 * we don't need to do the PF Reset 5463 */ 5464 if (cnt == 0) { 5465 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5466 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 5467 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 5468 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5469 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 5470 break; 5471 delaymsec(1); 5472 5473 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 5474 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5475 aprint_error(", Core reset upcoming." 5476 " Skipping PF reset reset request\n"); 5477 return -1; 5478 } 5479 } 5480 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 5481 aprint_error(", PF reset polling failed to complete" 5482 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 5483 return -1; 5484 } 5485 } 5486 5487 return 0; 5488 } 5489 5490 static int 5491 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 5492 bus_size_t size, bus_size_t align) 5493 { 5494 ixm->ixm_size = size; 5495 5496 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 5497 ixm->ixm_size, 0, 5498 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 5499 &ixm->ixm_map) != 0) 5500 return 1; 5501 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 5502 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 5503 BUS_DMA_WAITOK) != 0) 5504 goto destroy; 5505 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 5506 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 5507 goto free; 5508 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 5509 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 5510 goto unmap; 5511 5512 memset(ixm->ixm_kva, 0, ixm->ixm_size); 5513 5514 return 0; 5515 unmap: 5516 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5517 free: 5518 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5519 destroy: 5520 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5521 return 1; 5522 } 5523 5524 static void 5525 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 5526 { 5527 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 5528 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5529 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5530 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5531 } 5532 5533 static int 5534 ixl_setup_vlan_hwfilter(struct ixl_softc *sc) 5535 { 5536 struct ethercom *ec = &sc->sc_ec; 5537 struct vlanid_list *vlanidp; 5538 int rv; 5539 5540 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5541 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5542 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5543 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5544 5545 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5546 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5547 if (rv != 0) 5548 return rv; 5549 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5550 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5551 if (rv != 0) 5552 return rv; 5553 5554 ETHER_LOCK(ec); 5555 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5556 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 5557 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5558 if (rv != 0) 5559 break; 5560 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 5561 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5562 if (rv != 0) 5563 break; 5564 } 5565 ETHER_UNLOCK(ec); 5566 5567 return rv; 5568 } 5569 5570 static void 5571 ixl_teardown_vlan_hwfilter(struct ixl_softc *sc) 5572 { 5573 struct vlanid_list *vlanidp; 5574 struct ethercom *ec = &sc->sc_ec; 5575 5576 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5577 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5578 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5579 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5580 5581 ETHER_LOCK(ec); 5582 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5583 ixl_remove_macvlan(sc, sc->sc_enaddr, 5584 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5585 ixl_remove_macvlan(sc, etherbroadcastaddr, 5586 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5587 } 5588 ETHER_UNLOCK(ec); 5589 5590 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5591 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5592 ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5593 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5594 } 5595 5596 static int 5597 ixl_update_macvlan(struct ixl_softc *sc) 5598 { 5599 int rv = 0; 5600 int next_ec_capenable = sc->sc_ec.ec_capenable; 5601 5602 if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 5603 rv = ixl_setup_vlan_hwfilter(sc); 5604 if (rv != 0) 5605 ixl_teardown_vlan_hwfilter(sc); 5606 } else { 5607 ixl_teardown_vlan_hwfilter(sc); 5608 } 5609 5610 return rv; 5611 } 5612 5613 static int 5614 ixl_ifflags_cb(struct ethercom *ec) 5615 { 5616 struct ifnet *ifp = &ec->ec_if; 5617 struct ixl_softc *sc = ifp->if_softc; 5618 int rv, change; 5619 5620 mutex_enter(&sc->sc_cfg_lock); 5621 5622 change = ec->ec_capenable ^ sc->sc_cur_ec_capenable; 5623 5624 if (ISSET(change, ETHERCAP_VLAN_HWTAGGING)) { 5625 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 5626 rv = ENETRESET; 5627 goto out; 5628 } 5629 5630 if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) { 5631 rv = ixl_update_macvlan(sc); 5632 if (rv == 0) { 5633 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 5634 } else { 5635 CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER); 5636 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 5637 } 5638 } 5639 5640 rv = ixl_iff(sc); 5641 out: 5642 mutex_exit(&sc->sc_cfg_lock); 5643 5644 return rv; 5645 } 5646 5647 static int 5648 ixl_set_link_status_locked(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 5649 { 5650 const struct ixl_aq_link_status *status; 5651 const struct ixl_phy_type *itype; 5652 5653 uint64_t ifm_active = IFM_ETHER; 5654 uint64_t ifm_status = IFM_AVALID; 5655 int link_state = LINK_STATE_DOWN; 5656 uint64_t baudrate = 0; 5657 5658 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 5659 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) { 5660 ifm_active |= IFM_NONE; 5661 goto done; 5662 } 5663 5664 ifm_active |= IFM_FDX; 5665 ifm_status |= IFM_ACTIVE; 5666 link_state = LINK_STATE_UP; 5667 5668 itype = ixl_search_phy_type(status->phy_type); 5669 if (itype != NULL) 5670 ifm_active |= itype->ifm_type; 5671 5672 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 5673 ifm_active |= IFM_ETH_TXPAUSE; 5674 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 5675 ifm_active |= IFM_ETH_RXPAUSE; 5676 5677 baudrate = ixl_search_link_speed(status->link_speed); 5678 5679 done: 5680 /* sc->sc_cfg_lock held expect during attach */ 5681 sc->sc_media_active = ifm_active; 5682 sc->sc_media_status = ifm_status; 5683 5684 sc->sc_ec.ec_if.if_baudrate = baudrate; 5685 5686 return link_state; 5687 } 5688 5689 static int 5690 ixl_establish_intx(struct ixl_softc *sc) 5691 { 5692 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5693 pci_intr_handle_t *intr; 5694 char xnamebuf[32]; 5695 char intrbuf[PCI_INTRSTR_LEN]; 5696 char const *intrstr; 5697 5698 KASSERT(sc->sc_nintrs == 1); 5699 5700 intr = &sc->sc_ihp[0]; 5701 5702 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 5703 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 5704 device_xname(sc->sc_dev)); 5705 5706 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 5707 sc, xnamebuf); 5708 5709 if (sc->sc_ihs[0] == NULL) { 5710 aprint_error_dev(sc->sc_dev, 5711 "unable to establish interrupt at %s\n", intrstr); 5712 return -1; 5713 } 5714 5715 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 5716 return 0; 5717 } 5718 5719 static int 5720 ixl_establish_msix(struct ixl_softc *sc) 5721 { 5722 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5723 kcpuset_t *affinity; 5724 unsigned int vector = 0; 5725 unsigned int i; 5726 int affinity_to, r; 5727 char xnamebuf[32]; 5728 char intrbuf[PCI_INTRSTR_LEN]; 5729 char const *intrstr; 5730 5731 kcpuset_create(&affinity, false); 5732 5733 /* the "other" intr is mapped to vector 0 */ 5734 vector = 0; 5735 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5736 intrbuf, sizeof(intrbuf)); 5737 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 5738 device_xname(sc->sc_dev)); 5739 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5740 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 5741 sc, xnamebuf); 5742 if (sc->sc_ihs[vector] == NULL) { 5743 aprint_error_dev(sc->sc_dev, 5744 "unable to establish interrupt at %s\n", intrstr); 5745 goto fail; 5746 } 5747 5748 aprint_normal_dev(sc->sc_dev, "other interrupt at %s", intrstr); 5749 5750 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5751 affinity_to = (affinity_to + sc->sc_nqueue_pairs_max) % ncpu; 5752 5753 kcpuset_zero(affinity); 5754 kcpuset_set(affinity, affinity_to); 5755 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5756 if (r == 0) { 5757 aprint_normal(", affinity to %u", affinity_to); 5758 } 5759 aprint_normal("\n"); 5760 vector++; 5761 5762 sc->sc_msix_vector_queue = vector; 5763 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5764 5765 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5766 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5767 intrbuf, sizeof(intrbuf)); 5768 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 5769 device_xname(sc->sc_dev), i); 5770 5771 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5772 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 5773 (void *)&sc->sc_qps[i], xnamebuf); 5774 5775 if (sc->sc_ihs[vector] == NULL) { 5776 aprint_error_dev(sc->sc_dev, 5777 "unable to establish interrupt at %s\n", intrstr); 5778 goto fail; 5779 } 5780 5781 aprint_normal_dev(sc->sc_dev, 5782 "for TXRX%d interrupt at %s", i, intrstr); 5783 5784 kcpuset_zero(affinity); 5785 kcpuset_set(affinity, affinity_to); 5786 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5787 if (r == 0) { 5788 aprint_normal(", affinity to %u", affinity_to); 5789 affinity_to = (affinity_to + 1) % ncpu; 5790 } 5791 aprint_normal("\n"); 5792 vector++; 5793 } 5794 5795 kcpuset_destroy(affinity); 5796 5797 return 0; 5798 fail: 5799 for (i = 0; i < vector; i++) { 5800 pci_intr_disestablish(pc, sc->sc_ihs[i]); 5801 } 5802 5803 sc->sc_msix_vector_queue = 0; 5804 sc->sc_msix_vector_queue = 0; 5805 kcpuset_destroy(affinity); 5806 5807 return -1; 5808 } 5809 5810 static void 5811 ixl_config_queue_intr(struct ixl_softc *sc) 5812 { 5813 unsigned int i, vector; 5814 5815 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5816 vector = sc->sc_msix_vector_queue; 5817 } else { 5818 vector = I40E_INTR_NOTX_INTR; 5819 5820 ixl_wr(sc, I40E_PFINT_LNKLST0, 5821 (I40E_INTR_NOTX_QUEUE << 5822 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5823 (I40E_QUEUE_TYPE_RX << 5824 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5825 } 5826 5827 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5828 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5829 ixl_flush(sc); 5830 5831 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5832 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5833 (I40E_QUEUE_TYPE_RX << 5834 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5835 5836 ixl_wr(sc, I40E_QINT_RQCTL(i), 5837 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5838 (I40E_ITR_INDEX_RX << 5839 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5840 (I40E_INTR_NOTX_RX_QUEUE << 5841 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5842 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5843 (I40E_QUEUE_TYPE_TX << 5844 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5845 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5846 5847 ixl_wr(sc, I40E_QINT_TQCTL(i), 5848 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5849 (I40E_ITR_INDEX_TX << 5850 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5851 (I40E_INTR_NOTX_TX_QUEUE << 5852 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5853 (I40E_QUEUE_TYPE_EOL << 5854 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5855 (I40E_QUEUE_TYPE_RX << 5856 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5857 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5858 5859 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5860 ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_RX, i), 5861 sc->sc_itr_rx); 5862 ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_TX, i), 5863 sc->sc_itr_tx); 5864 vector++; 5865 } 5866 } 5867 ixl_flush(sc); 5868 5869 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), sc->sc_itr_rx); 5870 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), sc->sc_itr_tx); 5871 ixl_flush(sc); 5872 } 5873 5874 static void 5875 ixl_config_other_intr(struct ixl_softc *sc) 5876 { 5877 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5878 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5879 5880 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5881 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5882 I40E_PFINT_ICR0_ENA_GRST_MASK | 5883 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5884 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5885 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5886 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5887 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5888 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5889 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5890 5891 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5892 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5893 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5894 (I40E_ITR_INDEX_OTHER << 5895 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5896 ixl_flush(sc); 5897 } 5898 5899 static int 5900 ixl_setup_interrupts(struct ixl_softc *sc) 5901 { 5902 struct pci_attach_args *pa = &sc->sc_pa; 5903 pci_intr_type_t max_type, intr_type; 5904 int counts[PCI_INTR_TYPE_SIZE]; 5905 int error; 5906 unsigned int i; 5907 bool retry; 5908 5909 memset(counts, 0, sizeof(counts)); 5910 max_type = PCI_INTR_TYPE_MSIX; 5911 /* QPs + other interrupt */ 5912 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5913 counts[PCI_INTR_TYPE_INTX] = 1; 5914 5915 if (ixl_param_nomsix) 5916 counts[PCI_INTR_TYPE_MSIX] = 0; 5917 5918 do { 5919 retry = false; 5920 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5921 if (error != 0) { 5922 aprint_error_dev(sc->sc_dev, 5923 "couldn't map interrupt\n"); 5924 break; 5925 } 5926 5927 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5928 sc->sc_nintrs = counts[intr_type]; 5929 KASSERT(sc->sc_nintrs > 0); 5930 5931 for (i = 0; i < sc->sc_nintrs; i++) { 5932 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5933 PCI_INTR_MPSAFE, true); 5934 } 5935 5936 sc->sc_ihs = kmem_zalloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5937 KM_SLEEP); 5938 5939 if (intr_type == PCI_INTR_TYPE_MSIX) { 5940 error = ixl_establish_msix(sc); 5941 if (error) { 5942 counts[PCI_INTR_TYPE_MSIX] = 0; 5943 retry = true; 5944 } 5945 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5946 error = ixl_establish_intx(sc); 5947 } else { 5948 error = -1; 5949 } 5950 5951 if (error) { 5952 kmem_free(sc->sc_ihs, 5953 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5954 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5955 } else { 5956 sc->sc_intrtype = intr_type; 5957 } 5958 } while (retry); 5959 5960 return error; 5961 } 5962 5963 static void 5964 ixl_teardown_interrupts(struct ixl_softc *sc) 5965 { 5966 struct pci_attach_args *pa = &sc->sc_pa; 5967 unsigned int i; 5968 5969 for (i = 0; i < sc->sc_nintrs; i++) { 5970 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 5971 } 5972 5973 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5974 5975 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5976 sc->sc_ihs = NULL; 5977 sc->sc_nintrs = 0; 5978 } 5979 5980 static int 5981 ixl_setup_stats(struct ixl_softc *sc) 5982 { 5983 struct ixl_queue_pair *qp; 5984 struct ixl_tx_ring *txr; 5985 struct ixl_rx_ring *rxr; 5986 struct ixl_stats_counters *isc; 5987 unsigned int i; 5988 5989 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5990 qp = &sc->sc_qps[i]; 5991 txr = qp->qp_txr; 5992 rxr = qp->qp_rxr; 5993 5994 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 5995 NULL, qp->qp_name, "m_defrag successed"); 5996 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 5997 NULL, qp->qp_name, "m_defrag_failed"); 5998 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 5999 NULL, qp->qp_name, "Dropped in pcq"); 6000 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 6001 NULL, qp->qp_name, "Deferred transmit"); 6002 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 6003 NULL, qp->qp_name, "Interrupt on queue"); 6004 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 6005 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6006 6007 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 6008 NULL, qp->qp_name, "MGETHDR failed"); 6009 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 6010 NULL, qp->qp_name, "MCLGET failed"); 6011 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 6012 EVCNT_TYPE_MISC, NULL, qp->qp_name, 6013 "bus_dmamap_load_mbuf failed"); 6014 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 6015 NULL, qp->qp_name, "Interrupt on queue"); 6016 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 6017 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6018 } 6019 6020 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 6021 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 6022 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 6023 NULL, device_xname(sc->sc_dev), "Link status event"); 6024 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 6025 NULL, device_xname(sc->sc_dev), "ECC error"); 6026 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 6027 NULL, device_xname(sc->sc_dev), "PCI exception"); 6028 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 6029 NULL, device_xname(sc->sc_dev), "Critical error"); 6030 6031 isc = &sc->sc_stats_counters; 6032 evcnt_attach_dynamic(&isc->isc_crc_errors, EVCNT_TYPE_MISC, 6033 NULL, device_xname(sc->sc_dev), "CRC errors"); 6034 evcnt_attach_dynamic(&isc->isc_illegal_bytes, EVCNT_TYPE_MISC, 6035 NULL, device_xname(sc->sc_dev), "Illegal bytes"); 6036 evcnt_attach_dynamic(&isc->isc_mac_local_faults, EVCNT_TYPE_MISC, 6037 NULL, device_xname(sc->sc_dev), "Mac local faults"); 6038 evcnt_attach_dynamic(&isc->isc_mac_remote_faults, EVCNT_TYPE_MISC, 6039 NULL, device_xname(sc->sc_dev), "Mac remote faults"); 6040 evcnt_attach_dynamic(&isc->isc_link_xon_rx, EVCNT_TYPE_MISC, 6041 NULL, device_xname(sc->sc_dev), "Rx xon"); 6042 evcnt_attach_dynamic(&isc->isc_link_xon_tx, EVCNT_TYPE_MISC, 6043 NULL, device_xname(sc->sc_dev), "Tx xon"); 6044 evcnt_attach_dynamic(&isc->isc_link_xoff_rx, EVCNT_TYPE_MISC, 6045 NULL, device_xname(sc->sc_dev), "Rx xoff"); 6046 evcnt_attach_dynamic(&isc->isc_link_xoff_tx, EVCNT_TYPE_MISC, 6047 NULL, device_xname(sc->sc_dev), "Tx xoff"); 6048 evcnt_attach_dynamic(&isc->isc_rx_fragments, EVCNT_TYPE_MISC, 6049 NULL, device_xname(sc->sc_dev), "Rx fragments"); 6050 evcnt_attach_dynamic(&isc->isc_rx_jabber, EVCNT_TYPE_MISC, 6051 NULL, device_xname(sc->sc_dev), "Rx jabber"); 6052 6053 evcnt_attach_dynamic(&isc->isc_rx_size_64, EVCNT_TYPE_MISC, 6054 NULL, device_xname(sc->sc_dev), "Rx size 64"); 6055 evcnt_attach_dynamic(&isc->isc_rx_size_127, EVCNT_TYPE_MISC, 6056 NULL, device_xname(sc->sc_dev), "Rx size 127"); 6057 evcnt_attach_dynamic(&isc->isc_rx_size_255, EVCNT_TYPE_MISC, 6058 NULL, device_xname(sc->sc_dev), "Rx size 255"); 6059 evcnt_attach_dynamic(&isc->isc_rx_size_511, EVCNT_TYPE_MISC, 6060 NULL, device_xname(sc->sc_dev), "Rx size 511"); 6061 evcnt_attach_dynamic(&isc->isc_rx_size_1023, EVCNT_TYPE_MISC, 6062 NULL, device_xname(sc->sc_dev), "Rx size 1023"); 6063 evcnt_attach_dynamic(&isc->isc_rx_size_1522, EVCNT_TYPE_MISC, 6064 NULL, device_xname(sc->sc_dev), "Rx size 1522"); 6065 evcnt_attach_dynamic(&isc->isc_rx_size_big, EVCNT_TYPE_MISC, 6066 NULL, device_xname(sc->sc_dev), "Rx jumbo packets"); 6067 evcnt_attach_dynamic(&isc->isc_rx_undersize, EVCNT_TYPE_MISC, 6068 NULL, device_xname(sc->sc_dev), "Rx under size"); 6069 evcnt_attach_dynamic(&isc->isc_rx_oversize, EVCNT_TYPE_MISC, 6070 NULL, device_xname(sc->sc_dev), "Rx over size"); 6071 6072 evcnt_attach_dynamic(&isc->isc_rx_bytes, EVCNT_TYPE_MISC, 6073 NULL, device_xname(sc->sc_dev), "Rx bytes / port"); 6074 evcnt_attach_dynamic(&isc->isc_rx_discards, EVCNT_TYPE_MISC, 6075 NULL, device_xname(sc->sc_dev), "Rx discards / port"); 6076 evcnt_attach_dynamic(&isc->isc_rx_unicast, EVCNT_TYPE_MISC, 6077 NULL, device_xname(sc->sc_dev), "Rx unicast / port"); 6078 evcnt_attach_dynamic(&isc->isc_rx_multicast, EVCNT_TYPE_MISC, 6079 NULL, device_xname(sc->sc_dev), "Rx multicast / port"); 6080 evcnt_attach_dynamic(&isc->isc_rx_broadcast, EVCNT_TYPE_MISC, 6081 NULL, device_xname(sc->sc_dev), "Rx broadcast / port"); 6082 6083 evcnt_attach_dynamic(&isc->isc_vsi_rx_bytes, EVCNT_TYPE_MISC, 6084 NULL, device_xname(sc->sc_dev), "Rx bytes / vsi"); 6085 evcnt_attach_dynamic(&isc->isc_vsi_rx_discards, EVCNT_TYPE_MISC, 6086 NULL, device_xname(sc->sc_dev), "Rx discard / vsi"); 6087 evcnt_attach_dynamic(&isc->isc_vsi_rx_unicast, EVCNT_TYPE_MISC, 6088 NULL, device_xname(sc->sc_dev), "Rx unicast / vsi"); 6089 evcnt_attach_dynamic(&isc->isc_vsi_rx_multicast, EVCNT_TYPE_MISC, 6090 NULL, device_xname(sc->sc_dev), "Rx multicast / vsi"); 6091 evcnt_attach_dynamic(&isc->isc_vsi_rx_broadcast, EVCNT_TYPE_MISC, 6092 NULL, device_xname(sc->sc_dev), "Rx broadcast / vsi"); 6093 6094 evcnt_attach_dynamic(&isc->isc_tx_size_64, EVCNT_TYPE_MISC, 6095 NULL, device_xname(sc->sc_dev), "Tx size 64"); 6096 evcnt_attach_dynamic(&isc->isc_tx_size_127, EVCNT_TYPE_MISC, 6097 NULL, device_xname(sc->sc_dev), "Tx size 127"); 6098 evcnt_attach_dynamic(&isc->isc_tx_size_255, EVCNT_TYPE_MISC, 6099 NULL, device_xname(sc->sc_dev), "Tx size 255"); 6100 evcnt_attach_dynamic(&isc->isc_tx_size_511, EVCNT_TYPE_MISC, 6101 NULL, device_xname(sc->sc_dev), "Tx size 511"); 6102 evcnt_attach_dynamic(&isc->isc_tx_size_1023, EVCNT_TYPE_MISC, 6103 NULL, device_xname(sc->sc_dev), "Tx size 1023"); 6104 evcnt_attach_dynamic(&isc->isc_tx_size_1522, EVCNT_TYPE_MISC, 6105 NULL, device_xname(sc->sc_dev), "Tx size 1522"); 6106 evcnt_attach_dynamic(&isc->isc_tx_size_big, EVCNT_TYPE_MISC, 6107 NULL, device_xname(sc->sc_dev), "Tx jumbo packets"); 6108 6109 evcnt_attach_dynamic(&isc->isc_tx_bytes, EVCNT_TYPE_MISC, 6110 NULL, device_xname(sc->sc_dev), "Tx bytes / port"); 6111 evcnt_attach_dynamic(&isc->isc_tx_dropped_link_down, EVCNT_TYPE_MISC, 6112 NULL, device_xname(sc->sc_dev), 6113 "Tx dropped due to link down / port"); 6114 evcnt_attach_dynamic(&isc->isc_tx_unicast, EVCNT_TYPE_MISC, 6115 NULL, device_xname(sc->sc_dev), "Tx unicast / port"); 6116 evcnt_attach_dynamic(&isc->isc_tx_multicast, EVCNT_TYPE_MISC, 6117 NULL, device_xname(sc->sc_dev), "Tx multicast / port"); 6118 evcnt_attach_dynamic(&isc->isc_tx_broadcast, EVCNT_TYPE_MISC, 6119 NULL, device_xname(sc->sc_dev), "Tx broadcast / port"); 6120 6121 evcnt_attach_dynamic(&isc->isc_vsi_tx_bytes, EVCNT_TYPE_MISC, 6122 NULL, device_xname(sc->sc_dev), "Tx bytes / vsi"); 6123 evcnt_attach_dynamic(&isc->isc_vsi_tx_errors, EVCNT_TYPE_MISC, 6124 NULL, device_xname(sc->sc_dev), "Tx errors / vsi"); 6125 evcnt_attach_dynamic(&isc->isc_vsi_tx_unicast, EVCNT_TYPE_MISC, 6126 NULL, device_xname(sc->sc_dev), "Tx unicast / vsi"); 6127 evcnt_attach_dynamic(&isc->isc_vsi_tx_multicast, EVCNT_TYPE_MISC, 6128 NULL, device_xname(sc->sc_dev), "Tx multicast / vsi"); 6129 evcnt_attach_dynamic(&isc->isc_vsi_tx_broadcast, EVCNT_TYPE_MISC, 6130 NULL, device_xname(sc->sc_dev), "Tx broadcast / vsi"); 6131 6132 sc->sc_stats_intval = ixl_param_stats_interval; 6133 callout_init(&sc->sc_stats_callout, CALLOUT_MPSAFE); 6134 callout_setfunc(&sc->sc_stats_callout, ixl_stats_callout, sc); 6135 ixl_work_set(&sc->sc_stats_task, ixl_stats_update, sc); 6136 6137 return 0; 6138 } 6139 6140 static void 6141 ixl_teardown_stats(struct ixl_softc *sc) 6142 { 6143 struct ixl_tx_ring *txr; 6144 struct ixl_rx_ring *rxr; 6145 struct ixl_stats_counters *isc; 6146 unsigned int i; 6147 6148 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6149 txr = sc->sc_qps[i].qp_txr; 6150 rxr = sc->sc_qps[i].qp_rxr; 6151 6152 evcnt_detach(&txr->txr_defragged); 6153 evcnt_detach(&txr->txr_defrag_failed); 6154 evcnt_detach(&txr->txr_pcqdrop); 6155 evcnt_detach(&txr->txr_transmitdef); 6156 evcnt_detach(&txr->txr_intr); 6157 evcnt_detach(&txr->txr_defer); 6158 6159 evcnt_detach(&rxr->rxr_mgethdr_failed); 6160 evcnt_detach(&rxr->rxr_mgetcl_failed); 6161 evcnt_detach(&rxr->rxr_mbuf_load_failed); 6162 evcnt_detach(&rxr->rxr_intr); 6163 evcnt_detach(&rxr->rxr_defer); 6164 } 6165 6166 isc = &sc->sc_stats_counters; 6167 evcnt_detach(&isc->isc_crc_errors); 6168 evcnt_detach(&isc->isc_illegal_bytes); 6169 evcnt_detach(&isc->isc_mac_local_faults); 6170 evcnt_detach(&isc->isc_mac_remote_faults); 6171 evcnt_detach(&isc->isc_link_xon_rx); 6172 evcnt_detach(&isc->isc_link_xon_tx); 6173 evcnt_detach(&isc->isc_link_xoff_rx); 6174 evcnt_detach(&isc->isc_link_xoff_tx); 6175 evcnt_detach(&isc->isc_rx_fragments); 6176 evcnt_detach(&isc->isc_rx_jabber); 6177 evcnt_detach(&isc->isc_rx_bytes); 6178 evcnt_detach(&isc->isc_rx_discards); 6179 evcnt_detach(&isc->isc_rx_unicast); 6180 evcnt_detach(&isc->isc_rx_multicast); 6181 evcnt_detach(&isc->isc_rx_broadcast); 6182 evcnt_detach(&isc->isc_rx_size_64); 6183 evcnt_detach(&isc->isc_rx_size_127); 6184 evcnt_detach(&isc->isc_rx_size_255); 6185 evcnt_detach(&isc->isc_rx_size_511); 6186 evcnt_detach(&isc->isc_rx_size_1023); 6187 evcnt_detach(&isc->isc_rx_size_1522); 6188 evcnt_detach(&isc->isc_rx_size_big); 6189 evcnt_detach(&isc->isc_rx_undersize); 6190 evcnt_detach(&isc->isc_rx_oversize); 6191 evcnt_detach(&isc->isc_tx_bytes); 6192 evcnt_detach(&isc->isc_tx_dropped_link_down); 6193 evcnt_detach(&isc->isc_tx_unicast); 6194 evcnt_detach(&isc->isc_tx_multicast); 6195 evcnt_detach(&isc->isc_tx_broadcast); 6196 evcnt_detach(&isc->isc_tx_size_64); 6197 evcnt_detach(&isc->isc_tx_size_127); 6198 evcnt_detach(&isc->isc_tx_size_255); 6199 evcnt_detach(&isc->isc_tx_size_511); 6200 evcnt_detach(&isc->isc_tx_size_1023); 6201 evcnt_detach(&isc->isc_tx_size_1522); 6202 evcnt_detach(&isc->isc_tx_size_big); 6203 evcnt_detach(&isc->isc_vsi_rx_discards); 6204 evcnt_detach(&isc->isc_vsi_rx_bytes); 6205 evcnt_detach(&isc->isc_vsi_rx_unicast); 6206 evcnt_detach(&isc->isc_vsi_rx_multicast); 6207 evcnt_detach(&isc->isc_vsi_rx_broadcast); 6208 evcnt_detach(&isc->isc_vsi_tx_errors); 6209 evcnt_detach(&isc->isc_vsi_tx_bytes); 6210 evcnt_detach(&isc->isc_vsi_tx_unicast); 6211 evcnt_detach(&isc->isc_vsi_tx_multicast); 6212 evcnt_detach(&isc->isc_vsi_tx_broadcast); 6213 6214 evcnt_detach(&sc->sc_event_atq); 6215 evcnt_detach(&sc->sc_event_link); 6216 evcnt_detach(&sc->sc_event_ecc_err); 6217 evcnt_detach(&sc->sc_event_pci_exception); 6218 evcnt_detach(&sc->sc_event_crit_err); 6219 6220 callout_destroy(&sc->sc_stats_callout); 6221 } 6222 6223 static void 6224 ixl_stats_callout(void *xsc) 6225 { 6226 struct ixl_softc *sc = xsc; 6227 6228 ixl_work_add(sc->sc_workq, &sc->sc_stats_task); 6229 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 6230 } 6231 6232 static uint64_t 6233 ixl_stat_delta(struct ixl_softc *sc, uint32_t reg_hi, uint32_t reg_lo, 6234 uint64_t *offset, bool has_offset) 6235 { 6236 uint64_t value, delta; 6237 int bitwidth; 6238 6239 bitwidth = reg_hi == 0 ? 32 : 48; 6240 6241 value = ixl_rd(sc, reg_lo); 6242 6243 if (bitwidth > 32) { 6244 value |= ((uint64_t)ixl_rd(sc, reg_hi) << 32); 6245 } 6246 6247 if (__predict_true(has_offset)) { 6248 delta = value; 6249 if (value < *offset) 6250 delta += ((uint64_t)1 << bitwidth); 6251 delta -= *offset; 6252 } else { 6253 delta = 0; 6254 } 6255 atomic_swap_64(offset, value); 6256 6257 return delta; 6258 } 6259 6260 static void 6261 ixl_stats_update(void *xsc) 6262 { 6263 struct ixl_softc *sc = xsc; 6264 struct ixl_stats_counters *isc; 6265 uint64_t delta; 6266 6267 isc = &sc->sc_stats_counters; 6268 6269 /* errors */ 6270 delta = ixl_stat_delta(sc, 6271 0, I40E_GLPRT_CRCERRS(sc->sc_port), 6272 &isc->isc_crc_errors_offset, isc->isc_has_offset); 6273 atomic_add_64(&isc->isc_crc_errors.ev_count, delta); 6274 6275 delta = ixl_stat_delta(sc, 6276 0, I40E_GLPRT_ILLERRC(sc->sc_port), 6277 &isc->isc_illegal_bytes_offset, isc->isc_has_offset); 6278 atomic_add_64(&isc->isc_illegal_bytes.ev_count, delta); 6279 6280 /* rx */ 6281 delta = ixl_stat_delta(sc, 6282 I40E_GLPRT_GORCH(sc->sc_port), I40E_GLPRT_GORCL(sc->sc_port), 6283 &isc->isc_rx_bytes_offset, isc->isc_has_offset); 6284 atomic_add_64(&isc->isc_rx_bytes.ev_count, delta); 6285 6286 delta = ixl_stat_delta(sc, 6287 0, I40E_GLPRT_RDPC(sc->sc_port), 6288 &isc->isc_rx_discards_offset, isc->isc_has_offset); 6289 atomic_add_64(&isc->isc_rx_discards.ev_count, delta); 6290 6291 delta = ixl_stat_delta(sc, 6292 I40E_GLPRT_UPRCH(sc->sc_port), I40E_GLPRT_UPRCL(sc->sc_port), 6293 &isc->isc_rx_unicast_offset, isc->isc_has_offset); 6294 atomic_add_64(&isc->isc_rx_unicast.ev_count, delta); 6295 6296 delta = ixl_stat_delta(sc, 6297 I40E_GLPRT_MPRCH(sc->sc_port), I40E_GLPRT_MPRCL(sc->sc_port), 6298 &isc->isc_rx_multicast_offset, isc->isc_has_offset); 6299 atomic_add_64(&isc->isc_rx_multicast.ev_count, delta); 6300 6301 delta = ixl_stat_delta(sc, 6302 I40E_GLPRT_BPRCH(sc->sc_port), I40E_GLPRT_BPRCL(sc->sc_port), 6303 &isc->isc_rx_broadcast_offset, isc->isc_has_offset); 6304 atomic_add_64(&isc->isc_rx_broadcast.ev_count, delta); 6305 6306 /* Packet size stats rx */ 6307 delta = ixl_stat_delta(sc, 6308 I40E_GLPRT_PRC64H(sc->sc_port), I40E_GLPRT_PRC64L(sc->sc_port), 6309 &isc->isc_rx_size_64_offset, isc->isc_has_offset); 6310 atomic_add_64(&isc->isc_rx_size_64.ev_count, delta); 6311 6312 delta = ixl_stat_delta(sc, 6313 I40E_GLPRT_PRC127H(sc->sc_port), I40E_GLPRT_PRC127L(sc->sc_port), 6314 &isc->isc_rx_size_127_offset, isc->isc_has_offset); 6315 atomic_add_64(&isc->isc_rx_size_127.ev_count, delta); 6316 6317 delta = ixl_stat_delta(sc, 6318 I40E_GLPRT_PRC255H(sc->sc_port), I40E_GLPRT_PRC255L(sc->sc_port), 6319 &isc->isc_rx_size_255_offset, isc->isc_has_offset); 6320 atomic_add_64(&isc->isc_rx_size_255.ev_count, delta); 6321 6322 delta = ixl_stat_delta(sc, 6323 I40E_GLPRT_PRC511H(sc->sc_port), I40E_GLPRT_PRC511L(sc->sc_port), 6324 &isc->isc_rx_size_511_offset, isc->isc_has_offset); 6325 atomic_add_64(&isc->isc_rx_size_511.ev_count, delta); 6326 6327 delta = ixl_stat_delta(sc, 6328 I40E_GLPRT_PRC1023H(sc->sc_port), I40E_GLPRT_PRC1023L(sc->sc_port), 6329 &isc->isc_rx_size_1023_offset, isc->isc_has_offset); 6330 atomic_add_64(&isc->isc_rx_size_1023.ev_count, delta); 6331 6332 delta = ixl_stat_delta(sc, 6333 I40E_GLPRT_PRC1522H(sc->sc_port), I40E_GLPRT_PRC1522L(sc->sc_port), 6334 &isc->isc_rx_size_1522_offset, isc->isc_has_offset); 6335 atomic_add_64(&isc->isc_rx_size_1522.ev_count, delta); 6336 6337 delta = ixl_stat_delta(sc, 6338 I40E_GLPRT_PRC9522H(sc->sc_port), I40E_GLPRT_PRC9522L(sc->sc_port), 6339 &isc->isc_rx_size_big_offset, isc->isc_has_offset); 6340 atomic_add_64(&isc->isc_rx_size_big.ev_count, delta); 6341 6342 delta = ixl_stat_delta(sc, 6343 0, I40E_GLPRT_RUC(sc->sc_port), 6344 &isc->isc_rx_undersize_offset, isc->isc_has_offset); 6345 atomic_add_64(&isc->isc_rx_undersize.ev_count, delta); 6346 6347 delta = ixl_stat_delta(sc, 6348 0, I40E_GLPRT_ROC(sc->sc_port), 6349 &isc->isc_rx_oversize_offset, isc->isc_has_offset); 6350 atomic_add_64(&isc->isc_rx_oversize.ev_count, delta); 6351 6352 /* tx */ 6353 delta = ixl_stat_delta(sc, 6354 I40E_GLPRT_GOTCH(sc->sc_port), I40E_GLPRT_GOTCL(sc->sc_port), 6355 &isc->isc_tx_bytes_offset, isc->isc_has_offset); 6356 atomic_add_64(&isc->isc_tx_bytes.ev_count, delta); 6357 6358 delta = ixl_stat_delta(sc, 6359 0, I40E_GLPRT_TDOLD(sc->sc_port), 6360 &isc->isc_tx_dropped_link_down_offset, isc->isc_has_offset); 6361 atomic_add_64(&isc->isc_tx_dropped_link_down.ev_count, delta); 6362 6363 delta = ixl_stat_delta(sc, 6364 I40E_GLPRT_UPTCH(sc->sc_port), I40E_GLPRT_UPTCL(sc->sc_port), 6365 &isc->isc_tx_unicast_offset, isc->isc_has_offset); 6366 atomic_add_64(&isc->isc_tx_unicast.ev_count, delta); 6367 6368 delta = ixl_stat_delta(sc, 6369 I40E_GLPRT_MPTCH(sc->sc_port), I40E_GLPRT_MPTCL(sc->sc_port), 6370 &isc->isc_tx_multicast_offset, isc->isc_has_offset); 6371 atomic_add_64(&isc->isc_tx_multicast.ev_count, delta); 6372 6373 delta = ixl_stat_delta(sc, 6374 I40E_GLPRT_BPTCH(sc->sc_port), I40E_GLPRT_BPTCL(sc->sc_port), 6375 &isc->isc_tx_broadcast_offset, isc->isc_has_offset); 6376 atomic_add_64(&isc->isc_tx_broadcast.ev_count, delta); 6377 6378 /* Packet size stats tx */ 6379 delta = ixl_stat_delta(sc, 6380 I40E_GLPRT_PTC64L(sc->sc_port), I40E_GLPRT_PTC64L(sc->sc_port), 6381 &isc->isc_tx_size_64_offset, isc->isc_has_offset); 6382 atomic_add_64(&isc->isc_tx_size_64.ev_count, delta); 6383 6384 delta = ixl_stat_delta(sc, 6385 I40E_GLPRT_PTC127H(sc->sc_port), I40E_GLPRT_PTC127L(sc->sc_port), 6386 &isc->isc_tx_size_127_offset, isc->isc_has_offset); 6387 atomic_add_64(&isc->isc_tx_size_127.ev_count, delta); 6388 6389 delta = ixl_stat_delta(sc, 6390 I40E_GLPRT_PTC255H(sc->sc_port), I40E_GLPRT_PTC255L(sc->sc_port), 6391 &isc->isc_tx_size_255_offset, isc->isc_has_offset); 6392 atomic_add_64(&isc->isc_tx_size_255.ev_count, delta); 6393 6394 delta = ixl_stat_delta(sc, 6395 I40E_GLPRT_PTC511H(sc->sc_port), I40E_GLPRT_PTC511L(sc->sc_port), 6396 &isc->isc_tx_size_511_offset, isc->isc_has_offset); 6397 atomic_add_64(&isc->isc_tx_size_511.ev_count, delta); 6398 6399 delta = ixl_stat_delta(sc, 6400 I40E_GLPRT_PTC1023H(sc->sc_port), I40E_GLPRT_PTC1023L(sc->sc_port), 6401 &isc->isc_tx_size_1023_offset, isc->isc_has_offset); 6402 atomic_add_64(&isc->isc_tx_size_1023.ev_count, delta); 6403 6404 delta = ixl_stat_delta(sc, 6405 I40E_GLPRT_PTC1522H(sc->sc_port), I40E_GLPRT_PTC1522L(sc->sc_port), 6406 &isc->isc_tx_size_1522_offset, isc->isc_has_offset); 6407 atomic_add_64(&isc->isc_tx_size_1522.ev_count, delta); 6408 6409 delta = ixl_stat_delta(sc, 6410 I40E_GLPRT_PTC9522H(sc->sc_port), I40E_GLPRT_PTC9522L(sc->sc_port), 6411 &isc->isc_tx_size_big_offset, isc->isc_has_offset); 6412 atomic_add_64(&isc->isc_tx_size_big.ev_count, delta); 6413 6414 /* mac faults */ 6415 delta = ixl_stat_delta(sc, 6416 0, I40E_GLPRT_MLFC(sc->sc_port), 6417 &isc->isc_mac_local_faults_offset, isc->isc_has_offset); 6418 atomic_add_64(&isc->isc_mac_local_faults.ev_count, delta); 6419 6420 delta = ixl_stat_delta(sc, 6421 0, I40E_GLPRT_MRFC(sc->sc_port), 6422 &isc->isc_mac_remote_faults_offset, isc->isc_has_offset); 6423 atomic_add_64(&isc->isc_mac_remote_faults.ev_count, delta); 6424 6425 /* Flow control (LFC) stats */ 6426 delta = ixl_stat_delta(sc, 6427 0, I40E_GLPRT_LXONRXC(sc->sc_port), 6428 &isc->isc_link_xon_rx_offset, isc->isc_has_offset); 6429 atomic_add_64(&isc->isc_link_xon_rx.ev_count, delta); 6430 6431 delta = ixl_stat_delta(sc, 6432 0, I40E_GLPRT_LXONTXC(sc->sc_port), 6433 &isc->isc_link_xon_tx_offset, isc->isc_has_offset); 6434 atomic_add_64(&isc->isc_link_xon_tx.ev_count, delta); 6435 6436 delta = ixl_stat_delta(sc, 6437 0, I40E_GLPRT_LXOFFRXC(sc->sc_port), 6438 &isc->isc_link_xoff_rx_offset, isc->isc_has_offset); 6439 atomic_add_64(&isc->isc_link_xoff_rx.ev_count, delta); 6440 6441 delta = ixl_stat_delta(sc, 6442 0, I40E_GLPRT_LXOFFTXC(sc->sc_port), 6443 &isc->isc_link_xoff_tx_offset, isc->isc_has_offset); 6444 atomic_add_64(&isc->isc_link_xoff_tx.ev_count, delta); 6445 6446 /* fragments */ 6447 delta = ixl_stat_delta(sc, 6448 0, I40E_GLPRT_RFC(sc->sc_port), 6449 &isc->isc_rx_fragments_offset, isc->isc_has_offset); 6450 atomic_add_64(&isc->isc_rx_fragments.ev_count, delta); 6451 6452 delta = ixl_stat_delta(sc, 6453 0, I40E_GLPRT_RJC(sc->sc_port), 6454 &isc->isc_rx_jabber_offset, isc->isc_has_offset); 6455 atomic_add_64(&isc->isc_rx_jabber.ev_count, delta); 6456 6457 /* VSI rx counters */ 6458 delta = ixl_stat_delta(sc, 6459 0, I40E_GLV_RDPC(sc->sc_vsi_stat_counter_idx), 6460 &isc->isc_vsi_rx_discards_offset, isc->isc_has_offset); 6461 atomic_add_64(&isc->isc_vsi_rx_discards.ev_count, delta); 6462 6463 delta = ixl_stat_delta(sc, 6464 I40E_GLV_GORCH(sc->sc_vsi_stat_counter_idx), 6465 I40E_GLV_GORCL(sc->sc_vsi_stat_counter_idx), 6466 &isc->isc_vsi_rx_bytes_offset, isc->isc_has_offset); 6467 atomic_add_64(&isc->isc_vsi_rx_bytes.ev_count, delta); 6468 6469 delta = ixl_stat_delta(sc, 6470 I40E_GLV_UPRCH(sc->sc_vsi_stat_counter_idx), 6471 I40E_GLV_UPRCL(sc->sc_vsi_stat_counter_idx), 6472 &isc->isc_vsi_rx_unicast_offset, isc->isc_has_offset); 6473 atomic_add_64(&isc->isc_vsi_rx_unicast.ev_count, delta); 6474 6475 delta = ixl_stat_delta(sc, 6476 I40E_GLV_MPRCH(sc->sc_vsi_stat_counter_idx), 6477 I40E_GLV_MPRCL(sc->sc_vsi_stat_counter_idx), 6478 &isc->isc_vsi_rx_multicast_offset, isc->isc_has_offset); 6479 atomic_add_64(&isc->isc_vsi_rx_multicast.ev_count, delta); 6480 6481 delta = ixl_stat_delta(sc, 6482 I40E_GLV_BPRCH(sc->sc_vsi_stat_counter_idx), 6483 I40E_GLV_BPRCL(sc->sc_vsi_stat_counter_idx), 6484 &isc->isc_vsi_rx_broadcast_offset, isc->isc_has_offset); 6485 atomic_add_64(&isc->isc_vsi_rx_broadcast.ev_count, delta); 6486 6487 /* VSI tx counters */ 6488 delta = ixl_stat_delta(sc, 6489 0, I40E_GLV_TEPC(sc->sc_vsi_stat_counter_idx), 6490 &isc->isc_vsi_tx_errors_offset, isc->isc_has_offset); 6491 atomic_add_64(&isc->isc_vsi_tx_errors.ev_count, delta); 6492 6493 delta = ixl_stat_delta(sc, 6494 I40E_GLV_GOTCH(sc->sc_vsi_stat_counter_idx), 6495 I40E_GLV_GOTCL(sc->sc_vsi_stat_counter_idx), 6496 &isc->isc_vsi_tx_bytes_offset, isc->isc_has_offset); 6497 atomic_add_64(&isc->isc_vsi_tx_bytes.ev_count, delta); 6498 6499 delta = ixl_stat_delta(sc, 6500 I40E_GLV_UPTCH(sc->sc_vsi_stat_counter_idx), 6501 I40E_GLV_UPTCL(sc->sc_vsi_stat_counter_idx), 6502 &isc->isc_vsi_tx_unicast_offset, isc->isc_has_offset); 6503 atomic_add_64(&isc->isc_vsi_tx_unicast.ev_count, delta); 6504 6505 delta = ixl_stat_delta(sc, 6506 I40E_GLV_MPTCH(sc->sc_vsi_stat_counter_idx), 6507 I40E_GLV_MPTCL(sc->sc_vsi_stat_counter_idx), 6508 &isc->isc_vsi_tx_multicast_offset, isc->isc_has_offset); 6509 atomic_add_64(&isc->isc_vsi_tx_multicast.ev_count, delta); 6510 6511 delta = ixl_stat_delta(sc, 6512 I40E_GLV_BPTCH(sc->sc_vsi_stat_counter_idx), 6513 I40E_GLV_BPTCL(sc->sc_vsi_stat_counter_idx), 6514 &isc->isc_vsi_tx_broadcast_offset, isc->isc_has_offset); 6515 atomic_add_64(&isc->isc_vsi_tx_broadcast.ev_count, delta); 6516 } 6517 6518 static int 6519 ixl_setup_sysctls(struct ixl_softc *sc) 6520 { 6521 const char *devname; 6522 struct sysctllog **log; 6523 const struct sysctlnode *rnode, *rxnode, *txnode; 6524 int error; 6525 6526 log = &sc->sc_sysctllog; 6527 devname = device_xname(sc->sc_dev); 6528 6529 error = sysctl_createv(log, 0, NULL, &rnode, 6530 0, CTLTYPE_NODE, devname, 6531 SYSCTL_DESCR("ixl information and settings"), 6532 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 6533 if (error) 6534 goto out; 6535 6536 error = sysctl_createv(log, 0, &rnode, NULL, 6537 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 6538 SYSCTL_DESCR("Use workqueue for packet processing"), 6539 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 6540 if (error) 6541 goto out; 6542 6543 error = sysctl_createv(log, 0, &rnode, NULL, 6544 CTLFLAG_READONLY, CTLTYPE_INT, "stats_interval", 6545 SYSCTL_DESCR("Statistics collection interval in milliseconds"), 6546 NULL, 0, &sc->sc_stats_intval, 0, CTL_CREATE, CTL_EOL); 6547 6548 error = sysctl_createv(log, 0, &rnode, &rxnode, 6549 0, CTLTYPE_NODE, "rx", 6550 SYSCTL_DESCR("ixl information and settings for Rx"), 6551 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6552 if (error) 6553 goto out; 6554 6555 error = sysctl_createv(log, 0, &rxnode, NULL, 6556 CTLFLAG_READWRITE, CTLTYPE_INT, "itr", 6557 SYSCTL_DESCR("Interrupt Throttling"), 6558 ixl_sysctl_itr_handler, 0, 6559 (void *)sc, 0, CTL_CREATE, CTL_EOL); 6560 if (error) 6561 goto out; 6562 6563 error = sysctl_createv(log, 0, &rxnode, NULL, 6564 CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num", 6565 SYSCTL_DESCR("the number of rx descriptors"), 6566 NULL, 0, &sc->sc_rx_ring_ndescs, 0, CTL_CREATE, CTL_EOL); 6567 if (error) 6568 goto out; 6569 6570 error = sysctl_createv(log, 0, &rxnode, NULL, 6571 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6572 SYSCTL_DESCR("max number of Rx packets" 6573 " to process for interrupt processing"), 6574 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6575 if (error) 6576 goto out; 6577 6578 error = sysctl_createv(log, 0, &rxnode, NULL, 6579 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6580 SYSCTL_DESCR("max number of Rx packets" 6581 " to process for deferred processing"), 6582 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 6583 if (error) 6584 goto out; 6585 6586 error = sysctl_createv(log, 0, &rnode, &txnode, 6587 0, CTLTYPE_NODE, "tx", 6588 SYSCTL_DESCR("ixl information and settings for Tx"), 6589 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6590 if (error) 6591 goto out; 6592 6593 error = sysctl_createv(log, 0, &txnode, NULL, 6594 CTLFLAG_READWRITE, CTLTYPE_INT, "itr", 6595 SYSCTL_DESCR("Interrupt Throttling"), 6596 ixl_sysctl_itr_handler, 0, 6597 (void *)sc, 0, CTL_CREATE, CTL_EOL); 6598 if (error) 6599 goto out; 6600 6601 error = sysctl_createv(log, 0, &txnode, NULL, 6602 CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num", 6603 SYSCTL_DESCR("the number of tx descriptors"), 6604 NULL, 0, &sc->sc_tx_ring_ndescs, 0, CTL_CREATE, CTL_EOL); 6605 if (error) 6606 goto out; 6607 6608 error = sysctl_createv(log, 0, &txnode, NULL, 6609 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6610 SYSCTL_DESCR("max number of Tx packets" 6611 " to process for interrupt processing"), 6612 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6613 if (error) 6614 goto out; 6615 6616 error = sysctl_createv(log, 0, &txnode, NULL, 6617 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6618 SYSCTL_DESCR("max number of Tx packets" 6619 " to process for deferred processing"), 6620 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 6621 if (error) 6622 goto out; 6623 6624 out: 6625 if (error) { 6626 aprint_error_dev(sc->sc_dev, 6627 "unable to create sysctl node\n"); 6628 sysctl_teardown(log); 6629 } 6630 6631 return error; 6632 } 6633 6634 static void 6635 ixl_teardown_sysctls(struct ixl_softc *sc) 6636 { 6637 6638 sysctl_teardown(&sc->sc_sysctllog); 6639 } 6640 6641 static bool 6642 ixl_sysctlnode_is_rx(struct sysctlnode *node) 6643 { 6644 6645 if (strstr(node->sysctl_parent->sysctl_name, "rx") != NULL) 6646 return true; 6647 6648 return false; 6649 } 6650 6651 static int 6652 ixl_sysctl_itr_handler(SYSCTLFN_ARGS) 6653 { 6654 struct sysctlnode node = *rnode; 6655 struct ixl_softc *sc = (struct ixl_softc *)node.sysctl_data; 6656 struct ifnet *ifp = &sc->sc_ec.ec_if; 6657 uint32_t newitr, *itrptr; 6658 int error; 6659 6660 if (ixl_sysctlnode_is_rx(&node)) { 6661 itrptr = &sc->sc_itr_rx; 6662 } else { 6663 itrptr = &sc->sc_itr_tx; 6664 } 6665 6666 newitr = *itrptr; 6667 node.sysctl_data = &newitr; 6668 node.sysctl_size = sizeof(newitr); 6669 6670 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6671 6672 if (error || newp == NULL) 6673 return error; 6674 6675 /* ITRs are applied in ixl_init() for simple implementaion */ 6676 if (ISSET(ifp->if_flags, IFF_RUNNING)) 6677 return EBUSY; 6678 6679 if (newitr > 0x07ff) 6680 return EINVAL; 6681 6682 *itrptr = newitr; 6683 6684 return 0; 6685 } 6686 6687 static struct workqueue * 6688 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 6689 { 6690 struct workqueue *wq; 6691 int error; 6692 6693 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 6694 prio, ipl, flags); 6695 6696 if (error) 6697 return NULL; 6698 6699 return wq; 6700 } 6701 6702 static void 6703 ixl_workq_destroy(struct workqueue *wq) 6704 { 6705 6706 workqueue_destroy(wq); 6707 } 6708 6709 static void 6710 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 6711 { 6712 6713 memset(work, 0, sizeof(*work)); 6714 work->ixw_func = func; 6715 work->ixw_arg = arg; 6716 } 6717 6718 static void 6719 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 6720 { 6721 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 6722 return; 6723 6724 kpreempt_disable(); 6725 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 6726 kpreempt_enable(); 6727 } 6728 6729 static void 6730 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 6731 { 6732 6733 workqueue_wait(wq, &work->ixw_cookie); 6734 } 6735 6736 static void 6737 ixl_workq_work(struct work *wk, void *context) 6738 { 6739 struct ixl_work *work; 6740 6741 work = container_of(wk, struct ixl_work, ixw_cookie); 6742 6743 atomic_swap_uint(&work->ixw_added, 0); 6744 work->ixw_func(work->ixw_arg); 6745 } 6746 6747 static int 6748 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 6749 { 6750 struct ixl_aq_desc iaq; 6751 6752 memset(&iaq, 0, sizeof(iaq)); 6753 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 6754 iaq.iaq_param[1] = htole32(reg); 6755 6756 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6757 return ETIMEDOUT; 6758 6759 switch (htole16(iaq.iaq_retval)) { 6760 case IXL_AQ_RC_OK: 6761 /* success */ 6762 break; 6763 case IXL_AQ_RC_EACCES: 6764 return EPERM; 6765 case IXL_AQ_RC_EAGAIN: 6766 return EAGAIN; 6767 default: 6768 return EIO; 6769 } 6770 6771 *rv = htole32(iaq.iaq_param[3]); 6772 return 0; 6773 } 6774 6775 static uint32_t 6776 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 6777 { 6778 uint32_t val; 6779 int rv, retry, retry_limit; 6780 6781 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6782 retry_limit = 5; 6783 } else { 6784 retry_limit = 0; 6785 } 6786 6787 for (retry = 0; retry < retry_limit; retry++) { 6788 rv = ixl_rx_ctl_read(sc, reg, &val); 6789 if (rv == 0) 6790 return val; 6791 else if (rv == EAGAIN) 6792 delaymsec(1); 6793 else 6794 break; 6795 } 6796 6797 val = ixl_rd(sc, reg); 6798 6799 return val; 6800 } 6801 6802 static int 6803 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6804 { 6805 struct ixl_aq_desc iaq; 6806 6807 memset(&iaq, 0, sizeof(iaq)); 6808 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 6809 iaq.iaq_param[1] = htole32(reg); 6810 iaq.iaq_param[3] = htole32(value); 6811 6812 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6813 return ETIMEDOUT; 6814 6815 switch (htole16(iaq.iaq_retval)) { 6816 case IXL_AQ_RC_OK: 6817 /* success */ 6818 break; 6819 case IXL_AQ_RC_EACCES: 6820 return EPERM; 6821 case IXL_AQ_RC_EAGAIN: 6822 return EAGAIN; 6823 default: 6824 return EIO; 6825 } 6826 6827 return 0; 6828 } 6829 6830 static void 6831 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6832 { 6833 int rv, retry, retry_limit; 6834 6835 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6836 retry_limit = 5; 6837 } else { 6838 retry_limit = 0; 6839 } 6840 6841 for (retry = 0; retry < retry_limit; retry++) { 6842 rv = ixl_rx_ctl_write(sc, reg, value); 6843 if (rv == 0) 6844 return; 6845 else if (rv == EAGAIN) 6846 delaymsec(1); 6847 else 6848 break; 6849 } 6850 6851 ixl_wr(sc, reg, value); 6852 } 6853 6854 static int 6855 ixl_nvm_lock(struct ixl_softc *sc, char rw) 6856 { 6857 struct ixl_aq_desc iaq; 6858 struct ixl_aq_req_resource_param *param; 6859 int rv; 6860 6861 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6862 return 0; 6863 6864 memset(&iaq, 0, sizeof(iaq)); 6865 iaq.iaq_opcode = htole16(IXL_AQ_OP_REQUEST_RESOURCE); 6866 6867 param = (struct ixl_aq_req_resource_param *)&iaq.iaq_param; 6868 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6869 if (rw == 'R') { 6870 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_READ); 6871 } else { 6872 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_WRITE); 6873 } 6874 6875 rv = ixl_atq_poll(sc, &iaq, 250); 6876 6877 if (rv != 0) 6878 return ETIMEDOUT; 6879 6880 switch (le16toh(iaq.iaq_retval)) { 6881 case IXL_AQ_RC_OK: 6882 break; 6883 case IXL_AQ_RC_EACCES: 6884 return EACCES; 6885 case IXL_AQ_RC_EBUSY: 6886 return EBUSY; 6887 case IXL_AQ_RC_EPERM: 6888 return EPERM; 6889 } 6890 6891 return 0; 6892 } 6893 6894 static int 6895 ixl_nvm_unlock(struct ixl_softc *sc) 6896 { 6897 struct ixl_aq_desc iaq; 6898 struct ixl_aq_rel_resource_param *param; 6899 int rv; 6900 6901 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6902 return 0; 6903 6904 memset(&iaq, 0, sizeof(iaq)); 6905 iaq.iaq_opcode = htole16(IXL_AQ_OP_RELEASE_RESOURCE); 6906 6907 param = (struct ixl_aq_rel_resource_param *)&iaq.iaq_param; 6908 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6909 6910 rv = ixl_atq_poll(sc, &iaq, 250); 6911 6912 if (rv != 0) 6913 return ETIMEDOUT; 6914 6915 switch (le16toh(iaq.iaq_retval)) { 6916 case IXL_AQ_RC_OK: 6917 break; 6918 default: 6919 return EIO; 6920 } 6921 return 0; 6922 } 6923 6924 static int 6925 ixl_srdone_poll(struct ixl_softc *sc) 6926 { 6927 int wait_count; 6928 uint32_t reg; 6929 6930 for (wait_count = 0; wait_count < IXL_SRRD_SRCTL_ATTEMPTS; 6931 wait_count++) { 6932 reg = ixl_rd(sc, I40E_GLNVM_SRCTL); 6933 if (ISSET(reg, I40E_GLNVM_SRCTL_DONE_MASK)) 6934 break; 6935 6936 delaymsec(5); 6937 } 6938 6939 if (wait_count == IXL_SRRD_SRCTL_ATTEMPTS) 6940 return -1; 6941 6942 return 0; 6943 } 6944 6945 static int 6946 ixl_nvm_read_srctl(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 6947 { 6948 uint32_t reg; 6949 6950 if (ixl_srdone_poll(sc) != 0) 6951 return ETIMEDOUT; 6952 6953 reg = ((uint32_t)offset << I40E_GLNVM_SRCTL_ADDR_SHIFT) | 6954 __BIT(I40E_GLNVM_SRCTL_START_SHIFT); 6955 ixl_wr(sc, I40E_GLNVM_SRCTL, reg); 6956 6957 if (ixl_srdone_poll(sc) != 0) { 6958 aprint_debug("NVM read error: couldn't access " 6959 "Shadow RAM address: 0x%x\n", offset); 6960 return ETIMEDOUT; 6961 } 6962 6963 reg = ixl_rd(sc, I40E_GLNVM_SRDATA); 6964 *data = (uint16_t)__SHIFTOUT(reg, I40E_GLNVM_SRDATA_RDDATA_MASK); 6965 6966 return 0; 6967 } 6968 6969 static int 6970 ixl_nvm_read_aq(struct ixl_softc *sc, uint16_t offset_word, 6971 void *data, size_t len) 6972 { 6973 struct ixl_dmamem *idm; 6974 struct ixl_aq_desc iaq; 6975 struct ixl_aq_nvm_param *param; 6976 uint32_t offset_bytes; 6977 int rv; 6978 6979 idm = &sc->sc_aqbuf; 6980 if (len > IXL_DMA_LEN(idm)) 6981 return ENOMEM; 6982 6983 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 6984 memset(&iaq, 0, sizeof(iaq)); 6985 iaq.iaq_opcode = htole16(IXL_AQ_OP_NVM_READ); 6986 iaq.iaq_flags = htole16(IXL_AQ_BUF | 6987 ((len > I40E_AQ_LARGE_BUF) ? IXL_AQ_LB : 0)); 6988 iaq.iaq_datalen = htole16(len); 6989 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 6990 6991 param = (struct ixl_aq_nvm_param *)iaq.iaq_param; 6992 param->command_flags = IXL_AQ_NVM_LAST_CMD; 6993 param->module_pointer = 0; 6994 param->length = htole16(len); 6995 offset_bytes = (uint32_t)offset_word * 2; 6996 offset_bytes &= 0x00FFFFFF; 6997 param->offset = htole32(offset_bytes); 6998 6999 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 7000 BUS_DMASYNC_PREREAD); 7001 7002 rv = ixl_atq_poll(sc, &iaq, 250); 7003 7004 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 7005 BUS_DMASYNC_POSTREAD); 7006 7007 if (rv != 0) { 7008 return ETIMEDOUT; 7009 } 7010 7011 switch (le16toh(iaq.iaq_retval)) { 7012 case IXL_AQ_RC_OK: 7013 break; 7014 case IXL_AQ_RC_EPERM: 7015 return EPERM; 7016 case IXL_AQ_RC_EINVAL: 7017 return EINVAL; 7018 case IXL_AQ_RC_EBUSY: 7019 return EBUSY; 7020 case IXL_AQ_RC_EIO: 7021 default: 7022 return EIO; 7023 } 7024 7025 memcpy(data, IXL_DMA_KVA(idm), len); 7026 7027 return 0; 7028 } 7029 7030 static int 7031 ixl_rd16_nvm(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 7032 { 7033 int error; 7034 uint16_t buf; 7035 7036 error = ixl_nvm_lock(sc, 'R'); 7037 if (error) 7038 return error; 7039 7040 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMREAD)) { 7041 error = ixl_nvm_read_aq(sc, offset, 7042 &buf, sizeof(buf)); 7043 if (error == 0) 7044 *data = le16toh(buf); 7045 } else { 7046 error = ixl_nvm_read_srctl(sc, offset, &buf); 7047 if (error == 0) 7048 *data = buf; 7049 } 7050 7051 ixl_nvm_unlock(sc); 7052 7053 return error; 7054 } 7055 7056 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 7057 7058 #ifdef _MODULE 7059 #include "ioconf.c" 7060 #endif 7061 7062 #ifdef _MODULE 7063 static void 7064 ixl_parse_modprop(prop_dictionary_t dict) 7065 { 7066 prop_object_t obj; 7067 int64_t val; 7068 uint64_t uval; 7069 7070 if (dict == NULL) 7071 return; 7072 7073 obj = prop_dictionary_get(dict, "nomsix"); 7074 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_BOOL) { 7075 ixl_param_nomsix = prop_bool_true((prop_bool_t)obj); 7076 } 7077 7078 obj = prop_dictionary_get(dict, "stats_interval"); 7079 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7080 val = prop_number_signed_value((prop_number_t)obj); 7081 7082 /* the range has no reason */ 7083 if (100 < val && val < 180000) { 7084 ixl_param_stats_interval = val; 7085 } 7086 } 7087 7088 obj = prop_dictionary_get(dict, "nqps_limit"); 7089 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7090 val = prop_number_signed_value((prop_number_t)obj); 7091 7092 if (val <= INT32_MAX) 7093 ixl_param_nqps_limit = val; 7094 } 7095 7096 obj = prop_dictionary_get(dict, "rx_ndescs"); 7097 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7098 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7099 7100 if (uval > 8) 7101 ixl_param_rx_ndescs = uval; 7102 } 7103 7104 obj = prop_dictionary_get(dict, "tx_ndescs"); 7105 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7106 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7107 7108 if (uval > IXL_TX_PKT_DESCS) 7109 ixl_param_tx_ndescs = uval; 7110 } 7111 7112 } 7113 #endif 7114 7115 static int 7116 if_ixl_modcmd(modcmd_t cmd, void *opaque) 7117 { 7118 int error = 0; 7119 7120 #ifdef _MODULE 7121 switch (cmd) { 7122 case MODULE_CMD_INIT: 7123 ixl_parse_modprop((prop_dictionary_t)opaque); 7124 error = config_init_component(cfdriver_ioconf_if_ixl, 7125 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7126 break; 7127 case MODULE_CMD_FINI: 7128 error = config_fini_component(cfdriver_ioconf_if_ixl, 7129 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7130 break; 7131 default: 7132 error = ENOTTY; 7133 break; 7134 } 7135 #endif 7136 7137 return error; 7138 } 7139
Indexes created Sat Sep 20 22:09:52 GMT 2025