if_ixl.c revision 1.91
1 /* $NetBSD: if_ixl.c,v 1.91 2023/10/11 04:29:47 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 __KERNEL_RCSID(0, "$NetBSD: if_ixl.c,v 1.91 2023/10/11 04:29:47 yamaguchi Exp $"); 78 79 #ifdef _KERNEL_OPT 80 #include "opt_net_mpsafe.h" 81 #include "opt_if_ixl.h" 82 #endif 83 84 #include <sys/param.h> 85 #include <sys/types.h> 86 87 #include <sys/bitops.h> 88 #include <sys/cpu.h> 89 #include <sys/device.h> 90 #include <sys/evcnt.h> 91 #include <sys/interrupt.h> 92 #include <sys/kmem.h> 93 #include <sys/module.h> 94 #include <sys/mutex.h> 95 #include <sys/pcq.h> 96 #include <sys/syslog.h> 97 #include <sys/workqueue.h> 98 #include <sys/xcall.h> 99 100 #include <sys/bus.h> 101 102 #include <net/bpf.h> 103 #include <net/if.h> 104 #include <net/if_dl.h> 105 #include <net/if_media.h> 106 #include <net/if_ether.h> 107 #include <net/rss_config.h> 108 109 #include <netinet/tcp.h> /* for struct tcphdr */ 110 #include <netinet/udp.h> /* for struct udphdr */ 111 112 #include <dev/pci/pcivar.h> 113 #include <dev/pci/pcidevs.h> 114 115 #include <dev/pci/if_ixlreg.h> 116 #include <dev/pci/if_ixlvar.h> 117 118 #include <prop/proplib.h> 119 120 struct ixl_softc; /* defined */ 121 122 #define I40E_PF_RESET_WAIT_COUNT 200 123 #define I40E_AQ_LARGE_BUF 512 124 125 /* bitfields for Tx queue mapping in QTX_CTL */ 126 #define I40E_QTX_CTL_VF_QUEUE 0x0 127 #define I40E_QTX_CTL_VM_QUEUE 0x1 128 #define I40E_QTX_CTL_PF_QUEUE 0x2 129 130 #define I40E_QUEUE_TYPE_EOL 0x7ff 131 #define I40E_INTR_NOTX_QUEUE 0 132 133 #define I40E_QUEUE_TYPE_RX 0x0 134 #define I40E_QUEUE_TYPE_TX 0x1 135 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 136 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 137 138 #define I40E_ITR_INDEX_RX 0x0 139 #define I40E_ITR_INDEX_TX 0x1 140 #define I40E_ITR_INDEX_OTHER 0x2 141 #define I40E_ITR_INDEX_NONE 0x3 142 #define IXL_ITR_RX 0x7a /* 4K intrs/sec */ 143 #define IXL_ITR_TX 0x7a /* 4K intrs/sec */ 144 145 #define I40E_INTR_NOTX_QUEUE 0 146 #define I40E_INTR_NOTX_INTR 0 147 #define I40E_INTR_NOTX_RX_QUEUE 0 148 #define I40E_INTR_NOTX_TX_QUEUE 1 149 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 150 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 151 152 #define I40E_HASH_LUT_SIZE_128 0 153 154 #define IXL_ICR0_CRIT_ERR_MASK \ 155 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 156 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 157 I40E_PFINT_ICR0_PE_CRITERR_MASK) 158 159 #define IXL_QUEUE_MAX_XL710 64 160 #define IXL_QUEUE_MAX_X722 128 161 162 #define IXL_TX_PKT_DESCS 8 163 #define IXL_TX_PKT_MAXSIZE (MCLBYTES * IXL_TX_PKT_DESCS) 164 #define IXL_TX_QUEUE_ALIGN 128 165 #define IXL_RX_QUEUE_ALIGN 128 166 167 #define IXL_MCLBYTES (MCLBYTES - ETHER_ALIGN) 168 #define IXL_MTU_ETHERLEN ETHER_HDR_LEN \ 169 + ETHER_CRC_LEN 170 #if 0 171 #define IXL_MAX_MTU (9728 - IXL_MTU_ETHERLEN) 172 #else 173 /* (dbuff * 5) - ETHER_HDR_LEN - ETHER_CRC_LEN */ 174 #define IXL_MAX_MTU (9600 - IXL_MTU_ETHERLEN) 175 #endif 176 #define IXL_MIN_MTU (ETHER_MIN_LEN - ETHER_CRC_LEN) 177 178 #define IXL_PCIREG PCI_MAPREG_START 179 180 #define IXL_ITR0 0x0 181 #define IXL_ITR1 0x1 182 #define IXL_ITR2 0x2 183 #define IXL_NOITR 0x3 184 185 #define IXL_AQ_NUM 256 186 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 187 #define IXL_AQ_ALIGN 64 /* lol */ 188 #define IXL_AQ_BUFLEN 4096 189 190 #define IXL_HMC_ROUNDUP 512 191 #define IXL_HMC_PGSIZE 4096 192 #define IXL_HMC_DVASZ sizeof(uint64_t) 193 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 194 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 195 #define IXL_HMC_PDVALID 1ULL 196 197 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 198 199 #define IXL_SRRD_SRCTL_ATTEMPTS 100000 200 201 struct ixl_aq_regs { 202 bus_size_t atq_tail; 203 bus_size_t atq_head; 204 bus_size_t atq_len; 205 bus_size_t atq_bal; 206 bus_size_t atq_bah; 207 208 bus_size_t arq_tail; 209 bus_size_t arq_head; 210 bus_size_t arq_len; 211 bus_size_t arq_bal; 212 bus_size_t arq_bah; 213 214 uint32_t atq_len_enable; 215 uint32_t atq_tail_mask; 216 uint32_t atq_head_mask; 217 218 uint32_t arq_len_enable; 219 uint32_t arq_tail_mask; 220 uint32_t arq_head_mask; 221 }; 222 223 struct ixl_phy_type { 224 uint64_t phy_type; 225 uint64_t ifm_type; 226 }; 227 228 struct ixl_speed_type { 229 uint8_t dev_speed; 230 uint64_t net_speed; 231 }; 232 233 struct ixl_hmc_entry { 234 uint64_t hmc_base; 235 uint32_t hmc_count; 236 uint64_t hmc_size; 237 }; 238 239 enum ixl_hmc_types { 240 IXL_HMC_LAN_TX = 0, 241 IXL_HMC_LAN_RX, 242 IXL_HMC_FCOE_CTX, 243 IXL_HMC_FCOE_FILTER, 244 IXL_HMC_COUNT 245 }; 246 247 struct ixl_hmc_pack { 248 uint16_t offset; 249 uint16_t width; 250 uint16_t lsb; 251 }; 252 253 /* 254 * these hmc objects have weird sizes and alignments, so these are abstract 255 * representations of them that are nice for c to populate. 256 * 257 * the packing code relies on little-endian values being stored in the fields, 258 * no high bits in the fields being set, and the fields must be packed in the 259 * same order as they are in the ctx structure. 260 */ 261 262 struct ixl_hmc_rxq { 263 uint16_t head; 264 uint8_t cpuid; 265 uint64_t base; 266 #define IXL_HMC_RXQ_BASE_UNIT 128 267 uint16_t qlen; 268 uint16_t dbuff; 269 #define IXL_HMC_RXQ_DBUFF_UNIT 128 270 uint8_t hbuff; 271 #define IXL_HMC_RXQ_HBUFF_UNIT 64 272 uint8_t dtype; 273 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 274 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 275 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 276 uint8_t dsize; 277 #define IXL_HMC_RXQ_DSIZE_16 0 278 #define IXL_HMC_RXQ_DSIZE_32 1 279 uint8_t crcstrip; 280 uint8_t fc_ena; 281 uint8_t l2sel; 282 uint8_t hsplit_0; 283 uint8_t hsplit_1; 284 uint8_t showiv; 285 uint16_t rxmax; 286 uint8_t tphrdesc_ena; 287 uint8_t tphwdesc_ena; 288 uint8_t tphdata_ena; 289 uint8_t tphhead_ena; 290 uint8_t lrxqthresh; 291 uint8_t prefena; 292 }; 293 294 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 295 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 296 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 297 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 298 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 299 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 300 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 301 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 302 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 303 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 304 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 305 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 306 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 307 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 308 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 309 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 310 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 311 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 312 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 313 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 314 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 315 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 316 }; 317 318 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 319 320 struct ixl_hmc_txq { 321 uint16_t head; 322 uint8_t new_context; 323 uint64_t base; 324 #define IXL_HMC_TXQ_BASE_UNIT 128 325 uint8_t fc_ena; 326 uint8_t timesync_ena; 327 uint8_t fd_ena; 328 uint8_t alt_vlan_ena; 329 uint8_t cpuid; 330 uint16_t thead_wb; 331 uint8_t head_wb_ena; 332 #define IXL_HMC_TXQ_DESC_WB 0 333 #define IXL_HMC_TXQ_HEAD_WB 1 334 uint16_t qlen; 335 uint8_t tphrdesc_ena; 336 uint8_t tphrpacket_ena; 337 uint8_t tphwdesc_ena; 338 uint64_t head_wb_addr; 339 uint32_t crc; 340 uint16_t rdylist; 341 uint8_t rdylist_act; 342 }; 343 344 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 345 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 346 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 347 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 348 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 349 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 350 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 351 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 352 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 353 /* line 1 */ 354 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 355 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 356 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 357 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 358 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 359 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 360 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 361 /* line 7 */ 362 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 363 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 364 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 365 }; 366 367 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 368 369 struct ixl_work { 370 struct work ixw_cookie; 371 void (*ixw_func)(void *); 372 void *ixw_arg; 373 unsigned int ixw_added; 374 }; 375 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 376 377 struct ixl_tx_map { 378 struct mbuf *txm_m; 379 bus_dmamap_t txm_map; 380 unsigned int txm_eop; 381 }; 382 383 struct ixl_tx_ring { 384 kmutex_t txr_lock; 385 struct ixl_softc *txr_sc; 386 387 unsigned int txr_prod; 388 unsigned int txr_cons; 389 390 struct ixl_tx_map *txr_maps; 391 struct ixl_dmamem txr_mem; 392 393 bus_size_t txr_tail; 394 unsigned int txr_qid; 395 pcq_t *txr_intrq; 396 void *txr_si; 397 398 struct evcnt txr_defragged; 399 struct evcnt txr_defrag_failed; 400 struct evcnt txr_pcqdrop; 401 struct evcnt txr_transmitdef; 402 struct evcnt txr_intr; 403 struct evcnt txr_defer; 404 }; 405 406 struct ixl_rx_map { 407 struct mbuf *rxm_m; 408 bus_dmamap_t rxm_map; 409 }; 410 411 struct ixl_rx_ring { 412 kmutex_t rxr_lock; 413 414 unsigned int rxr_prod; 415 unsigned int rxr_cons; 416 417 struct ixl_rx_map *rxr_maps; 418 struct ixl_dmamem rxr_mem; 419 420 struct mbuf *rxr_m_head; 421 struct mbuf **rxr_m_tail; 422 423 bus_size_t rxr_tail; 424 unsigned int rxr_qid; 425 426 struct evcnt rxr_mgethdr_failed; 427 struct evcnt rxr_mgetcl_failed; 428 struct evcnt rxr_mbuf_load_failed; 429 struct evcnt rxr_intr; 430 struct evcnt rxr_defer; 431 }; 432 433 struct ixl_queue_pair { 434 struct ixl_softc *qp_sc; 435 struct ixl_tx_ring *qp_txr; 436 struct ixl_rx_ring *qp_rxr; 437 438 char qp_name[16]; 439 440 void *qp_si; 441 struct work qp_work; 442 bool qp_workqueue; 443 }; 444 445 struct ixl_atq { 446 struct ixl_aq_desc iatq_desc; 447 void (*iatq_fn)(struct ixl_softc *, 448 const struct ixl_aq_desc *); 449 bool iatq_inuse; 450 }; 451 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 452 453 struct ixl_product { 454 unsigned int vendor_id; 455 unsigned int product_id; 456 }; 457 458 struct ixl_stats_counters { 459 bool isc_has_offset; 460 struct evcnt isc_crc_errors; 461 uint64_t isc_crc_errors_offset; 462 struct evcnt isc_illegal_bytes; 463 uint64_t isc_illegal_bytes_offset; 464 struct evcnt isc_rx_bytes; 465 uint64_t isc_rx_bytes_offset; 466 struct evcnt isc_rx_discards; 467 uint64_t isc_rx_discards_offset; 468 struct evcnt isc_rx_unicast; 469 uint64_t isc_rx_unicast_offset; 470 struct evcnt isc_rx_multicast; 471 uint64_t isc_rx_multicast_offset; 472 struct evcnt isc_rx_broadcast; 473 uint64_t isc_rx_broadcast_offset; 474 struct evcnt isc_rx_size_64; 475 uint64_t isc_rx_size_64_offset; 476 struct evcnt isc_rx_size_127; 477 uint64_t isc_rx_size_127_offset; 478 struct evcnt isc_rx_size_255; 479 uint64_t isc_rx_size_255_offset; 480 struct evcnt isc_rx_size_511; 481 uint64_t isc_rx_size_511_offset; 482 struct evcnt isc_rx_size_1023; 483 uint64_t isc_rx_size_1023_offset; 484 struct evcnt isc_rx_size_1522; 485 uint64_t isc_rx_size_1522_offset; 486 struct evcnt isc_rx_size_big; 487 uint64_t isc_rx_size_big_offset; 488 struct evcnt isc_rx_undersize; 489 uint64_t isc_rx_undersize_offset; 490 struct evcnt isc_rx_oversize; 491 uint64_t isc_rx_oversize_offset; 492 struct evcnt isc_rx_fragments; 493 uint64_t isc_rx_fragments_offset; 494 struct evcnt isc_rx_jabber; 495 uint64_t isc_rx_jabber_offset; 496 struct evcnt isc_tx_bytes; 497 uint64_t isc_tx_bytes_offset; 498 struct evcnt isc_tx_dropped_link_down; 499 uint64_t isc_tx_dropped_link_down_offset; 500 struct evcnt isc_tx_unicast; 501 uint64_t isc_tx_unicast_offset; 502 struct evcnt isc_tx_multicast; 503 uint64_t isc_tx_multicast_offset; 504 struct evcnt isc_tx_broadcast; 505 uint64_t isc_tx_broadcast_offset; 506 struct evcnt isc_tx_size_64; 507 uint64_t isc_tx_size_64_offset; 508 struct evcnt isc_tx_size_127; 509 uint64_t isc_tx_size_127_offset; 510 struct evcnt isc_tx_size_255; 511 uint64_t isc_tx_size_255_offset; 512 struct evcnt isc_tx_size_511; 513 uint64_t isc_tx_size_511_offset; 514 struct evcnt isc_tx_size_1023; 515 uint64_t isc_tx_size_1023_offset; 516 struct evcnt isc_tx_size_1522; 517 uint64_t isc_tx_size_1522_offset; 518 struct evcnt isc_tx_size_big; 519 uint64_t isc_tx_size_big_offset; 520 struct evcnt isc_mac_local_faults; 521 uint64_t isc_mac_local_faults_offset; 522 struct evcnt isc_mac_remote_faults; 523 uint64_t isc_mac_remote_faults_offset; 524 struct evcnt isc_link_xon_rx; 525 uint64_t isc_link_xon_rx_offset; 526 struct evcnt isc_link_xon_tx; 527 uint64_t isc_link_xon_tx_offset; 528 struct evcnt isc_link_xoff_rx; 529 uint64_t isc_link_xoff_rx_offset; 530 struct evcnt isc_link_xoff_tx; 531 uint64_t isc_link_xoff_tx_offset; 532 struct evcnt isc_vsi_rx_discards; 533 uint64_t isc_vsi_rx_discards_offset; 534 struct evcnt isc_vsi_rx_bytes; 535 uint64_t isc_vsi_rx_bytes_offset; 536 struct evcnt isc_vsi_rx_unicast; 537 uint64_t isc_vsi_rx_unicast_offset; 538 struct evcnt isc_vsi_rx_multicast; 539 uint64_t isc_vsi_rx_multicast_offset; 540 struct evcnt isc_vsi_rx_broadcast; 541 uint64_t isc_vsi_rx_broadcast_offset; 542 struct evcnt isc_vsi_tx_errors; 543 uint64_t isc_vsi_tx_errors_offset; 544 struct evcnt isc_vsi_tx_bytes; 545 uint64_t isc_vsi_tx_bytes_offset; 546 struct evcnt isc_vsi_tx_unicast; 547 uint64_t isc_vsi_tx_unicast_offset; 548 struct evcnt isc_vsi_tx_multicast; 549 uint64_t isc_vsi_tx_multicast_offset; 550 struct evcnt isc_vsi_tx_broadcast; 551 uint64_t isc_vsi_tx_broadcast_offset; 552 }; 553 554 /* 555 * Locking notes: 556 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 557 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 558 * - more than one lock of them cannot be held at once. 559 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 560 * (a spin mutex). 561 * - the lock cannot held with txr_lock or rxr_lock. 562 * + a field named sc_arq_* is not protected by any lock. 563 * - operations for sc_arq_* is done in one context related to 564 * sc_arq_task. 565 * + other fields in ixl_softc is protected by sc_cfg_lock 566 * (an adaptive mutex) 567 * - It must be held before another lock is held, and It can be 568 * released after the other lock is released. 569 * */ 570 571 struct ixl_softc { 572 device_t sc_dev; 573 struct ethercom sc_ec; 574 bool sc_attached; 575 bool sc_dead; 576 uint32_t sc_port; 577 struct sysctllog *sc_sysctllog; 578 struct workqueue *sc_workq; 579 struct workqueue *sc_workq_txrx; 580 int sc_stats_intval; 581 callout_t sc_stats_callout; 582 struct ixl_work sc_stats_task; 583 struct ixl_stats_counters 584 sc_stats_counters; 585 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 586 struct ifmedia sc_media; 587 uint64_t sc_media_status; 588 uint64_t sc_media_active; 589 uint64_t sc_phy_types; 590 uint8_t sc_phy_abilities; 591 uint8_t sc_phy_linkspeed; 592 uint8_t sc_phy_fec_cfg; 593 uint16_t sc_eee_cap; 594 uint32_t sc_eeer_val; 595 uint8_t sc_d3_lpan; 596 kmutex_t sc_cfg_lock; 597 enum i40e_mac_type sc_mac_type; 598 uint32_t sc_rss_table_size; 599 uint32_t sc_rss_table_entry_width; 600 bool sc_txrx_workqueue; 601 u_int sc_tx_process_limit; 602 u_int sc_rx_process_limit; 603 u_int sc_tx_intr_process_limit; 604 u_int sc_rx_intr_process_limit; 605 606 int sc_cur_ec_capenable; 607 608 struct pci_attach_args sc_pa; 609 pci_intr_handle_t *sc_ihp; 610 void **sc_ihs; 611 unsigned int sc_nintrs; 612 613 bus_dma_tag_t sc_dmat; 614 bus_space_tag_t sc_memt; 615 bus_space_handle_t sc_memh; 616 bus_size_t sc_mems; 617 618 uint8_t sc_pf_id; 619 uint16_t sc_uplink_seid; /* le */ 620 uint16_t sc_downlink_seid; /* le */ 621 uint16_t sc_vsi_number; 622 uint16_t sc_vsi_stat_counter_idx; 623 uint16_t sc_seid; 624 unsigned int sc_base_queue; 625 626 pci_intr_type_t sc_intrtype; 627 unsigned int sc_msix_vector_queue; 628 629 struct ixl_dmamem sc_scratch; 630 struct ixl_dmamem sc_aqbuf; 631 632 const struct ixl_aq_regs * 633 sc_aq_regs; 634 uint32_t sc_aq_flags; 635 #define IXL_SC_AQ_FLAG_RXCTL __BIT(0) 636 #define IXL_SC_AQ_FLAG_NVMLOCK __BIT(1) 637 #define IXL_SC_AQ_FLAG_NVMREAD __BIT(2) 638 #define IXL_SC_AQ_FLAG_RSS __BIT(3) 639 640 kmutex_t sc_atq_lock; 641 kcondvar_t sc_atq_cv; 642 struct ixl_dmamem sc_atq; 643 unsigned int sc_atq_prod; 644 unsigned int sc_atq_cons; 645 646 struct ixl_dmamem sc_arq; 647 struct ixl_work sc_arq_task; 648 struct ixl_aq_bufs sc_arq_idle; 649 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 650 unsigned int sc_arq_prod; 651 unsigned int sc_arq_cons; 652 653 struct ixl_work sc_link_state_task; 654 struct ixl_atq sc_link_state_atq; 655 656 struct ixl_dmamem sc_hmc_sd; 657 struct ixl_dmamem sc_hmc_pd; 658 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 659 660 struct if_percpuq *sc_ipq; 661 unsigned int sc_tx_ring_ndescs; 662 unsigned int sc_rx_ring_ndescs; 663 unsigned int sc_nqueue_pairs; 664 unsigned int sc_nqueue_pairs_max; 665 unsigned int sc_nqueue_pairs_device; 666 struct ixl_queue_pair *sc_qps; 667 uint32_t sc_itr_rx; 668 uint32_t sc_itr_tx; 669 670 struct evcnt sc_event_atq; 671 struct evcnt sc_event_link; 672 struct evcnt sc_event_ecc_err; 673 struct evcnt sc_event_pci_exception; 674 struct evcnt sc_event_crit_err; 675 }; 676 677 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 678 #define IXL_TX_PROCESS_LIMIT 256 679 #define IXL_RX_PROCESS_LIMIT 256 680 #define IXL_TX_INTR_PROCESS_LIMIT 256 681 #define IXL_RX_INTR_PROCESS_LIMIT 0U 682 683 #define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx | \ 684 IFCAP_CSUM_TCPv4_Rx | \ 685 IFCAP_CSUM_UDPv4_Rx | \ 686 IFCAP_CSUM_TCPv6_Rx | \ 687 IFCAP_CSUM_UDPv6_Rx) 688 #define IXL_IFCAP_TXCSUM (IFCAP_CSUM_IPv4_Tx | \ 689 IFCAP_CSUM_TCPv4_Tx | \ 690 IFCAP_CSUM_UDPv4_Tx | \ 691 IFCAP_CSUM_TCPv6_Tx | \ 692 IFCAP_CSUM_UDPv6_Tx) 693 #define IXL_CSUM_ALL_OFFLOAD (M_CSUM_IPv4 | \ 694 M_CSUM_TCPv4 | M_CSUM_TCPv6 | \ 695 M_CSUM_UDPv4 | M_CSUM_UDPv6) 696 697 #define delaymsec(_x) DELAY(1000 * (_x)) 698 #ifdef IXL_DEBUG 699 #define DDPRINTF(sc, fmt, args...) \ 700 do { \ 701 if ((sc) != NULL) { \ 702 device_printf( \ 703 ((struct ixl_softc *)(sc))->sc_dev, \ 704 ""); \ 705 } \ 706 printf("%s:\t" fmt, __func__, ##args); \ 707 } while (0) 708 #else 709 #define DDPRINTF(sc, fmt, args...) __nothing 710 #endif 711 #ifndef IXL_STATS_INTERVAL_MSEC 712 #define IXL_STATS_INTERVAL_MSEC 10000 713 #endif 714 #ifndef IXL_QUEUE_NUM 715 #define IXL_QUEUE_NUM 0 716 #endif 717 718 static bool ixl_param_nomsix = false; 719 static int ixl_param_stats_interval = IXL_STATS_INTERVAL_MSEC; 720 static int ixl_param_nqps_limit = IXL_QUEUE_NUM; 721 static unsigned int ixl_param_tx_ndescs = 512; 722 static unsigned int ixl_param_rx_ndescs = 512; 723 724 static enum i40e_mac_type 725 ixl_mactype(pci_product_id_t); 726 static void ixl_pci_csr_setup(pci_chipset_tag_t, pcitag_t); 727 static void ixl_clear_hw(struct ixl_softc *); 728 static int ixl_pf_reset(struct ixl_softc *); 729 730 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 731 bus_size_t, bus_size_t); 732 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 733 734 static int ixl_arq_fill(struct ixl_softc *); 735 static void ixl_arq_unfill(struct ixl_softc *); 736 737 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 738 unsigned int); 739 static void ixl_atq_set(struct ixl_atq *, 740 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 741 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 742 static void ixl_atq_done(struct ixl_softc *); 743 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 744 static int ixl_atq_exec_locked(struct ixl_softc *, struct ixl_atq *); 745 static int ixl_get_version(struct ixl_softc *); 746 static int ixl_get_nvm_version(struct ixl_softc *); 747 static int ixl_get_hw_capabilities(struct ixl_softc *); 748 static int ixl_pxe_clear(struct ixl_softc *); 749 static int ixl_lldp_shut(struct ixl_softc *); 750 static int ixl_get_mac(struct ixl_softc *); 751 static int ixl_get_switch_config(struct ixl_softc *); 752 static int ixl_phy_mask_ints(struct ixl_softc *); 753 static int ixl_get_phy_info(struct ixl_softc *); 754 static int ixl_set_phy_config(struct ixl_softc *, uint8_t, uint8_t, bool); 755 static int ixl_set_phy_autoselect(struct ixl_softc *); 756 static int ixl_restart_an(struct ixl_softc *); 757 static int ixl_hmc(struct ixl_softc *); 758 static void ixl_hmc_free(struct ixl_softc *); 759 static int ixl_get_vsi(struct ixl_softc *); 760 static int ixl_set_vsi(struct ixl_softc *); 761 static void ixl_set_filter_control(struct ixl_softc *); 762 static void ixl_get_link_status(void *); 763 static int ixl_get_link_status_poll(struct ixl_softc *, int *); 764 static void ixl_get_link_status_done(struct ixl_softc *, 765 const struct ixl_aq_desc *); 766 static int ixl_set_link_status_locked(struct ixl_softc *, 767 const struct ixl_aq_desc *); 768 static uint64_t ixl_search_link_speed(uint8_t); 769 static uint8_t ixl_search_baudrate(uint64_t); 770 static void ixl_config_rss(struct ixl_softc *); 771 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 772 uint16_t, uint16_t); 773 static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *, 774 uint16_t, uint16_t); 775 static void ixl_arq(void *); 776 static void ixl_hmc_pack(void *, const void *, 777 const struct ixl_hmc_pack *, unsigned int); 778 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 779 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 780 static int ixl_rd16_nvm(struct ixl_softc *, uint16_t, uint16_t *); 781 782 static int ixl_match(device_t, cfdata_t, void *); 783 static void ixl_attach(device_t, device_t, void *); 784 static int ixl_detach(device_t, int); 785 786 static void ixl_media_add(struct ixl_softc *); 787 static int ixl_media_change(struct ifnet *); 788 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 789 static int ixl_ioctl(struct ifnet *, u_long, void *); 790 static void ixl_start(struct ifnet *); 791 static int ixl_transmit(struct ifnet *, struct mbuf *); 792 static void ixl_deferred_transmit(void *); 793 static int ixl_intr(void *); 794 static int ixl_queue_intr(void *); 795 static int ixl_other_intr(void *); 796 static void ixl_handle_queue(void *); 797 static void ixl_handle_queue_wk(struct work *, void *); 798 static void ixl_sched_handle_queue(struct ixl_softc *, 799 struct ixl_queue_pair *); 800 static int ixl_init(struct ifnet *); 801 static int ixl_init_locked(struct ixl_softc *); 802 static void ixl_stop(struct ifnet *, int); 803 static void ixl_stop_locked(struct ixl_softc *); 804 static int ixl_iff(struct ixl_softc *); 805 static int ixl_ifflags_cb(struct ethercom *); 806 static int ixl_setup_interrupts(struct ixl_softc *); 807 static int ixl_establish_intx(struct ixl_softc *); 808 static int ixl_establish_msix(struct ixl_softc *); 809 static void ixl_enable_queue_intr(struct ixl_softc *, 810 struct ixl_queue_pair *); 811 static void ixl_disable_queue_intr(struct ixl_softc *, 812 struct ixl_queue_pair *); 813 static void ixl_enable_other_intr(struct ixl_softc *); 814 static void ixl_disable_other_intr(struct ixl_softc *); 815 static void ixl_config_queue_intr(struct ixl_softc *); 816 static void ixl_config_other_intr(struct ixl_softc *); 817 818 static struct ixl_tx_ring * 819 ixl_txr_alloc(struct ixl_softc *, unsigned int); 820 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 821 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 822 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 823 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 824 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 825 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 826 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 827 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 828 829 static struct ixl_rx_ring * 830 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 831 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 832 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 833 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 834 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 835 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 836 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 837 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 838 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 839 840 static struct workqueue * 841 ixl_workq_create(const char *, pri_t, int, int); 842 static void ixl_workq_destroy(struct workqueue *); 843 static int ixl_workqs_teardown(device_t); 844 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 845 static void ixl_work_add(struct workqueue *, struct ixl_work *); 846 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 847 static void ixl_workq_work(struct work *, void *); 848 static const struct ixl_product * 849 ixl_lookup(const struct pci_attach_args *pa); 850 static void ixl_link_state_update(struct ixl_softc *, 851 const struct ixl_aq_desc *); 852 static int ixl_vlan_cb(struct ethercom *, uint16_t, bool); 853 static int ixl_setup_vlan_hwfilter(struct ixl_softc *); 854 static void ixl_teardown_vlan_hwfilter(struct ixl_softc *); 855 static int ixl_update_macvlan(struct ixl_softc *); 856 static int ixl_setup_interrupts(struct ixl_softc *); 857 static void ixl_teardown_interrupts(struct ixl_softc *); 858 static int ixl_setup_stats(struct ixl_softc *); 859 static void ixl_teardown_stats(struct ixl_softc *); 860 static void ixl_stats_callout(void *); 861 static void ixl_stats_update(void *); 862 static int ixl_setup_sysctls(struct ixl_softc *); 863 static void ixl_teardown_sysctls(struct ixl_softc *); 864 static int ixl_sysctl_itr_handler(SYSCTLFN_PROTO); 865 static int ixl_queue_pairs_alloc(struct ixl_softc *); 866 static void ixl_queue_pairs_free(struct ixl_softc *); 867 868 static const struct ixl_phy_type ixl_phy_type_map[] = { 869 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 870 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 871 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 872 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 873 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 874 { 1ULL << IXL_PHY_TYPE_XAUI | 875 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 876 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 877 { 1ULL << IXL_PHY_TYPE_XLAUI | 878 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 879 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 880 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 881 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 882 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 883 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 884 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 885 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 886 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 887 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 888 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 889 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 890 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 891 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 892 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 893 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 894 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 895 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 896 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 897 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 898 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 899 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 900 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 901 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 902 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_ACC }, 903 { 1ULL << IXL_PHY_TYPE_2500BASE_T_1, IFM_2500_T }, 904 { 1ULL << IXL_PHY_TYPE_5000BASE_T_1, IFM_5000_T }, 905 { 1ULL << IXL_PHY_TYPE_2500BASE_T_2, IFM_2500_T }, 906 { 1ULL << IXL_PHY_TYPE_5000BASE_T_2, IFM_5000_T }, 907 }; 908 909 static const struct ixl_speed_type ixl_speed_type_map[] = { 910 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 911 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 912 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 913 { IXL_AQ_LINK_SPEED_5000MB, IF_Mbps(5000) }, 914 { IXL_AQ_LINK_SPEED_2500MB, IF_Mbps(2500) }, 915 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 916 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 917 }; 918 919 static const struct ixl_aq_regs ixl_pf_aq_regs = { 920 .atq_tail = I40E_PF_ATQT, 921 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 922 .atq_head = I40E_PF_ATQH, 923 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 924 .atq_len = I40E_PF_ATQLEN, 925 .atq_bal = I40E_PF_ATQBAL, 926 .atq_bah = I40E_PF_ATQBAH, 927 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 928 929 .arq_tail = I40E_PF_ARQT, 930 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 931 .arq_head = I40E_PF_ARQH, 932 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 933 .arq_len = I40E_PF_ARQLEN, 934 .arq_bal = I40E_PF_ARQBAL, 935 .arq_bah = I40E_PF_ARQBAH, 936 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 937 }; 938 939 #define ixl_rd(_s, _r) \ 940 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 941 #define ixl_wr(_s, _r, _v) \ 942 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 943 #define ixl_barrier(_s, _r, _l, _o) \ 944 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 945 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 946 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 947 948 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 949 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 950 DVF_DETACH_SHUTDOWN); 951 952 static const struct ixl_product ixl_products[] = { 953 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 954 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 955 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 956 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 957 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 958 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 959 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_1 }, 960 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_2 }, 961 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 962 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 963 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 964 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 965 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 966 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 967 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 968 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 969 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 970 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 971 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 972 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_SFP }, 973 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_BP }, 974 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_V710_5G_T}, 975 /* required last entry */ 976 {0, 0} 977 }; 978 979 static const struct ixl_product * 980 ixl_lookup(const struct pci_attach_args *pa) 981 { 982 const struct ixl_product *ixlp; 983 984 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 985 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 986 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 987 return ixlp; 988 } 989 990 return NULL; 991 } 992 993 static void 994 ixl_intr_barrier(void) 995 { 996 997 /* wait for finish of all handler */ 998 xc_barrier(0); 999 } 1000 1001 static int 1002 ixl_match(device_t parent, cfdata_t match, void *aux) 1003 { 1004 const struct pci_attach_args *pa = aux; 1005 1006 return (ixl_lookup(pa) != NULL) ? 1 : 0; 1007 } 1008 1009 static void 1010 ixl_attach(device_t parent, device_t self, void *aux) 1011 { 1012 struct ixl_softc *sc; 1013 struct pci_attach_args *pa = aux; 1014 struct ifnet *ifp; 1015 pcireg_t memtype; 1016 uint32_t firstq, port, ari, func; 1017 char xnamebuf[32]; 1018 int tries, rv, link; 1019 1020 sc = device_private(self); 1021 sc->sc_dev = self; 1022 ifp = &sc->sc_ec.ec_if; 1023 1024 sc->sc_pa = *pa; 1025 sc->sc_dmat = (pci_dma64_available(pa)) ? 1026 pa->pa_dmat64 : pa->pa_dmat; 1027 sc->sc_aq_regs = &ixl_pf_aq_regs; 1028 1029 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(pa->pa_id)); 1030 1031 ixl_pci_csr_setup(pa->pa_pc, pa->pa_tag); 1032 1033 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 1034 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 1035 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 1036 aprint_error(": unable to map registers\n"); 1037 return; 1038 } 1039 1040 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 1041 1042 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 1043 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 1044 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 1045 sc->sc_base_queue = firstq; 1046 1047 ixl_clear_hw(sc); 1048 if (ixl_pf_reset(sc) == -1) { 1049 /* error printed by ixl pf_reset */ 1050 goto unmap; 1051 } 1052 1053 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 1054 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 1055 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 1056 sc->sc_port = port; 1057 aprint_normal(": port %u", sc->sc_port); 1058 1059 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 1060 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 1061 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 1062 1063 func = ixl_rd(sc, I40E_PF_FUNC_RID); 1064 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 1065 1066 /* initialise the adminq */ 1067 1068 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 1069 1070 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 1071 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1072 aprint_error("\n" "%s: unable to allocate atq\n", 1073 device_xname(self)); 1074 goto unmap; 1075 } 1076 1077 SIMPLEQ_INIT(&sc->sc_arq_idle); 1078 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 1079 sc->sc_arq_cons = 0; 1080 sc->sc_arq_prod = 0; 1081 1082 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 1083 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1084 aprint_error("\n" "%s: unable to allocate arq\n", 1085 device_xname(self)); 1086 goto free_atq; 1087 } 1088 1089 if (!ixl_arq_fill(sc)) { 1090 aprint_error("\n" "%s: unable to fill arq descriptors\n", 1091 device_xname(self)); 1092 goto free_arq; 1093 } 1094 1095 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1096 0, IXL_DMA_LEN(&sc->sc_atq), 1097 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1098 1099 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1100 0, IXL_DMA_LEN(&sc->sc_arq), 1101 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1102 1103 for (tries = 0; tries < 10; tries++) { 1104 sc->sc_atq_cons = 0; 1105 sc->sc_atq_prod = 0; 1106 1107 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1108 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1109 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1110 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1111 1112 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1113 1114 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1115 ixl_dmamem_lo(&sc->sc_atq)); 1116 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1117 ixl_dmamem_hi(&sc->sc_atq)); 1118 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1119 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1120 1121 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1122 ixl_dmamem_lo(&sc->sc_arq)); 1123 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1124 ixl_dmamem_hi(&sc->sc_arq)); 1125 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1126 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1127 1128 rv = ixl_get_version(sc); 1129 if (rv == 0) 1130 break; 1131 if (rv != ETIMEDOUT) { 1132 aprint_error(", unable to get firmware version\n"); 1133 goto shutdown; 1134 } 1135 1136 delaymsec(100); 1137 } 1138 1139 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1140 1141 if (ixl_dmamem_alloc(sc, &sc->sc_aqbuf, IXL_AQ_BUFLEN, 0) != 0) { 1142 aprint_error_dev(self, ", unable to allocate nvm buffer\n"); 1143 goto shutdown; 1144 } 1145 1146 ixl_get_nvm_version(sc); 1147 1148 if (sc->sc_mac_type == I40E_MAC_X722) 1149 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_X722; 1150 else 1151 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_XL710; 1152 1153 rv = ixl_get_hw_capabilities(sc); 1154 if (rv != 0) { 1155 aprint_error(", GET HW CAPABILITIES %s\n", 1156 rv == ETIMEDOUT ? "timeout" : "error"); 1157 goto free_aqbuf; 1158 } 1159 1160 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1161 if (ixl_param_nqps_limit > 0) { 1162 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_max, 1163 ixl_param_nqps_limit); 1164 } 1165 1166 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1167 sc->sc_tx_ring_ndescs = ixl_param_tx_ndescs; 1168 sc->sc_rx_ring_ndescs = ixl_param_rx_ndescs; 1169 1170 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1171 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1172 KASSERT(sc->sc_rx_ring_ndescs == 1173 (1U << (fls32(sc->sc_rx_ring_ndescs) - 1))); 1174 KASSERT(sc->sc_tx_ring_ndescs == 1175 (1U << (fls32(sc->sc_tx_ring_ndescs) - 1))); 1176 1177 if (ixl_get_mac(sc) != 0) { 1178 /* error printed by ixl_get_mac */ 1179 goto free_aqbuf; 1180 } 1181 1182 aprint_normal("\n"); 1183 aprint_naive("\n"); 1184 1185 aprint_normal_dev(self, "Ethernet address %s\n", 1186 ether_sprintf(sc->sc_enaddr)); 1187 1188 rv = ixl_pxe_clear(sc); 1189 if (rv != 0) { 1190 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1191 rv == ETIMEDOUT ? "timeout" : "error"); 1192 } 1193 1194 ixl_set_filter_control(sc); 1195 1196 if (ixl_hmc(sc) != 0) { 1197 /* error printed by ixl_hmc */ 1198 goto free_aqbuf; 1199 } 1200 1201 if (ixl_lldp_shut(sc) != 0) { 1202 /* error printed by ixl_lldp_shut */ 1203 goto free_hmc; 1204 } 1205 1206 if (ixl_phy_mask_ints(sc) != 0) { 1207 /* error printed by ixl_phy_mask_ints */ 1208 goto free_hmc; 1209 } 1210 1211 if (ixl_restart_an(sc) != 0) { 1212 /* error printed by ixl_restart_an */ 1213 goto free_hmc; 1214 } 1215 1216 if (ixl_get_switch_config(sc) != 0) { 1217 /* error printed by ixl_get_switch_config */ 1218 goto free_hmc; 1219 } 1220 1221 rv = ixl_get_link_status_poll(sc, NULL); 1222 if (rv != 0) { 1223 aprint_error_dev(self, "GET LINK STATUS %s\n", 1224 rv == ETIMEDOUT ? "timeout" : "error"); 1225 goto free_hmc; 1226 } 1227 1228 /* 1229 * The FW often returns EIO in "Get PHY Abilities" command 1230 * if there is no delay 1231 */ 1232 DELAY(500); 1233 if (ixl_get_phy_info(sc) != 0) { 1234 /* error printed by ixl_get_phy_info */ 1235 goto free_hmc; 1236 } 1237 1238 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1239 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1240 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1241 goto free_hmc; 1242 } 1243 1244 rv = ixl_get_vsi(sc); 1245 if (rv != 0) { 1246 aprint_error_dev(self, "GET VSI %s %d\n", 1247 rv == ETIMEDOUT ? "timeout" : "error", rv); 1248 goto free_scratch; 1249 } 1250 1251 rv = ixl_set_vsi(sc); 1252 if (rv != 0) { 1253 aprint_error_dev(self, "UPDATE VSI error %s %d\n", 1254 rv == ETIMEDOUT ? "timeout" : "error", rv); 1255 goto free_scratch; 1256 } 1257 1258 if (ixl_queue_pairs_alloc(sc) != 0) { 1259 /* error printed by ixl_queue_pairs_alloc */ 1260 goto free_scratch; 1261 } 1262 1263 if (ixl_setup_interrupts(sc) != 0) { 1264 /* error printed by ixl_setup_interrupts */ 1265 goto free_queue_pairs; 1266 } 1267 1268 if (ixl_setup_stats(sc) != 0) { 1269 aprint_error_dev(self, "failed to setup event counters\n"); 1270 goto teardown_intrs; 1271 } 1272 1273 if (ixl_setup_sysctls(sc) != 0) { 1274 /* error printed by ixl_setup_sysctls */ 1275 goto teardown_stats; 1276 } 1277 1278 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1279 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1280 IPL_NET, WQ_MPSAFE); 1281 if (sc->sc_workq == NULL) 1282 goto teardown_sysctls; 1283 1284 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1285 rv = workqueue_create(&sc->sc_workq_txrx, xnamebuf, ixl_handle_queue_wk, 1286 sc, IXL_WORKQUEUE_PRI, IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1287 if (rv != 0) { 1288 sc->sc_workq_txrx = NULL; 1289 goto teardown_wqs; 1290 } 1291 1292 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1293 cv_init(&sc->sc_atq_cv, xnamebuf); 1294 1295 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1296 1297 ifp->if_softc = sc; 1298 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1299 ifp->if_extflags = IFEF_MPSAFE; 1300 ifp->if_ioctl = ixl_ioctl; 1301 ifp->if_start = ixl_start; 1302 ifp->if_transmit = ixl_transmit; 1303 ifp->if_init = ixl_init; 1304 ifp->if_stop = ixl_stop; 1305 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1306 IFQ_SET_READY(&ifp->if_snd); 1307 ifp->if_capabilities |= IXL_IFCAP_RXCSUM; 1308 ifp->if_capabilities |= IXL_IFCAP_TXCSUM; 1309 #if 0 1310 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1311 #endif 1312 ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb); 1313 sc->sc_ec.ec_capabilities |= ETHERCAP_JUMBO_MTU; 1314 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1315 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1316 1317 sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities; 1318 /* Disable VLAN_HWFILTER by default */ 1319 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1320 1321 sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable; 1322 1323 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1324 ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, ixl_media_change, 1325 ixl_media_status, &sc->sc_cfg_lock); 1326 1327 ixl_media_add(sc); 1328 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1329 if (ISSET(sc->sc_phy_abilities, 1330 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1331 ifmedia_add(&sc->sc_media, 1332 IFM_ETHER | IFM_AUTO | IFM_FLOW, 0, NULL); 1333 } 1334 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_NONE, 0, NULL); 1335 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1336 1337 if_initialize(ifp); 1338 1339 sc->sc_ipq = if_percpuq_create(ifp); 1340 if_deferred_start_init(ifp, NULL); 1341 ether_ifattach(ifp, sc->sc_enaddr); 1342 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1343 1344 rv = ixl_get_link_status_poll(sc, &link); 1345 if (rv != 0) 1346 link = LINK_STATE_UNKNOWN; 1347 if_link_state_change(ifp, link); 1348 1349 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 1350 ixl_work_set(&sc->sc_link_state_task, ixl_get_link_status, sc); 1351 1352 ixl_config_other_intr(sc); 1353 ixl_enable_other_intr(sc); 1354 1355 ixl_set_phy_autoselect(sc); 1356 1357 /* remove default mac filter and replace it so we can see vlans */ 1358 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 1359 if (rv != ENOENT) { 1360 aprint_debug_dev(self, 1361 "unable to remove macvlan %u\n", rv); 1362 } 1363 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1364 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1365 if (rv != ENOENT) { 1366 aprint_debug_dev(self, 1367 "unable to remove macvlan, ignore vlan %u\n", rv); 1368 } 1369 1370 if (ixl_update_macvlan(sc) != 0) { 1371 aprint_debug_dev(self, 1372 "couldn't enable vlan hardware filter\n"); 1373 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1374 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1375 } 1376 1377 sc->sc_txrx_workqueue = true; 1378 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1379 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1380 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1381 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1382 1383 ixl_stats_update(sc); 1384 sc->sc_stats_counters.isc_has_offset = true; 1385 1386 if (pmf_device_register(self, NULL, NULL) != true) 1387 aprint_debug_dev(self, "couldn't establish power handler\n"); 1388 sc->sc_itr_rx = IXL_ITR_RX; 1389 sc->sc_itr_tx = IXL_ITR_TX; 1390 sc->sc_attached = true; 1391 if_register(ifp); 1392 1393 return; 1394 1395 teardown_wqs: 1396 config_finalize_register(self, ixl_workqs_teardown); 1397 teardown_sysctls: 1398 ixl_teardown_sysctls(sc); 1399 teardown_stats: 1400 ixl_teardown_stats(sc); 1401 teardown_intrs: 1402 ixl_teardown_interrupts(sc); 1403 free_queue_pairs: 1404 ixl_queue_pairs_free(sc); 1405 free_scratch: 1406 ixl_dmamem_free(sc, &sc->sc_scratch); 1407 free_hmc: 1408 ixl_hmc_free(sc); 1409 free_aqbuf: 1410 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1411 shutdown: 1412 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1413 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1414 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1415 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1416 1417 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1418 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1419 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1420 1421 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1422 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1423 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1424 1425 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1426 0, IXL_DMA_LEN(&sc->sc_arq), 1427 BUS_DMASYNC_POSTREAD); 1428 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1429 0, IXL_DMA_LEN(&sc->sc_atq), 1430 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1431 1432 ixl_arq_unfill(sc); 1433 free_arq: 1434 ixl_dmamem_free(sc, &sc->sc_arq); 1435 free_atq: 1436 ixl_dmamem_free(sc, &sc->sc_atq); 1437 unmap: 1438 mutex_destroy(&sc->sc_atq_lock); 1439 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1440 mutex_destroy(&sc->sc_cfg_lock); 1441 sc->sc_mems = 0; 1442 1443 sc->sc_attached = false; 1444 } 1445 1446 static int 1447 ixl_detach(device_t self, int flags) 1448 { 1449 struct ixl_softc *sc = device_private(self); 1450 struct ifnet *ifp = &sc->sc_ec.ec_if; 1451 1452 if (!sc->sc_attached) 1453 return 0; 1454 1455 ixl_stop(ifp, 1); 1456 1457 callout_halt(&sc->sc_stats_callout, NULL); 1458 ixl_work_wait(sc->sc_workq, &sc->sc_stats_task); 1459 1460 /* detach the I/F before stop adminq due to callbacks */ 1461 ether_ifdetach(ifp); 1462 if_detach(ifp); 1463 ifmedia_fini(&sc->sc_media); 1464 if_percpuq_destroy(sc->sc_ipq); 1465 1466 ixl_disable_other_intr(sc); 1467 ixl_intr_barrier(); 1468 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1469 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1470 1471 if (sc->sc_workq != NULL) { 1472 ixl_workq_destroy(sc->sc_workq); 1473 sc->sc_workq = NULL; 1474 } 1475 1476 if (sc->sc_workq_txrx != NULL) { 1477 workqueue_destroy(sc->sc_workq_txrx); 1478 sc->sc_workq_txrx = NULL; 1479 } 1480 1481 ixl_teardown_interrupts(sc); 1482 ixl_teardown_stats(sc); 1483 ixl_teardown_sysctls(sc); 1484 1485 ixl_queue_pairs_free(sc); 1486 1487 ixl_dmamem_free(sc, &sc->sc_scratch); 1488 ixl_hmc_free(sc); 1489 1490 /* shutdown */ 1491 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1492 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1493 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1494 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1495 1496 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1497 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1498 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1499 1500 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1501 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1502 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1503 1504 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1505 0, IXL_DMA_LEN(&sc->sc_arq), 1506 BUS_DMASYNC_POSTREAD); 1507 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1508 0, IXL_DMA_LEN(&sc->sc_atq), 1509 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1510 1511 ixl_arq_unfill(sc); 1512 1513 ixl_dmamem_free(sc, &sc->sc_arq); 1514 ixl_dmamem_free(sc, &sc->sc_atq); 1515 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1516 1517 cv_destroy(&sc->sc_atq_cv); 1518 mutex_destroy(&sc->sc_atq_lock); 1519 1520 if (sc->sc_mems != 0) { 1521 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1522 sc->sc_mems = 0; 1523 } 1524 1525 mutex_destroy(&sc->sc_cfg_lock); 1526 1527 return 0; 1528 } 1529 1530 static int 1531 ixl_workqs_teardown(device_t self) 1532 { 1533 struct ixl_softc *sc = device_private(self); 1534 1535 if (sc->sc_workq != NULL) { 1536 ixl_workq_destroy(sc->sc_workq); 1537 sc->sc_workq = NULL; 1538 } 1539 1540 if (sc->sc_workq_txrx != NULL) { 1541 workqueue_destroy(sc->sc_workq_txrx); 1542 sc->sc_workq_txrx = NULL; 1543 } 1544 1545 return 0; 1546 } 1547 1548 static int 1549 ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 1550 { 1551 struct ifnet *ifp = &ec->ec_if; 1552 struct ixl_softc *sc = ifp->if_softc; 1553 int rv; 1554 1555 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1556 return 0; 1557 } 1558 1559 if (set) { 1560 rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid, 1561 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1562 if (rv == 0) { 1563 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 1564 vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1565 } 1566 } else { 1567 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid, 1568 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1569 (void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid, 1570 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1571 } 1572 1573 return rv; 1574 } 1575 1576 static void 1577 ixl_media_add(struct ixl_softc *sc) 1578 { 1579 struct ifmedia *ifm = &sc->sc_media; 1580 const struct ixl_phy_type *itype; 1581 unsigned int i; 1582 bool flow; 1583 1584 if (ISSET(sc->sc_phy_abilities, 1585 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1586 flow = true; 1587 } else { 1588 flow = false; 1589 } 1590 1591 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1592 itype = &ixl_phy_type_map[i]; 1593 1594 if (ISSET(sc->sc_phy_types, itype->phy_type)) { 1595 ifmedia_add(ifm, 1596 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1597 1598 if (flow) { 1599 ifmedia_add(ifm, 1600 IFM_ETHER | IFM_FDX | IFM_FLOW | 1601 itype->ifm_type, 0, NULL); 1602 } 1603 1604 if (itype->ifm_type != IFM_100_TX) 1605 continue; 1606 1607 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1608 0, NULL); 1609 if (flow) { 1610 ifmedia_add(ifm, 1611 IFM_ETHER | IFM_FLOW | itype->ifm_type, 1612 0, NULL); 1613 } 1614 } 1615 } 1616 } 1617 1618 static void 1619 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1620 { 1621 struct ixl_softc *sc = ifp->if_softc; 1622 1623 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1624 1625 ifmr->ifm_status = sc->sc_media_status; 1626 ifmr->ifm_active = sc->sc_media_active; 1627 } 1628 1629 static int 1630 ixl_media_change(struct ifnet *ifp) 1631 { 1632 struct ixl_softc *sc = ifp->if_softc; 1633 struct ifmedia *ifm = &sc->sc_media; 1634 uint64_t ifm_active = sc->sc_media_active; 1635 uint8_t link_speed, abilities; 1636 1637 switch (IFM_SUBTYPE(ifm_active)) { 1638 case IFM_1000_SGMII: 1639 case IFM_1000_KX: 1640 case IFM_10G_KX4: 1641 case IFM_10G_KR: 1642 case IFM_40G_KR4: 1643 case IFM_20G_KR2: 1644 case IFM_25G_KR: 1645 /* backplanes */ 1646 return EINVAL; 1647 } 1648 1649 abilities = IXL_PHY_ABILITY_AUTONEGO | IXL_PHY_ABILITY_LINKUP; 1650 1651 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1652 case IFM_AUTO: 1653 link_speed = sc->sc_phy_linkspeed; 1654 break; 1655 case IFM_NONE: 1656 link_speed = 0; 1657 CLR(abilities, IXL_PHY_ABILITY_LINKUP); 1658 break; 1659 default: 1660 link_speed = ixl_search_baudrate( 1661 ifmedia_baudrate(ifm->ifm_media)); 1662 } 1663 1664 if (ISSET(abilities, IXL_PHY_ABILITY_LINKUP)) { 1665 if (ISSET(link_speed, sc->sc_phy_linkspeed) == 0) 1666 return EINVAL; 1667 } 1668 1669 if (ifm->ifm_media & IFM_FLOW) { 1670 abilities |= sc->sc_phy_abilities & 1671 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX); 1672 } 1673 1674 return ixl_set_phy_config(sc, link_speed, abilities, false); 1675 } 1676 1677 1678 static void 1679 ixl_del_all_multiaddr(struct ixl_softc *sc) 1680 { 1681 struct ethercom *ec = &sc->sc_ec; 1682 struct ether_multi *enm; 1683 struct ether_multistep step; 1684 1685 ETHER_LOCK(ec); 1686 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1687 ETHER_NEXT_MULTI(step, enm)) { 1688 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1689 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1690 } 1691 ETHER_UNLOCK(ec); 1692 } 1693 1694 static int 1695 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1696 { 1697 struct ifnet *ifp = &sc->sc_ec.ec_if; 1698 int rv; 1699 1700 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1701 return 0; 1702 1703 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1704 ixl_del_all_multiaddr(sc); 1705 SET(ifp->if_flags, IFF_ALLMULTI); 1706 return ENETRESET; 1707 } 1708 1709 /* multicast address can not use VLAN HWFILTER */ 1710 rv = ixl_add_macvlan(sc, addrlo, 0, 1711 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1712 1713 if (rv == ENOSPC) { 1714 ixl_del_all_multiaddr(sc); 1715 SET(ifp->if_flags, IFF_ALLMULTI); 1716 return ENETRESET; 1717 } 1718 1719 return rv; 1720 } 1721 1722 static int 1723 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1724 { 1725 struct ifnet *ifp = &sc->sc_ec.ec_if; 1726 struct ethercom *ec = &sc->sc_ec; 1727 struct ether_multi *enm, *enm_last; 1728 struct ether_multistep step; 1729 int error, rv = 0; 1730 1731 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1732 ixl_remove_macvlan(sc, addrlo, 0, 1733 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1734 return 0; 1735 } 1736 1737 ETHER_LOCK(ec); 1738 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1739 ETHER_NEXT_MULTI(step, enm)) { 1740 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1741 ETHER_ADDR_LEN) != 0) { 1742 goto out; 1743 } 1744 } 1745 1746 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1747 ETHER_NEXT_MULTI(step, enm)) { 1748 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1749 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1750 if (error != 0) 1751 break; 1752 } 1753 1754 if (enm != NULL) { 1755 enm_last = enm; 1756 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1757 ETHER_NEXT_MULTI(step, enm)) { 1758 if (enm == enm_last) 1759 break; 1760 1761 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1762 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1763 } 1764 } else { 1765 CLR(ifp->if_flags, IFF_ALLMULTI); 1766 rv = ENETRESET; 1767 } 1768 1769 out: 1770 ETHER_UNLOCK(ec); 1771 return rv; 1772 } 1773 1774 static int 1775 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1776 { 1777 struct ifreq *ifr = (struct ifreq *)data; 1778 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1779 const struct sockaddr *sa; 1780 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1781 int s, error = 0; 1782 unsigned int nmtu; 1783 1784 switch (cmd) { 1785 case SIOCSIFMTU: 1786 nmtu = ifr->ifr_mtu; 1787 1788 if (nmtu < IXL_MIN_MTU || nmtu > IXL_MAX_MTU) { 1789 error = EINVAL; 1790 break; 1791 } 1792 if (ifp->if_mtu != nmtu) { 1793 s = splnet(); 1794 error = ether_ioctl(ifp, cmd, data); 1795 splx(s); 1796 if (error == ENETRESET) 1797 error = ixl_init(ifp); 1798 } 1799 break; 1800 case SIOCADDMULTI: 1801 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1802 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1803 error = ether_multiaddr(sa, addrlo, addrhi); 1804 if (error != 0) 1805 return error; 1806 1807 error = ixl_add_multi(sc, addrlo, addrhi); 1808 if (error != 0 && error != ENETRESET) { 1809 ether_delmulti(sa, &sc->sc_ec); 1810 error = EIO; 1811 } 1812 } 1813 break; 1814 1815 case SIOCDELMULTI: 1816 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1817 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1818 error = ether_multiaddr(sa, addrlo, addrhi); 1819 if (error != 0) 1820 return error; 1821 1822 error = ixl_del_multi(sc, addrlo, addrhi); 1823 } 1824 break; 1825 1826 default: 1827 s = splnet(); 1828 error = ether_ioctl(ifp, cmd, data); 1829 splx(s); 1830 } 1831 1832 if (error == ENETRESET) 1833 error = ixl_iff(sc); 1834 1835 return error; 1836 } 1837 1838 static enum i40e_mac_type 1839 ixl_mactype(pci_product_id_t id) 1840 { 1841 1842 switch (id) { 1843 case PCI_PRODUCT_INTEL_XL710_SFP: 1844 case PCI_PRODUCT_INTEL_XL710_KX_B: 1845 case PCI_PRODUCT_INTEL_XL710_KX_C: 1846 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1847 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1848 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1849 case PCI_PRODUCT_INTEL_X710_10G_T_1: 1850 case PCI_PRODUCT_INTEL_X710_10G_T_2: 1851 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1852 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1853 case PCI_PRODUCT_INTEL_X710_T4_10G: 1854 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1855 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1856 case PCI_PRODUCT_INTEL_X710_10G_SFP: 1857 case PCI_PRODUCT_INTEL_X710_10G_BP: 1858 return I40E_MAC_XL710; 1859 1860 case PCI_PRODUCT_INTEL_X722_KX: 1861 case PCI_PRODUCT_INTEL_X722_QSFP: 1862 case PCI_PRODUCT_INTEL_X722_SFP: 1863 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1864 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1865 case PCI_PRODUCT_INTEL_X722_I_SFP: 1866 return I40E_MAC_X722; 1867 } 1868 1869 return I40E_MAC_GENERIC; 1870 } 1871 1872 static void 1873 ixl_pci_csr_setup(pci_chipset_tag_t pc, pcitag_t tag) 1874 { 1875 pcireg_t csr; 1876 1877 csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 1878 csr |= (PCI_COMMAND_MASTER_ENABLE | 1879 PCI_COMMAND_MEM_ENABLE); 1880 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr); 1881 } 1882 1883 static inline void * 1884 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1885 { 1886 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1887 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1888 1889 if (i >= e->hmc_count) 1890 return NULL; 1891 1892 kva += e->hmc_base; 1893 kva += i * e->hmc_size; 1894 1895 return kva; 1896 } 1897 1898 static inline size_t 1899 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1900 { 1901 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1902 1903 return e->hmc_size; 1904 } 1905 1906 static void 1907 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1908 { 1909 struct ixl_rx_ring *rxr = qp->qp_rxr; 1910 1911 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1912 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1913 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1914 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1915 ixl_flush(sc); 1916 } 1917 1918 static void 1919 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1920 { 1921 struct ixl_rx_ring *rxr = qp->qp_rxr; 1922 1923 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1924 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1925 ixl_flush(sc); 1926 } 1927 1928 static void 1929 ixl_enable_other_intr(struct ixl_softc *sc) 1930 { 1931 1932 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1933 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1934 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1935 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1936 ixl_flush(sc); 1937 } 1938 1939 static void 1940 ixl_disable_other_intr(struct ixl_softc *sc) 1941 { 1942 1943 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1944 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1945 ixl_flush(sc); 1946 } 1947 1948 static int 1949 ixl_reinit(struct ixl_softc *sc) 1950 { 1951 struct ixl_rx_ring *rxr; 1952 struct ixl_tx_ring *txr; 1953 unsigned int i; 1954 uint32_t reg; 1955 1956 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1957 1958 if (ixl_get_vsi(sc) != 0) 1959 return EIO; 1960 1961 if (ixl_set_vsi(sc) != 0) 1962 return EIO; 1963 1964 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1965 txr = sc->sc_qps[i].qp_txr; 1966 rxr = sc->sc_qps[i].qp_rxr; 1967 1968 ixl_txr_config(sc, txr); 1969 ixl_rxr_config(sc, rxr); 1970 } 1971 1972 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1973 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 1974 1975 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1976 txr = sc->sc_qps[i].qp_txr; 1977 rxr = sc->sc_qps[i].qp_rxr; 1978 1979 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 1980 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 1981 ixl_flush(sc); 1982 1983 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 1984 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 1985 1986 /* ixl_rxfill() needs lock held */ 1987 mutex_enter(&rxr->rxr_lock); 1988 ixl_rxfill(sc, rxr); 1989 mutex_exit(&rxr->rxr_lock); 1990 1991 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 1992 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 1993 ixl_wr(sc, I40E_QRX_ENA(i), reg); 1994 if (ixl_rxr_enabled(sc, rxr) != 0) 1995 goto stop; 1996 1997 ixl_txr_qdis(sc, txr, 1); 1998 1999 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2000 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2001 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2002 2003 if (ixl_txr_enabled(sc, txr) != 0) 2004 goto stop; 2005 } 2006 2007 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2008 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2009 2010 return 0; 2011 2012 stop: 2013 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2014 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2015 2016 return ETIMEDOUT; 2017 } 2018 2019 static int 2020 ixl_init_locked(struct ixl_softc *sc) 2021 { 2022 struct ifnet *ifp = &sc->sc_ec.ec_if; 2023 unsigned int i; 2024 int error, eccap_change; 2025 2026 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2027 2028 if (ISSET(ifp->if_flags, IFF_RUNNING)) 2029 ixl_stop_locked(sc); 2030 2031 if (sc->sc_dead) { 2032 return ENXIO; 2033 } 2034 2035 eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable; 2036 if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING)) 2037 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 2038 2039 if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) { 2040 if (ixl_update_macvlan(sc) == 0) { 2041 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 2042 } else { 2043 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 2044 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 2045 } 2046 } 2047 2048 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 2049 sc->sc_nqueue_pairs = 1; 2050 else 2051 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 2052 2053 error = ixl_reinit(sc); 2054 if (error) { 2055 ixl_stop_locked(sc); 2056 return error; 2057 } 2058 2059 SET(ifp->if_flags, IFF_RUNNING); 2060 CLR(ifp->if_flags, IFF_OACTIVE); 2061 2062 ixl_config_rss(sc); 2063 ixl_config_queue_intr(sc); 2064 2065 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2066 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 2067 } 2068 2069 error = ixl_iff(sc); 2070 if (error) { 2071 ixl_stop_locked(sc); 2072 return error; 2073 } 2074 2075 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 2076 2077 return 0; 2078 } 2079 2080 static int 2081 ixl_init(struct ifnet *ifp) 2082 { 2083 struct ixl_softc *sc = ifp->if_softc; 2084 int error; 2085 2086 mutex_enter(&sc->sc_cfg_lock); 2087 error = ixl_init_locked(sc); 2088 mutex_exit(&sc->sc_cfg_lock); 2089 2090 if (error == 0) 2091 (void)ixl_get_link_status(sc); 2092 2093 return error; 2094 } 2095 2096 static int 2097 ixl_iff(struct ixl_softc *sc) 2098 { 2099 struct ifnet *ifp = &sc->sc_ec.ec_if; 2100 struct ixl_atq iatq; 2101 struct ixl_aq_desc *iaq; 2102 struct ixl_aq_vsi_promisc_param *param; 2103 uint16_t flag_add, flag_del; 2104 int error; 2105 2106 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2107 return 0; 2108 2109 memset(&iatq, 0, sizeof(iatq)); 2110 2111 iaq = &iatq.iatq_desc; 2112 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 2113 2114 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 2115 param->flags = htole16(0); 2116 2117 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER) 2118 || ISSET(ifp->if_flags, IFF_PROMISC)) { 2119 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2120 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2121 } 2122 2123 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 2124 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2125 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2126 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 2127 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2128 } 2129 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2130 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2131 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2132 param->seid = sc->sc_seid; 2133 2134 error = ixl_atq_exec(sc, &iatq); 2135 if (error) 2136 return error; 2137 2138 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 2139 return EIO; 2140 2141 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 2142 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 2143 flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH; 2144 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH; 2145 } else { 2146 flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN; 2147 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN; 2148 } 2149 2150 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del); 2151 2152 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2153 ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add); 2154 } 2155 return 0; 2156 } 2157 2158 static void 2159 ixl_stop_locked(struct ixl_softc *sc) 2160 { 2161 struct ifnet *ifp = &sc->sc_ec.ec_if; 2162 struct ixl_rx_ring *rxr; 2163 struct ixl_tx_ring *txr; 2164 unsigned int i; 2165 uint32_t reg; 2166 2167 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2168 2169 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 2170 callout_stop(&sc->sc_stats_callout); 2171 2172 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2173 txr = sc->sc_qps[i].qp_txr; 2174 rxr = sc->sc_qps[i].qp_rxr; 2175 2176 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 2177 2178 mutex_enter(&txr->txr_lock); 2179 ixl_txr_qdis(sc, txr, 0); 2180 mutex_exit(&txr->txr_lock); 2181 } 2182 2183 /* XXX wait at least 400 usec for all tx queues in one go */ 2184 ixl_flush(sc); 2185 DELAY(500); 2186 2187 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2188 txr = sc->sc_qps[i].qp_txr; 2189 rxr = sc->sc_qps[i].qp_rxr; 2190 2191 mutex_enter(&txr->txr_lock); 2192 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2193 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2194 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2195 mutex_exit(&txr->txr_lock); 2196 2197 mutex_enter(&rxr->rxr_lock); 2198 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2199 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2200 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2201 mutex_exit(&rxr->rxr_lock); 2202 } 2203 2204 /* XXX short wait for all queue disables to settle */ 2205 ixl_flush(sc); 2206 DELAY(50); 2207 2208 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2209 txr = sc->sc_qps[i].qp_txr; 2210 rxr = sc->sc_qps[i].qp_rxr; 2211 2212 mutex_enter(&txr->txr_lock); 2213 if (ixl_txr_disabled(sc, txr) != 0) { 2214 mutex_exit(&txr->txr_lock); 2215 goto die; 2216 } 2217 mutex_exit(&txr->txr_lock); 2218 2219 mutex_enter(&rxr->rxr_lock); 2220 if (ixl_rxr_disabled(sc, rxr) != 0) { 2221 mutex_exit(&rxr->rxr_lock); 2222 goto die; 2223 } 2224 mutex_exit(&rxr->rxr_lock); 2225 } 2226 2227 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2228 sc->sc_qps[i].qp_workqueue = false; 2229 workqueue_wait(sc->sc_workq_txrx, 2230 &sc->sc_qps[i].qp_work); 2231 } 2232 2233 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2234 txr = sc->sc_qps[i].qp_txr; 2235 rxr = sc->sc_qps[i].qp_rxr; 2236 2237 mutex_enter(&txr->txr_lock); 2238 ixl_txr_unconfig(sc, txr); 2239 mutex_exit(&txr->txr_lock); 2240 2241 mutex_enter(&rxr->rxr_lock); 2242 ixl_rxr_unconfig(sc, rxr); 2243 mutex_exit(&rxr->rxr_lock); 2244 2245 ixl_txr_clean(sc, txr); 2246 ixl_rxr_clean(sc, rxr); 2247 } 2248 2249 return; 2250 die: 2251 sc->sc_dead = true; 2252 log(LOG_CRIT, "%s: failed to shut down rings", 2253 device_xname(sc->sc_dev)); 2254 return; 2255 } 2256 2257 static void 2258 ixl_stop(struct ifnet *ifp, int disable) 2259 { 2260 struct ixl_softc *sc = ifp->if_softc; 2261 2262 mutex_enter(&sc->sc_cfg_lock); 2263 ixl_stop_locked(sc); 2264 mutex_exit(&sc->sc_cfg_lock); 2265 } 2266 2267 static int 2268 ixl_queue_pairs_alloc(struct ixl_softc *sc) 2269 { 2270 struct ixl_queue_pair *qp; 2271 unsigned int i; 2272 size_t sz; 2273 2274 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2275 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 2276 2277 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2278 qp = &sc->sc_qps[i]; 2279 2280 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2281 ixl_handle_queue, qp); 2282 if (qp->qp_si == NULL) 2283 goto free; 2284 2285 qp->qp_txr = ixl_txr_alloc(sc, i); 2286 if (qp->qp_txr == NULL) 2287 goto free; 2288 2289 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2290 if (qp->qp_rxr == NULL) 2291 goto free; 2292 2293 qp->qp_sc = sc; 2294 snprintf(qp->qp_name, sizeof(qp->qp_name), 2295 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2296 } 2297 2298 return 0; 2299 free: 2300 if (sc->sc_qps != NULL) { 2301 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2302 qp = &sc->sc_qps[i]; 2303 2304 if (qp->qp_txr != NULL) 2305 ixl_txr_free(sc, qp->qp_txr); 2306 if (qp->qp_rxr != NULL) 2307 ixl_rxr_free(sc, qp->qp_rxr); 2308 if (qp->qp_si != NULL) 2309 softint_disestablish(qp->qp_si); 2310 } 2311 2312 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2313 kmem_free(sc->sc_qps, sz); 2314 sc->sc_qps = NULL; 2315 } 2316 2317 return -1; 2318 } 2319 2320 static void 2321 ixl_queue_pairs_free(struct ixl_softc *sc) 2322 { 2323 struct ixl_queue_pair *qp; 2324 unsigned int i; 2325 size_t sz; 2326 2327 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2328 qp = &sc->sc_qps[i]; 2329 ixl_txr_free(sc, qp->qp_txr); 2330 ixl_rxr_free(sc, qp->qp_rxr); 2331 softint_disestablish(qp->qp_si); 2332 } 2333 2334 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2335 kmem_free(sc->sc_qps, sz); 2336 sc->sc_qps = NULL; 2337 } 2338 2339 static struct ixl_tx_ring * 2340 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2341 { 2342 struct ixl_tx_ring *txr = NULL; 2343 struct ixl_tx_map *maps = NULL, *txm; 2344 unsigned int i; 2345 2346 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2347 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2348 KM_SLEEP); 2349 2350 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2351 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2352 IXL_TX_QUEUE_ALIGN) != 0) 2353 goto free; 2354 2355 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2356 txm = &maps[i]; 2357 2358 if (bus_dmamap_create(sc->sc_dmat, IXL_TX_PKT_MAXSIZE, 2359 IXL_TX_PKT_DESCS, IXL_TX_PKT_MAXSIZE, 0, 2360 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2361 goto uncreate; 2362 2363 txm->txm_eop = -1; 2364 txm->txm_m = NULL; 2365 } 2366 2367 txr->txr_cons = txr->txr_prod = 0; 2368 txr->txr_maps = maps; 2369 2370 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2371 if (txr->txr_intrq == NULL) 2372 goto uncreate; 2373 2374 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2375 ixl_deferred_transmit, txr); 2376 if (txr->txr_si == NULL) 2377 goto destroy_pcq; 2378 2379 txr->txr_tail = I40E_QTX_TAIL(qid); 2380 txr->txr_qid = qid; 2381 txr->txr_sc = sc; 2382 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2383 2384 return txr; 2385 2386 destroy_pcq: 2387 pcq_destroy(txr->txr_intrq); 2388 uncreate: 2389 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2390 txm = &maps[i]; 2391 2392 if (txm->txm_map == NULL) 2393 continue; 2394 2395 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2396 } 2397 2398 ixl_dmamem_free(sc, &txr->txr_mem); 2399 free: 2400 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2401 kmem_free(txr, sizeof(*txr)); 2402 2403 return NULL; 2404 } 2405 2406 static void 2407 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2408 { 2409 unsigned int qid; 2410 bus_size_t reg; 2411 uint32_t r; 2412 2413 qid = txr->txr_qid + sc->sc_base_queue; 2414 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2415 qid %= 128; 2416 2417 r = ixl_rd(sc, reg); 2418 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2419 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2420 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2421 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2422 ixl_wr(sc, reg, r); 2423 } 2424 2425 static void 2426 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2427 { 2428 struct ixl_hmc_txq txq; 2429 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2430 void *hmc; 2431 2432 memset(&txq, 0, sizeof(txq)); 2433 txq.head = htole16(txr->txr_cons); 2434 txq.new_context = 1; 2435 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2436 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2437 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2438 txq.tphrdesc_ena = 0; 2439 txq.tphrpacket_ena = 0; 2440 txq.tphwdesc_ena = 0; 2441 txq.rdylist = data->qs_handle[0]; 2442 2443 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2444 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2445 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2446 __arraycount(ixl_hmc_pack_txq)); 2447 } 2448 2449 static void 2450 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2451 { 2452 void *hmc; 2453 2454 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2455 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2456 txr->txr_cons = txr->txr_prod = 0; 2457 } 2458 2459 static void 2460 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2461 { 2462 struct ixl_tx_map *maps, *txm; 2463 bus_dmamap_t map; 2464 unsigned int i; 2465 2466 maps = txr->txr_maps; 2467 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2468 txm = &maps[i]; 2469 2470 if (txm->txm_m == NULL) 2471 continue; 2472 2473 map = txm->txm_map; 2474 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2475 BUS_DMASYNC_POSTWRITE); 2476 bus_dmamap_unload(sc->sc_dmat, map); 2477 2478 m_freem(txm->txm_m); 2479 txm->txm_m = NULL; 2480 } 2481 } 2482 2483 static int 2484 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2485 { 2486 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2487 uint32_t reg; 2488 int i; 2489 2490 for (i = 0; i < 10; i++) { 2491 reg = ixl_rd(sc, ena); 2492 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2493 return 0; 2494 2495 delaymsec(10); 2496 } 2497 2498 return ETIMEDOUT; 2499 } 2500 2501 static int 2502 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2503 { 2504 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2505 uint32_t reg; 2506 int i; 2507 2508 KASSERT(mutex_owned(&txr->txr_lock)); 2509 2510 for (i = 0; i < 10; i++) { 2511 reg = ixl_rd(sc, ena); 2512 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2513 return 0; 2514 2515 delaymsec(10); 2516 } 2517 2518 return ETIMEDOUT; 2519 } 2520 2521 static void 2522 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2523 { 2524 struct ixl_tx_map *maps, *txm; 2525 struct mbuf *m; 2526 unsigned int i; 2527 2528 softint_disestablish(txr->txr_si); 2529 2530 maps = txr->txr_maps; 2531 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2532 txm = &maps[i]; 2533 2534 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2535 } 2536 2537 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2538 m_freem(m); 2539 pcq_destroy(txr->txr_intrq); 2540 2541 ixl_dmamem_free(sc, &txr->txr_mem); 2542 mutex_destroy(&txr->txr_lock); 2543 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2544 kmem_free(txr, sizeof(*txr)); 2545 } 2546 2547 static inline int 2548 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2549 struct ixl_tx_ring *txr) 2550 { 2551 struct mbuf *m; 2552 int error; 2553 2554 KASSERT(mutex_owned(&txr->txr_lock)); 2555 2556 m = *m0; 2557 2558 error = bus_dmamap_load_mbuf(dmat, map, m, 2559 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2560 if (error != EFBIG) 2561 return error; 2562 2563 m = m_defrag(m, M_DONTWAIT); 2564 if (m != NULL) { 2565 *m0 = m; 2566 txr->txr_defragged.ev_count++; 2567 2568 error = bus_dmamap_load_mbuf(dmat, map, m, 2569 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2570 } else { 2571 txr->txr_defrag_failed.ev_count++; 2572 error = ENOBUFS; 2573 } 2574 2575 return error; 2576 } 2577 2578 static inline int 2579 ixl_tx_setup_offloads(struct mbuf *m, uint64_t *cmd_txd) 2580 { 2581 struct ether_header *eh; 2582 size_t len; 2583 uint64_t cmd; 2584 2585 cmd = 0; 2586 2587 eh = mtod(m, struct ether_header *); 2588 switch (htons(eh->ether_type)) { 2589 case ETHERTYPE_IP: 2590 case ETHERTYPE_IPV6: 2591 len = ETHER_HDR_LEN; 2592 break; 2593 case ETHERTYPE_VLAN: 2594 len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2595 break; 2596 default: 2597 len = 0; 2598 } 2599 cmd |= ((len >> 1) << IXL_TX_DESC_MACLEN_SHIFT); 2600 2601 if (m->m_pkthdr.csum_flags & 2602 (M_CSUM_TSOv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) { 2603 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4; 2604 } 2605 if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) { 2606 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4_CSUM; 2607 } 2608 2609 if (m->m_pkthdr.csum_flags & 2610 (M_CSUM_TSOv6 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) { 2611 cmd |= IXL_TX_DESC_CMD_IIPT_IPV6; 2612 } 2613 2614 switch (cmd & IXL_TX_DESC_CMD_IIPT_MASK) { 2615 case IXL_TX_DESC_CMD_IIPT_IPV4: 2616 case IXL_TX_DESC_CMD_IIPT_IPV4_CSUM: 2617 len = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); 2618 break; 2619 case IXL_TX_DESC_CMD_IIPT_IPV6: 2620 len = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); 2621 break; 2622 default: 2623 len = 0; 2624 } 2625 cmd |= ((len >> 2) << IXL_TX_DESC_IPLEN_SHIFT); 2626 2627 if (m->m_pkthdr.csum_flags & 2628 (M_CSUM_TSOv4 | M_CSUM_TSOv6 | M_CSUM_TCPv4 | M_CSUM_TCPv6)) { 2629 len = sizeof(struct tcphdr); 2630 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_TCP; 2631 } else if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) { 2632 len = sizeof(struct udphdr); 2633 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_UDP; 2634 } else { 2635 len = 0; 2636 } 2637 cmd |= ((len >> 2) << IXL_TX_DESC_L4LEN_SHIFT); 2638 2639 *cmd_txd |= cmd; 2640 return 0; 2641 } 2642 2643 static void 2644 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2645 bool is_transmit) 2646 { 2647 struct ixl_softc *sc = ifp->if_softc; 2648 struct ixl_tx_desc *ring, *txd; 2649 struct ixl_tx_map *txm; 2650 bus_dmamap_t map; 2651 struct mbuf *m; 2652 uint64_t cmd, cmd_txd; 2653 unsigned int prod, free, last, i; 2654 unsigned int mask; 2655 int post = 0; 2656 2657 KASSERT(mutex_owned(&txr->txr_lock)); 2658 2659 if (!ISSET(ifp->if_flags, IFF_RUNNING) 2660 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2661 if (!is_transmit) 2662 IFQ_PURGE(&ifp->if_snd); 2663 return; 2664 } 2665 2666 prod = txr->txr_prod; 2667 free = txr->txr_cons; 2668 if (free <= prod) 2669 free += sc->sc_tx_ring_ndescs; 2670 free -= prod; 2671 2672 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2673 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2674 2675 ring = IXL_DMA_KVA(&txr->txr_mem); 2676 mask = sc->sc_tx_ring_ndescs - 1; 2677 last = prod; 2678 cmd = 0; 2679 txd = NULL; 2680 2681 for (;;) { 2682 if (free <= IXL_TX_PKT_DESCS) { 2683 if (!is_transmit) 2684 SET(ifp->if_flags, IFF_OACTIVE); 2685 break; 2686 } 2687 2688 if (is_transmit) 2689 m = pcq_get(txr->txr_intrq); 2690 else 2691 IFQ_DEQUEUE(&ifp->if_snd, m); 2692 2693 if (m == NULL) 2694 break; 2695 2696 txm = &txr->txr_maps[prod]; 2697 map = txm->txm_map; 2698 2699 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2700 if_statinc(ifp, if_oerrors); 2701 m_freem(m); 2702 continue; 2703 } 2704 2705 cmd_txd = 0; 2706 if (m->m_pkthdr.csum_flags & IXL_CSUM_ALL_OFFLOAD) { 2707 ixl_tx_setup_offloads(m, &cmd_txd); 2708 } 2709 2710 if (vlan_has_tag(m)) { 2711 uint16_t vtag; 2712 vtag = htole16(vlan_get_tag(m)); 2713 cmd_txd |= (uint64_t)vtag << 2714 IXL_TX_DESC_L2TAG1_SHIFT; 2715 cmd_txd |= IXL_TX_DESC_CMD_IL2TAG1; 2716 } 2717 2718 bus_dmamap_sync(sc->sc_dmat, map, 0, 2719 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2720 2721 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2722 txd = &ring[prod]; 2723 2724 cmd = (uint64_t)map->dm_segs[i].ds_len << 2725 IXL_TX_DESC_BSIZE_SHIFT; 2726 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2727 cmd |= cmd_txd; 2728 2729 txd->addr = htole64(map->dm_segs[i].ds_addr); 2730 txd->cmd = htole64(cmd); 2731 2732 last = prod; 2733 2734 prod++; 2735 prod &= mask; 2736 } 2737 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2738 txd->cmd = htole64(cmd); 2739 2740 txm->txm_m = m; 2741 txm->txm_eop = last; 2742 2743 bpf_mtap(ifp, m, BPF_D_OUT); 2744 2745 free -= i; 2746 post = 1; 2747 } 2748 2749 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2750 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2751 2752 if (post) { 2753 txr->txr_prod = prod; 2754 ixl_wr(sc, txr->txr_tail, prod); 2755 } 2756 } 2757 2758 static int 2759 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2760 { 2761 struct ifnet *ifp = &sc->sc_ec.ec_if; 2762 struct ixl_tx_desc *ring, *txd; 2763 struct ixl_tx_map *txm; 2764 struct mbuf *m; 2765 bus_dmamap_t map; 2766 unsigned int cons, prod, last; 2767 unsigned int mask; 2768 uint64_t dtype; 2769 int done = 0, more = 0; 2770 2771 KASSERT(mutex_owned(&txr->txr_lock)); 2772 2773 prod = txr->txr_prod; 2774 cons = txr->txr_cons; 2775 2776 if (cons == prod) 2777 return 0; 2778 2779 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2780 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2781 2782 ring = IXL_DMA_KVA(&txr->txr_mem); 2783 mask = sc->sc_tx_ring_ndescs - 1; 2784 2785 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2786 2787 do { 2788 if (txlimit-- <= 0) { 2789 more = 1; 2790 break; 2791 } 2792 2793 txm = &txr->txr_maps[cons]; 2794 last = txm->txm_eop; 2795 txd = &ring[last]; 2796 2797 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2798 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2799 break; 2800 2801 map = txm->txm_map; 2802 2803 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2804 BUS_DMASYNC_POSTWRITE); 2805 bus_dmamap_unload(sc->sc_dmat, map); 2806 2807 m = txm->txm_m; 2808 if (m != NULL) { 2809 if_statinc_ref(nsr, if_opackets); 2810 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 2811 if (ISSET(m->m_flags, M_MCAST)) 2812 if_statinc_ref(nsr, if_omcasts); 2813 m_freem(m); 2814 } 2815 2816 txm->txm_m = NULL; 2817 txm->txm_eop = -1; 2818 2819 cons = last + 1; 2820 cons &= mask; 2821 done = 1; 2822 } while (cons != prod); 2823 2824 IF_STAT_PUTREF(ifp); 2825 2826 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2827 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2828 2829 txr->txr_cons = cons; 2830 2831 if (done) { 2832 softint_schedule(txr->txr_si); 2833 if (txr->txr_qid == 0) { 2834 CLR(ifp->if_flags, IFF_OACTIVE); 2835 if_schedule_deferred_start(ifp); 2836 } 2837 } 2838 2839 return more; 2840 } 2841 2842 static void 2843 ixl_start(struct ifnet *ifp) 2844 { 2845 struct ixl_softc *sc; 2846 struct ixl_tx_ring *txr; 2847 2848 sc = ifp->if_softc; 2849 txr = sc->sc_qps[0].qp_txr; 2850 2851 mutex_enter(&txr->txr_lock); 2852 ixl_tx_common_locked(ifp, txr, false); 2853 mutex_exit(&txr->txr_lock); 2854 } 2855 2856 static inline unsigned int 2857 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2858 { 2859 u_int cpuid; 2860 2861 cpuid = cpu_index(curcpu()); 2862 2863 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2864 } 2865 2866 static int 2867 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2868 { 2869 struct ixl_softc *sc; 2870 struct ixl_tx_ring *txr; 2871 unsigned int qid; 2872 2873 sc = ifp->if_softc; 2874 qid = ixl_select_txqueue(sc, m); 2875 2876 txr = sc->sc_qps[qid].qp_txr; 2877 2878 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2879 mutex_enter(&txr->txr_lock); 2880 txr->txr_pcqdrop.ev_count++; 2881 mutex_exit(&txr->txr_lock); 2882 2883 m_freem(m); 2884 return ENOBUFS; 2885 } 2886 2887 #ifdef IXL_ALWAYS_TXDEFER 2888 kpreempt_disable(); 2889 softint_schedule(txr->txr_si); 2890 kpreempt_enable(); 2891 #else 2892 if (mutex_tryenter(&txr->txr_lock)) { 2893 ixl_tx_common_locked(ifp, txr, true); 2894 mutex_exit(&txr->txr_lock); 2895 } else { 2896 kpreempt_disable(); 2897 softint_schedule(txr->txr_si); 2898 kpreempt_enable(); 2899 } 2900 #endif 2901 2902 return 0; 2903 } 2904 2905 static void 2906 ixl_deferred_transmit(void *xtxr) 2907 { 2908 struct ixl_tx_ring *txr = xtxr; 2909 struct ixl_softc *sc = txr->txr_sc; 2910 struct ifnet *ifp = &sc->sc_ec.ec_if; 2911 2912 mutex_enter(&txr->txr_lock); 2913 txr->txr_transmitdef.ev_count++; 2914 if (pcq_peek(txr->txr_intrq) != NULL) 2915 ixl_tx_common_locked(ifp, txr, true); 2916 mutex_exit(&txr->txr_lock); 2917 } 2918 2919 static struct ixl_rx_ring * 2920 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2921 { 2922 struct ixl_rx_ring *rxr = NULL; 2923 struct ixl_rx_map *maps = NULL, *rxm; 2924 unsigned int i; 2925 2926 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2927 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2928 KM_SLEEP); 2929 2930 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2931 sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs, 2932 IXL_RX_QUEUE_ALIGN) != 0) 2933 goto free; 2934 2935 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2936 rxm = &maps[i]; 2937 2938 if (bus_dmamap_create(sc->sc_dmat, 2939 IXL_MCLBYTES, 1, IXL_MCLBYTES, 0, 2940 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2941 goto uncreate; 2942 2943 rxm->rxm_m = NULL; 2944 } 2945 2946 rxr->rxr_cons = rxr->rxr_prod = 0; 2947 rxr->rxr_m_head = NULL; 2948 rxr->rxr_m_tail = &rxr->rxr_m_head; 2949 rxr->rxr_maps = maps; 2950 2951 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2952 rxr->rxr_qid = qid; 2953 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2954 2955 return rxr; 2956 2957 uncreate: 2958 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2959 rxm = &maps[i]; 2960 2961 if (rxm->rxm_map == NULL) 2962 continue; 2963 2964 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2965 } 2966 2967 ixl_dmamem_free(sc, &rxr->rxr_mem); 2968 free: 2969 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2970 kmem_free(rxr, sizeof(*rxr)); 2971 2972 return NULL; 2973 } 2974 2975 static void 2976 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2977 { 2978 struct ixl_rx_map *maps, *rxm; 2979 bus_dmamap_t map; 2980 unsigned int i; 2981 2982 maps = rxr->rxr_maps; 2983 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2984 rxm = &maps[i]; 2985 2986 if (rxm->rxm_m == NULL) 2987 continue; 2988 2989 map = rxm->rxm_map; 2990 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2991 BUS_DMASYNC_POSTWRITE); 2992 bus_dmamap_unload(sc->sc_dmat, map); 2993 2994 m_freem(rxm->rxm_m); 2995 rxm->rxm_m = NULL; 2996 } 2997 2998 m_freem(rxr->rxr_m_head); 2999 rxr->rxr_m_head = NULL; 3000 rxr->rxr_m_tail = &rxr->rxr_m_head; 3001 3002 rxr->rxr_prod = rxr->rxr_cons = 0; 3003 } 3004 3005 static int 3006 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3007 { 3008 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3009 uint32_t reg; 3010 int i; 3011 3012 for (i = 0; i < 10; i++) { 3013 reg = ixl_rd(sc, ena); 3014 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 3015 return 0; 3016 3017 delaymsec(10); 3018 } 3019 3020 return ETIMEDOUT; 3021 } 3022 3023 static int 3024 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3025 { 3026 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3027 uint32_t reg; 3028 int i; 3029 3030 KASSERT(mutex_owned(&rxr->rxr_lock)); 3031 3032 for (i = 0; i < 10; i++) { 3033 reg = ixl_rd(sc, ena); 3034 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 3035 return 0; 3036 3037 delaymsec(10); 3038 } 3039 3040 return ETIMEDOUT; 3041 } 3042 3043 static void 3044 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3045 { 3046 struct ixl_hmc_rxq rxq; 3047 struct ifnet *ifp = &sc->sc_ec.ec_if; 3048 uint16_t rxmax; 3049 void *hmc; 3050 3051 memset(&rxq, 0, sizeof(rxq)); 3052 rxmax = ifp->if_mtu + IXL_MTU_ETHERLEN; 3053 3054 rxq.head = htole16(rxr->rxr_cons); 3055 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 3056 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 3057 rxq.dbuff = htole16(IXL_MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 3058 rxq.hbuff = 0; 3059 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 3060 rxq.dsize = IXL_HMC_RXQ_DSIZE_32; 3061 rxq.crcstrip = 1; 3062 rxq.l2sel = 1; 3063 rxq.showiv = 1; 3064 rxq.rxmax = htole16(rxmax); 3065 rxq.tphrdesc_ena = 0; 3066 rxq.tphwdesc_ena = 0; 3067 rxq.tphdata_ena = 0; 3068 rxq.tphhead_ena = 0; 3069 rxq.lrxqthresh = 0; 3070 rxq.prefena = 1; 3071 3072 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3073 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3074 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 3075 __arraycount(ixl_hmc_pack_rxq)); 3076 } 3077 3078 static void 3079 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3080 { 3081 void *hmc; 3082 3083 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3084 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3085 rxr->rxr_cons = rxr->rxr_prod = 0; 3086 } 3087 3088 static void 3089 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3090 { 3091 struct ixl_rx_map *maps, *rxm; 3092 unsigned int i; 3093 3094 maps = rxr->rxr_maps; 3095 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3096 rxm = &maps[i]; 3097 3098 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 3099 } 3100 3101 ixl_dmamem_free(sc, &rxr->rxr_mem); 3102 mutex_destroy(&rxr->rxr_lock); 3103 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3104 kmem_free(rxr, sizeof(*rxr)); 3105 } 3106 3107 static inline void 3108 ixl_rx_csum(struct mbuf *m, uint64_t qword) 3109 { 3110 int flags_mask; 3111 3112 if (!ISSET(qword, IXL_RX_DESC_L3L4P)) { 3113 /* No L3 or L4 checksum was calculated */ 3114 return; 3115 } 3116 3117 switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) { 3118 case IXL_RX_DESC_PTYPE_IPV4FRAG: 3119 case IXL_RX_DESC_PTYPE_IPV4: 3120 case IXL_RX_DESC_PTYPE_SCTPV4: 3121 case IXL_RX_DESC_PTYPE_ICMPV4: 3122 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3123 break; 3124 case IXL_RX_DESC_PTYPE_TCPV4: 3125 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3126 flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD; 3127 break; 3128 case IXL_RX_DESC_PTYPE_UDPV4: 3129 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3130 flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD; 3131 break; 3132 case IXL_RX_DESC_PTYPE_TCPV6: 3133 flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD; 3134 break; 3135 case IXL_RX_DESC_PTYPE_UDPV6: 3136 flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD; 3137 break; 3138 default: 3139 flags_mask = 0; 3140 } 3141 3142 m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 | 3143 M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6)); 3144 3145 if (ISSET(qword, IXL_RX_DESC_IPE)) { 3146 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD); 3147 } 3148 3149 if (ISSET(qword, IXL_RX_DESC_L4E)) { 3150 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD); 3151 } 3152 } 3153 3154 static int 3155 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 3156 { 3157 struct ifnet *ifp = &sc->sc_ec.ec_if; 3158 struct ixl_rx_wb_desc_32 *ring, *rxd; 3159 struct ixl_rx_map *rxm; 3160 bus_dmamap_t map; 3161 unsigned int cons, prod; 3162 struct mbuf *m; 3163 uint64_t word, word0; 3164 unsigned int len; 3165 unsigned int mask; 3166 int done = 0, more = 0; 3167 3168 KASSERT(mutex_owned(&rxr->rxr_lock)); 3169 3170 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3171 return 0; 3172 3173 prod = rxr->rxr_prod; 3174 cons = rxr->rxr_cons; 3175 3176 if (cons == prod) 3177 return 0; 3178 3179 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3180 0, IXL_DMA_LEN(&rxr->rxr_mem), 3181 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3182 3183 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3184 mask = sc->sc_rx_ring_ndescs - 1; 3185 3186 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 3187 3188 do { 3189 if (rxlimit-- <= 0) { 3190 more = 1; 3191 break; 3192 } 3193 3194 rxd = &ring[cons]; 3195 3196 word = le64toh(rxd->qword1); 3197 3198 if (!ISSET(word, IXL_RX_DESC_DD)) 3199 break; 3200 3201 rxm = &rxr->rxr_maps[cons]; 3202 3203 map = rxm->rxm_map; 3204 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3205 BUS_DMASYNC_POSTREAD); 3206 bus_dmamap_unload(sc->sc_dmat, map); 3207 3208 m = rxm->rxm_m; 3209 rxm->rxm_m = NULL; 3210 3211 KASSERT(m != NULL); 3212 3213 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 3214 m->m_len = len; 3215 m->m_pkthdr.len = 0; 3216 3217 m->m_next = NULL; 3218 *rxr->rxr_m_tail = m; 3219 rxr->rxr_m_tail = &m->m_next; 3220 3221 m = rxr->rxr_m_head; 3222 m->m_pkthdr.len += len; 3223 3224 if (ISSET(word, IXL_RX_DESC_EOP)) { 3225 word0 = le64toh(rxd->qword0); 3226 3227 if (ISSET(word, IXL_RX_DESC_L2TAG1P)) { 3228 uint16_t vtag; 3229 vtag = __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK); 3230 vlan_set_tag(m, le16toh(vtag)); 3231 } 3232 3233 if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0) 3234 ixl_rx_csum(m, word); 3235 3236 if (!ISSET(word, 3237 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 3238 m_set_rcvif(m, ifp); 3239 if_statinc_ref(nsr, if_ipackets); 3240 if_statadd_ref(nsr, if_ibytes, 3241 m->m_pkthdr.len); 3242 if_percpuq_enqueue(sc->sc_ipq, m); 3243 } else { 3244 if_statinc_ref(nsr, if_ierrors); 3245 m_freem(m); 3246 } 3247 3248 rxr->rxr_m_head = NULL; 3249 rxr->rxr_m_tail = &rxr->rxr_m_head; 3250 } 3251 3252 cons++; 3253 cons &= mask; 3254 3255 done = 1; 3256 } while (cons != prod); 3257 3258 if (done) { 3259 rxr->rxr_cons = cons; 3260 if (ixl_rxfill(sc, rxr) == -1) 3261 if_statinc_ref(nsr, if_iqdrops); 3262 } 3263 3264 IF_STAT_PUTREF(ifp); 3265 3266 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3267 0, IXL_DMA_LEN(&rxr->rxr_mem), 3268 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3269 3270 return more; 3271 } 3272 3273 static int 3274 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3275 { 3276 struct ixl_rx_rd_desc_32 *ring, *rxd; 3277 struct ixl_rx_map *rxm; 3278 bus_dmamap_t map; 3279 struct mbuf *m; 3280 unsigned int prod; 3281 unsigned int slots; 3282 unsigned int mask; 3283 int post = 0, error = 0; 3284 3285 KASSERT(mutex_owned(&rxr->rxr_lock)); 3286 3287 prod = rxr->rxr_prod; 3288 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 3289 sc->sc_rx_ring_ndescs); 3290 3291 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3292 mask = sc->sc_rx_ring_ndescs - 1; 3293 3294 if (__predict_false(slots <= 0)) 3295 return -1; 3296 3297 do { 3298 rxm = &rxr->rxr_maps[prod]; 3299 3300 MGETHDR(m, M_DONTWAIT, MT_DATA); 3301 if (m == NULL) { 3302 rxr->rxr_mgethdr_failed.ev_count++; 3303 error = -1; 3304 break; 3305 } 3306 3307 MCLGET(m, M_DONTWAIT); 3308 if (!ISSET(m->m_flags, M_EXT)) { 3309 rxr->rxr_mgetcl_failed.ev_count++; 3310 error = -1; 3311 m_freem(m); 3312 break; 3313 } 3314 3315 m->m_len = m->m_pkthdr.len = MCLBYTES; 3316 m_adj(m, ETHER_ALIGN); 3317 3318 map = rxm->rxm_map; 3319 3320 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 3321 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 3322 rxr->rxr_mbuf_load_failed.ev_count++; 3323 error = -1; 3324 m_freem(m); 3325 break; 3326 } 3327 3328 rxm->rxm_m = m; 3329 3330 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3331 BUS_DMASYNC_PREREAD); 3332 3333 rxd = &ring[prod]; 3334 3335 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 3336 rxd->haddr = htole64(0); 3337 3338 prod++; 3339 prod &= mask; 3340 3341 post = 1; 3342 3343 } while (--slots); 3344 3345 if (post) { 3346 rxr->rxr_prod = prod; 3347 ixl_wr(sc, rxr->rxr_tail, prod); 3348 } 3349 3350 return error; 3351 } 3352 3353 static inline int 3354 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 3355 u_int txlimit, struct evcnt *txevcnt, 3356 u_int rxlimit, struct evcnt *rxevcnt) 3357 { 3358 struct ixl_tx_ring *txr = qp->qp_txr; 3359 struct ixl_rx_ring *rxr = qp->qp_rxr; 3360 int txmore, rxmore; 3361 int rv; 3362 3363 mutex_enter(&txr->txr_lock); 3364 txevcnt->ev_count++; 3365 txmore = ixl_txeof(sc, txr, txlimit); 3366 mutex_exit(&txr->txr_lock); 3367 3368 mutex_enter(&rxr->rxr_lock); 3369 rxevcnt->ev_count++; 3370 rxmore = ixl_rxeof(sc, rxr, rxlimit); 3371 mutex_exit(&rxr->rxr_lock); 3372 3373 rv = txmore | (rxmore << 1); 3374 3375 return rv; 3376 } 3377 3378 static void 3379 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 3380 { 3381 3382 if (qp->qp_workqueue) 3383 workqueue_enqueue(sc->sc_workq_txrx, &qp->qp_work, NULL); 3384 else 3385 softint_schedule(qp->qp_si); 3386 } 3387 3388 static int 3389 ixl_intr(void *xsc) 3390 { 3391 struct ixl_softc *sc = xsc; 3392 struct ixl_tx_ring *txr; 3393 struct ixl_rx_ring *rxr; 3394 uint32_t icr, rxintr, txintr; 3395 int rv = 0; 3396 unsigned int i; 3397 3398 KASSERT(sc != NULL); 3399 3400 ixl_enable_other_intr(sc); 3401 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3402 3403 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3404 atomic_inc_64(&sc->sc_event_atq.ev_count); 3405 ixl_atq_done(sc); 3406 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3407 rv = 1; 3408 } 3409 3410 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3411 atomic_inc_64(&sc->sc_event_link.ev_count); 3412 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3413 rv = 1; 3414 } 3415 3416 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 3417 txintr = icr & I40E_INTR_NOTX_TX_MASK; 3418 3419 if (txintr || rxintr) { 3420 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 3421 txr = sc->sc_qps[i].qp_txr; 3422 rxr = sc->sc_qps[i].qp_rxr; 3423 3424 ixl_handle_queue_common(sc, &sc->sc_qps[i], 3425 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 3426 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 3427 } 3428 rv = 1; 3429 } 3430 3431 return rv; 3432 } 3433 3434 static int 3435 ixl_queue_intr(void *xqp) 3436 { 3437 struct ixl_queue_pair *qp = xqp; 3438 struct ixl_tx_ring *txr = qp->qp_txr; 3439 struct ixl_rx_ring *rxr = qp->qp_rxr; 3440 struct ixl_softc *sc = qp->qp_sc; 3441 u_int txlimit, rxlimit; 3442 int more; 3443 3444 txlimit = sc->sc_tx_intr_process_limit; 3445 rxlimit = sc->sc_rx_intr_process_limit; 3446 qp->qp_workqueue = sc->sc_txrx_workqueue; 3447 3448 more = ixl_handle_queue_common(sc, qp, 3449 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3450 3451 if (more != 0) { 3452 ixl_sched_handle_queue(sc, qp); 3453 } else { 3454 /* for ALTQ */ 3455 if (txr->txr_qid == 0) 3456 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3457 softint_schedule(txr->txr_si); 3458 3459 ixl_enable_queue_intr(sc, qp); 3460 } 3461 3462 return 1; 3463 } 3464 3465 static void 3466 ixl_handle_queue_wk(struct work *wk, void *xsc) 3467 { 3468 struct ixl_queue_pair *qp; 3469 3470 qp = container_of(wk, struct ixl_queue_pair, qp_work); 3471 ixl_handle_queue(qp); 3472 } 3473 3474 static void 3475 ixl_handle_queue(void *xqp) 3476 { 3477 struct ixl_queue_pair *qp = xqp; 3478 struct ixl_softc *sc = qp->qp_sc; 3479 struct ixl_tx_ring *txr = qp->qp_txr; 3480 struct ixl_rx_ring *rxr = qp->qp_rxr; 3481 u_int txlimit, rxlimit; 3482 int more; 3483 3484 txlimit = sc->sc_tx_process_limit; 3485 rxlimit = sc->sc_rx_process_limit; 3486 3487 more = ixl_handle_queue_common(sc, qp, 3488 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3489 3490 if (more != 0) 3491 ixl_sched_handle_queue(sc, qp); 3492 else 3493 ixl_enable_queue_intr(sc, qp); 3494 } 3495 3496 static inline void 3497 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3498 { 3499 uint32_t hmc_idx, hmc_isvf; 3500 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3501 3502 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3503 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3504 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3505 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3506 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3507 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3508 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3509 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3510 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3511 3512 device_printf(sc->sc_dev, 3513 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3514 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3515 } 3516 3517 static int 3518 ixl_other_intr(void *xsc) 3519 { 3520 struct ixl_softc *sc = xsc; 3521 uint32_t icr, mask, reg; 3522 int rv; 3523 3524 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3525 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3526 3527 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3528 atomic_inc_64(&sc->sc_event_atq.ev_count); 3529 ixl_atq_done(sc); 3530 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3531 rv = 1; 3532 } 3533 3534 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3535 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3536 device_printf(sc->sc_dev, "link stat changed\n"); 3537 3538 atomic_inc_64(&sc->sc_event_link.ev_count); 3539 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3540 rv = 1; 3541 } 3542 3543 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3544 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3545 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3546 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3547 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3548 3549 device_printf(sc->sc_dev, "GRST: %s\n", 3550 reg == I40E_RESET_CORER ? "CORER" : 3551 reg == I40E_RESET_GLOBR ? "GLOBR" : 3552 reg == I40E_RESET_EMPR ? "EMPR" : 3553 "POR"); 3554 } 3555 3556 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3557 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3558 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3559 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3560 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3561 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3562 3563 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3564 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3565 device_printf(sc->sc_dev, "critical error\n"); 3566 } 3567 3568 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3569 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3570 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3571 ixl_print_hmc_error(sc, reg); 3572 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3573 } 3574 3575 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3576 ixl_flush(sc); 3577 ixl_enable_other_intr(sc); 3578 return rv; 3579 } 3580 3581 static void 3582 ixl_get_link_status_done(struct ixl_softc *sc, 3583 const struct ixl_aq_desc *iaq) 3584 { 3585 struct ixl_aq_desc iaq_buf; 3586 3587 memcpy(&iaq_buf, iaq, sizeof(iaq_buf)); 3588 3589 /* 3590 * The lock can be released here 3591 * because there is no post processing about ATQ 3592 */ 3593 mutex_exit(&sc->sc_atq_lock); 3594 ixl_link_state_update(sc, &iaq_buf); 3595 mutex_enter(&sc->sc_atq_lock); 3596 } 3597 3598 static void 3599 ixl_get_link_status(void *xsc) 3600 { 3601 struct ixl_softc *sc = xsc; 3602 struct ixl_aq_desc *iaq; 3603 struct ixl_aq_link_param *param; 3604 int error; 3605 3606 mutex_enter(&sc->sc_atq_lock); 3607 3608 if (sc->sc_link_state_atq.iatq_inuse) 3609 goto done; 3610 3611 iaq = &sc->sc_link_state_atq.iatq_desc; 3612 memset(iaq, 0, sizeof(*iaq)); 3613 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3614 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3615 param->notify = IXL_AQ_LINK_NOTIFY; 3616 3617 error = ixl_atq_exec_locked(sc, &sc->sc_link_state_atq); 3618 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 3619 3620 if (error == 0) { 3621 ixl_get_link_status_done(sc, iaq); 3622 } 3623 3624 done: 3625 mutex_exit(&sc->sc_atq_lock); 3626 } 3627 3628 static void 3629 ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3630 { 3631 struct ifnet *ifp = &sc->sc_ec.ec_if; 3632 int link_state; 3633 3634 mutex_enter(&sc->sc_cfg_lock); 3635 link_state = ixl_set_link_status_locked(sc, iaq); 3636 mutex_exit(&sc->sc_cfg_lock); 3637 3638 if (ifp->if_link_state != link_state) 3639 if_link_state_change(ifp, link_state); 3640 3641 if (link_state != LINK_STATE_DOWN) { 3642 kpreempt_disable(); 3643 if_schedule_deferred_start(ifp); 3644 kpreempt_enable(); 3645 } 3646 } 3647 3648 static void 3649 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3650 const char *msg) 3651 { 3652 char buf[512]; 3653 size_t len; 3654 3655 len = sizeof(buf); 3656 buf[--len] = '\0'; 3657 3658 device_printf(sc->sc_dev, "%s\n", msg); 3659 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3660 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3661 buf, le16toh(iaq->iaq_opcode)); 3662 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3663 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3664 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3665 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3666 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3667 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3668 } 3669 3670 static void 3671 ixl_arq(void *xsc) 3672 { 3673 struct ixl_softc *sc = xsc; 3674 struct ixl_aq_desc *arq, *iaq; 3675 struct ixl_aq_buf *aqb; 3676 unsigned int cons = sc->sc_arq_cons; 3677 unsigned int prod; 3678 int done = 0; 3679 3680 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3681 sc->sc_aq_regs->arq_head_mask; 3682 3683 if (cons == prod) 3684 goto done; 3685 3686 arq = IXL_DMA_KVA(&sc->sc_arq); 3687 3688 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3689 0, IXL_DMA_LEN(&sc->sc_arq), 3690 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3691 3692 do { 3693 iaq = &arq[cons]; 3694 aqb = sc->sc_arq_live[cons]; 3695 3696 KASSERT(aqb != NULL); 3697 3698 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3699 BUS_DMASYNC_POSTREAD); 3700 3701 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3702 ixl_aq_dump(sc, iaq, "arq event"); 3703 3704 switch (iaq->iaq_opcode) { 3705 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3706 ixl_link_state_update(sc, iaq); 3707 break; 3708 } 3709 3710 memset(iaq, 0, sizeof(*iaq)); 3711 sc->sc_arq_live[cons] = NULL; 3712 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3713 3714 cons++; 3715 cons &= IXL_AQ_MASK; 3716 3717 done = 1; 3718 } while (cons != prod); 3719 3720 if (done) { 3721 sc->sc_arq_cons = cons; 3722 ixl_arq_fill(sc); 3723 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3724 0, IXL_DMA_LEN(&sc->sc_arq), 3725 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3726 } 3727 3728 done: 3729 ixl_enable_other_intr(sc); 3730 } 3731 3732 static void 3733 ixl_atq_set(struct ixl_atq *iatq, 3734 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3735 { 3736 3737 iatq->iatq_fn = fn; 3738 } 3739 3740 static int 3741 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3742 { 3743 struct ixl_aq_desc *atq, *slot; 3744 unsigned int prod, cons, prod_next; 3745 3746 /* assert locked */ 3747 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3748 3749 atq = IXL_DMA_KVA(&sc->sc_atq); 3750 prod = sc->sc_atq_prod; 3751 cons = sc->sc_atq_cons; 3752 prod_next = (prod +1) & IXL_AQ_MASK; 3753 3754 if (cons == prod_next) 3755 return ENOMEM; 3756 3757 slot = &atq[prod]; 3758 3759 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3760 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3761 3762 KASSERT(iatq->iatq_fn != NULL); 3763 *slot = iatq->iatq_desc; 3764 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3765 3766 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3767 ixl_aq_dump(sc, slot, "atq command"); 3768 3769 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3770 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3771 3772 sc->sc_atq_prod = prod_next; 3773 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3774 iatq->iatq_inuse = true; 3775 3776 return 0; 3777 } 3778 3779 static void 3780 ixl_atq_done_locked(struct ixl_softc *sc) 3781 { 3782 struct ixl_aq_desc *atq, *slot; 3783 struct ixl_atq *iatq; 3784 unsigned int cons; 3785 unsigned int prod; 3786 3787 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3788 3789 prod = sc->sc_atq_prod; 3790 cons = sc->sc_atq_cons; 3791 3792 if (prod == cons) 3793 return; 3794 3795 atq = IXL_DMA_KVA(&sc->sc_atq); 3796 3797 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3798 0, IXL_DMA_LEN(&sc->sc_atq), 3799 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3800 3801 do { 3802 slot = &atq[cons]; 3803 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3804 break; 3805 3806 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3807 iatq->iatq_desc = *slot; 3808 iatq->iatq_inuse = false; 3809 3810 memset(slot, 0, sizeof(*slot)); 3811 3812 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3813 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3814 3815 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3816 3817 cons++; 3818 cons &= IXL_AQ_MASK; 3819 } while (cons != prod); 3820 3821 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3822 0, IXL_DMA_LEN(&sc->sc_atq), 3823 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3824 3825 sc->sc_atq_cons = cons; 3826 } 3827 3828 static void 3829 ixl_atq_done(struct ixl_softc *sc) 3830 { 3831 3832 mutex_enter(&sc->sc_atq_lock); 3833 ixl_atq_done_locked(sc); 3834 mutex_exit(&sc->sc_atq_lock); 3835 } 3836 3837 static void 3838 ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3839 { 3840 3841 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3842 3843 cv_broadcast(&sc->sc_atq_cv); 3844 } 3845 3846 static int 3847 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3848 { 3849 int error; 3850 3851 mutex_enter(&sc->sc_atq_lock); 3852 error = ixl_atq_exec_locked(sc, iatq); 3853 mutex_exit(&sc->sc_atq_lock); 3854 3855 return error; 3856 } 3857 3858 static int 3859 ixl_atq_exec_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3860 { 3861 int error; 3862 3863 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3864 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3865 3866 ixl_atq_set(iatq, ixl_wakeup); 3867 3868 error = ixl_atq_post_locked(sc, iatq); 3869 if (error) 3870 return error; 3871 3872 do { 3873 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3874 IXL_ATQ_EXEC_TIMEOUT); 3875 if (error == EWOULDBLOCK) 3876 break; 3877 } while (iatq->iatq_inuse); 3878 3879 return error; 3880 } 3881 3882 static int 3883 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3884 { 3885 struct ixl_aq_desc *atq, *slot; 3886 unsigned int prod; 3887 unsigned int t = 0; 3888 3889 mutex_enter(&sc->sc_atq_lock); 3890 3891 atq = IXL_DMA_KVA(&sc->sc_atq); 3892 prod = sc->sc_atq_prod; 3893 slot = atq + prod; 3894 3895 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3896 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3897 3898 *slot = *iaq; 3899 slot->iaq_flags |= htole16(IXL_AQ_SI); 3900 3901 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3902 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3903 3904 prod++; 3905 prod &= IXL_AQ_MASK; 3906 sc->sc_atq_prod = prod; 3907 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3908 3909 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3910 delaymsec(1); 3911 3912 if (t++ > tm) { 3913 mutex_exit(&sc->sc_atq_lock); 3914 return ETIMEDOUT; 3915 } 3916 } 3917 3918 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3919 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3920 *iaq = *slot; 3921 memset(slot, 0, sizeof(*slot)); 3922 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3923 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3924 3925 sc->sc_atq_cons = prod; 3926 3927 mutex_exit(&sc->sc_atq_lock); 3928 3929 return 0; 3930 } 3931 3932 static int 3933 ixl_get_version(struct ixl_softc *sc) 3934 { 3935 struct ixl_aq_desc iaq; 3936 uint32_t fwbuild, fwver, apiver; 3937 uint16_t api_maj_ver, api_min_ver; 3938 3939 memset(&iaq, 0, sizeof(iaq)); 3940 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3941 3942 iaq.iaq_retval = le16toh(23); 3943 3944 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3945 return ETIMEDOUT; 3946 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3947 return EIO; 3948 3949 fwbuild = le32toh(iaq.iaq_param[1]); 3950 fwver = le32toh(iaq.iaq_param[2]); 3951 apiver = le32toh(iaq.iaq_param[3]); 3952 3953 api_maj_ver = (uint16_t)apiver; 3954 api_min_ver = (uint16_t)(apiver >> 16); 3955 3956 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 3957 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 3958 3959 if (sc->sc_mac_type == I40E_MAC_X722) { 3960 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK | 3961 IXL_SC_AQ_FLAG_NVMREAD); 3962 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 3963 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS); 3964 } 3965 3966 #define IXL_API_VER(maj, min) (((uint32_t)(maj) << 16) | (min)) 3967 if (IXL_API_VER(api_maj_ver, api_min_ver) >= IXL_API_VER(1, 5)) { 3968 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 3969 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK); 3970 } 3971 #undef IXL_API_VER 3972 3973 return 0; 3974 } 3975 3976 static int 3977 ixl_get_nvm_version(struct ixl_softc *sc) 3978 { 3979 uint16_t nvmver, cfg_ptr, eetrack_hi, eetrack_lo, oem_hi, oem_lo; 3980 uint32_t eetrack, oem; 3981 uint16_t nvm_maj_ver, nvm_min_ver, oem_build; 3982 uint8_t oem_ver, oem_patch; 3983 3984 nvmver = cfg_ptr = eetrack_hi = eetrack_lo = oem_hi = oem_lo = 0; 3985 ixl_rd16_nvm(sc, I40E_SR_NVM_DEV_STARTER_VERSION, &nvmver); 3986 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_HI, &eetrack_hi); 3987 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_LO, &eetrack_lo); 3988 ixl_rd16_nvm(sc, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr); 3989 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF, &oem_hi); 3990 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF + 1, &oem_lo); 3991 3992 nvm_maj_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_HI_MASK); 3993 nvm_min_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_LO_MASK); 3994 eetrack = ((uint32_t)eetrack_hi << 16) | eetrack_lo; 3995 oem = ((uint32_t)oem_hi << 16) | oem_lo; 3996 oem_ver = __SHIFTOUT(oem, IXL_NVM_OEMVERSION_MASK); 3997 oem_build = __SHIFTOUT(oem, IXL_NVM_OEMBUILD_MASK); 3998 oem_patch = __SHIFTOUT(oem, IXL_NVM_OEMPATCH_MASK); 3999 4000 aprint_normal(" nvm %x.%02x etid %08x oem %d.%d.%d", 4001 nvm_maj_ver, nvm_min_ver, eetrack, 4002 oem_ver, oem_build, oem_patch); 4003 4004 return 0; 4005 } 4006 4007 static int 4008 ixl_pxe_clear(struct ixl_softc *sc) 4009 { 4010 struct ixl_aq_desc iaq; 4011 int rv; 4012 4013 memset(&iaq, 0, sizeof(iaq)); 4014 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 4015 iaq.iaq_param[0] = htole32(0x2); 4016 4017 rv = ixl_atq_poll(sc, &iaq, 250); 4018 4019 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 4020 4021 if (rv != 0) 4022 return ETIMEDOUT; 4023 4024 switch (iaq.iaq_retval) { 4025 case htole16(IXL_AQ_RC_OK): 4026 case htole16(IXL_AQ_RC_EEXIST): 4027 break; 4028 default: 4029 return EIO; 4030 } 4031 4032 return 0; 4033 } 4034 4035 static int 4036 ixl_lldp_shut(struct ixl_softc *sc) 4037 { 4038 struct ixl_aq_desc iaq; 4039 4040 memset(&iaq, 0, sizeof(iaq)); 4041 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 4042 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 4043 4044 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4045 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 4046 return -1; 4047 } 4048 4049 switch (iaq.iaq_retval) { 4050 case htole16(IXL_AQ_RC_EMODE): 4051 case htole16(IXL_AQ_RC_EPERM): 4052 /* ignore silently */ 4053 default: 4054 break; 4055 } 4056 4057 return 0; 4058 } 4059 4060 static void 4061 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 4062 { 4063 uint16_t id; 4064 uint32_t number, logical_id; 4065 4066 id = le16toh(cap->cap_id); 4067 number = le32toh(cap->number); 4068 logical_id = le32toh(cap->logical_id); 4069 4070 switch (id) { 4071 case IXL_AQ_CAP_RSS: 4072 sc->sc_rss_table_size = number; 4073 sc->sc_rss_table_entry_width = logical_id; 4074 break; 4075 case IXL_AQ_CAP_RXQ: 4076 case IXL_AQ_CAP_TXQ: 4077 sc->sc_nqueue_pairs_device = MIN(number, 4078 sc->sc_nqueue_pairs_device); 4079 break; 4080 } 4081 } 4082 4083 static int 4084 ixl_get_hw_capabilities(struct ixl_softc *sc) 4085 { 4086 struct ixl_dmamem idm; 4087 struct ixl_aq_desc iaq; 4088 struct ixl_aq_capability *caps; 4089 size_t i, ncaps; 4090 bus_size_t caps_size; 4091 uint16_t status; 4092 int rv; 4093 4094 caps_size = sizeof(caps[0]) * 40; 4095 memset(&iaq, 0, sizeof(iaq)); 4096 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 4097 4098 do { 4099 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 4100 return -1; 4101 } 4102 4103 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4104 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4105 iaq.iaq_datalen = htole16(caps_size); 4106 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4107 4108 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4109 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 4110 4111 rv = ixl_atq_poll(sc, &iaq, 250); 4112 4113 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4114 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 4115 4116 if (rv != 0) { 4117 aprint_error(", HW capabilities timeout\n"); 4118 goto done; 4119 } 4120 4121 status = le16toh(iaq.iaq_retval); 4122 4123 if (status == IXL_AQ_RC_ENOMEM) { 4124 caps_size = le16toh(iaq.iaq_datalen); 4125 ixl_dmamem_free(sc, &idm); 4126 } 4127 } while (status == IXL_AQ_RC_ENOMEM); 4128 4129 if (status != IXL_AQ_RC_OK) { 4130 aprint_error(", HW capabilities error\n"); 4131 goto done; 4132 } 4133 4134 caps = IXL_DMA_KVA(&idm); 4135 ncaps = le16toh(iaq.iaq_param[1]); 4136 4137 for (i = 0; i < ncaps; i++) { 4138 ixl_parse_hw_capability(sc, &caps[i]); 4139 } 4140 4141 done: 4142 ixl_dmamem_free(sc, &idm); 4143 return rv; 4144 } 4145 4146 static int 4147 ixl_get_mac(struct ixl_softc *sc) 4148 { 4149 struct ixl_dmamem idm; 4150 struct ixl_aq_desc iaq; 4151 struct ixl_aq_mac_addresses *addrs; 4152 int rv; 4153 4154 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 4155 aprint_error(", unable to allocate mac addresses\n"); 4156 return -1; 4157 } 4158 4159 memset(&iaq, 0, sizeof(iaq)); 4160 iaq.iaq_flags = htole16(IXL_AQ_BUF); 4161 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 4162 iaq.iaq_datalen = htole16(sizeof(*addrs)); 4163 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4164 4165 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4166 BUS_DMASYNC_PREREAD); 4167 4168 rv = ixl_atq_poll(sc, &iaq, 250); 4169 4170 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4171 BUS_DMASYNC_POSTREAD); 4172 4173 if (rv != 0) { 4174 aprint_error(", MAC ADDRESS READ timeout\n"); 4175 rv = -1; 4176 goto done; 4177 } 4178 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4179 aprint_error(", MAC ADDRESS READ error\n"); 4180 rv = -1; 4181 goto done; 4182 } 4183 4184 addrs = IXL_DMA_KVA(&idm); 4185 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 4186 printf(", port address is not valid\n"); 4187 goto done; 4188 } 4189 4190 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 4191 rv = 0; 4192 4193 done: 4194 ixl_dmamem_free(sc, &idm); 4195 return rv; 4196 } 4197 4198 static int 4199 ixl_get_switch_config(struct ixl_softc *sc) 4200 { 4201 struct ixl_dmamem idm; 4202 struct ixl_aq_desc iaq; 4203 struct ixl_aq_switch_config *hdr; 4204 struct ixl_aq_switch_config_element *elms, *elm; 4205 unsigned int nelm, i; 4206 int rv; 4207 4208 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4209 aprint_error_dev(sc->sc_dev, 4210 "unable to allocate switch config buffer\n"); 4211 return -1; 4212 } 4213 4214 memset(&iaq, 0, sizeof(iaq)); 4215 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4216 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4217 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 4218 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 4219 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4220 4221 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4222 BUS_DMASYNC_PREREAD); 4223 4224 rv = ixl_atq_poll(sc, &iaq, 250); 4225 4226 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4227 BUS_DMASYNC_POSTREAD); 4228 4229 if (rv != 0) { 4230 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 4231 rv = -1; 4232 goto done; 4233 } 4234 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4235 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 4236 rv = -1; 4237 goto done; 4238 } 4239 4240 hdr = IXL_DMA_KVA(&idm); 4241 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 4242 4243 nelm = le16toh(hdr->num_reported); 4244 if (nelm < 1) { 4245 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 4246 rv = -1; 4247 goto done; 4248 } 4249 4250 for (i = 0; i < nelm; i++) { 4251 elm = &elms[i]; 4252 4253 aprint_debug_dev(sc->sc_dev, 4254 "type %x revision %u seid %04x\n", 4255 elm->type, elm->revision, le16toh(elm->seid)); 4256 aprint_debug_dev(sc->sc_dev, 4257 "uplink %04x downlink %04x\n", 4258 le16toh(elm->uplink_seid), 4259 le16toh(elm->downlink_seid)); 4260 aprint_debug_dev(sc->sc_dev, 4261 "conntype %x scheduler %04x extra %04x\n", 4262 elm->connection_type, 4263 le16toh(elm->scheduler_id), 4264 le16toh(elm->element_info)); 4265 } 4266 4267 elm = &elms[0]; 4268 4269 sc->sc_uplink_seid = elm->uplink_seid; 4270 sc->sc_downlink_seid = elm->downlink_seid; 4271 sc->sc_seid = elm->seid; 4272 4273 if ((sc->sc_uplink_seid == htole16(0)) != 4274 (sc->sc_downlink_seid == htole16(0))) { 4275 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 4276 rv = -1; 4277 goto done; 4278 } 4279 4280 done: 4281 ixl_dmamem_free(sc, &idm); 4282 return rv; 4283 } 4284 4285 static int 4286 ixl_phy_mask_ints(struct ixl_softc *sc) 4287 { 4288 struct ixl_aq_desc iaq; 4289 4290 memset(&iaq, 0, sizeof(iaq)); 4291 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 4292 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 4293 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 4294 IXL_AQ_PHY_EV_MEDIA_NA)); 4295 4296 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4297 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 4298 return -1; 4299 } 4300 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4301 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 4302 return -1; 4303 } 4304 4305 return 0; 4306 } 4307 4308 static int 4309 ixl_get_phy_abilities(struct ixl_softc *sc, struct ixl_dmamem *idm) 4310 { 4311 struct ixl_aq_desc iaq; 4312 int rv; 4313 4314 memset(&iaq, 0, sizeof(iaq)); 4315 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4316 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4317 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 4318 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 4319 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 4320 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 4321 4322 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4323 BUS_DMASYNC_PREREAD); 4324 4325 rv = ixl_atq_poll(sc, &iaq, 250); 4326 4327 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4328 BUS_DMASYNC_POSTREAD); 4329 4330 if (rv != 0) 4331 return -1; 4332 4333 return le16toh(iaq.iaq_retval); 4334 } 4335 4336 static int 4337 ixl_get_phy_info(struct ixl_softc *sc) 4338 { 4339 struct ixl_dmamem idm; 4340 struct ixl_aq_phy_abilities *phy; 4341 int rv; 4342 4343 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4344 aprint_error_dev(sc->sc_dev, 4345 "unable to allocate phy abilities buffer\n"); 4346 return -1; 4347 } 4348 4349 rv = ixl_get_phy_abilities(sc, &idm); 4350 switch (rv) { 4351 case -1: 4352 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 4353 goto done; 4354 case IXL_AQ_RC_OK: 4355 break; 4356 case IXL_AQ_RC_EIO: 4357 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 4358 goto done; 4359 default: 4360 aprint_error_dev(sc->sc_dev, 4361 "GET PHY ABILITIIES error %u\n", rv); 4362 goto done; 4363 } 4364 4365 phy = IXL_DMA_KVA(&idm); 4366 4367 sc->sc_phy_types = le32toh(phy->phy_type); 4368 sc->sc_phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 4369 4370 sc->sc_phy_abilities = phy->abilities; 4371 sc->sc_phy_linkspeed = phy->link_speed; 4372 sc->sc_phy_fec_cfg = phy->fec_cfg_curr_mod_ext_info & 4373 (IXL_AQ_ENABLE_FEC_KR | IXL_AQ_ENABLE_FEC_RS | 4374 IXL_AQ_REQUEST_FEC_KR | IXL_AQ_REQUEST_FEC_RS); 4375 sc->sc_eee_cap = phy->eee_capability; 4376 sc->sc_eeer_val = phy->eeer_val; 4377 sc->sc_d3_lpan = phy->d3_lpan; 4378 4379 rv = 0; 4380 4381 done: 4382 ixl_dmamem_free(sc, &idm); 4383 return rv; 4384 } 4385 4386 static int 4387 ixl_set_phy_config(struct ixl_softc *sc, 4388 uint8_t link_speed, uint8_t abilities, bool polling) 4389 { 4390 struct ixl_aq_phy_param *param; 4391 struct ixl_atq iatq; 4392 struct ixl_aq_desc *iaq; 4393 int error; 4394 4395 memset(&iatq, 0, sizeof(iatq)); 4396 4397 iaq = &iatq.iatq_desc; 4398 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_CONFIG); 4399 param = (struct ixl_aq_phy_param *)&iaq->iaq_param; 4400 param->phy_types = htole32((uint32_t)sc->sc_phy_types); 4401 param->phy_type_ext = (uint8_t)(sc->sc_phy_types >> 32); 4402 param->link_speed = link_speed; 4403 param->abilities = abilities | IXL_AQ_PHY_ABILITY_AUTO_LINK; 4404 param->fec_cfg = sc->sc_phy_fec_cfg; 4405 param->eee_capability = sc->sc_eee_cap; 4406 param->eeer_val = sc->sc_eeer_val; 4407 param->d3_lpan = sc->sc_d3_lpan; 4408 4409 if (polling) 4410 error = ixl_atq_poll(sc, iaq, 250); 4411 else 4412 error = ixl_atq_exec(sc, &iatq); 4413 4414 if (error != 0) 4415 return error; 4416 4417 switch (le16toh(iaq->iaq_retval)) { 4418 case IXL_AQ_RC_OK: 4419 break; 4420 case IXL_AQ_RC_EPERM: 4421 return EPERM; 4422 default: 4423 return EIO; 4424 } 4425 4426 return 0; 4427 } 4428 4429 static int 4430 ixl_set_phy_autoselect(struct ixl_softc *sc) 4431 { 4432 uint8_t link_speed, abilities; 4433 4434 link_speed = sc->sc_phy_linkspeed; 4435 abilities = IXL_PHY_ABILITY_LINKUP | IXL_PHY_ABILITY_AUTONEGO; 4436 4437 return ixl_set_phy_config(sc, link_speed, abilities, true); 4438 } 4439 4440 static int 4441 ixl_get_link_status_poll(struct ixl_softc *sc, int *l) 4442 { 4443 struct ixl_aq_desc iaq; 4444 struct ixl_aq_link_param *param; 4445 int link; 4446 4447 memset(&iaq, 0, sizeof(iaq)); 4448 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 4449 param = (struct ixl_aq_link_param *)iaq.iaq_param; 4450 param->notify = IXL_AQ_LINK_NOTIFY; 4451 4452 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4453 return ETIMEDOUT; 4454 } 4455 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4456 return EIO; 4457 } 4458 4459 /* It is unneccessary to hold lock */ 4460 link = ixl_set_link_status_locked(sc, &iaq); 4461 4462 if (l != NULL) 4463 *l = link; 4464 4465 return 0; 4466 } 4467 4468 static int 4469 ixl_get_vsi(struct ixl_softc *sc) 4470 { 4471 struct ixl_dmamem *vsi = &sc->sc_scratch; 4472 struct ixl_aq_desc iaq; 4473 struct ixl_aq_vsi_param *param; 4474 struct ixl_aq_vsi_reply *reply; 4475 struct ixl_aq_vsi_data *data; 4476 int rv; 4477 4478 /* grumble, vsi info isn't "known" at compile time */ 4479 4480 memset(&iaq, 0, sizeof(iaq)); 4481 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4482 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4483 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 4484 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4485 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4486 4487 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4488 param->uplink_seid = sc->sc_seid; 4489 4490 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4491 BUS_DMASYNC_PREREAD); 4492 4493 rv = ixl_atq_poll(sc, &iaq, 250); 4494 4495 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4496 BUS_DMASYNC_POSTREAD); 4497 4498 if (rv != 0) { 4499 return ETIMEDOUT; 4500 } 4501 4502 switch (le16toh(iaq.iaq_retval)) { 4503 case IXL_AQ_RC_OK: 4504 break; 4505 case IXL_AQ_RC_ENOENT: 4506 return ENOENT; 4507 case IXL_AQ_RC_EACCES: 4508 return EACCES; 4509 default: 4510 return EIO; 4511 } 4512 4513 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 4514 sc->sc_vsi_number = le16toh(reply->vsi_number); 4515 data = IXL_DMA_KVA(vsi); 4516 sc->sc_vsi_stat_counter_idx = le16toh(data->stat_counter_idx); 4517 4518 return 0; 4519 } 4520 4521 static int 4522 ixl_set_vsi(struct ixl_softc *sc) 4523 { 4524 struct ixl_dmamem *vsi = &sc->sc_scratch; 4525 struct ixl_aq_desc iaq; 4526 struct ixl_aq_vsi_param *param; 4527 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 4528 unsigned int qnum; 4529 uint16_t val; 4530 int rv; 4531 4532 qnum = sc->sc_nqueue_pairs - 1; 4533 4534 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 4535 IXL_AQ_VSI_VALID_VLAN); 4536 4537 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 4538 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 4539 data->queue_mapping[0] = htole16(0); 4540 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 4541 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 4542 4543 val = le16toh(data->port_vlan_flags); 4544 CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK); 4545 SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL); 4546 4547 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { 4548 SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH); 4549 } else { 4550 SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING); 4551 } 4552 4553 data->port_vlan_flags = htole16(val); 4554 4555 /* grumble, vsi info isn't "known" at compile time */ 4556 4557 memset(&iaq, 0, sizeof(iaq)); 4558 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4559 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4560 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 4561 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4562 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4563 4564 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4565 param->uplink_seid = sc->sc_seid; 4566 4567 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4568 BUS_DMASYNC_PREWRITE); 4569 4570 rv = ixl_atq_poll(sc, &iaq, 250); 4571 4572 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4573 BUS_DMASYNC_POSTWRITE); 4574 4575 if (rv != 0) { 4576 return ETIMEDOUT; 4577 } 4578 4579 switch (le16toh(iaq.iaq_retval)) { 4580 case IXL_AQ_RC_OK: 4581 break; 4582 case IXL_AQ_RC_ENOENT: 4583 return ENOENT; 4584 case IXL_AQ_RC_EACCES: 4585 return EACCES; 4586 default: 4587 return EIO; 4588 } 4589 4590 return 0; 4591 } 4592 4593 static void 4594 ixl_set_filter_control(struct ixl_softc *sc) 4595 { 4596 uint32_t reg; 4597 4598 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4599 4600 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4601 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4602 4603 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4604 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4605 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4606 4607 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4608 } 4609 4610 static inline void 4611 ixl_get_default_rss_key(uint32_t *buf, size_t len) 4612 { 4613 size_t cplen; 4614 uint8_t rss_seed[RSS_KEYSIZE]; 4615 4616 rss_getkey(rss_seed); 4617 memset(buf, 0, len); 4618 4619 cplen = MIN(len, sizeof(rss_seed)); 4620 memcpy(buf, rss_seed, cplen); 4621 } 4622 4623 static int 4624 ixl_set_rss_key(struct ixl_softc *sc, uint8_t *key, size_t keylen) 4625 { 4626 struct ixl_dmamem *idm; 4627 struct ixl_atq iatq; 4628 struct ixl_aq_desc *iaq; 4629 struct ixl_aq_rss_key_param *param; 4630 struct ixl_aq_rss_key_data *data; 4631 size_t len, datalen, stdlen, extlen; 4632 uint16_t vsi_id; 4633 int rv; 4634 4635 memset(&iatq, 0, sizeof(iatq)); 4636 iaq = &iatq.iatq_desc; 4637 idm = &sc->sc_aqbuf; 4638 4639 datalen = sizeof(*data); 4640 4641 /*XXX The buf size has to be less than the size of the register */ 4642 datalen = MIN(IXL_RSS_KEY_SIZE_REG * sizeof(uint32_t), datalen); 4643 4644 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4645 (datalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4646 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_KEY); 4647 iaq->iaq_datalen = htole16(datalen); 4648 4649 param = (struct ixl_aq_rss_key_param *)iaq->iaq_param; 4650 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSKEY_VSI_ID_SHIFT) | 4651 IXL_AQ_RSSKEY_VSI_VALID; 4652 param->vsi_id = htole16(vsi_id); 4653 4654 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4655 data = IXL_DMA_KVA(idm); 4656 4657 len = MIN(keylen, datalen); 4658 stdlen = MIN(sizeof(data->standard_rss_key), len); 4659 memcpy(data->standard_rss_key, key, stdlen); 4660 len = (len > stdlen) ? (len - stdlen) : 0; 4661 4662 extlen = MIN(sizeof(data->extended_hash_key), len); 4663 extlen = (stdlen < keylen) ? 0 : keylen - stdlen; 4664 memcpy(data->extended_hash_key, key + stdlen, extlen); 4665 4666 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4667 4668 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4669 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4670 4671 rv = ixl_atq_exec(sc, &iatq); 4672 4673 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4674 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4675 4676 if (rv != 0) { 4677 return ETIMEDOUT; 4678 } 4679 4680 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4681 return EIO; 4682 } 4683 4684 return 0; 4685 } 4686 4687 static int 4688 ixl_set_rss_lut(struct ixl_softc *sc, uint8_t *lut, size_t lutlen) 4689 { 4690 struct ixl_dmamem *idm; 4691 struct ixl_atq iatq; 4692 struct ixl_aq_desc *iaq; 4693 struct ixl_aq_rss_lut_param *param; 4694 uint16_t vsi_id; 4695 uint8_t *data; 4696 size_t dmalen; 4697 int rv; 4698 4699 memset(&iatq, 0, sizeof(iatq)); 4700 iaq = &iatq.iatq_desc; 4701 idm = &sc->sc_aqbuf; 4702 4703 dmalen = MIN(lutlen, IXL_DMA_LEN(idm)); 4704 4705 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4706 (dmalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4707 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_LUT); 4708 iaq->iaq_datalen = htole16(dmalen); 4709 4710 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4711 data = IXL_DMA_KVA(idm); 4712 memcpy(data, lut, dmalen); 4713 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4714 4715 param = (struct ixl_aq_rss_lut_param *)iaq->iaq_param; 4716 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSLUT_VSI_ID_SHIFT) | 4717 IXL_AQ_RSSLUT_VSI_VALID; 4718 param->vsi_id = htole16(vsi_id); 4719 param->flags = htole16(IXL_AQ_RSSLUT_TABLE_TYPE_PF << 4720 IXL_AQ_RSSLUT_TABLE_TYPE_SHIFT); 4721 4722 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4723 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4724 4725 rv = ixl_atq_exec(sc, &iatq); 4726 4727 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4728 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4729 4730 if (rv != 0) { 4731 return ETIMEDOUT; 4732 } 4733 4734 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4735 return EIO; 4736 } 4737 4738 return 0; 4739 } 4740 4741 static int 4742 ixl_register_rss_key(struct ixl_softc *sc) 4743 { 4744 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4745 int rv; 4746 size_t i; 4747 4748 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4749 4750 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4751 rv = ixl_set_rss_key(sc, (uint8_t*)rss_seed, 4752 sizeof(rss_seed)); 4753 } else { 4754 rv = 0; 4755 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4756 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4757 } 4758 } 4759 4760 return rv; 4761 } 4762 4763 static void 4764 ixl_register_rss_pctype(struct ixl_softc *sc) 4765 { 4766 uint64_t set_hena = 0; 4767 uint32_t hena0, hena1; 4768 4769 /* 4770 * We use TCP/UDP with IPv4/IPv6 by default. 4771 * Note: the device can not use just IP header in each 4772 * TCP/UDP packets for the RSS hash calculation. 4773 */ 4774 if (sc->sc_mac_type == I40E_MAC_X722) 4775 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4776 else 4777 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4778 4779 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4780 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4781 4782 SET(hena0, set_hena); 4783 SET(hena1, set_hena >> 32); 4784 4785 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4786 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4787 } 4788 4789 static int 4790 ixl_register_rss_hlut(struct ixl_softc *sc) 4791 { 4792 unsigned int qid; 4793 uint8_t hlut_buf[512], lut_mask; 4794 uint32_t *hluts; 4795 size_t i, hluts_num; 4796 int rv; 4797 4798 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4799 4800 for (i = 0; i < sc->sc_rss_table_size; i++) { 4801 qid = i % sc->sc_nqueue_pairs; 4802 hlut_buf[i] = qid & lut_mask; 4803 } 4804 4805 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4806 rv = ixl_set_rss_lut(sc, hlut_buf, sizeof(hlut_buf)); 4807 } else { 4808 rv = 0; 4809 hluts = (uint32_t *)hlut_buf; 4810 hluts_num = sc->sc_rss_table_size >> 2; 4811 for (i = 0; i < hluts_num; i++) { 4812 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4813 } 4814 ixl_flush(sc); 4815 } 4816 4817 return rv; 4818 } 4819 4820 static void 4821 ixl_config_rss(struct ixl_softc *sc) 4822 { 4823 4824 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4825 4826 ixl_register_rss_key(sc); 4827 ixl_register_rss_pctype(sc); 4828 ixl_register_rss_hlut(sc); 4829 } 4830 4831 static const struct ixl_phy_type * 4832 ixl_search_phy_type(uint8_t phy_type) 4833 { 4834 const struct ixl_phy_type *itype; 4835 uint64_t mask; 4836 unsigned int i; 4837 4838 if (phy_type >= 64) 4839 return NULL; 4840 4841 mask = 1ULL << phy_type; 4842 4843 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4844 itype = &ixl_phy_type_map[i]; 4845 4846 if (ISSET(itype->phy_type, mask)) 4847 return itype; 4848 } 4849 4850 return NULL; 4851 } 4852 4853 static uint64_t 4854 ixl_search_link_speed(uint8_t link_speed) 4855 { 4856 const struct ixl_speed_type *type; 4857 unsigned int i; 4858 4859 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4860 type = &ixl_speed_type_map[i]; 4861 4862 if (ISSET(type->dev_speed, link_speed)) 4863 return type->net_speed; 4864 } 4865 4866 return 0; 4867 } 4868 4869 static uint8_t 4870 ixl_search_baudrate(uint64_t baudrate) 4871 { 4872 const struct ixl_speed_type *type; 4873 unsigned int i; 4874 4875 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4876 type = &ixl_speed_type_map[i]; 4877 4878 if (type->net_speed == baudrate) { 4879 return type->dev_speed; 4880 } 4881 } 4882 4883 return 0; 4884 } 4885 4886 static int 4887 ixl_restart_an(struct ixl_softc *sc) 4888 { 4889 struct ixl_aq_desc iaq; 4890 4891 memset(&iaq, 0, sizeof(iaq)); 4892 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4893 iaq.iaq_param[0] = 4894 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4895 4896 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4897 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4898 return -1; 4899 } 4900 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4901 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4902 return -1; 4903 } 4904 4905 return 0; 4906 } 4907 4908 static int 4909 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4910 uint16_t vlan, uint16_t flags) 4911 { 4912 struct ixl_aq_desc iaq; 4913 struct ixl_aq_add_macvlan *param; 4914 struct ixl_aq_add_macvlan_elem *elem; 4915 4916 memset(&iaq, 0, sizeof(iaq)); 4917 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4918 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4919 iaq.iaq_datalen = htole16(sizeof(*elem)); 4920 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4921 4922 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4923 param->num_addrs = htole16(1); 4924 param->seid0 = htole16(0x8000) | sc->sc_seid; 4925 param->seid1 = 0; 4926 param->seid2 = 0; 4927 4928 elem = IXL_DMA_KVA(&sc->sc_scratch); 4929 memset(elem, 0, sizeof(*elem)); 4930 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4931 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4932 elem->vlan = htole16(vlan); 4933 4934 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4935 return IXL_AQ_RC_EINVAL; 4936 } 4937 4938 switch (le16toh(iaq.iaq_retval)) { 4939 case IXL_AQ_RC_OK: 4940 break; 4941 case IXL_AQ_RC_ENOSPC: 4942 return ENOSPC; 4943 case IXL_AQ_RC_ENOENT: 4944 return ENOENT; 4945 case IXL_AQ_RC_EACCES: 4946 return EACCES; 4947 case IXL_AQ_RC_EEXIST: 4948 return EEXIST; 4949 case IXL_AQ_RC_EINVAL: 4950 return EINVAL; 4951 default: 4952 return EIO; 4953 } 4954 4955 return 0; 4956 } 4957 4958 static int 4959 ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4960 uint16_t vlan, uint16_t flags) 4961 { 4962 struct ixl_aq_desc iaq; 4963 struct ixl_aq_remove_macvlan *param; 4964 struct ixl_aq_remove_macvlan_elem *elem; 4965 4966 memset(&iaq, 0, sizeof(iaq)); 4967 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4968 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 4969 iaq.iaq_datalen = htole16(sizeof(*elem)); 4970 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4971 4972 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 4973 param->num_addrs = htole16(1); 4974 param->seid0 = htole16(0x8000) | sc->sc_seid; 4975 param->seid1 = 0; 4976 param->seid2 = 0; 4977 4978 elem = IXL_DMA_KVA(&sc->sc_scratch); 4979 memset(elem, 0, sizeof(*elem)); 4980 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4981 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 4982 elem->vlan = htole16(vlan); 4983 4984 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4985 return EINVAL; 4986 } 4987 4988 switch (le16toh(iaq.iaq_retval)) { 4989 case IXL_AQ_RC_OK: 4990 break; 4991 case IXL_AQ_RC_ENOENT: 4992 return ENOENT; 4993 case IXL_AQ_RC_EACCES: 4994 return EACCES; 4995 case IXL_AQ_RC_EINVAL: 4996 return EINVAL; 4997 default: 4998 return EIO; 4999 } 5000 5001 return 0; 5002 } 5003 5004 static int 5005 ixl_hmc(struct ixl_softc *sc) 5006 { 5007 struct { 5008 uint32_t count; 5009 uint32_t minsize; 5010 bus_size_t objsiz; 5011 bus_size_t setoff; 5012 bus_size_t setcnt; 5013 } regs[] = { 5014 { 5015 0, 5016 IXL_HMC_TXQ_MINSIZE, 5017 I40E_GLHMC_LANTXOBJSZ, 5018 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 5019 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 5020 }, 5021 { 5022 0, 5023 IXL_HMC_RXQ_MINSIZE, 5024 I40E_GLHMC_LANRXOBJSZ, 5025 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 5026 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 5027 }, 5028 { 5029 0, 5030 0, 5031 I40E_GLHMC_FCOEDDPOBJSZ, 5032 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 5033 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 5034 }, 5035 { 5036 0, 5037 0, 5038 I40E_GLHMC_FCOEFOBJSZ, 5039 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 5040 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 5041 }, 5042 }; 5043 struct ixl_hmc_entry *e; 5044 uint64_t size, dva; 5045 uint8_t *kva; 5046 uint64_t *sdpage; 5047 unsigned int i; 5048 int npages, tables; 5049 uint32_t reg; 5050 5051 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 5052 5053 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 5054 ixl_rd(sc, I40E_GLHMC_LANQMAX); 5055 5056 size = 0; 5057 for (i = 0; i < __arraycount(regs); i++) { 5058 e = &sc->sc_hmc_entries[i]; 5059 5060 e->hmc_count = regs[i].count; 5061 reg = ixl_rd(sc, regs[i].objsiz); 5062 e->hmc_size = IXL_BIT_ULL(0x3F & reg); 5063 e->hmc_base = size; 5064 5065 if ((e->hmc_size * 8) < regs[i].minsize) { 5066 aprint_error_dev(sc->sc_dev, 5067 "kernel hmc entry is too big\n"); 5068 return -1; 5069 } 5070 5071 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 5072 } 5073 size = roundup(size, IXL_HMC_PGSIZE); 5074 npages = size / IXL_HMC_PGSIZE; 5075 5076 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 5077 5078 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 5079 aprint_error_dev(sc->sc_dev, 5080 "unable to allocate hmc pd memory\n"); 5081 return -1; 5082 } 5083 5084 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 5085 IXL_HMC_PGSIZE) != 0) { 5086 aprint_error_dev(sc->sc_dev, 5087 "unable to allocate hmc sd memory\n"); 5088 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5089 return -1; 5090 } 5091 5092 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 5093 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 5094 5095 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 5096 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 5097 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5098 5099 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 5100 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 5101 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 5102 5103 for (i = 0; (int)i < npages; i++) { 5104 *sdpage = htole64(dva | IXL_HMC_PDVALID); 5105 sdpage++; 5106 5107 dva += IXL_HMC_PGSIZE; 5108 } 5109 5110 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 5111 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 5112 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5113 5114 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 5115 for (i = 0; (int)i < tables; i++) { 5116 uint32_t count; 5117 5118 KASSERT(npages >= 0); 5119 5120 count = ((unsigned int)npages > IXL_HMC_PGS) ? 5121 IXL_HMC_PGS : (unsigned int)npages; 5122 5123 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 5124 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 5125 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 5126 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 5127 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 5128 ixl_wr(sc, I40E_PFHMC_SDCMD, 5129 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 5130 5131 npages -= IXL_HMC_PGS; 5132 dva += IXL_HMC_PGSIZE; 5133 } 5134 5135 for (i = 0; i < __arraycount(regs); i++) { 5136 e = &sc->sc_hmc_entries[i]; 5137 5138 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 5139 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 5140 } 5141 5142 return 0; 5143 } 5144 5145 static void 5146 ixl_hmc_free(struct ixl_softc *sc) 5147 { 5148 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 5149 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5150 } 5151 5152 static void 5153 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 5154 unsigned int npacking) 5155 { 5156 uint8_t *dst = d; 5157 const uint8_t *src = s; 5158 unsigned int i; 5159 5160 for (i = 0; i < npacking; i++) { 5161 const struct ixl_hmc_pack *pack = &packing[i]; 5162 unsigned int offset = pack->lsb / 8; 5163 unsigned int align = pack->lsb % 8; 5164 const uint8_t *in = src + pack->offset; 5165 uint8_t *out = dst + offset; 5166 int width = pack->width; 5167 unsigned int inbits = 0; 5168 5169 if (align) { 5170 inbits = (*in++) << align; 5171 *out++ |= (inbits & 0xff); 5172 inbits >>= 8; 5173 5174 width -= 8 - align; 5175 } 5176 5177 while (width >= 8) { 5178 inbits |= (*in++) << align; 5179 *out++ = (inbits & 0xff); 5180 inbits >>= 8; 5181 5182 width -= 8; 5183 } 5184 5185 if (width > 0) { 5186 inbits |= (*in) << align; 5187 *out |= (inbits & ((1 << width) - 1)); 5188 } 5189 } 5190 } 5191 5192 static struct ixl_aq_buf * 5193 ixl_aqb_alloc(struct ixl_softc *sc) 5194 { 5195 struct ixl_aq_buf *aqb; 5196 5197 aqb = kmem_alloc(sizeof(*aqb), KM_SLEEP); 5198 5199 aqb->aqb_size = IXL_AQ_BUFLEN; 5200 5201 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 5202 aqb->aqb_size, 0, 5203 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 5204 goto free; 5205 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 5206 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 5207 BUS_DMA_WAITOK) != 0) 5208 goto destroy; 5209 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 5210 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 5211 goto dma_free; 5212 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 5213 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 5214 goto unmap; 5215 5216 return aqb; 5217 unmap: 5218 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5219 dma_free: 5220 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5221 destroy: 5222 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5223 free: 5224 kmem_free(aqb, sizeof(*aqb)); 5225 5226 return NULL; 5227 } 5228 5229 static void 5230 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 5231 { 5232 5233 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 5234 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5235 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5236 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5237 kmem_free(aqb, sizeof(*aqb)); 5238 } 5239 5240 static int 5241 ixl_arq_fill(struct ixl_softc *sc) 5242 { 5243 struct ixl_aq_buf *aqb; 5244 struct ixl_aq_desc *arq, *iaq; 5245 unsigned int prod = sc->sc_arq_prod; 5246 unsigned int n; 5247 int post = 0; 5248 5249 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 5250 IXL_AQ_NUM); 5251 arq = IXL_DMA_KVA(&sc->sc_arq); 5252 5253 if (__predict_false(n <= 0)) 5254 return 0; 5255 5256 do { 5257 aqb = sc->sc_arq_live[prod]; 5258 iaq = &arq[prod]; 5259 5260 if (aqb == NULL) { 5261 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 5262 if (aqb != NULL) { 5263 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5264 ixl_aq_buf, aqb_entry); 5265 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 5266 break; 5267 } 5268 5269 sc->sc_arq_live[prod] = aqb; 5270 memset(aqb->aqb_data, 0, aqb->aqb_size); 5271 5272 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 5273 aqb->aqb_size, BUS_DMASYNC_PREREAD); 5274 5275 iaq->iaq_flags = htole16(IXL_AQ_BUF | 5276 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 5277 IXL_AQ_LB : 0)); 5278 iaq->iaq_opcode = 0; 5279 iaq->iaq_datalen = htole16(aqb->aqb_size); 5280 iaq->iaq_retval = 0; 5281 iaq->iaq_cookie = 0; 5282 iaq->iaq_param[0] = 0; 5283 iaq->iaq_param[1] = 0; 5284 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 5285 } 5286 5287 prod++; 5288 prod &= IXL_AQ_MASK; 5289 5290 post = 1; 5291 5292 } while (--n); 5293 5294 if (post) { 5295 sc->sc_arq_prod = prod; 5296 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 5297 } 5298 5299 return post; 5300 } 5301 5302 static void 5303 ixl_arq_unfill(struct ixl_softc *sc) 5304 { 5305 struct ixl_aq_buf *aqb; 5306 unsigned int i; 5307 5308 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 5309 aqb = sc->sc_arq_live[i]; 5310 if (aqb == NULL) 5311 continue; 5312 5313 sc->sc_arq_live[i] = NULL; 5314 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 5315 BUS_DMASYNC_POSTREAD); 5316 ixl_aqb_free(sc, aqb); 5317 } 5318 5319 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 5320 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5321 ixl_aq_buf, aqb_entry); 5322 ixl_aqb_free(sc, aqb); 5323 } 5324 } 5325 5326 static void 5327 ixl_clear_hw(struct ixl_softc *sc) 5328 { 5329 uint32_t num_queues, base_queue; 5330 uint32_t num_pf_int; 5331 uint32_t num_vf_int; 5332 uint32_t num_vfs; 5333 uint32_t i, j; 5334 uint32_t val; 5335 uint32_t eol = 0x7ff; 5336 5337 /* get number of interrupts, queues, and vfs */ 5338 val = ixl_rd(sc, I40E_GLPCI_CNF2); 5339 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 5340 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 5341 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 5342 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 5343 5344 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 5345 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 5346 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 5347 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 5348 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 5349 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 5350 num_queues = (j - base_queue) + 1; 5351 else 5352 num_queues = 0; 5353 5354 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 5355 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 5356 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 5357 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 5358 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 5359 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 5360 num_vfs = (j - i) + 1; 5361 else 5362 num_vfs = 0; 5363 5364 /* stop all the interrupts */ 5365 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5366 ixl_flush(sc); 5367 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 5368 for (i = 0; i < num_pf_int - 2; i++) 5369 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 5370 ixl_flush(sc); 5371 5372 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 5373 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5374 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 5375 for (i = 0; i < num_pf_int - 2; i++) 5376 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 5377 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5378 for (i = 0; i < num_vfs; i++) 5379 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 5380 for (i = 0; i < num_vf_int - 2; i++) 5381 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 5382 5383 /* warn the HW of the coming Tx disables */ 5384 for (i = 0; i < num_queues; i++) { 5385 uint32_t abs_queue_idx = base_queue + i; 5386 uint32_t reg_block = 0; 5387 5388 if (abs_queue_idx >= 128) { 5389 reg_block = abs_queue_idx / 128; 5390 abs_queue_idx %= 128; 5391 } 5392 5393 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 5394 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 5395 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 5396 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 5397 5398 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 5399 } 5400 delaymsec(400); 5401 5402 /* stop all the queues */ 5403 for (i = 0; i < num_queues; i++) { 5404 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 5405 ixl_wr(sc, I40E_QTX_ENA(i), 0); 5406 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 5407 ixl_wr(sc, I40E_QRX_ENA(i), 0); 5408 } 5409 5410 /* short wait for all queue disables to settle */ 5411 delaymsec(50); 5412 } 5413 5414 static int 5415 ixl_pf_reset(struct ixl_softc *sc) 5416 { 5417 uint32_t cnt = 0; 5418 uint32_t cnt1 = 0; 5419 uint32_t reg = 0, reg0 = 0; 5420 uint32_t grst_del; 5421 5422 /* 5423 * Poll for Global Reset steady state in case of recent GRST. 5424 * The grst delay value is in 100ms units, and we'll wait a 5425 * couple counts longer to be sure we don't just miss the end. 5426 */ 5427 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 5428 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 5429 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 5430 5431 grst_del = grst_del * 20; 5432 5433 for (cnt = 0; cnt < grst_del; cnt++) { 5434 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 5435 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 5436 break; 5437 delaymsec(100); 5438 } 5439 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5440 aprint_error(", Global reset polling failed to complete\n"); 5441 return -1; 5442 } 5443 5444 /* Now Wait for the FW to be ready */ 5445 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 5446 reg = ixl_rd(sc, I40E_GLNVM_ULD); 5447 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5448 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 5449 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5450 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 5451 break; 5452 5453 delaymsec(10); 5454 } 5455 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5456 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 5457 aprint_error(", wait for FW Reset complete timed out " 5458 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 5459 return -1; 5460 } 5461 5462 /* 5463 * If there was a Global Reset in progress when we got here, 5464 * we don't need to do the PF Reset 5465 */ 5466 if (cnt == 0) { 5467 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5468 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 5469 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 5470 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5471 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 5472 break; 5473 delaymsec(1); 5474 5475 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 5476 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5477 aprint_error(", Core reset upcoming." 5478 " Skipping PF reset reset request\n"); 5479 return -1; 5480 } 5481 } 5482 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 5483 aprint_error(", PF reset polling failed to complete" 5484 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 5485 return -1; 5486 } 5487 } 5488 5489 return 0; 5490 } 5491 5492 static int 5493 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 5494 bus_size_t size, bus_size_t align) 5495 { 5496 ixm->ixm_size = size; 5497 5498 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 5499 ixm->ixm_size, 0, 5500 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 5501 &ixm->ixm_map) != 0) 5502 return 1; 5503 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 5504 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 5505 BUS_DMA_WAITOK) != 0) 5506 goto destroy; 5507 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 5508 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 5509 goto free; 5510 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 5511 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 5512 goto unmap; 5513 5514 memset(ixm->ixm_kva, 0, ixm->ixm_size); 5515 5516 return 0; 5517 unmap: 5518 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5519 free: 5520 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5521 destroy: 5522 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5523 return 1; 5524 } 5525 5526 static void 5527 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 5528 { 5529 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 5530 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5531 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5532 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5533 } 5534 5535 static int 5536 ixl_setup_vlan_hwfilter(struct ixl_softc *sc) 5537 { 5538 struct ethercom *ec = &sc->sc_ec; 5539 struct vlanid_list *vlanidp; 5540 int rv; 5541 5542 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5543 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5544 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5545 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5546 5547 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5548 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5549 if (rv != 0) 5550 return rv; 5551 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5552 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5553 if (rv != 0) 5554 return rv; 5555 5556 ETHER_LOCK(ec); 5557 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5558 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 5559 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5560 if (rv != 0) 5561 break; 5562 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 5563 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5564 if (rv != 0) 5565 break; 5566 } 5567 ETHER_UNLOCK(ec); 5568 5569 return rv; 5570 } 5571 5572 static void 5573 ixl_teardown_vlan_hwfilter(struct ixl_softc *sc) 5574 { 5575 struct vlanid_list *vlanidp; 5576 struct ethercom *ec = &sc->sc_ec; 5577 5578 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5579 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5580 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5581 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5582 5583 ETHER_LOCK(ec); 5584 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5585 ixl_remove_macvlan(sc, sc->sc_enaddr, 5586 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5587 ixl_remove_macvlan(sc, etherbroadcastaddr, 5588 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5589 } 5590 ETHER_UNLOCK(ec); 5591 5592 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5593 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5594 ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5595 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5596 } 5597 5598 static int 5599 ixl_update_macvlan(struct ixl_softc *sc) 5600 { 5601 int rv = 0; 5602 int next_ec_capenable = sc->sc_ec.ec_capenable; 5603 5604 if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 5605 rv = ixl_setup_vlan_hwfilter(sc); 5606 if (rv != 0) 5607 ixl_teardown_vlan_hwfilter(sc); 5608 } else { 5609 ixl_teardown_vlan_hwfilter(sc); 5610 } 5611 5612 return rv; 5613 } 5614 5615 static int 5616 ixl_ifflags_cb(struct ethercom *ec) 5617 { 5618 struct ifnet *ifp = &ec->ec_if; 5619 struct ixl_softc *sc = ifp->if_softc; 5620 int rv, change; 5621 5622 mutex_enter(&sc->sc_cfg_lock); 5623 5624 change = ec->ec_capenable ^ sc->sc_cur_ec_capenable; 5625 5626 if (ISSET(change, ETHERCAP_VLAN_HWTAGGING)) { 5627 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 5628 rv = ENETRESET; 5629 goto out; 5630 } 5631 5632 if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) { 5633 rv = ixl_update_macvlan(sc); 5634 if (rv == 0) { 5635 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 5636 } else { 5637 CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER); 5638 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 5639 } 5640 } 5641 5642 rv = ixl_iff(sc); 5643 out: 5644 mutex_exit(&sc->sc_cfg_lock); 5645 5646 return rv; 5647 } 5648 5649 static int 5650 ixl_set_link_status_locked(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 5651 { 5652 const struct ixl_aq_link_status *status; 5653 const struct ixl_phy_type *itype; 5654 5655 uint64_t ifm_active = IFM_ETHER; 5656 uint64_t ifm_status = IFM_AVALID; 5657 int link_state = LINK_STATE_DOWN; 5658 uint64_t baudrate = 0; 5659 5660 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 5661 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) { 5662 ifm_active |= IFM_NONE; 5663 goto done; 5664 } 5665 5666 ifm_active |= IFM_FDX; 5667 ifm_status |= IFM_ACTIVE; 5668 link_state = LINK_STATE_UP; 5669 5670 itype = ixl_search_phy_type(status->phy_type); 5671 if (itype != NULL) 5672 ifm_active |= itype->ifm_type; 5673 5674 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 5675 ifm_active |= IFM_ETH_TXPAUSE; 5676 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 5677 ifm_active |= IFM_ETH_RXPAUSE; 5678 5679 baudrate = ixl_search_link_speed(status->link_speed); 5680 5681 done: 5682 /* sc->sc_cfg_lock held expect during attach */ 5683 sc->sc_media_active = ifm_active; 5684 sc->sc_media_status = ifm_status; 5685 5686 sc->sc_ec.ec_if.if_baudrate = baudrate; 5687 5688 return link_state; 5689 } 5690 5691 static int 5692 ixl_establish_intx(struct ixl_softc *sc) 5693 { 5694 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5695 pci_intr_handle_t *intr; 5696 char xnamebuf[32]; 5697 char intrbuf[PCI_INTRSTR_LEN]; 5698 char const *intrstr; 5699 5700 KASSERT(sc->sc_nintrs == 1); 5701 5702 intr = &sc->sc_ihp[0]; 5703 5704 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 5705 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 5706 device_xname(sc->sc_dev)); 5707 5708 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 5709 sc, xnamebuf); 5710 5711 if (sc->sc_ihs[0] == NULL) { 5712 aprint_error_dev(sc->sc_dev, 5713 "unable to establish interrupt at %s\n", intrstr); 5714 return -1; 5715 } 5716 5717 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 5718 return 0; 5719 } 5720 5721 static int 5722 ixl_establish_msix(struct ixl_softc *sc) 5723 { 5724 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5725 kcpuset_t *affinity; 5726 unsigned int vector = 0; 5727 unsigned int i; 5728 int affinity_to, r; 5729 char xnamebuf[32]; 5730 char intrbuf[PCI_INTRSTR_LEN]; 5731 char const *intrstr; 5732 5733 kcpuset_create(&affinity, false); 5734 5735 /* the "other" intr is mapped to vector 0 */ 5736 vector = 0; 5737 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5738 intrbuf, sizeof(intrbuf)); 5739 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 5740 device_xname(sc->sc_dev)); 5741 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5742 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 5743 sc, xnamebuf); 5744 if (sc->sc_ihs[vector] == NULL) { 5745 aprint_error_dev(sc->sc_dev, 5746 "unable to establish interrupt at %s\n", intrstr); 5747 goto fail; 5748 } 5749 5750 aprint_normal_dev(sc->sc_dev, "other interrupt at %s", intrstr); 5751 5752 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5753 affinity_to = (affinity_to + sc->sc_nqueue_pairs_max) % ncpu; 5754 5755 kcpuset_zero(affinity); 5756 kcpuset_set(affinity, affinity_to); 5757 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5758 if (r == 0) { 5759 aprint_normal(", affinity to %u", affinity_to); 5760 } 5761 aprint_normal("\n"); 5762 vector++; 5763 5764 sc->sc_msix_vector_queue = vector; 5765 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5766 5767 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5768 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5769 intrbuf, sizeof(intrbuf)); 5770 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 5771 device_xname(sc->sc_dev), i); 5772 5773 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5774 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 5775 (void *)&sc->sc_qps[i], xnamebuf); 5776 5777 if (sc->sc_ihs[vector] == NULL) { 5778 aprint_error_dev(sc->sc_dev, 5779 "unable to establish interrupt at %s\n", intrstr); 5780 goto fail; 5781 } 5782 5783 aprint_normal_dev(sc->sc_dev, 5784 "for TXRX%d interrupt at %s", i, intrstr); 5785 5786 kcpuset_zero(affinity); 5787 kcpuset_set(affinity, affinity_to); 5788 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5789 if (r == 0) { 5790 aprint_normal(", affinity to %u", affinity_to); 5791 affinity_to = (affinity_to + 1) % ncpu; 5792 } 5793 aprint_normal("\n"); 5794 vector++; 5795 } 5796 5797 kcpuset_destroy(affinity); 5798 5799 return 0; 5800 fail: 5801 for (i = 0; i < vector; i++) { 5802 pci_intr_disestablish(pc, sc->sc_ihs[i]); 5803 } 5804 5805 sc->sc_msix_vector_queue = 0; 5806 sc->sc_msix_vector_queue = 0; 5807 kcpuset_destroy(affinity); 5808 5809 return -1; 5810 } 5811 5812 static void 5813 ixl_config_queue_intr(struct ixl_softc *sc) 5814 { 5815 unsigned int i, vector; 5816 5817 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5818 vector = sc->sc_msix_vector_queue; 5819 } else { 5820 vector = I40E_INTR_NOTX_INTR; 5821 5822 ixl_wr(sc, I40E_PFINT_LNKLST0, 5823 (I40E_INTR_NOTX_QUEUE << 5824 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5825 (I40E_QUEUE_TYPE_RX << 5826 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5827 } 5828 5829 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5830 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5831 ixl_flush(sc); 5832 5833 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5834 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5835 (I40E_QUEUE_TYPE_RX << 5836 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5837 5838 ixl_wr(sc, I40E_QINT_RQCTL(i), 5839 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5840 (I40E_ITR_INDEX_RX << 5841 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5842 (I40E_INTR_NOTX_RX_QUEUE << 5843 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5844 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5845 (I40E_QUEUE_TYPE_TX << 5846 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5847 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5848 5849 ixl_wr(sc, I40E_QINT_TQCTL(i), 5850 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5851 (I40E_ITR_INDEX_TX << 5852 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5853 (I40E_INTR_NOTX_TX_QUEUE << 5854 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5855 (I40E_QUEUE_TYPE_EOL << 5856 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5857 (I40E_QUEUE_TYPE_RX << 5858 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5859 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5860 5861 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5862 ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_RX, i), 5863 sc->sc_itr_rx); 5864 ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_TX, i), 5865 sc->sc_itr_tx); 5866 vector++; 5867 } 5868 } 5869 ixl_flush(sc); 5870 5871 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), sc->sc_itr_rx); 5872 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), sc->sc_itr_tx); 5873 ixl_flush(sc); 5874 } 5875 5876 static void 5877 ixl_config_other_intr(struct ixl_softc *sc) 5878 { 5879 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5880 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5881 5882 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5883 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5884 I40E_PFINT_ICR0_ENA_GRST_MASK | 5885 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5886 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5887 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5888 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5889 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5890 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5891 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5892 5893 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5894 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5895 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5896 (I40E_ITR_INDEX_OTHER << 5897 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5898 ixl_flush(sc); 5899 } 5900 5901 static int 5902 ixl_setup_interrupts(struct ixl_softc *sc) 5903 { 5904 struct pci_attach_args *pa = &sc->sc_pa; 5905 pci_intr_type_t max_type, intr_type; 5906 int counts[PCI_INTR_TYPE_SIZE]; 5907 int error; 5908 unsigned int i; 5909 bool retry; 5910 5911 memset(counts, 0, sizeof(counts)); 5912 max_type = PCI_INTR_TYPE_MSIX; 5913 /* QPs + other interrupt */ 5914 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5915 counts[PCI_INTR_TYPE_INTX] = 1; 5916 5917 if (ixl_param_nomsix) 5918 counts[PCI_INTR_TYPE_MSIX] = 0; 5919 5920 do { 5921 retry = false; 5922 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5923 if (error != 0) { 5924 aprint_error_dev(sc->sc_dev, 5925 "couldn't map interrupt\n"); 5926 break; 5927 } 5928 5929 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5930 sc->sc_nintrs = counts[intr_type]; 5931 KASSERT(sc->sc_nintrs > 0); 5932 5933 for (i = 0; i < sc->sc_nintrs; i++) { 5934 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5935 PCI_INTR_MPSAFE, true); 5936 } 5937 5938 sc->sc_ihs = kmem_zalloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5939 KM_SLEEP); 5940 5941 if (intr_type == PCI_INTR_TYPE_MSIX) { 5942 error = ixl_establish_msix(sc); 5943 if (error) { 5944 counts[PCI_INTR_TYPE_MSIX] = 0; 5945 retry = true; 5946 } 5947 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5948 error = ixl_establish_intx(sc); 5949 } else { 5950 error = -1; 5951 } 5952 5953 if (error) { 5954 kmem_free(sc->sc_ihs, 5955 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5956 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5957 } else { 5958 sc->sc_intrtype = intr_type; 5959 } 5960 } while (retry); 5961 5962 return error; 5963 } 5964 5965 static void 5966 ixl_teardown_interrupts(struct ixl_softc *sc) 5967 { 5968 struct pci_attach_args *pa = &sc->sc_pa; 5969 unsigned int i; 5970 5971 for (i = 0; i < sc->sc_nintrs; i++) { 5972 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 5973 } 5974 5975 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5976 5977 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5978 sc->sc_ihs = NULL; 5979 sc->sc_nintrs = 0; 5980 } 5981 5982 static int 5983 ixl_setup_stats(struct ixl_softc *sc) 5984 { 5985 struct ixl_queue_pair *qp; 5986 struct ixl_tx_ring *txr; 5987 struct ixl_rx_ring *rxr; 5988 struct ixl_stats_counters *isc; 5989 unsigned int i; 5990 5991 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5992 qp = &sc->sc_qps[i]; 5993 txr = qp->qp_txr; 5994 rxr = qp->qp_rxr; 5995 5996 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 5997 NULL, qp->qp_name, "m_defrag successed"); 5998 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 5999 NULL, qp->qp_name, "m_defrag_failed"); 6000 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 6001 NULL, qp->qp_name, "Dropped in pcq"); 6002 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 6003 NULL, qp->qp_name, "Deferred transmit"); 6004 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 6005 NULL, qp->qp_name, "Interrupt on queue"); 6006 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 6007 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6008 6009 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 6010 NULL, qp->qp_name, "MGETHDR failed"); 6011 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 6012 NULL, qp->qp_name, "MCLGET failed"); 6013 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 6014 EVCNT_TYPE_MISC, NULL, qp->qp_name, 6015 "bus_dmamap_load_mbuf failed"); 6016 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 6017 NULL, qp->qp_name, "Interrupt on queue"); 6018 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 6019 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6020 } 6021 6022 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 6023 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 6024 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 6025 NULL, device_xname(sc->sc_dev), "Link status event"); 6026 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 6027 NULL, device_xname(sc->sc_dev), "ECC error"); 6028 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 6029 NULL, device_xname(sc->sc_dev), "PCI exception"); 6030 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 6031 NULL, device_xname(sc->sc_dev), "Critical error"); 6032 6033 isc = &sc->sc_stats_counters; 6034 evcnt_attach_dynamic(&isc->isc_crc_errors, EVCNT_TYPE_MISC, 6035 NULL, device_xname(sc->sc_dev), "CRC errors"); 6036 evcnt_attach_dynamic(&isc->isc_illegal_bytes, EVCNT_TYPE_MISC, 6037 NULL, device_xname(sc->sc_dev), "Illegal bytes"); 6038 evcnt_attach_dynamic(&isc->isc_mac_local_faults, EVCNT_TYPE_MISC, 6039 NULL, device_xname(sc->sc_dev), "Mac local faults"); 6040 evcnt_attach_dynamic(&isc->isc_mac_remote_faults, EVCNT_TYPE_MISC, 6041 NULL, device_xname(sc->sc_dev), "Mac remote faults"); 6042 evcnt_attach_dynamic(&isc->isc_link_xon_rx, EVCNT_TYPE_MISC, 6043 NULL, device_xname(sc->sc_dev), "Rx xon"); 6044 evcnt_attach_dynamic(&isc->isc_link_xon_tx, EVCNT_TYPE_MISC, 6045 NULL, device_xname(sc->sc_dev), "Tx xon"); 6046 evcnt_attach_dynamic(&isc->isc_link_xoff_rx, EVCNT_TYPE_MISC, 6047 NULL, device_xname(sc->sc_dev), "Rx xoff"); 6048 evcnt_attach_dynamic(&isc->isc_link_xoff_tx, EVCNT_TYPE_MISC, 6049 NULL, device_xname(sc->sc_dev), "Tx xoff"); 6050 evcnt_attach_dynamic(&isc->isc_rx_fragments, EVCNT_TYPE_MISC, 6051 NULL, device_xname(sc->sc_dev), "Rx fragments"); 6052 evcnt_attach_dynamic(&isc->isc_rx_jabber, EVCNT_TYPE_MISC, 6053 NULL, device_xname(sc->sc_dev), "Rx jabber"); 6054 6055 evcnt_attach_dynamic(&isc->isc_rx_size_64, EVCNT_TYPE_MISC, 6056 NULL, device_xname(sc->sc_dev), "Rx size 64"); 6057 evcnt_attach_dynamic(&isc->isc_rx_size_127, EVCNT_TYPE_MISC, 6058 NULL, device_xname(sc->sc_dev), "Rx size 127"); 6059 evcnt_attach_dynamic(&isc->isc_rx_size_255, EVCNT_TYPE_MISC, 6060 NULL, device_xname(sc->sc_dev), "Rx size 255"); 6061 evcnt_attach_dynamic(&isc->isc_rx_size_511, EVCNT_TYPE_MISC, 6062 NULL, device_xname(sc->sc_dev), "Rx size 511"); 6063 evcnt_attach_dynamic(&isc->isc_rx_size_1023, EVCNT_TYPE_MISC, 6064 NULL, device_xname(sc->sc_dev), "Rx size 1023"); 6065 evcnt_attach_dynamic(&isc->isc_rx_size_1522, EVCNT_TYPE_MISC, 6066 NULL, device_xname(sc->sc_dev), "Rx size 1522"); 6067 evcnt_attach_dynamic(&isc->isc_rx_size_big, EVCNT_TYPE_MISC, 6068 NULL, device_xname(sc->sc_dev), "Rx jumbo packets"); 6069 evcnt_attach_dynamic(&isc->isc_rx_undersize, EVCNT_TYPE_MISC, 6070 NULL, device_xname(sc->sc_dev), "Rx under size"); 6071 evcnt_attach_dynamic(&isc->isc_rx_oversize, EVCNT_TYPE_MISC, 6072 NULL, device_xname(sc->sc_dev), "Rx over size"); 6073 6074 evcnt_attach_dynamic(&isc->isc_rx_bytes, EVCNT_TYPE_MISC, 6075 NULL, device_xname(sc->sc_dev), "Rx bytes / port"); 6076 evcnt_attach_dynamic(&isc->isc_rx_discards, EVCNT_TYPE_MISC, 6077 NULL, device_xname(sc->sc_dev), "Rx discards / port"); 6078 evcnt_attach_dynamic(&isc->isc_rx_unicast, EVCNT_TYPE_MISC, 6079 NULL, device_xname(sc->sc_dev), "Rx unicast / port"); 6080 evcnt_attach_dynamic(&isc->isc_rx_multicast, EVCNT_TYPE_MISC, 6081 NULL, device_xname(sc->sc_dev), "Rx multicast / port"); 6082 evcnt_attach_dynamic(&isc->isc_rx_broadcast, EVCNT_TYPE_MISC, 6083 NULL, device_xname(sc->sc_dev), "Rx broadcast / port"); 6084 6085 evcnt_attach_dynamic(&isc->isc_vsi_rx_bytes, EVCNT_TYPE_MISC, 6086 NULL, device_xname(sc->sc_dev), "Rx bytes / vsi"); 6087 evcnt_attach_dynamic(&isc->isc_vsi_rx_discards, EVCNT_TYPE_MISC, 6088 NULL, device_xname(sc->sc_dev), "Rx discards / vsi"); 6089 evcnt_attach_dynamic(&isc->isc_vsi_rx_unicast, EVCNT_TYPE_MISC, 6090 NULL, device_xname(sc->sc_dev), "Rx unicast / vsi"); 6091 evcnt_attach_dynamic(&isc->isc_vsi_rx_multicast, EVCNT_TYPE_MISC, 6092 NULL, device_xname(sc->sc_dev), "Rx multicast / vsi"); 6093 evcnt_attach_dynamic(&isc->isc_vsi_rx_broadcast, EVCNT_TYPE_MISC, 6094 NULL, device_xname(sc->sc_dev), "Rx broadcast / vsi"); 6095 6096 evcnt_attach_dynamic(&isc->isc_tx_size_64, EVCNT_TYPE_MISC, 6097 NULL, device_xname(sc->sc_dev), "Tx size 64"); 6098 evcnt_attach_dynamic(&isc->isc_tx_size_127, EVCNT_TYPE_MISC, 6099 NULL, device_xname(sc->sc_dev), "Tx size 127"); 6100 evcnt_attach_dynamic(&isc->isc_tx_size_255, EVCNT_TYPE_MISC, 6101 NULL, device_xname(sc->sc_dev), "Tx size 255"); 6102 evcnt_attach_dynamic(&isc->isc_tx_size_511, EVCNT_TYPE_MISC, 6103 NULL, device_xname(sc->sc_dev), "Tx size 511"); 6104 evcnt_attach_dynamic(&isc->isc_tx_size_1023, EVCNT_TYPE_MISC, 6105 NULL, device_xname(sc->sc_dev), "Tx size 1023"); 6106 evcnt_attach_dynamic(&isc->isc_tx_size_1522, EVCNT_TYPE_MISC, 6107 NULL, device_xname(sc->sc_dev), "Tx size 1522"); 6108 evcnt_attach_dynamic(&isc->isc_tx_size_big, EVCNT_TYPE_MISC, 6109 NULL, device_xname(sc->sc_dev), "Tx jumbo packets"); 6110 6111 evcnt_attach_dynamic(&isc->isc_tx_bytes, EVCNT_TYPE_MISC, 6112 NULL, device_xname(sc->sc_dev), "Tx bytes / port"); 6113 evcnt_attach_dynamic(&isc->isc_tx_dropped_link_down, EVCNT_TYPE_MISC, 6114 NULL, device_xname(sc->sc_dev), 6115 "Tx dropped due to link down / port"); 6116 evcnt_attach_dynamic(&isc->isc_tx_unicast, EVCNT_TYPE_MISC, 6117 NULL, device_xname(sc->sc_dev), "Tx unicast / port"); 6118 evcnt_attach_dynamic(&isc->isc_tx_multicast, EVCNT_TYPE_MISC, 6119 NULL, device_xname(sc->sc_dev), "Tx multicast / port"); 6120 evcnt_attach_dynamic(&isc->isc_tx_broadcast, EVCNT_TYPE_MISC, 6121 NULL, device_xname(sc->sc_dev), "Tx broadcast / port"); 6122 6123 evcnt_attach_dynamic(&isc->isc_vsi_tx_bytes, EVCNT_TYPE_MISC, 6124 NULL, device_xname(sc->sc_dev), "Tx bytes / vsi"); 6125 evcnt_attach_dynamic(&isc->isc_vsi_tx_errors, EVCNT_TYPE_MISC, 6126 NULL, device_xname(sc->sc_dev), "Tx errors / vsi"); 6127 evcnt_attach_dynamic(&isc->isc_vsi_tx_unicast, EVCNT_TYPE_MISC, 6128 NULL, device_xname(sc->sc_dev), "Tx unicast / vsi"); 6129 evcnt_attach_dynamic(&isc->isc_vsi_tx_multicast, EVCNT_TYPE_MISC, 6130 NULL, device_xname(sc->sc_dev), "Tx multicast / vsi"); 6131 evcnt_attach_dynamic(&isc->isc_vsi_tx_broadcast, EVCNT_TYPE_MISC, 6132 NULL, device_xname(sc->sc_dev), "Tx broadcast / vsi"); 6133 6134 sc->sc_stats_intval = ixl_param_stats_interval; 6135 callout_init(&sc->sc_stats_callout, CALLOUT_MPSAFE); 6136 callout_setfunc(&sc->sc_stats_callout, ixl_stats_callout, sc); 6137 ixl_work_set(&sc->sc_stats_task, ixl_stats_update, sc); 6138 6139 return 0; 6140 } 6141 6142 static void 6143 ixl_teardown_stats(struct ixl_softc *sc) 6144 { 6145 struct ixl_tx_ring *txr; 6146 struct ixl_rx_ring *rxr; 6147 struct ixl_stats_counters *isc; 6148 unsigned int i; 6149 6150 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6151 txr = sc->sc_qps[i].qp_txr; 6152 rxr = sc->sc_qps[i].qp_rxr; 6153 6154 evcnt_detach(&txr->txr_defragged); 6155 evcnt_detach(&txr->txr_defrag_failed); 6156 evcnt_detach(&txr->txr_pcqdrop); 6157 evcnt_detach(&txr->txr_transmitdef); 6158 evcnt_detach(&txr->txr_intr); 6159 evcnt_detach(&txr->txr_defer); 6160 6161 evcnt_detach(&rxr->rxr_mgethdr_failed); 6162 evcnt_detach(&rxr->rxr_mgetcl_failed); 6163 evcnt_detach(&rxr->rxr_mbuf_load_failed); 6164 evcnt_detach(&rxr->rxr_intr); 6165 evcnt_detach(&rxr->rxr_defer); 6166 } 6167 6168 isc = &sc->sc_stats_counters; 6169 evcnt_detach(&isc->isc_crc_errors); 6170 evcnt_detach(&isc->isc_illegal_bytes); 6171 evcnt_detach(&isc->isc_mac_local_faults); 6172 evcnt_detach(&isc->isc_mac_remote_faults); 6173 evcnt_detach(&isc->isc_link_xon_rx); 6174 evcnt_detach(&isc->isc_link_xon_tx); 6175 evcnt_detach(&isc->isc_link_xoff_rx); 6176 evcnt_detach(&isc->isc_link_xoff_tx); 6177 evcnt_detach(&isc->isc_rx_fragments); 6178 evcnt_detach(&isc->isc_rx_jabber); 6179 evcnt_detach(&isc->isc_rx_bytes); 6180 evcnt_detach(&isc->isc_rx_discards); 6181 evcnt_detach(&isc->isc_rx_unicast); 6182 evcnt_detach(&isc->isc_rx_multicast); 6183 evcnt_detach(&isc->isc_rx_broadcast); 6184 evcnt_detach(&isc->isc_rx_size_64); 6185 evcnt_detach(&isc->isc_rx_size_127); 6186 evcnt_detach(&isc->isc_rx_size_255); 6187 evcnt_detach(&isc->isc_rx_size_511); 6188 evcnt_detach(&isc->isc_rx_size_1023); 6189 evcnt_detach(&isc->isc_rx_size_1522); 6190 evcnt_detach(&isc->isc_rx_size_big); 6191 evcnt_detach(&isc->isc_rx_undersize); 6192 evcnt_detach(&isc->isc_rx_oversize); 6193 evcnt_detach(&isc->isc_tx_bytes); 6194 evcnt_detach(&isc->isc_tx_dropped_link_down); 6195 evcnt_detach(&isc->isc_tx_unicast); 6196 evcnt_detach(&isc->isc_tx_multicast); 6197 evcnt_detach(&isc->isc_tx_broadcast); 6198 evcnt_detach(&isc->isc_tx_size_64); 6199 evcnt_detach(&isc->isc_tx_size_127); 6200 evcnt_detach(&isc->isc_tx_size_255); 6201 evcnt_detach(&isc->isc_tx_size_511); 6202 evcnt_detach(&isc->isc_tx_size_1023); 6203 evcnt_detach(&isc->isc_tx_size_1522); 6204 evcnt_detach(&isc->isc_tx_size_big); 6205 evcnt_detach(&isc->isc_vsi_rx_discards); 6206 evcnt_detach(&isc->isc_vsi_rx_bytes); 6207 evcnt_detach(&isc->isc_vsi_rx_unicast); 6208 evcnt_detach(&isc->isc_vsi_rx_multicast); 6209 evcnt_detach(&isc->isc_vsi_rx_broadcast); 6210 evcnt_detach(&isc->isc_vsi_tx_errors); 6211 evcnt_detach(&isc->isc_vsi_tx_bytes); 6212 evcnt_detach(&isc->isc_vsi_tx_unicast); 6213 evcnt_detach(&isc->isc_vsi_tx_multicast); 6214 evcnt_detach(&isc->isc_vsi_tx_broadcast); 6215 6216 evcnt_detach(&sc->sc_event_atq); 6217 evcnt_detach(&sc->sc_event_link); 6218 evcnt_detach(&sc->sc_event_ecc_err); 6219 evcnt_detach(&sc->sc_event_pci_exception); 6220 evcnt_detach(&sc->sc_event_crit_err); 6221 6222 callout_destroy(&sc->sc_stats_callout); 6223 } 6224 6225 static void 6226 ixl_stats_callout(void *xsc) 6227 { 6228 struct ixl_softc *sc = xsc; 6229 6230 ixl_work_add(sc->sc_workq, &sc->sc_stats_task); 6231 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 6232 } 6233 6234 static uint64_t 6235 ixl_stat_delta(struct ixl_softc *sc, uint32_t reg_hi, uint32_t reg_lo, 6236 uint64_t *offset, bool has_offset) 6237 { 6238 uint64_t value, delta; 6239 int bitwidth; 6240 6241 bitwidth = reg_hi == 0 ? 32 : 48; 6242 6243 value = ixl_rd(sc, reg_lo); 6244 6245 if (bitwidth > 32) { 6246 value |= ((uint64_t)ixl_rd(sc, reg_hi) << 32); 6247 } 6248 6249 if (__predict_true(has_offset)) { 6250 delta = value; 6251 if (value < *offset) 6252 delta += ((uint64_t)1 << bitwidth); 6253 delta -= *offset; 6254 } else { 6255 delta = 0; 6256 } 6257 atomic_swap_64(offset, value); 6258 6259 return delta; 6260 } 6261 6262 static void 6263 ixl_stats_update(void *xsc) 6264 { 6265 struct ixl_softc *sc = xsc; 6266 struct ixl_stats_counters *isc; 6267 uint64_t delta; 6268 6269 isc = &sc->sc_stats_counters; 6270 6271 /* errors */ 6272 delta = ixl_stat_delta(sc, 6273 0, I40E_GLPRT_CRCERRS(sc->sc_port), 6274 &isc->isc_crc_errors_offset, isc->isc_has_offset); 6275 atomic_add_64(&isc->isc_crc_errors.ev_count, delta); 6276 6277 delta = ixl_stat_delta(sc, 6278 0, I40E_GLPRT_ILLERRC(sc->sc_port), 6279 &isc->isc_illegal_bytes_offset, isc->isc_has_offset); 6280 atomic_add_64(&isc->isc_illegal_bytes.ev_count, delta); 6281 6282 /* rx */ 6283 delta = ixl_stat_delta(sc, 6284 I40E_GLPRT_GORCH(sc->sc_port), I40E_GLPRT_GORCL(sc->sc_port), 6285 &isc->isc_rx_bytes_offset, isc->isc_has_offset); 6286 atomic_add_64(&isc->isc_rx_bytes.ev_count, delta); 6287 6288 delta = ixl_stat_delta(sc, 6289 0, I40E_GLPRT_RDPC(sc->sc_port), 6290 &isc->isc_rx_discards_offset, isc->isc_has_offset); 6291 atomic_add_64(&isc->isc_rx_discards.ev_count, delta); 6292 6293 delta = ixl_stat_delta(sc, 6294 I40E_GLPRT_UPRCH(sc->sc_port), I40E_GLPRT_UPRCL(sc->sc_port), 6295 &isc->isc_rx_unicast_offset, isc->isc_has_offset); 6296 atomic_add_64(&isc->isc_rx_unicast.ev_count, delta); 6297 6298 delta = ixl_stat_delta(sc, 6299 I40E_GLPRT_MPRCH(sc->sc_port), I40E_GLPRT_MPRCL(sc->sc_port), 6300 &isc->isc_rx_multicast_offset, isc->isc_has_offset); 6301 atomic_add_64(&isc->isc_rx_multicast.ev_count, delta); 6302 6303 delta = ixl_stat_delta(sc, 6304 I40E_GLPRT_BPRCH(sc->sc_port), I40E_GLPRT_BPRCL(sc->sc_port), 6305 &isc->isc_rx_broadcast_offset, isc->isc_has_offset); 6306 atomic_add_64(&isc->isc_rx_broadcast.ev_count, delta); 6307 6308 /* Packet size stats rx */ 6309 delta = ixl_stat_delta(sc, 6310 I40E_GLPRT_PRC64H(sc->sc_port), I40E_GLPRT_PRC64L(sc->sc_port), 6311 &isc->isc_rx_size_64_offset, isc->isc_has_offset); 6312 atomic_add_64(&isc->isc_rx_size_64.ev_count, delta); 6313 6314 delta = ixl_stat_delta(sc, 6315 I40E_GLPRT_PRC127H(sc->sc_port), I40E_GLPRT_PRC127L(sc->sc_port), 6316 &isc->isc_rx_size_127_offset, isc->isc_has_offset); 6317 atomic_add_64(&isc->isc_rx_size_127.ev_count, delta); 6318 6319 delta = ixl_stat_delta(sc, 6320 I40E_GLPRT_PRC255H(sc->sc_port), I40E_GLPRT_PRC255L(sc->sc_port), 6321 &isc->isc_rx_size_255_offset, isc->isc_has_offset); 6322 atomic_add_64(&isc->isc_rx_size_255.ev_count, delta); 6323 6324 delta = ixl_stat_delta(sc, 6325 I40E_GLPRT_PRC511H(sc->sc_port), I40E_GLPRT_PRC511L(sc->sc_port), 6326 &isc->isc_rx_size_511_offset, isc->isc_has_offset); 6327 atomic_add_64(&isc->isc_rx_size_511.ev_count, delta); 6328 6329 delta = ixl_stat_delta(sc, 6330 I40E_GLPRT_PRC1023H(sc->sc_port), I40E_GLPRT_PRC1023L(sc->sc_port), 6331 &isc->isc_rx_size_1023_offset, isc->isc_has_offset); 6332 atomic_add_64(&isc->isc_rx_size_1023.ev_count, delta); 6333 6334 delta = ixl_stat_delta(sc, 6335 I40E_GLPRT_PRC1522H(sc->sc_port), I40E_GLPRT_PRC1522L(sc->sc_port), 6336 &isc->isc_rx_size_1522_offset, isc->isc_has_offset); 6337 atomic_add_64(&isc->isc_rx_size_1522.ev_count, delta); 6338 6339 delta = ixl_stat_delta(sc, 6340 I40E_GLPRT_PRC9522H(sc->sc_port), I40E_GLPRT_PRC9522L(sc->sc_port), 6341 &isc->isc_rx_size_big_offset, isc->isc_has_offset); 6342 atomic_add_64(&isc->isc_rx_size_big.ev_count, delta); 6343 6344 delta = ixl_stat_delta(sc, 6345 0, I40E_GLPRT_RUC(sc->sc_port), 6346 &isc->isc_rx_undersize_offset, isc->isc_has_offset); 6347 atomic_add_64(&isc->isc_rx_undersize.ev_count, delta); 6348 6349 delta = ixl_stat_delta(sc, 6350 0, I40E_GLPRT_ROC(sc->sc_port), 6351 &isc->isc_rx_oversize_offset, isc->isc_has_offset); 6352 atomic_add_64(&isc->isc_rx_oversize.ev_count, delta); 6353 6354 /* tx */ 6355 delta = ixl_stat_delta(sc, 6356 I40E_GLPRT_GOTCH(sc->sc_port), I40E_GLPRT_GOTCL(sc->sc_port), 6357 &isc->isc_tx_bytes_offset, isc->isc_has_offset); 6358 atomic_add_64(&isc->isc_tx_bytes.ev_count, delta); 6359 6360 delta = ixl_stat_delta(sc, 6361 0, I40E_GLPRT_TDOLD(sc->sc_port), 6362 &isc->isc_tx_dropped_link_down_offset, isc->isc_has_offset); 6363 atomic_add_64(&isc->isc_tx_dropped_link_down.ev_count, delta); 6364 6365 delta = ixl_stat_delta(sc, 6366 I40E_GLPRT_UPTCH(sc->sc_port), I40E_GLPRT_UPTCL(sc->sc_port), 6367 &isc->isc_tx_unicast_offset, isc->isc_has_offset); 6368 atomic_add_64(&isc->isc_tx_unicast.ev_count, delta); 6369 6370 delta = ixl_stat_delta(sc, 6371 I40E_GLPRT_MPTCH(sc->sc_port), I40E_GLPRT_MPTCL(sc->sc_port), 6372 &isc->isc_tx_multicast_offset, isc->isc_has_offset); 6373 atomic_add_64(&isc->isc_tx_multicast.ev_count, delta); 6374 6375 delta = ixl_stat_delta(sc, 6376 I40E_GLPRT_BPTCH(sc->sc_port), I40E_GLPRT_BPTCL(sc->sc_port), 6377 &isc->isc_tx_broadcast_offset, isc->isc_has_offset); 6378 atomic_add_64(&isc->isc_tx_broadcast.ev_count, delta); 6379 6380 /* Packet size stats tx */ 6381 delta = ixl_stat_delta(sc, 6382 I40E_GLPRT_PTC64L(sc->sc_port), I40E_GLPRT_PTC64L(sc->sc_port), 6383 &isc->isc_tx_size_64_offset, isc->isc_has_offset); 6384 atomic_add_64(&isc->isc_tx_size_64.ev_count, delta); 6385 6386 delta = ixl_stat_delta(sc, 6387 I40E_GLPRT_PTC127H(sc->sc_port), I40E_GLPRT_PTC127L(sc->sc_port), 6388 &isc->isc_tx_size_127_offset, isc->isc_has_offset); 6389 atomic_add_64(&isc->isc_tx_size_127.ev_count, delta); 6390 6391 delta = ixl_stat_delta(sc, 6392 I40E_GLPRT_PTC255H(sc->sc_port), I40E_GLPRT_PTC255L(sc->sc_port), 6393 &isc->isc_tx_size_255_offset, isc->isc_has_offset); 6394 atomic_add_64(&isc->isc_tx_size_255.ev_count, delta); 6395 6396 delta = ixl_stat_delta(sc, 6397 I40E_GLPRT_PTC511H(sc->sc_port), I40E_GLPRT_PTC511L(sc->sc_port), 6398 &isc->isc_tx_size_511_offset, isc->isc_has_offset); 6399 atomic_add_64(&isc->isc_tx_size_511.ev_count, delta); 6400 6401 delta = ixl_stat_delta(sc, 6402 I40E_GLPRT_PTC1023H(sc->sc_port), I40E_GLPRT_PTC1023L(sc->sc_port), 6403 &isc->isc_tx_size_1023_offset, isc->isc_has_offset); 6404 atomic_add_64(&isc->isc_tx_size_1023.ev_count, delta); 6405 6406 delta = ixl_stat_delta(sc, 6407 I40E_GLPRT_PTC1522H(sc->sc_port), I40E_GLPRT_PTC1522L(sc->sc_port), 6408 &isc->isc_tx_size_1522_offset, isc->isc_has_offset); 6409 atomic_add_64(&isc->isc_tx_size_1522.ev_count, delta); 6410 6411 delta = ixl_stat_delta(sc, 6412 I40E_GLPRT_PTC9522H(sc->sc_port), I40E_GLPRT_PTC9522L(sc->sc_port), 6413 &isc->isc_tx_size_big_offset, isc->isc_has_offset); 6414 atomic_add_64(&isc->isc_tx_size_big.ev_count, delta); 6415 6416 /* mac faults */ 6417 delta = ixl_stat_delta(sc, 6418 0, I40E_GLPRT_MLFC(sc->sc_port), 6419 &isc->isc_mac_local_faults_offset, isc->isc_has_offset); 6420 atomic_add_64(&isc->isc_mac_local_faults.ev_count, delta); 6421 6422 delta = ixl_stat_delta(sc, 6423 0, I40E_GLPRT_MRFC(sc->sc_port), 6424 &isc->isc_mac_remote_faults_offset, isc->isc_has_offset); 6425 atomic_add_64(&isc->isc_mac_remote_faults.ev_count, delta); 6426 6427 /* Flow control (LFC) stats */ 6428 delta = ixl_stat_delta(sc, 6429 0, I40E_GLPRT_LXONRXC(sc->sc_port), 6430 &isc->isc_link_xon_rx_offset, isc->isc_has_offset); 6431 atomic_add_64(&isc->isc_link_xon_rx.ev_count, delta); 6432 6433 delta = ixl_stat_delta(sc, 6434 0, I40E_GLPRT_LXONTXC(sc->sc_port), 6435 &isc->isc_link_xon_tx_offset, isc->isc_has_offset); 6436 atomic_add_64(&isc->isc_link_xon_tx.ev_count, delta); 6437 6438 delta = ixl_stat_delta(sc, 6439 0, I40E_GLPRT_LXOFFRXC(sc->sc_port), 6440 &isc->isc_link_xoff_rx_offset, isc->isc_has_offset); 6441 atomic_add_64(&isc->isc_link_xoff_rx.ev_count, delta); 6442 6443 delta = ixl_stat_delta(sc, 6444 0, I40E_GLPRT_LXOFFTXC(sc->sc_port), 6445 &isc->isc_link_xoff_tx_offset, isc->isc_has_offset); 6446 atomic_add_64(&isc->isc_link_xoff_tx.ev_count, delta); 6447 6448 /* fragments */ 6449 delta = ixl_stat_delta(sc, 6450 0, I40E_GLPRT_RFC(sc->sc_port), 6451 &isc->isc_rx_fragments_offset, isc->isc_has_offset); 6452 atomic_add_64(&isc->isc_rx_fragments.ev_count, delta); 6453 6454 delta = ixl_stat_delta(sc, 6455 0, I40E_GLPRT_RJC(sc->sc_port), 6456 &isc->isc_rx_jabber_offset, isc->isc_has_offset); 6457 atomic_add_64(&isc->isc_rx_jabber.ev_count, delta); 6458 6459 /* VSI rx counters */ 6460 delta = ixl_stat_delta(sc, 6461 0, I40E_GLV_RDPC(sc->sc_vsi_stat_counter_idx), 6462 &isc->isc_vsi_rx_discards_offset, isc->isc_has_offset); 6463 atomic_add_64(&isc->isc_vsi_rx_discards.ev_count, delta); 6464 6465 delta = ixl_stat_delta(sc, 6466 I40E_GLV_GORCH(sc->sc_vsi_stat_counter_idx), 6467 I40E_GLV_GORCL(sc->sc_vsi_stat_counter_idx), 6468 &isc->isc_vsi_rx_bytes_offset, isc->isc_has_offset); 6469 atomic_add_64(&isc->isc_vsi_rx_bytes.ev_count, delta); 6470 6471 delta = ixl_stat_delta(sc, 6472 I40E_GLV_UPRCH(sc->sc_vsi_stat_counter_idx), 6473 I40E_GLV_UPRCL(sc->sc_vsi_stat_counter_idx), 6474 &isc->isc_vsi_rx_unicast_offset, isc->isc_has_offset); 6475 atomic_add_64(&isc->isc_vsi_rx_unicast.ev_count, delta); 6476 6477 delta = ixl_stat_delta(sc, 6478 I40E_GLV_MPRCH(sc->sc_vsi_stat_counter_idx), 6479 I40E_GLV_MPRCL(sc->sc_vsi_stat_counter_idx), 6480 &isc->isc_vsi_rx_multicast_offset, isc->isc_has_offset); 6481 atomic_add_64(&isc->isc_vsi_rx_multicast.ev_count, delta); 6482 6483 delta = ixl_stat_delta(sc, 6484 I40E_GLV_BPRCH(sc->sc_vsi_stat_counter_idx), 6485 I40E_GLV_BPRCL(sc->sc_vsi_stat_counter_idx), 6486 &isc->isc_vsi_rx_broadcast_offset, isc->isc_has_offset); 6487 atomic_add_64(&isc->isc_vsi_rx_broadcast.ev_count, delta); 6488 6489 /* VSI tx counters */ 6490 delta = ixl_stat_delta(sc, 6491 0, I40E_GLV_TEPC(sc->sc_vsi_stat_counter_idx), 6492 &isc->isc_vsi_tx_errors_offset, isc->isc_has_offset); 6493 atomic_add_64(&isc->isc_vsi_tx_errors.ev_count, delta); 6494 6495 delta = ixl_stat_delta(sc, 6496 I40E_GLV_GOTCH(sc->sc_vsi_stat_counter_idx), 6497 I40E_GLV_GOTCL(sc->sc_vsi_stat_counter_idx), 6498 &isc->isc_vsi_tx_bytes_offset, isc->isc_has_offset); 6499 atomic_add_64(&isc->isc_vsi_tx_bytes.ev_count, delta); 6500 6501 delta = ixl_stat_delta(sc, 6502 I40E_GLV_UPTCH(sc->sc_vsi_stat_counter_idx), 6503 I40E_GLV_UPTCL(sc->sc_vsi_stat_counter_idx), 6504 &isc->isc_vsi_tx_unicast_offset, isc->isc_has_offset); 6505 atomic_add_64(&isc->isc_vsi_tx_unicast.ev_count, delta); 6506 6507 delta = ixl_stat_delta(sc, 6508 I40E_GLV_MPTCH(sc->sc_vsi_stat_counter_idx), 6509 I40E_GLV_MPTCL(sc->sc_vsi_stat_counter_idx), 6510 &isc->isc_vsi_tx_multicast_offset, isc->isc_has_offset); 6511 atomic_add_64(&isc->isc_vsi_tx_multicast.ev_count, delta); 6512 6513 delta = ixl_stat_delta(sc, 6514 I40E_GLV_BPTCH(sc->sc_vsi_stat_counter_idx), 6515 I40E_GLV_BPTCL(sc->sc_vsi_stat_counter_idx), 6516 &isc->isc_vsi_tx_broadcast_offset, isc->isc_has_offset); 6517 atomic_add_64(&isc->isc_vsi_tx_broadcast.ev_count, delta); 6518 } 6519 6520 static int 6521 ixl_setup_sysctls(struct ixl_softc *sc) 6522 { 6523 const char *devname; 6524 struct sysctllog **log; 6525 const struct sysctlnode *rnode, *rxnode, *txnode; 6526 int error; 6527 6528 log = &sc->sc_sysctllog; 6529 devname = device_xname(sc->sc_dev); 6530 6531 error = sysctl_createv(log, 0, NULL, &rnode, 6532 0, CTLTYPE_NODE, devname, 6533 SYSCTL_DESCR("ixl information and settings"), 6534 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 6535 if (error) 6536 goto out; 6537 6538 error = sysctl_createv(log, 0, &rnode, NULL, 6539 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 6540 SYSCTL_DESCR("Use workqueue for packet processing"), 6541 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 6542 if (error) 6543 goto out; 6544 6545 error = sysctl_createv(log, 0, &rnode, NULL, 6546 CTLFLAG_READONLY, CTLTYPE_INT, "stats_interval", 6547 SYSCTL_DESCR("Statistics collection interval in milliseconds"), 6548 NULL, 0, &sc->sc_stats_intval, 0, CTL_CREATE, CTL_EOL); 6549 6550 error = sysctl_createv(log, 0, &rnode, &rxnode, 6551 0, CTLTYPE_NODE, "rx", 6552 SYSCTL_DESCR("ixl information and settings for Rx"), 6553 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6554 if (error) 6555 goto out; 6556 6557 error = sysctl_createv(log, 0, &rxnode, NULL, 6558 CTLFLAG_READWRITE, CTLTYPE_INT, "itr", 6559 SYSCTL_DESCR("Interrupt Throttling"), 6560 ixl_sysctl_itr_handler, 0, 6561 (void *)sc, 0, CTL_CREATE, CTL_EOL); 6562 if (error) 6563 goto out; 6564 6565 error = sysctl_createv(log, 0, &rxnode, NULL, 6566 CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num", 6567 SYSCTL_DESCR("the number of rx descriptors"), 6568 NULL, 0, &sc->sc_rx_ring_ndescs, 0, CTL_CREATE, CTL_EOL); 6569 if (error) 6570 goto out; 6571 6572 error = sysctl_createv(log, 0, &rxnode, NULL, 6573 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6574 SYSCTL_DESCR("max number of Rx packets" 6575 " to process for interrupt processing"), 6576 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6577 if (error) 6578 goto out; 6579 6580 error = sysctl_createv(log, 0, &rxnode, NULL, 6581 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6582 SYSCTL_DESCR("max number of Rx packets" 6583 " to process for deferred processing"), 6584 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 6585 if (error) 6586 goto out; 6587 6588 error = sysctl_createv(log, 0, &rnode, &txnode, 6589 0, CTLTYPE_NODE, "tx", 6590 SYSCTL_DESCR("ixl information and settings for Tx"), 6591 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6592 if (error) 6593 goto out; 6594 6595 error = sysctl_createv(log, 0, &txnode, NULL, 6596 CTLFLAG_READWRITE, CTLTYPE_INT, "itr", 6597 SYSCTL_DESCR("Interrupt Throttling"), 6598 ixl_sysctl_itr_handler, 0, 6599 (void *)sc, 0, CTL_CREATE, CTL_EOL); 6600 if (error) 6601 goto out; 6602 6603 error = sysctl_createv(log, 0, &txnode, NULL, 6604 CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num", 6605 SYSCTL_DESCR("the number of tx descriptors"), 6606 NULL, 0, &sc->sc_tx_ring_ndescs, 0, CTL_CREATE, CTL_EOL); 6607 if (error) 6608 goto out; 6609 6610 error = sysctl_createv(log, 0, &txnode, NULL, 6611 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6612 SYSCTL_DESCR("max number of Tx packets" 6613 " to process for interrupt processing"), 6614 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6615 if (error) 6616 goto out; 6617 6618 error = sysctl_createv(log, 0, &txnode, NULL, 6619 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6620 SYSCTL_DESCR("max number of Tx packets" 6621 " to process for deferred processing"), 6622 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 6623 if (error) 6624 goto out; 6625 6626 out: 6627 if (error) { 6628 aprint_error_dev(sc->sc_dev, 6629 "unable to create sysctl node\n"); 6630 sysctl_teardown(log); 6631 } 6632 6633 return error; 6634 } 6635 6636 static void 6637 ixl_teardown_sysctls(struct ixl_softc *sc) 6638 { 6639 6640 sysctl_teardown(&sc->sc_sysctllog); 6641 } 6642 6643 static bool 6644 ixl_sysctlnode_is_rx(struct sysctlnode *node) 6645 { 6646 6647 if (strstr(node->sysctl_parent->sysctl_name, "rx") != NULL) 6648 return true; 6649 6650 return false; 6651 } 6652 6653 static int 6654 ixl_sysctl_itr_handler(SYSCTLFN_ARGS) 6655 { 6656 struct sysctlnode node = *rnode; 6657 struct ixl_softc *sc = (struct ixl_softc *)node.sysctl_data; 6658 struct ifnet *ifp = &sc->sc_ec.ec_if; 6659 uint32_t newitr, *itrptr; 6660 int error; 6661 6662 if (ixl_sysctlnode_is_rx(&node)) { 6663 itrptr = &sc->sc_itr_rx; 6664 } else { 6665 itrptr = &sc->sc_itr_tx; 6666 } 6667 6668 newitr = *itrptr; 6669 node.sysctl_data = &newitr; 6670 node.sysctl_size = sizeof(newitr); 6671 6672 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6673 6674 if (error || newp == NULL) 6675 return error; 6676 6677 /* ITRs are applied in ixl_init() for simple implementation */ 6678 if (ISSET(ifp->if_flags, IFF_RUNNING)) 6679 return EBUSY; 6680 6681 if (newitr > 0x07ff) 6682 return EINVAL; 6683 6684 *itrptr = newitr; 6685 6686 return 0; 6687 } 6688 6689 static struct workqueue * 6690 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 6691 { 6692 struct workqueue *wq; 6693 int error; 6694 6695 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 6696 prio, ipl, flags); 6697 6698 if (error) 6699 return NULL; 6700 6701 return wq; 6702 } 6703 6704 static void 6705 ixl_workq_destroy(struct workqueue *wq) 6706 { 6707 6708 workqueue_destroy(wq); 6709 } 6710 6711 static void 6712 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 6713 { 6714 6715 memset(work, 0, sizeof(*work)); 6716 work->ixw_func = func; 6717 work->ixw_arg = arg; 6718 } 6719 6720 static void 6721 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 6722 { 6723 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 6724 return; 6725 6726 kpreempt_disable(); 6727 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 6728 kpreempt_enable(); 6729 } 6730 6731 static void 6732 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 6733 { 6734 6735 workqueue_wait(wq, &work->ixw_cookie); 6736 } 6737 6738 static void 6739 ixl_workq_work(struct work *wk, void *context) 6740 { 6741 struct ixl_work *work; 6742 6743 work = container_of(wk, struct ixl_work, ixw_cookie); 6744 6745 atomic_swap_uint(&work->ixw_added, 0); 6746 work->ixw_func(work->ixw_arg); 6747 } 6748 6749 static int 6750 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 6751 { 6752 struct ixl_aq_desc iaq; 6753 6754 memset(&iaq, 0, sizeof(iaq)); 6755 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 6756 iaq.iaq_param[1] = htole32(reg); 6757 6758 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6759 return ETIMEDOUT; 6760 6761 switch (htole16(iaq.iaq_retval)) { 6762 case IXL_AQ_RC_OK: 6763 /* success */ 6764 break; 6765 case IXL_AQ_RC_EACCES: 6766 return EPERM; 6767 case IXL_AQ_RC_EAGAIN: 6768 return EAGAIN; 6769 default: 6770 return EIO; 6771 } 6772 6773 *rv = htole32(iaq.iaq_param[3]); 6774 return 0; 6775 } 6776 6777 static uint32_t 6778 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 6779 { 6780 uint32_t val; 6781 int rv, retry, retry_limit; 6782 6783 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6784 retry_limit = 5; 6785 } else { 6786 retry_limit = 0; 6787 } 6788 6789 for (retry = 0; retry < retry_limit; retry++) { 6790 rv = ixl_rx_ctl_read(sc, reg, &val); 6791 if (rv == 0) 6792 return val; 6793 else if (rv == EAGAIN) 6794 delaymsec(1); 6795 else 6796 break; 6797 } 6798 6799 val = ixl_rd(sc, reg); 6800 6801 return val; 6802 } 6803 6804 static int 6805 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6806 { 6807 struct ixl_aq_desc iaq; 6808 6809 memset(&iaq, 0, sizeof(iaq)); 6810 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 6811 iaq.iaq_param[1] = htole32(reg); 6812 iaq.iaq_param[3] = htole32(value); 6813 6814 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6815 return ETIMEDOUT; 6816 6817 switch (htole16(iaq.iaq_retval)) { 6818 case IXL_AQ_RC_OK: 6819 /* success */ 6820 break; 6821 case IXL_AQ_RC_EACCES: 6822 return EPERM; 6823 case IXL_AQ_RC_EAGAIN: 6824 return EAGAIN; 6825 default: 6826 return EIO; 6827 } 6828 6829 return 0; 6830 } 6831 6832 static void 6833 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6834 { 6835 int rv, retry, retry_limit; 6836 6837 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6838 retry_limit = 5; 6839 } else { 6840 retry_limit = 0; 6841 } 6842 6843 for (retry = 0; retry < retry_limit; retry++) { 6844 rv = ixl_rx_ctl_write(sc, reg, value); 6845 if (rv == 0) 6846 return; 6847 else if (rv == EAGAIN) 6848 delaymsec(1); 6849 else 6850 break; 6851 } 6852 6853 ixl_wr(sc, reg, value); 6854 } 6855 6856 static int 6857 ixl_nvm_lock(struct ixl_softc *sc, char rw) 6858 { 6859 struct ixl_aq_desc iaq; 6860 struct ixl_aq_req_resource_param *param; 6861 int rv; 6862 6863 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6864 return 0; 6865 6866 memset(&iaq, 0, sizeof(iaq)); 6867 iaq.iaq_opcode = htole16(IXL_AQ_OP_REQUEST_RESOURCE); 6868 6869 param = (struct ixl_aq_req_resource_param *)&iaq.iaq_param; 6870 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6871 if (rw == 'R') { 6872 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_READ); 6873 } else { 6874 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_WRITE); 6875 } 6876 6877 rv = ixl_atq_poll(sc, &iaq, 250); 6878 6879 if (rv != 0) 6880 return ETIMEDOUT; 6881 6882 switch (le16toh(iaq.iaq_retval)) { 6883 case IXL_AQ_RC_OK: 6884 break; 6885 case IXL_AQ_RC_EACCES: 6886 return EACCES; 6887 case IXL_AQ_RC_EBUSY: 6888 return EBUSY; 6889 case IXL_AQ_RC_EPERM: 6890 return EPERM; 6891 } 6892 6893 return 0; 6894 } 6895 6896 static int 6897 ixl_nvm_unlock(struct ixl_softc *sc) 6898 { 6899 struct ixl_aq_desc iaq; 6900 struct ixl_aq_rel_resource_param *param; 6901 int rv; 6902 6903 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6904 return 0; 6905 6906 memset(&iaq, 0, sizeof(iaq)); 6907 iaq.iaq_opcode = htole16(IXL_AQ_OP_RELEASE_RESOURCE); 6908 6909 param = (struct ixl_aq_rel_resource_param *)&iaq.iaq_param; 6910 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6911 6912 rv = ixl_atq_poll(sc, &iaq, 250); 6913 6914 if (rv != 0) 6915 return ETIMEDOUT; 6916 6917 switch (le16toh(iaq.iaq_retval)) { 6918 case IXL_AQ_RC_OK: 6919 break; 6920 default: 6921 return EIO; 6922 } 6923 return 0; 6924 } 6925 6926 static int 6927 ixl_srdone_poll(struct ixl_softc *sc) 6928 { 6929 int wait_count; 6930 uint32_t reg; 6931 6932 for (wait_count = 0; wait_count < IXL_SRRD_SRCTL_ATTEMPTS; 6933 wait_count++) { 6934 reg = ixl_rd(sc, I40E_GLNVM_SRCTL); 6935 if (ISSET(reg, I40E_GLNVM_SRCTL_DONE_MASK)) 6936 break; 6937 6938 delaymsec(5); 6939 } 6940 6941 if (wait_count == IXL_SRRD_SRCTL_ATTEMPTS) 6942 return -1; 6943 6944 return 0; 6945 } 6946 6947 static int 6948 ixl_nvm_read_srctl(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 6949 { 6950 uint32_t reg; 6951 6952 if (ixl_srdone_poll(sc) != 0) 6953 return ETIMEDOUT; 6954 6955 reg = ((uint32_t)offset << I40E_GLNVM_SRCTL_ADDR_SHIFT) | 6956 __BIT(I40E_GLNVM_SRCTL_START_SHIFT); 6957 ixl_wr(sc, I40E_GLNVM_SRCTL, reg); 6958 6959 if (ixl_srdone_poll(sc) != 0) { 6960 aprint_debug("NVM read error: couldn't access " 6961 "Shadow RAM address: 0x%x\n", offset); 6962 return ETIMEDOUT; 6963 } 6964 6965 reg = ixl_rd(sc, I40E_GLNVM_SRDATA); 6966 *data = (uint16_t)__SHIFTOUT(reg, I40E_GLNVM_SRDATA_RDDATA_MASK); 6967 6968 return 0; 6969 } 6970 6971 static int 6972 ixl_nvm_read_aq(struct ixl_softc *sc, uint16_t offset_word, 6973 void *data, size_t len) 6974 { 6975 struct ixl_dmamem *idm; 6976 struct ixl_aq_desc iaq; 6977 struct ixl_aq_nvm_param *param; 6978 uint32_t offset_bytes; 6979 int rv; 6980 6981 idm = &sc->sc_aqbuf; 6982 if (len > IXL_DMA_LEN(idm)) 6983 return ENOMEM; 6984 6985 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 6986 memset(&iaq, 0, sizeof(iaq)); 6987 iaq.iaq_opcode = htole16(IXL_AQ_OP_NVM_READ); 6988 iaq.iaq_flags = htole16(IXL_AQ_BUF | 6989 ((len > I40E_AQ_LARGE_BUF) ? IXL_AQ_LB : 0)); 6990 iaq.iaq_datalen = htole16(len); 6991 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 6992 6993 param = (struct ixl_aq_nvm_param *)iaq.iaq_param; 6994 param->command_flags = IXL_AQ_NVM_LAST_CMD; 6995 param->module_pointer = 0; 6996 param->length = htole16(len); 6997 offset_bytes = (uint32_t)offset_word * 2; 6998 offset_bytes &= 0x00FFFFFF; 6999 param->offset = htole32(offset_bytes); 7000 7001 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 7002 BUS_DMASYNC_PREREAD); 7003 7004 rv = ixl_atq_poll(sc, &iaq, 250); 7005 7006 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 7007 BUS_DMASYNC_POSTREAD); 7008 7009 if (rv != 0) { 7010 return ETIMEDOUT; 7011 } 7012 7013 switch (le16toh(iaq.iaq_retval)) { 7014 case IXL_AQ_RC_OK: 7015 break; 7016 case IXL_AQ_RC_EPERM: 7017 return EPERM; 7018 case IXL_AQ_RC_EINVAL: 7019 return EINVAL; 7020 case IXL_AQ_RC_EBUSY: 7021 return EBUSY; 7022 case IXL_AQ_RC_EIO: 7023 default: 7024 return EIO; 7025 } 7026 7027 memcpy(data, IXL_DMA_KVA(idm), len); 7028 7029 return 0; 7030 } 7031 7032 static int 7033 ixl_rd16_nvm(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 7034 { 7035 int error; 7036 uint16_t buf; 7037 7038 error = ixl_nvm_lock(sc, 'R'); 7039 if (error) 7040 return error; 7041 7042 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMREAD)) { 7043 error = ixl_nvm_read_aq(sc, offset, 7044 &buf, sizeof(buf)); 7045 if (error == 0) 7046 *data = le16toh(buf); 7047 } else { 7048 error = ixl_nvm_read_srctl(sc, offset, &buf); 7049 if (error == 0) 7050 *data = buf; 7051 } 7052 7053 ixl_nvm_unlock(sc); 7054 7055 return error; 7056 } 7057 7058 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 7059 7060 #ifdef _MODULE 7061 #include "ioconf.c" 7062 #endif 7063 7064 #ifdef _MODULE 7065 static void 7066 ixl_parse_modprop(prop_dictionary_t dict) 7067 { 7068 prop_object_t obj; 7069 int64_t val; 7070 uint64_t uval; 7071 7072 if (dict == NULL) 7073 return; 7074 7075 obj = prop_dictionary_get(dict, "nomsix"); 7076 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_BOOL) { 7077 ixl_param_nomsix = prop_bool_true((prop_bool_t)obj); 7078 } 7079 7080 obj = prop_dictionary_get(dict, "stats_interval"); 7081 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7082 val = prop_number_signed_value((prop_number_t)obj); 7083 7084 /* the range has no reason */ 7085 if (100 < val && val < 180000) { 7086 ixl_param_stats_interval = val; 7087 } 7088 } 7089 7090 obj = prop_dictionary_get(dict, "nqps_limit"); 7091 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7092 val = prop_number_signed_value((prop_number_t)obj); 7093 7094 if (val <= INT32_MAX) 7095 ixl_param_nqps_limit = val; 7096 } 7097 7098 obj = prop_dictionary_get(dict, "rx_ndescs"); 7099 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7100 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7101 7102 if (uval > 8) 7103 ixl_param_rx_ndescs = uval; 7104 } 7105 7106 obj = prop_dictionary_get(dict, "tx_ndescs"); 7107 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7108 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7109 7110 if (uval > IXL_TX_PKT_DESCS) 7111 ixl_param_tx_ndescs = uval; 7112 } 7113 7114 } 7115 #endif 7116 7117 static int 7118 if_ixl_modcmd(modcmd_t cmd, void *opaque) 7119 { 7120 int error = 0; 7121 7122 #ifdef _MODULE 7123 switch (cmd) { 7124 case MODULE_CMD_INIT: 7125 ixl_parse_modprop((prop_dictionary_t)opaque); 7126 error = config_init_component(cfdriver_ioconf_if_ixl, 7127 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7128 break; 7129 case MODULE_CMD_FINI: 7130 error = config_fini_component(cfdriver_ioconf_if_ixl, 7131 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7132 break; 7133 default: 7134 error = ENOTTY; 7135 break; 7136 } 7137 #endif 7138 7139 return error; 7140 } 7141
Indexes created Sun Sep 21 16:09:51 GMT 2025