if_ixl.c revision 1.10
1 /* $NetBSD: if_ixl.c,v 1.10 2019/12/20 02:04:26 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 78 #ifdef _KERNEL_OPT 79 #include "opt_net_mpsafe.h" 80 #endif 81 82 #include <sys/param.h> 83 #include <sys/types.h> 84 85 #include <sys/cpu.h> 86 #include <sys/device.h> 87 #include <sys/evcnt.h> 88 #include <sys/interrupt.h> 89 #include <sys/kmem.h> 90 #include <sys/malloc.h> 91 #include <sys/module.h> 92 #include <sys/mutex.h> 93 #include <sys/pcq.h> 94 #include <sys/syslog.h> 95 #include <sys/workqueue.h> 96 97 #include <sys/bus.h> 98 99 #include <net/bpf.h> 100 #include <net/if.h> 101 #include <net/if_dl.h> 102 #include <net/if_media.h> 103 #include <net/if_ether.h> 104 #include <net/rss_config.h> 105 106 #include <dev/pci/pcivar.h> 107 #include <dev/pci/pcidevs.h> 108 109 #include <dev/pci/if_ixlreg.h> 110 #include <dev/pci/if_ixlvar.h> 111 112 struct ixl_softc; /* defined */ 113 114 #define I40E_PF_RESET_WAIT_COUNT 200 115 #define I40E_AQ_LARGE_BUF 512 116 117 /* bitfields for Tx queue mapping in QTX_CTL */ 118 #define I40E_QTX_CTL_VF_QUEUE 0x0 119 #define I40E_QTX_CTL_VM_QUEUE 0x1 120 #define I40E_QTX_CTL_PF_QUEUE 0x2 121 122 #define I40E_QUEUE_TYPE_EOL 0x7ff 123 #define I40E_INTR_NOTX_QUEUE 0 124 125 #define I40E_QUEUE_TYPE_RX 0x0 126 #define I40E_QUEUE_TYPE_TX 0x1 127 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 128 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 129 130 #define I40E_ITR_INDEX_RX 0x0 131 #define I40E_ITR_INDEX_TX 0x1 132 #define I40E_ITR_INDEX_OTHER 0x2 133 #define I40E_ITR_INDEX_NONE 0x3 134 135 #define I40E_INTR_NOTX_QUEUE 0 136 #define I40E_INTR_NOTX_INTR 0 137 #define I40E_INTR_NOTX_RX_QUEUE 0 138 #define I40E_INTR_NOTX_TX_QUEUE 1 139 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 140 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 141 142 #define BIT_ULL(a) (1ULL << (a)) 143 #define IXL_RSS_HENA_DEFAULT_BASE \ 144 (BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 145 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 146 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 147 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 148 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 149 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 150 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 151 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 152 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 153 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 154 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 155 #define IXL_RSS_HENA_DEFAULT_XL710 IXL_RSS_HENA_DEFAULT_BASE 156 #define IXL_RSS_HENA_DEFAULT_X722 (IXL_RSS_HENA_DEFAULT_XL710 | \ 157 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 158 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 159 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 160 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 161 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 162 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 163 #define I40E_HASH_LUT_SIZE_128 0 164 #define IXL_RSS_KEY_SIZE_REG 13 165 166 #define IXL_ICR0_CRIT_ERR_MASK \ 167 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 168 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 169 I40E_PFINT_ICR0_PE_CRITERR_MASK) 170 171 #define IXL_TX_PKT_DESCS 8 172 #define IXL_TX_QUEUE_ALIGN 128 173 #define IXL_RX_QUEUE_ALIGN 128 174 175 #define IXL_HARDMTU 9712 /* 9726 - ETHER_HDR_LEN */ 176 177 #define IXL_PCIREG PCI_MAPREG_START 178 179 #define IXL_ITR0 0x0 180 #define IXL_ITR1 0x1 181 #define IXL_ITR2 0x2 182 #define IXL_NOITR 0x3 183 184 #define IXL_AQ_NUM 256 185 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 186 #define IXL_AQ_ALIGN 64 /* lol */ 187 #define IXL_AQ_BUFLEN 4096 188 189 #define IXL_HMC_ROUNDUP 512 190 #define IXL_HMC_PGSIZE 4096 191 #define IXL_HMC_DVASZ sizeof(uint64_t) 192 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 193 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 194 #define IXL_HMC_PDVALID 1ULL 195 196 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 197 198 struct ixl_aq_regs { 199 bus_size_t atq_tail; 200 bus_size_t atq_head; 201 bus_size_t atq_len; 202 bus_size_t atq_bal; 203 bus_size_t atq_bah; 204 205 bus_size_t arq_tail; 206 bus_size_t arq_head; 207 bus_size_t arq_len; 208 bus_size_t arq_bal; 209 bus_size_t arq_bah; 210 211 uint32_t atq_len_enable; 212 uint32_t atq_tail_mask; 213 uint32_t atq_head_mask; 214 215 uint32_t arq_len_enable; 216 uint32_t arq_tail_mask; 217 uint32_t arq_head_mask; 218 }; 219 220 struct ixl_phy_type { 221 uint64_t phy_type; 222 uint64_t ifm_type; 223 }; 224 225 struct ixl_speed_type { 226 uint8_t dev_speed; 227 uint64_t net_speed; 228 }; 229 230 struct ixl_aq_buf { 231 SIMPLEQ_ENTRY(ixl_aq_buf) 232 aqb_entry; 233 void *aqb_data; 234 bus_dmamap_t aqb_map; 235 bus_dma_segment_t aqb_seg; 236 size_t aqb_size; 237 int aqb_nsegs; 238 }; 239 SIMPLEQ_HEAD(ixl_aq_bufs, ixl_aq_buf); 240 241 struct ixl_dmamem { 242 bus_dmamap_t ixm_map; 243 bus_dma_segment_t ixm_seg; 244 int ixm_nsegs; 245 size_t ixm_size; 246 void *ixm_kva; 247 }; 248 249 #define IXL_DMA_MAP(_ixm) ((_ixm)->ixm_map) 250 #define IXL_DMA_DVA(_ixm) ((_ixm)->ixm_map->dm_segs[0].ds_addr) 251 #define IXL_DMA_KVA(_ixm) ((void *)(_ixm)->ixm_kva) 252 #define IXL_DMA_LEN(_ixm) ((_ixm)->ixm_size) 253 254 struct ixl_hmc_entry { 255 uint64_t hmc_base; 256 uint32_t hmc_count; 257 uint64_t hmc_size; 258 }; 259 260 enum ixl_hmc_types { 261 IXL_HMC_LAN_TX = 0, 262 IXL_HMC_LAN_RX, 263 IXL_HMC_FCOE_CTX, 264 IXL_HMC_FCOE_FILTER, 265 IXL_HMC_COUNT 266 }; 267 268 struct ixl_hmc_pack { 269 uint16_t offset; 270 uint16_t width; 271 uint16_t lsb; 272 }; 273 274 /* 275 * these hmc objects have weird sizes and alignments, so these are abstract 276 * representations of them that are nice for c to populate. 277 * 278 * the packing code relies on little-endian values being stored in the fields, 279 * no high bits in the fields being set, and the fields must be packed in the 280 * same order as they are in the ctx structure. 281 */ 282 283 struct ixl_hmc_rxq { 284 uint16_t head; 285 uint8_t cpuid; 286 uint64_t base; 287 #define IXL_HMC_RXQ_BASE_UNIT 128 288 uint16_t qlen; 289 uint16_t dbuff; 290 #define IXL_HMC_RXQ_DBUFF_UNIT 128 291 uint8_t hbuff; 292 #define IXL_HMC_RXQ_HBUFF_UNIT 64 293 uint8_t dtype; 294 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 295 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 296 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 297 uint8_t dsize; 298 #define IXL_HMC_RXQ_DSIZE_16 0 299 #define IXL_HMC_RXQ_DSIZE_32 1 300 uint8_t crcstrip; 301 uint8_t fc_ena; 302 uint8_t l2sel; 303 uint8_t hsplit_0; 304 uint8_t hsplit_1; 305 uint8_t showiv; 306 uint16_t rxmax; 307 uint8_t tphrdesc_ena; 308 uint8_t tphwdesc_ena; 309 uint8_t tphdata_ena; 310 uint8_t tphhead_ena; 311 uint8_t lrxqthresh; 312 uint8_t prefena; 313 }; 314 315 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 316 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 317 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 318 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 319 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 320 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 321 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 322 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 323 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 324 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 325 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 326 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 327 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 328 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 329 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 330 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 331 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 332 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 333 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 334 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 335 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 336 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 337 }; 338 339 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 340 341 struct ixl_hmc_txq { 342 uint16_t head; 343 uint8_t new_context; 344 uint64_t base; 345 #define IXL_HMC_TXQ_BASE_UNIT 128 346 uint8_t fc_ena; 347 uint8_t timesync_ena; 348 uint8_t fd_ena; 349 uint8_t alt_vlan_ena; 350 uint16_t thead_wb; 351 uint8_t cpuid; 352 uint8_t head_wb_ena; 353 #define IXL_HMC_TXQ_DESC_WB 0 354 #define IXL_HMC_TXQ_HEAD_WB 1 355 uint16_t qlen; 356 uint8_t tphrdesc_ena; 357 uint8_t tphrpacket_ena; 358 uint8_t tphwdesc_ena; 359 uint64_t head_wb_addr; 360 uint32_t crc; 361 uint16_t rdylist; 362 uint8_t rdylist_act; 363 }; 364 365 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 366 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 367 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 368 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 369 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 370 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 371 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 372 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 373 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 374 /* line 1 */ 375 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 376 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 377 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 378 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 379 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 380 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 381 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 382 /* line 7 */ 383 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 384 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 385 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 386 }; 387 388 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 389 390 struct ixl_work { 391 struct work ixw_cookie; 392 void (*ixw_func)(void *); 393 void *ixw_arg; 394 unsigned int ixw_added; 395 }; 396 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 397 398 struct ixl_tx_map { 399 struct mbuf *txm_m; 400 bus_dmamap_t txm_map; 401 unsigned int txm_eop; 402 }; 403 404 struct ixl_tx_ring { 405 kmutex_t txr_lock; 406 struct ixl_softc *txr_sc; 407 408 unsigned int txr_prod; 409 unsigned int txr_cons; 410 411 struct ixl_tx_map *txr_maps; 412 struct ixl_dmamem txr_mem; 413 414 bus_size_t txr_tail; 415 unsigned int txr_qid; 416 pcq_t *txr_intrq; 417 void *txr_si; 418 419 uint64_t txr_oerrors; /* if_oerrors */ 420 uint64_t txr_opackets; /* if_opackets */ 421 uint64_t txr_obytes; /* if_obytes */ 422 uint64_t txr_omcasts; /* if_omcasts */ 423 424 struct evcnt txr_defragged; 425 struct evcnt txr_defrag_failed; 426 struct evcnt txr_pcqdrop; 427 struct evcnt txr_transmitdef; 428 struct evcnt txr_intr; 429 struct evcnt txr_defer; 430 }; 431 432 struct ixl_rx_map { 433 struct mbuf *rxm_m; 434 bus_dmamap_t rxm_map; 435 }; 436 437 struct ixl_rx_ring { 438 kmutex_t rxr_lock; 439 440 unsigned int rxr_prod; 441 unsigned int rxr_cons; 442 443 struct ixl_rx_map *rxr_maps; 444 struct ixl_dmamem rxr_mem; 445 446 struct mbuf *rxr_m_head; 447 struct mbuf **rxr_m_tail; 448 449 bus_size_t rxr_tail; 450 unsigned int rxr_qid; 451 452 uint64_t rxr_ipackets; /* if_ipackets */ 453 uint64_t rxr_ibytes; /* if_ibytes */ 454 uint64_t rxr_iqdrops; /* iqdrops */ 455 uint64_t rxr_ierrors; /* if_ierrors */ 456 457 struct evcnt rxr_mgethdr_failed; 458 struct evcnt rxr_mgetcl_failed; 459 struct evcnt rxr_mbuf_load_failed; 460 struct evcnt rxr_intr; 461 struct evcnt rxr_defer; 462 }; 463 464 struct ixl_queue_pair { 465 struct ixl_softc *qp_sc; 466 struct ixl_tx_ring *qp_txr; 467 struct ixl_rx_ring *qp_rxr; 468 469 char qp_name[16]; 470 471 void *qp_si; 472 struct ixl_work qp_task; 473 bool qp_workqueue; 474 }; 475 476 struct ixl_atq { 477 struct ixl_aq_desc iatq_desc; 478 void (*iatq_fn)(struct ixl_softc *, 479 const struct ixl_aq_desc *); 480 }; 481 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 482 483 struct ixl_product { 484 unsigned int vendor_id; 485 unsigned int product_id; 486 }; 487 488 /* 489 * Locking notes: 490 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 491 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 492 * - more than one lock of them cannot be held at once. 493 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 494 * (a spin mutex). 495 * - the lock cannot held with txr_lock or rxr_lock. 496 * + a field named sc_arq_* is not protected by any lock. 497 * - operations for sc_arq_* is done in one context related to 498 * sc_arq_task. 499 * + other fields in ixl_softc is protected by sc_cfg_lock 500 * (an adaptive mutex) 501 * - It must be held before another lock is held, and It can be 502 * released after the other lock is released. 503 * */ 504 505 struct ixl_softc { 506 device_t sc_dev; 507 struct ethercom sc_ec; 508 bool sc_attached; 509 bool sc_dead; 510 bool sc_rxctl_atq; 511 struct sysctllog *sc_sysctllog; 512 struct workqueue *sc_workq; 513 struct workqueue *sc_workq_txrx; 514 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 515 struct ifmedia sc_media; 516 uint64_t sc_media_status; 517 uint64_t sc_media_active; 518 kmutex_t sc_cfg_lock; 519 enum i40e_mac_type sc_mac_type; 520 uint32_t sc_rss_table_size; 521 uint32_t sc_rss_table_entry_width; 522 bool sc_txrx_workqueue; 523 u_int sc_tx_process_limit; 524 u_int sc_rx_process_limit; 525 u_int sc_tx_intr_process_limit; 526 u_int sc_rx_intr_process_limit; 527 528 struct pci_attach_args sc_pa; 529 pci_intr_handle_t *sc_ihp; 530 void **sc_ihs; 531 unsigned int sc_nintrs; 532 533 bus_dma_tag_t sc_dmat; 534 bus_space_tag_t sc_memt; 535 bus_space_handle_t sc_memh; 536 bus_size_t sc_mems; 537 538 uint8_t sc_pf_id; 539 uint16_t sc_uplink_seid; /* le */ 540 uint16_t sc_downlink_seid; /* le */ 541 uint16_t sc_vsi_number; /* le */ 542 uint16_t sc_seid; 543 unsigned int sc_base_queue; 544 545 pci_intr_type_t sc_intrtype; 546 unsigned int sc_msix_vector_queue; 547 548 struct ixl_dmamem sc_scratch; 549 550 const struct ixl_aq_regs * 551 sc_aq_regs; 552 553 kmutex_t sc_atq_lock; 554 kcondvar_t sc_atq_cv; 555 struct ixl_dmamem sc_atq; 556 unsigned int sc_atq_prod; 557 unsigned int sc_atq_cons; 558 559 struct ixl_dmamem sc_arq; 560 struct ixl_work sc_arq_task; 561 struct ixl_aq_bufs sc_arq_idle; 562 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 563 unsigned int sc_arq_prod; 564 unsigned int sc_arq_cons; 565 566 struct ixl_work sc_link_state_task; 567 struct ixl_atq sc_link_state_atq; 568 569 struct ixl_dmamem sc_hmc_sd; 570 struct ixl_dmamem sc_hmc_pd; 571 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 572 573 unsigned int sc_tx_ring_ndescs; 574 unsigned int sc_rx_ring_ndescs; 575 unsigned int sc_nqueue_pairs; 576 unsigned int sc_nqueue_pairs_max; 577 unsigned int sc_nqueue_pairs_device; 578 struct ixl_queue_pair *sc_qps; 579 580 struct evcnt sc_event_atq; 581 struct evcnt sc_event_link; 582 struct evcnt sc_event_ecc_err; 583 struct evcnt sc_event_pci_exception; 584 struct evcnt sc_event_crit_err; 585 }; 586 587 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 588 #define IXL_TX_PROCESS_LIMIT 256 589 #define IXL_RX_PROCESS_LIMIT 256 590 #define IXL_TX_INTR_PROCESS_LIMIT 256 591 #define IXL_RX_INTR_PROCESS_LIMIT 0U 592 593 #define delaymsec(_x) DELAY(1000 * (_x)) 594 #ifdef IXL_DEBUG 595 #define DDPRINTF(sc, fmt, args...) \ 596 do { \ 597 if (sc != NULL) \ 598 device_printf(sc->sc_dev, ""); \ 599 printf("%s:\t" fmt, __func__, ##args); \ 600 } while (0) 601 #else 602 #define DDPRINTF(sc, fmt, args...) __nothing 603 #endif 604 #define IXL_NOMSIX false 605 606 static enum i40e_mac_type 607 ixl_mactype(pci_product_id_t); 608 static void ixl_clear_hw(struct ixl_softc *); 609 static int ixl_pf_reset(struct ixl_softc *); 610 611 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 612 bus_size_t, bus_size_t); 613 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 614 615 static int ixl_arq_fill(struct ixl_softc *); 616 static void ixl_arq_unfill(struct ixl_softc *); 617 618 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 619 unsigned int); 620 static void ixl_atq_set(struct ixl_atq *, 621 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 622 static int ixl_atq_post(struct ixl_softc *, struct ixl_atq *); 623 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 624 static void ixl_atq_done(struct ixl_softc *); 625 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 626 static int ixl_get_version(struct ixl_softc *); 627 static int ixl_get_hw_capabilities(struct ixl_softc *); 628 static int ixl_pxe_clear(struct ixl_softc *); 629 static int ixl_lldp_shut(struct ixl_softc *); 630 static int ixl_get_mac(struct ixl_softc *); 631 static int ixl_get_switch_config(struct ixl_softc *); 632 static int ixl_phy_mask_ints(struct ixl_softc *); 633 static int ixl_get_phy_types(struct ixl_softc *, uint64_t *); 634 static int ixl_restart_an(struct ixl_softc *); 635 static int ixl_hmc(struct ixl_softc *); 636 static void ixl_hmc_free(struct ixl_softc *); 637 static int ixl_get_vsi(struct ixl_softc *); 638 static int ixl_set_vsi(struct ixl_softc *); 639 static void ixl_set_filter_control(struct ixl_softc *); 640 static void ixl_get_link_status(void *); 641 static int ixl_get_link_status_poll(struct ixl_softc *); 642 static int ixl_set_link_status(struct ixl_softc *, 643 const struct ixl_aq_desc *); 644 static void ixl_config_rss(struct ixl_softc *); 645 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 646 uint16_t, uint16_t); 647 static int ixl_remove_macvlan(struct ixl_softc *, uint8_t *, uint16_t, 648 uint16_t); 649 static void ixl_arq(void *); 650 static void ixl_hmc_pack(void *, const void *, 651 const struct ixl_hmc_pack *, unsigned int); 652 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 653 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 654 655 static int ixl_match(device_t, cfdata_t, void *); 656 static void ixl_attach(device_t, device_t, void *); 657 static int ixl_detach(device_t, int); 658 659 static void ixl_media_add(struct ixl_softc *, uint64_t); 660 static int ixl_media_change(struct ifnet *); 661 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 662 static void ixl_watchdog(struct ifnet *); 663 static int ixl_ioctl(struct ifnet *, u_long, void *); 664 static void ixl_start(struct ifnet *); 665 static int ixl_transmit(struct ifnet *, struct mbuf *); 666 static void ixl_deferred_transmit(void *); 667 static int ixl_intr(void *); 668 static int ixl_queue_intr(void *); 669 static int ixl_other_intr(void *); 670 static void ixl_handle_queue(void *); 671 static void ixl_sched_handle_queue(struct ixl_softc *, 672 struct ixl_queue_pair *); 673 static int ixl_init(struct ifnet *); 674 static int ixl_init_locked(struct ixl_softc *); 675 static void ixl_stop(struct ifnet *, int); 676 static void ixl_stop_locked(struct ixl_softc *); 677 static int ixl_iff(struct ixl_softc *); 678 static int ixl_ifflags_cb(struct ethercom *); 679 static int ixl_setup_interrupts(struct ixl_softc *); 680 static int ixl_establish_intx(struct ixl_softc *); 681 static int ixl_establish_msix(struct ixl_softc *); 682 static void ixl_set_affinity_msix(struct ixl_softc *); 683 static void ixl_enable_queue_intr(struct ixl_softc *, 684 struct ixl_queue_pair *); 685 static void ixl_disable_queue_intr(struct ixl_softc *, 686 struct ixl_queue_pair *); 687 static void ixl_enable_other_intr(struct ixl_softc *); 688 static void ixl_disable_other_intr(struct ixl_softc *); 689 static void ixl_config_queue_intr(struct ixl_softc *); 690 static void ixl_config_other_intr(struct ixl_softc *); 691 692 static struct ixl_tx_ring * 693 ixl_txr_alloc(struct ixl_softc *, unsigned int); 694 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 695 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 696 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 697 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 698 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 699 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 700 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 701 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 702 703 static struct ixl_rx_ring * 704 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 705 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 706 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 707 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 708 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 709 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 710 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 711 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 712 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 713 714 static struct workqueue * 715 ixl_workq_create(const char *, pri_t, int, int); 716 static void ixl_workq_destroy(struct workqueue *); 717 static int ixl_workqs_teardown(device_t); 718 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 719 static void ixl_work_add(struct workqueue *, struct ixl_work *); 720 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 721 static void ixl_workq_work(struct work *, void *); 722 static const struct ixl_product * 723 ixl_lookup(const struct pci_attach_args *pa); 724 static void ixl_link_state_update(struct ixl_softc *, 725 const struct ixl_aq_desc *); 726 static int ixl_set_macvlan(struct ixl_softc *); 727 static int ixl_setup_interrupts(struct ixl_softc *);; 728 static void ixl_teardown_interrupts(struct ixl_softc *); 729 static int ixl_setup_stats(struct ixl_softc *); 730 static void ixl_teardown_stats(struct ixl_softc *); 731 static int ixl_setup_sysctls(struct ixl_softc *); 732 static void ixl_teardown_sysctls(struct ixl_softc *); 733 static int ixl_queue_pairs_alloc(struct ixl_softc *); 734 static void ixl_queue_pairs_free(struct ixl_softc *); 735 736 static const struct ixl_phy_type ixl_phy_type_map[] = { 737 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 738 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 739 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 740 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 741 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 742 { 1ULL << IXL_PHY_TYPE_XAUI | 743 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 744 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 745 { 1ULL << IXL_PHY_TYPE_XLAUI | 746 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 747 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 748 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 749 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 750 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 751 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 752 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 753 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 754 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 755 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 756 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 757 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 758 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 759 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 760 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 761 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 762 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 763 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 764 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 765 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 766 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 767 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 768 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 769 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 770 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_CR }, 771 }; 772 773 static const struct ixl_speed_type ixl_speed_type_map[] = { 774 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 775 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 776 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 777 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 778 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 779 }; 780 781 static const struct ixl_aq_regs ixl_pf_aq_regs = { 782 .atq_tail = I40E_PF_ATQT, 783 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 784 .atq_head = I40E_PF_ATQH, 785 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 786 .atq_len = I40E_PF_ATQLEN, 787 .atq_bal = I40E_PF_ATQBAL, 788 .atq_bah = I40E_PF_ATQBAH, 789 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 790 791 .arq_tail = I40E_PF_ARQT, 792 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 793 .arq_head = I40E_PF_ARQH, 794 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 795 .arq_len = I40E_PF_ARQLEN, 796 .arq_bal = I40E_PF_ARQBAL, 797 .arq_bah = I40E_PF_ARQBAH, 798 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 799 }; 800 801 #define ixl_rd(_s, _r) \ 802 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 803 #define ixl_wr(_s, _r, _v) \ 804 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 805 #define ixl_barrier(_s, _r, _l, _o) \ 806 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 807 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 808 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 809 810 static inline uint32_t 811 ixl_dmamem_hi(struct ixl_dmamem *ixm) 812 { 813 uint32_t retval; 814 uint64_t val; 815 816 if (sizeof(IXL_DMA_DVA(ixm)) > 4) { 817 val = (intptr_t)IXL_DMA_DVA(ixm); 818 retval = (uint32_t)(val >> 32); 819 } else { 820 retval = 0; 821 } 822 823 return retval; 824 } 825 826 static inline uint32_t 827 ixl_dmamem_lo(struct ixl_dmamem *ixm) 828 { 829 830 return (uint32_t)IXL_DMA_DVA(ixm); 831 } 832 833 static inline void 834 ixl_aq_dva(struct ixl_aq_desc *iaq, bus_addr_t addr) 835 { 836 uint64_t val; 837 838 if (sizeof(addr) > 4) { 839 val = (intptr_t)addr; 840 iaq->iaq_param[2] = htole32(val >> 32); 841 } else { 842 iaq->iaq_param[2] = htole32(0); 843 } 844 845 iaq->iaq_param[3] = htole32(addr); 846 } 847 848 static inline unsigned int 849 ixl_rxr_unrefreshed(unsigned int prod, unsigned int cons, unsigned int ndescs) 850 { 851 unsigned int num; 852 853 if (prod < cons) 854 num = cons - prod; 855 else 856 num = (ndescs - prod) + cons; 857 858 if (__predict_true(num > 0)) { 859 /* device cannot receive packets if all descripter is filled */ 860 num -= 1; 861 } 862 863 return num; 864 } 865 866 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 867 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 868 DVF_DETACH_SHUTDOWN); 869 870 static const struct ixl_product ixl_products[] = { 871 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 872 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 873 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 874 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 875 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 876 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 877 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T }, 878 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 879 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 880 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 881 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 882 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 883 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 884 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 885 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 886 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 887 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 888 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 889 /* required last entry */ 890 {0, 0} 891 }; 892 893 static const struct ixl_product * 894 ixl_lookup(const struct pci_attach_args *pa) 895 { 896 const struct ixl_product *ixlp; 897 898 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 899 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 900 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 901 return ixlp; 902 } 903 904 return NULL; 905 } 906 907 static int 908 ixl_match(device_t parent, cfdata_t match, void *aux) 909 { 910 const struct pci_attach_args *pa = aux; 911 912 return (ixl_lookup(pa) != NULL) ? 1 : 0; 913 } 914 915 static void 916 ixl_attach(device_t parent, device_t self, void *aux) 917 { 918 struct ixl_softc *sc; 919 struct pci_attach_args *pa = aux; 920 struct ifnet *ifp; 921 pcireg_t memtype, reg; 922 uint32_t firstq, port, ari, func; 923 uint64_t phy_types = 0; 924 char xnamebuf[32]; 925 int tries, rv; 926 927 sc = device_private(self); 928 sc->sc_dev = self; 929 ifp = &sc->sc_ec.ec_if; 930 931 sc->sc_pa = *pa; 932 sc->sc_dmat = (pci_dma64_available(pa)) ? 933 pa->pa_dmat64 : pa->pa_dmat; 934 sc->sc_aq_regs = &ixl_pf_aq_regs; 935 936 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 937 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(reg)); 938 939 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 940 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 941 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 942 aprint_error(": unable to map registers\n"); 943 return; 944 } 945 946 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 947 948 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 949 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 950 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 951 sc->sc_base_queue = firstq; 952 953 ixl_clear_hw(sc); 954 if (ixl_pf_reset(sc) == -1) { 955 /* error printed by ixl pf_reset */ 956 goto unmap; 957 } 958 959 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 960 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 961 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 962 aprint_normal(": port %u", port); 963 964 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 965 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 966 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 967 968 func = ixl_rd(sc, I40E_PF_FUNC_RID); 969 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 970 971 /* initialise the adminq */ 972 973 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 974 975 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 976 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 977 aprint_error("\n" "%s: unable to allocate atq\n", 978 device_xname(self)); 979 goto unmap; 980 } 981 982 SIMPLEQ_INIT(&sc->sc_arq_idle); 983 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 984 sc->sc_arq_cons = 0; 985 sc->sc_arq_prod = 0; 986 987 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 988 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 989 aprint_error("\n" "%s: unable to allocate arq\n", 990 device_xname(self)); 991 goto free_atq; 992 } 993 994 if (!ixl_arq_fill(sc)) { 995 aprint_error("\n" "%s: unable to fill arq descriptors\n", 996 device_xname(self)); 997 goto free_arq; 998 } 999 1000 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1001 0, IXL_DMA_LEN(&sc->sc_atq), 1002 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1003 1004 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1005 0, IXL_DMA_LEN(&sc->sc_arq), 1006 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1007 1008 for (tries = 0; tries < 10; tries++) { 1009 sc->sc_atq_cons = 0; 1010 sc->sc_atq_prod = 0; 1011 1012 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1013 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1014 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1015 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1016 1017 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1018 1019 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1020 ixl_dmamem_lo(&sc->sc_atq)); 1021 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1022 ixl_dmamem_hi(&sc->sc_atq)); 1023 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1024 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1025 1026 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1027 ixl_dmamem_lo(&sc->sc_arq)); 1028 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1029 ixl_dmamem_hi(&sc->sc_arq)); 1030 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1031 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1032 1033 rv = ixl_get_version(sc); 1034 if (rv == 0) 1035 break; 1036 if (rv != ETIMEDOUT) { 1037 aprint_error(", unable to get firmware version\n"); 1038 goto shutdown; 1039 } 1040 1041 delaymsec(100); 1042 } 1043 1044 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1045 1046 if (sc->sc_mac_type == I40E_MAC_X722) 1047 sc->sc_nqueue_pairs_device = 128; 1048 else 1049 sc->sc_nqueue_pairs_device = 64; 1050 1051 rv = ixl_get_hw_capabilities(sc); 1052 if (rv != 0) { 1053 aprint_error(", GET HW CAPABILITIES %s\n", 1054 rv == ETIMEDOUT ? "timeout" : "error"); 1055 goto shutdown; 1056 } 1057 1058 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max = 1059 MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1060 sc->sc_tx_ring_ndescs = 1024; 1061 sc->sc_rx_ring_ndescs = 1024; 1062 1063 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1064 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1065 1066 if (ixl_get_mac(sc) != 0) { 1067 /* error printed by ixl_get_mac */ 1068 goto shutdown; 1069 } 1070 1071 aprint_normal("\n"); 1072 aprint_naive("\n"); 1073 1074 aprint_normal_dev(self, "Ethernet address %s\n", 1075 ether_sprintf(sc->sc_enaddr)); 1076 1077 rv = ixl_pxe_clear(sc); 1078 if (rv != 0) { 1079 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1080 rv == ETIMEDOUT ? "timeout" : "error"); 1081 } 1082 1083 ixl_set_filter_control(sc); 1084 1085 if (ixl_hmc(sc) != 0) { 1086 /* error printed by ixl_hmc */ 1087 goto shutdown; 1088 } 1089 1090 if (ixl_lldp_shut(sc) != 0) { 1091 /* error printed by ixl_lldp_shut */ 1092 goto free_hmc; 1093 } 1094 1095 if (ixl_phy_mask_ints(sc) != 0) { 1096 /* error printed by ixl_phy_mask_ints */ 1097 goto free_hmc; 1098 } 1099 1100 if (ixl_restart_an(sc) != 0) { 1101 /* error printed by ixl_restart_an */ 1102 goto free_hmc; 1103 } 1104 1105 if (ixl_get_switch_config(sc) != 0) { 1106 /* error printed by ixl_get_switch_config */ 1107 goto free_hmc; 1108 } 1109 1110 if (ixl_get_phy_types(sc, &phy_types) != 0) { 1111 /* error printed by ixl_get_phy_abilities */ 1112 goto free_hmc; 1113 } 1114 1115 rv = ixl_get_link_status_poll(sc); 1116 if (rv != 0) { 1117 aprint_error_dev(self, "GET LINK STATUS %s\n", 1118 rv == ETIMEDOUT ? "timeout" : "error"); 1119 goto free_hmc; 1120 } 1121 1122 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1123 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1124 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1125 goto free_hmc; 1126 } 1127 1128 if (ixl_get_vsi(sc) != 0) { 1129 /* error printed by ixl_get_vsi */ 1130 goto free_scratch; 1131 } 1132 1133 if (ixl_set_vsi(sc) != 0) { 1134 /* error printed by ixl_set_vsi */ 1135 goto free_scratch; 1136 } 1137 1138 if (ixl_queue_pairs_alloc(sc) != 0) { 1139 /* error printed by ixl_queue_pairs_alloc */ 1140 goto free_scratch; 1141 } 1142 1143 if (ixl_setup_interrupts(sc) != 0) { 1144 /* error printed by ixl_setup_interrupts */ 1145 goto free_queue_pairs; 1146 } 1147 1148 if (ixl_setup_stats(sc) != 0) { 1149 aprint_error_dev(self, "failed to setup event counters\n"); 1150 goto teardown_intrs; 1151 } 1152 1153 if (ixl_setup_sysctls(sc) != 0) { 1154 /* error printed by ixl_setup_sysctls */ 1155 goto teardown_stats; 1156 } 1157 1158 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1159 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1160 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1161 if (sc->sc_workq == NULL) 1162 goto teardown_sysctls; 1163 1164 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1165 sc->sc_workq_txrx = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1166 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1167 if (sc->sc_workq_txrx == NULL) 1168 goto teardown_wqs; 1169 1170 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1171 cv_init(&sc->sc_atq_cv, xnamebuf); 1172 1173 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1174 1175 ifp->if_softc = sc; 1176 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1177 ifp->if_extflags = IFEF_MPSAFE; 1178 ifp->if_ioctl = ixl_ioctl; 1179 ifp->if_start = ixl_start; 1180 ifp->if_transmit = ixl_transmit; 1181 ifp->if_watchdog = ixl_watchdog; 1182 ifp->if_init = ixl_init; 1183 ifp->if_stop = ixl_stop; 1184 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1185 IFQ_SET_READY(&ifp->if_snd); 1186 #if 0 1187 ifp->if_capabilities |= IFCAP_CSUM_IPv4_Rx | 1188 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1189 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx; 1190 #endif 1191 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_MTU; 1192 #if 0 1193 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1194 #endif 1195 1196 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1197 ifmedia_init(&sc->sc_media, IFM_IMASK, ixl_media_change, 1198 ixl_media_status); 1199 1200 ixl_media_add(sc, phy_types); 1201 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1202 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1203 1204 if_attach(ifp); 1205 if_deferred_start_init(ifp, NULL); 1206 ether_ifattach(ifp, sc->sc_enaddr); 1207 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1208 1209 (void)ixl_get_link_status_poll(sc); 1210 ixl_work_set(&sc->sc_link_state_task, ixl_get_link_status, sc); 1211 1212 ixl_config_other_intr(sc); 1213 ixl_enable_other_intr(sc); 1214 1215 ixl_set_macvlan(sc); 1216 1217 sc->sc_txrx_workqueue = true; 1218 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1219 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1220 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1221 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1222 1223 if (pmf_device_register(self, NULL, NULL) != true) 1224 aprint_debug_dev(self, "couldn't establish power handler\n"); 1225 sc->sc_attached = true; 1226 return; 1227 1228 teardown_wqs: 1229 config_finalize_register(self, ixl_workqs_teardown); 1230 teardown_sysctls: 1231 ixl_teardown_sysctls(sc); 1232 teardown_stats: 1233 ixl_teardown_stats(sc); 1234 teardown_intrs: 1235 ixl_teardown_interrupts(sc); 1236 free_queue_pairs: 1237 ixl_queue_pairs_free(sc); 1238 free_scratch: 1239 ixl_dmamem_free(sc, &sc->sc_scratch); 1240 free_hmc: 1241 ixl_hmc_free(sc); 1242 shutdown: 1243 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1244 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1245 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1246 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1247 1248 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1249 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1250 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1251 1252 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1253 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1254 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1255 1256 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1257 0, IXL_DMA_LEN(&sc->sc_arq), 1258 BUS_DMASYNC_POSTREAD); 1259 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1260 0, IXL_DMA_LEN(&sc->sc_atq), 1261 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1262 1263 ixl_arq_unfill(sc); 1264 free_arq: 1265 ixl_dmamem_free(sc, &sc->sc_arq); 1266 free_atq: 1267 ixl_dmamem_free(sc, &sc->sc_atq); 1268 unmap: 1269 mutex_destroy(&sc->sc_atq_lock); 1270 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1271 mutex_destroy(&sc->sc_cfg_lock); 1272 sc->sc_mems = 0; 1273 1274 sc->sc_attached = false; 1275 } 1276 1277 static int 1278 ixl_detach(device_t self, int flags) 1279 { 1280 struct ixl_softc *sc = device_private(self); 1281 struct ifnet *ifp = &sc->sc_ec.ec_if; 1282 1283 if (!sc->sc_attached) 1284 return 0; 1285 1286 ixl_stop(ifp, 1); 1287 1288 ixl_disable_other_intr(sc); 1289 1290 /* wait for ATQ handler */ 1291 mutex_enter(&sc->sc_atq_lock); 1292 mutex_exit(&sc->sc_atq_lock); 1293 1294 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1295 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1296 1297 if (sc->sc_workq != NULL) { 1298 ixl_workq_destroy(sc->sc_workq); 1299 sc->sc_workq = NULL; 1300 } 1301 1302 if (sc->sc_workq_txrx != NULL) { 1303 ixl_workq_destroy(sc->sc_workq_txrx); 1304 sc->sc_workq_txrx = NULL; 1305 } 1306 1307 ifmedia_delete_instance(&sc->sc_media, IFM_INST_ANY); 1308 ether_ifdetach(ifp); 1309 if_detach(ifp); 1310 1311 ixl_teardown_interrupts(sc); 1312 ixl_teardown_stats(sc); 1313 1314 ixl_queue_pairs_free(sc); 1315 1316 ixl_dmamem_free(sc, &sc->sc_scratch); 1317 ixl_hmc_free(sc); 1318 1319 /* shutdown */ 1320 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1321 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1322 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1323 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1324 1325 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1326 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1327 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1328 1329 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1330 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1331 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1332 1333 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1334 0, IXL_DMA_LEN(&sc->sc_arq), 1335 BUS_DMASYNC_POSTREAD); 1336 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1337 0, IXL_DMA_LEN(&sc->sc_atq), 1338 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1339 1340 ixl_arq_unfill(sc); 1341 1342 ixl_dmamem_free(sc, &sc->sc_arq); 1343 ixl_dmamem_free(sc, &sc->sc_atq); 1344 1345 cv_destroy(&sc->sc_atq_cv); 1346 mutex_destroy(&sc->sc_atq_lock); 1347 1348 if (sc->sc_mems != 0) { 1349 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1350 sc->sc_mems = 0; 1351 } 1352 1353 mutex_destroy(&sc->sc_cfg_lock); 1354 1355 return 0; 1356 } 1357 1358 static int 1359 ixl_workqs_teardown(device_t self) 1360 { 1361 struct ixl_softc *sc = device_private(self); 1362 1363 if (sc->sc_workq != NULL) { 1364 ixl_workq_destroy(sc->sc_workq); 1365 sc->sc_workq = NULL; 1366 } 1367 1368 if (sc->sc_workq_txrx != NULL) { 1369 ixl_workq_destroy(sc->sc_workq_txrx); 1370 sc->sc_workq_txrx = NULL; 1371 } 1372 1373 return 0; 1374 } 1375 1376 static void 1377 ixl_media_add(struct ixl_softc *sc, uint64_t phy_types) 1378 { 1379 struct ifmedia *ifm = &sc->sc_media; 1380 const struct ixl_phy_type *itype; 1381 unsigned int i; 1382 1383 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1384 itype = &ixl_phy_type_map[i]; 1385 1386 if (ISSET(phy_types, itype->phy_type)) { 1387 ifmedia_add(ifm, 1388 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1389 1390 if (itype->ifm_type == IFM_100_TX) { 1391 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1392 0, NULL); 1393 } 1394 } 1395 } 1396 } 1397 1398 static void 1399 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1400 { 1401 struct ixl_softc *sc = ifp->if_softc; 1402 1403 ifmr->ifm_status = sc->sc_media_status; 1404 ifmr->ifm_active = sc->sc_media_active; 1405 1406 mutex_enter(&sc->sc_cfg_lock); 1407 if (ifp->if_link_state == LINK_STATE_UP) 1408 SET(ifmr->ifm_status, IFM_ACTIVE); 1409 mutex_exit(&sc->sc_cfg_lock); 1410 } 1411 1412 static int 1413 ixl_media_change(struct ifnet *ifp) 1414 { 1415 1416 return 0; 1417 } 1418 1419 static void 1420 ixl_watchdog(struct ifnet *ifp) 1421 { 1422 1423 } 1424 1425 static void 1426 ixl_del_all_multiaddr(struct ixl_softc *sc) 1427 { 1428 struct ethercom *ec = &sc->sc_ec; 1429 struct ether_multi *enm; 1430 struct ether_multistep step; 1431 1432 ETHER_LOCK(ec); 1433 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1434 ETHER_NEXT_MULTI(step, enm)) { 1435 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1436 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1437 } 1438 ETHER_UNLOCK(ec); 1439 } 1440 1441 static int 1442 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1443 { 1444 struct ifnet *ifp = &sc->sc_ec.ec_if; 1445 int rv; 1446 1447 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1448 return 0; 1449 1450 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1451 ixl_del_all_multiaddr(sc); 1452 SET(ifp->if_flags, IFF_ALLMULTI); 1453 return ENETRESET; 1454 } 1455 1456 rv = ixl_add_macvlan(sc, addrlo, 0, 1457 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1458 1459 if (rv == ENOSPC) { 1460 ixl_del_all_multiaddr(sc); 1461 SET(ifp->if_flags, IFF_ALLMULTI); 1462 return ENETRESET; 1463 } 1464 1465 return rv; 1466 } 1467 1468 static int 1469 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1470 { 1471 struct ifnet *ifp = &sc->sc_ec.ec_if; 1472 struct ethercom *ec = &sc->sc_ec; 1473 struct ether_multi *enm, *enm_last; 1474 struct ether_multistep step; 1475 int error, rv = 0; 1476 1477 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1478 ixl_remove_macvlan(sc, addrlo, 0, 1479 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1480 return 0; 1481 } 1482 1483 ETHER_LOCK(ec); 1484 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1485 ETHER_NEXT_MULTI(step, enm)) { 1486 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1487 ETHER_ADDR_LEN) != 0) { 1488 goto out; 1489 } 1490 } 1491 1492 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1493 ETHER_NEXT_MULTI(step, enm)) { 1494 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1495 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1496 if (error != 0) 1497 break; 1498 } 1499 1500 if (enm != NULL) { 1501 enm_last = enm; 1502 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1503 ETHER_NEXT_MULTI(step, enm)) { 1504 if (enm == enm_last) 1505 break; 1506 1507 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1508 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1509 } 1510 } else { 1511 CLR(ifp->if_flags, IFF_ALLMULTI); 1512 rv = ENETRESET; 1513 } 1514 1515 out: 1516 ETHER_UNLOCK(ec); 1517 return rv; 1518 } 1519 1520 static int 1521 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1522 { 1523 struct ifreq *ifr = (struct ifreq *)data; 1524 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1525 struct ixl_tx_ring *txr; 1526 struct ixl_rx_ring *rxr; 1527 const struct sockaddr *sa; 1528 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1529 int s, error = 0; 1530 unsigned int i; 1531 1532 switch (cmd) { 1533 case SIOCADDMULTI: 1534 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1535 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1536 error = ether_multiaddr(sa, addrlo, addrhi); 1537 if (error != 0) 1538 return error; 1539 1540 error = ixl_add_multi(sc, addrlo, addrhi); 1541 if (error != 0 && error != ENETRESET) { 1542 ether_delmulti(sa, &sc->sc_ec); 1543 error = EIO; 1544 } 1545 } 1546 break; 1547 1548 case SIOCDELMULTI: 1549 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1550 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1551 error = ether_multiaddr(sa, addrlo, addrhi); 1552 if (error != 0) 1553 return error; 1554 1555 error = ixl_del_multi(sc, addrlo, addrhi); 1556 } 1557 break; 1558 1559 case SIOCGIFDATA: 1560 case SIOCZIFDATA: 1561 ifp->if_ipackets = 0; 1562 ifp->if_ibytes = 0; 1563 ifp->if_iqdrops = 0; 1564 ifp->if_ierrors = 0; 1565 ifp->if_opackets = 0; 1566 ifp->if_obytes = 0; 1567 ifp->if_omcasts = 0; 1568 1569 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 1570 txr = sc->sc_qps[i].qp_txr; 1571 rxr = sc->sc_qps[i].qp_rxr; 1572 1573 mutex_enter(&rxr->rxr_lock); 1574 ifp->if_ipackets += rxr->rxr_ipackets; 1575 ifp->if_ibytes += rxr->rxr_ibytes; 1576 ifp->if_iqdrops += rxr->rxr_iqdrops; 1577 ifp->if_ierrors += rxr->rxr_ierrors; 1578 if (cmd == SIOCZIFDATA) { 1579 rxr->rxr_ipackets = 0; 1580 rxr->rxr_ibytes = 0; 1581 rxr->rxr_iqdrops = 0; 1582 rxr->rxr_ierrors = 0; 1583 } 1584 mutex_exit(&rxr->rxr_lock); 1585 1586 mutex_enter(&txr->txr_lock); 1587 ifp->if_opackets += txr->txr_opackets; 1588 ifp->if_obytes += txr->txr_opackets; 1589 ifp->if_omcasts += txr->txr_omcasts; 1590 if (cmd == SIOCZIFDATA) { 1591 txr->txr_opackets = 0; 1592 txr->txr_opackets = 0; 1593 txr->txr_omcasts = 0; 1594 } 1595 mutex_exit(&txr->txr_lock); 1596 } 1597 /* FALLTHROUGH */ 1598 default: 1599 s = splnet(); 1600 error = ether_ioctl(ifp, cmd, data); 1601 splx(s); 1602 } 1603 1604 if (error == ENETRESET) 1605 error = ixl_iff(sc); 1606 1607 return error; 1608 } 1609 1610 static enum i40e_mac_type 1611 ixl_mactype(pci_product_id_t id) 1612 { 1613 1614 switch (id) { 1615 case PCI_PRODUCT_INTEL_XL710_SFP: 1616 case PCI_PRODUCT_INTEL_XL710_KX_B: 1617 case PCI_PRODUCT_INTEL_XL710_KX_C: 1618 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1619 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1620 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1621 case PCI_PRODUCT_INTEL_X710_10G_T: 1622 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1623 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1624 case PCI_PRODUCT_INTEL_X710_T4_10G: 1625 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1626 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1627 return I40E_MAC_XL710; 1628 1629 case PCI_PRODUCT_INTEL_X722_KX: 1630 case PCI_PRODUCT_INTEL_X722_QSFP: 1631 case PCI_PRODUCT_INTEL_X722_SFP: 1632 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1633 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1634 case PCI_PRODUCT_INTEL_X722_I_SFP: 1635 return I40E_MAC_X722; 1636 } 1637 1638 return I40E_MAC_GENERIC; 1639 } 1640 1641 static inline void * 1642 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1643 { 1644 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1645 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1646 1647 if (i >= e->hmc_count) 1648 return NULL; 1649 1650 kva += e->hmc_base; 1651 kva += i * e->hmc_size; 1652 1653 return kva; 1654 } 1655 1656 static inline size_t 1657 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1658 { 1659 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1660 1661 return e->hmc_size; 1662 } 1663 1664 static void 1665 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1666 { 1667 struct ixl_rx_ring *rxr = qp->qp_rxr; 1668 1669 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1670 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1671 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1672 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1673 ixl_flush(sc); 1674 } 1675 1676 static void 1677 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1678 { 1679 struct ixl_rx_ring *rxr = qp->qp_rxr; 1680 1681 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1682 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1683 ixl_flush(sc); 1684 } 1685 1686 static void 1687 ixl_enable_other_intr(struct ixl_softc *sc) 1688 { 1689 1690 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1691 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1692 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1693 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1694 ixl_flush(sc); 1695 } 1696 1697 static void 1698 ixl_disable_other_intr(struct ixl_softc *sc) 1699 { 1700 1701 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1702 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1703 ixl_flush(sc); 1704 } 1705 1706 static int 1707 ixl_reinit(struct ixl_softc *sc) 1708 { 1709 struct ixl_rx_ring *rxr; 1710 struct ixl_tx_ring *txr; 1711 unsigned int i; 1712 uint32_t reg; 1713 1714 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1715 1716 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1717 txr = sc->sc_qps[i].qp_txr; 1718 rxr = sc->sc_qps[i].qp_rxr; 1719 1720 txr->txr_cons = txr->txr_prod = 0; 1721 rxr->rxr_cons = rxr->rxr_prod = 0; 1722 1723 ixl_txr_config(sc, txr); 1724 ixl_rxr_config(sc, rxr); 1725 } 1726 1727 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1728 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 1729 1730 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1731 txr = sc->sc_qps[i].qp_txr; 1732 rxr = sc->sc_qps[i].qp_rxr; 1733 1734 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 1735 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 1736 ixl_flush(sc); 1737 1738 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 1739 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 1740 1741 1742 /* ixl_rxfill() needs lock held */ 1743 mutex_enter(&rxr->rxr_lock); 1744 ixl_rxfill(sc, rxr); 1745 mutex_exit(&rxr->rxr_lock); 1746 1747 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 1748 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 1749 ixl_wr(sc, I40E_QRX_ENA(i), reg); 1750 if (ixl_rxr_enabled(sc, rxr) != 0) 1751 goto stop; 1752 1753 ixl_txr_qdis(sc, txr, 1); 1754 1755 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 1756 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 1757 ixl_wr(sc, I40E_QTX_ENA(i), reg); 1758 1759 if (ixl_txr_enabled(sc, txr) != 0) 1760 goto stop; 1761 } 1762 1763 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1764 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 1765 1766 return 0; 1767 1768 stop: 1769 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1770 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 1771 1772 return ETIMEDOUT; 1773 } 1774 1775 static int 1776 ixl_init_locked(struct ixl_softc *sc) 1777 { 1778 struct ifnet *ifp = &sc->sc_ec.ec_if; 1779 unsigned int i; 1780 int error; 1781 1782 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1783 1784 if (sc->sc_dead) { 1785 return ENXIO; 1786 } 1787 1788 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 1789 sc->sc_nqueue_pairs = 1; 1790 else 1791 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1792 1793 error = ixl_reinit(sc); 1794 if (error) { 1795 ixl_stop_locked(sc); 1796 return error; 1797 } 1798 1799 SET(ifp->if_flags, IFF_RUNNING); 1800 CLR(ifp->if_flags, IFF_OACTIVE); 1801 1802 (void)ixl_get_link_status(sc); 1803 1804 ixl_config_rss(sc); 1805 ixl_config_queue_intr(sc); 1806 1807 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1808 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 1809 } 1810 1811 error = ixl_iff(sc); 1812 if (error) { 1813 ixl_stop_locked(sc); 1814 return error; 1815 } 1816 1817 return 0; 1818 } 1819 1820 static int 1821 ixl_init(struct ifnet *ifp) 1822 { 1823 struct ixl_softc *sc = ifp->if_softc; 1824 int error; 1825 1826 mutex_enter(&sc->sc_cfg_lock); 1827 error = ixl_init_locked(sc); 1828 mutex_exit(&sc->sc_cfg_lock); 1829 1830 return error; 1831 } 1832 1833 static int 1834 ixl_iff(struct ixl_softc *sc) 1835 { 1836 struct ifnet *ifp = &sc->sc_ec.ec_if; 1837 struct ixl_atq iatq; 1838 struct ixl_aq_desc *iaq; 1839 struct ixl_aq_vsi_promisc_param *param; 1840 int error; 1841 1842 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 1843 return 0; 1844 1845 memset(&iatq, 0, sizeof(iatq)); 1846 1847 iaq = &iatq.iatq_desc; 1848 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 1849 1850 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 1851 param->flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 1852 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 1853 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 1854 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 1855 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 1856 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1857 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 1858 } 1859 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 1860 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 1861 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 1862 param->seid = sc->sc_seid; 1863 1864 error = ixl_atq_exec(sc, &iatq); 1865 if (error) 1866 return error; 1867 1868 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 1869 return EIO; 1870 1871 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 1872 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1873 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1874 1875 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 1876 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 1877 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1878 } 1879 return 0; 1880 } 1881 1882 static void 1883 ixl_stop_rendezvous(struct ixl_softc *sc) 1884 { 1885 struct ixl_tx_ring *txr; 1886 struct ixl_rx_ring *rxr; 1887 unsigned int i; 1888 1889 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1890 txr = sc->sc_qps[i].qp_txr; 1891 rxr = sc->sc_qps[i].qp_rxr; 1892 1893 mutex_enter(&txr->txr_lock); 1894 mutex_exit(&txr->txr_lock); 1895 1896 mutex_enter(&rxr->rxr_lock); 1897 mutex_exit(&rxr->rxr_lock); 1898 1899 ixl_work_wait(sc->sc_workq_txrx, 1900 &sc->sc_qps[i].qp_task); 1901 } 1902 } 1903 1904 static void 1905 ixl_stop_locked(struct ixl_softc *sc) 1906 { 1907 struct ifnet *ifp = &sc->sc_ec.ec_if; 1908 struct ixl_rx_ring *rxr; 1909 struct ixl_tx_ring *txr; 1910 unsigned int i; 1911 uint32_t reg; 1912 1913 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1914 1915 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 1916 1917 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1918 txr = sc->sc_qps[i].qp_txr; 1919 rxr = sc->sc_qps[i].qp_rxr; 1920 1921 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 1922 1923 mutex_enter(&txr->txr_lock); 1924 ixl_txr_qdis(sc, txr, 0); 1925 /* XXX wait at least 400 usec for all tx queues in one go */ 1926 ixl_flush(sc); 1927 DELAY(500); 1928 1929 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 1930 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 1931 ixl_wr(sc, I40E_QTX_ENA(i), reg); 1932 /* XXX wait 50ms from completaion of the TX queue disable*/ 1933 ixl_flush(sc); 1934 DELAY(50); 1935 1936 if (ixl_txr_disabled(sc, txr) != 0) { 1937 mutex_exit(&txr->txr_lock); 1938 goto die; 1939 } 1940 mutex_exit(&txr->txr_lock); 1941 1942 mutex_enter(&rxr->rxr_lock); 1943 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 1944 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 1945 ixl_wr(sc, I40E_QRX_ENA(i), reg); 1946 /* XXX wait 50ms from completion of the RX queue disable */ 1947 ixl_flush(sc); 1948 DELAY(50); 1949 1950 if (ixl_rxr_disabled(sc, rxr) != 0) { 1951 mutex_exit(&rxr->rxr_lock); 1952 goto die; 1953 } 1954 mutex_exit(&rxr->rxr_lock); 1955 } 1956 1957 ixl_stop_rendezvous(sc); 1958 1959 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1960 txr = sc->sc_qps[i].qp_txr; 1961 rxr = sc->sc_qps[i].qp_rxr; 1962 1963 ixl_txr_unconfig(sc, txr); 1964 ixl_rxr_unconfig(sc, rxr); 1965 1966 ixl_txr_clean(sc, txr); 1967 ixl_rxr_clean(sc, rxr); 1968 } 1969 1970 return; 1971 die: 1972 sc->sc_dead = true; 1973 log(LOG_CRIT, "%s: failed to shut down rings", 1974 device_xname(sc->sc_dev)); 1975 return; 1976 } 1977 1978 static void 1979 ixl_stop(struct ifnet *ifp, int disable) 1980 { 1981 struct ixl_softc *sc = ifp->if_softc; 1982 1983 mutex_enter(&sc->sc_cfg_lock); 1984 ixl_stop_locked(sc); 1985 mutex_exit(&sc->sc_cfg_lock); 1986 } 1987 1988 static int 1989 ixl_queue_pairs_alloc(struct ixl_softc *sc) 1990 { 1991 struct ixl_queue_pair *qp; 1992 unsigned int i; 1993 size_t sz; 1994 1995 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 1996 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 1997 1998 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 1999 qp = &sc->sc_qps[i]; 2000 2001 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2002 ixl_handle_queue, qp); 2003 if (qp->qp_si == NULL) 2004 goto free; 2005 2006 qp->qp_txr = ixl_txr_alloc(sc, i); 2007 if (qp->qp_txr == NULL) 2008 goto free; 2009 2010 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2011 if (qp->qp_rxr == NULL) 2012 goto free; 2013 2014 qp->qp_sc = sc; 2015 ixl_work_set(&qp->qp_task, ixl_handle_queue, qp); 2016 snprintf(qp->qp_name, sizeof(qp->qp_name), 2017 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2018 } 2019 2020 return 0; 2021 free: 2022 if (sc->sc_qps != NULL) { 2023 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2024 qp = &sc->sc_qps[i]; 2025 2026 if (qp->qp_txr != NULL) 2027 ixl_txr_free(sc, qp->qp_txr); 2028 if (qp->qp_rxr != NULL) 2029 ixl_rxr_free(sc, qp->qp_rxr); 2030 if (qp->qp_si != NULL) 2031 softint_disestablish(qp->qp_si); 2032 } 2033 2034 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2035 kmem_free(sc->sc_qps, sz); 2036 sc->sc_qps = NULL; 2037 } 2038 2039 return -1; 2040 } 2041 2042 static void 2043 ixl_queue_pairs_free(struct ixl_softc *sc) 2044 { 2045 struct ixl_queue_pair *qp; 2046 unsigned int i; 2047 size_t sz; 2048 2049 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2050 qp = &sc->sc_qps[i]; 2051 ixl_txr_free(sc, qp->qp_txr); 2052 ixl_rxr_free(sc, qp->qp_rxr); 2053 softint_disestablish(qp->qp_si); 2054 } 2055 2056 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2057 kmem_free(sc->sc_qps, sz); 2058 sc->sc_qps = NULL; 2059 } 2060 2061 static struct ixl_tx_ring * 2062 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2063 { 2064 struct ixl_tx_ring *txr = NULL; 2065 struct ixl_tx_map *maps = NULL, *txm; 2066 unsigned int i; 2067 2068 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2069 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2070 KM_SLEEP); 2071 2072 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2073 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2074 IXL_TX_QUEUE_ALIGN) != 0) 2075 goto free; 2076 2077 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2078 txm = &maps[i]; 2079 2080 if (bus_dmamap_create(sc->sc_dmat, 2081 IXL_HARDMTU, IXL_TX_PKT_DESCS, IXL_HARDMTU, 0, 2082 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2083 goto uncreate; 2084 2085 txm->txm_eop = -1; 2086 txm->txm_m = NULL; 2087 } 2088 2089 txr->txr_cons = txr->txr_prod = 0; 2090 txr->txr_maps = maps; 2091 2092 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2093 if (txr->txr_intrq == NULL) 2094 goto uncreate; 2095 2096 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2097 ixl_deferred_transmit, txr); 2098 if (txr->txr_si == NULL) 2099 goto destroy_pcq; 2100 2101 txr->txr_tail = I40E_QTX_TAIL(qid); 2102 txr->txr_qid = qid; 2103 txr->txr_sc = sc; 2104 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2105 2106 return txr; 2107 2108 destroy_pcq: 2109 pcq_destroy(txr->txr_intrq); 2110 uncreate: 2111 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2112 txm = &maps[i]; 2113 2114 if (txm->txm_map == NULL) 2115 continue; 2116 2117 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2118 } 2119 2120 ixl_dmamem_free(sc, &txr->txr_mem); 2121 free: 2122 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2123 kmem_free(txr, sizeof(*txr)); 2124 2125 return NULL; 2126 } 2127 2128 static void 2129 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2130 { 2131 unsigned int qid; 2132 bus_size_t reg; 2133 uint32_t r; 2134 2135 qid = txr->txr_qid + sc->sc_base_queue; 2136 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2137 qid %= 128; 2138 2139 r = ixl_rd(sc, reg); 2140 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2141 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2142 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2143 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2144 ixl_wr(sc, reg, r); 2145 } 2146 2147 static void 2148 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2149 { 2150 struct ixl_hmc_txq txq; 2151 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2152 void *hmc; 2153 2154 memset(&txq, 0, sizeof(txq)); 2155 txq.head = htole16(txr->txr_cons); 2156 txq.new_context = 1; 2157 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2158 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2159 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2160 txq.tphrdesc_ena = 0; 2161 txq.tphrpacket_ena = 0; 2162 txq.tphwdesc_ena = 0; 2163 txq.rdylist = data->qs_handle[0]; 2164 2165 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2166 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2167 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2168 __arraycount(ixl_hmc_pack_txq)); 2169 } 2170 2171 static void 2172 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2173 { 2174 void *hmc; 2175 2176 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2177 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2178 } 2179 2180 static void 2181 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2182 { 2183 struct ixl_tx_map *maps, *txm; 2184 bus_dmamap_t map; 2185 unsigned int i; 2186 2187 maps = txr->txr_maps; 2188 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2189 txm = &maps[i]; 2190 2191 if (txm->txm_m == NULL) 2192 continue; 2193 2194 map = txm->txm_map; 2195 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2196 BUS_DMASYNC_POSTWRITE); 2197 bus_dmamap_unload(sc->sc_dmat, map); 2198 2199 m_freem(txm->txm_m); 2200 txm->txm_m = NULL; 2201 } 2202 } 2203 2204 static int 2205 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2206 { 2207 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2208 uint32_t reg; 2209 int i; 2210 2211 for (i = 0; i < 10; i++) { 2212 reg = ixl_rd(sc, ena); 2213 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2214 return 0; 2215 2216 delaymsec(10); 2217 } 2218 2219 return ETIMEDOUT; 2220 } 2221 2222 static int 2223 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2224 { 2225 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2226 uint32_t reg; 2227 int i; 2228 2229 KASSERT(mutex_owned(&txr->txr_lock)); 2230 2231 for (i = 0; i < 20; i++) { 2232 reg = ixl_rd(sc, ena); 2233 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2234 return 0; 2235 2236 delaymsec(10); 2237 } 2238 2239 return ETIMEDOUT; 2240 } 2241 2242 static void 2243 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2244 { 2245 struct ixl_tx_map *maps, *txm; 2246 struct mbuf *m; 2247 unsigned int i; 2248 2249 softint_disestablish(txr->txr_si); 2250 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2251 m_freem(m); 2252 pcq_destroy(txr->txr_intrq); 2253 2254 maps = txr->txr_maps; 2255 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2256 txm = &maps[i]; 2257 2258 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2259 } 2260 2261 ixl_dmamem_free(sc, &txr->txr_mem); 2262 mutex_destroy(&txr->txr_lock); 2263 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2264 kmem_free(txr, sizeof(*txr)); 2265 } 2266 2267 static inline int 2268 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2269 struct ixl_tx_ring *txr) 2270 { 2271 struct mbuf *m; 2272 int error; 2273 2274 KASSERT(mutex_owned(&txr->txr_lock)); 2275 2276 m = *m0; 2277 2278 error = bus_dmamap_load_mbuf(dmat, map, m, 2279 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2280 if (error != EFBIG) 2281 return error; 2282 2283 m = m_defrag(m, M_DONTWAIT); 2284 if (m != NULL) { 2285 *m0 = m; 2286 txr->txr_defragged.ev_count++; 2287 2288 error = bus_dmamap_load_mbuf(dmat, map, m, 2289 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2290 } else { 2291 txr->txr_defrag_failed.ev_count++; 2292 error = ENOBUFS; 2293 } 2294 2295 return error; 2296 } 2297 2298 static void 2299 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2300 bool is_transmit) 2301 { 2302 struct ixl_softc *sc = ifp->if_softc; 2303 struct ixl_tx_desc *ring, *txd; 2304 struct ixl_tx_map *txm; 2305 bus_dmamap_t map; 2306 struct mbuf *m; 2307 uint64_t cmd; 2308 unsigned int prod, free, last, i; 2309 unsigned int mask; 2310 int post = 0; 2311 2312 KASSERT(mutex_owned(&txr->txr_lock)); 2313 2314 if (ifp->if_link_state != LINK_STATE_UP 2315 || !ISSET(ifp->if_flags, IFF_RUNNING) 2316 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2317 if (!is_transmit) 2318 IFQ_PURGE(&ifp->if_snd); 2319 return; 2320 } 2321 2322 prod = txr->txr_prod; 2323 free = txr->txr_cons; 2324 if (free <= prod) 2325 free += sc->sc_tx_ring_ndescs; 2326 free -= prod; 2327 2328 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2329 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2330 2331 ring = IXL_DMA_KVA(&txr->txr_mem); 2332 mask = sc->sc_tx_ring_ndescs - 1; 2333 last = prod; 2334 cmd = 0; 2335 txd = NULL; 2336 2337 for (;;) { 2338 if (free <= IXL_TX_PKT_DESCS) { 2339 if (!is_transmit) 2340 SET(ifp->if_flags, IFF_OACTIVE); 2341 break; 2342 } 2343 2344 if (is_transmit) 2345 m = pcq_get(txr->txr_intrq); 2346 else 2347 IFQ_DEQUEUE(&ifp->if_snd, m); 2348 2349 if (m == NULL) 2350 break; 2351 2352 txm = &txr->txr_maps[prod]; 2353 map = txm->txm_map; 2354 2355 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2356 txr->txr_oerrors++; 2357 m_freem(m); 2358 continue; 2359 } 2360 2361 bus_dmamap_sync(sc->sc_dmat, map, 0, 2362 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2363 2364 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2365 txd = &ring[prod]; 2366 2367 cmd = (uint64_t)map->dm_segs[i].ds_len << 2368 IXL_TX_DESC_BSIZE_SHIFT; 2369 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2370 2371 txd->addr = htole64(map->dm_segs[i].ds_addr); 2372 txd->cmd = htole64(cmd); 2373 2374 last = prod; 2375 2376 prod++; 2377 prod &= mask; 2378 } 2379 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2380 txd->cmd = htole64(cmd); 2381 2382 txm->txm_m = m; 2383 txm->txm_eop = last; 2384 2385 bpf_mtap(ifp, m, BPF_D_OUT); 2386 2387 free -= i; 2388 post = 1; 2389 } 2390 2391 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2392 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2393 2394 if (post) { 2395 txr->txr_prod = prod; 2396 ixl_wr(sc, txr->txr_tail, prod); 2397 } 2398 } 2399 2400 static int 2401 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2402 { 2403 struct ifnet *ifp = &sc->sc_ec.ec_if; 2404 struct ixl_tx_desc *ring, *txd; 2405 struct ixl_tx_map *txm; 2406 struct mbuf *m; 2407 bus_dmamap_t map; 2408 unsigned int cons, prod, last; 2409 unsigned int mask; 2410 uint64_t dtype; 2411 int done = 0, more = 0; 2412 2413 KASSERT(mutex_owned(&txr->txr_lock)); 2414 2415 prod = txr->txr_prod; 2416 cons = txr->txr_cons; 2417 2418 if (cons == prod) 2419 return 0; 2420 2421 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2422 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2423 2424 ring = IXL_DMA_KVA(&txr->txr_mem); 2425 mask = sc->sc_tx_ring_ndescs - 1; 2426 2427 do { 2428 if (txlimit-- <= 0) { 2429 more = 1; 2430 break; 2431 } 2432 2433 txm = &txr->txr_maps[cons]; 2434 last = txm->txm_eop; 2435 txd = &ring[last]; 2436 2437 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2438 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2439 break; 2440 2441 map = txm->txm_map; 2442 2443 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2444 BUS_DMASYNC_POSTWRITE); 2445 bus_dmamap_unload(sc->sc_dmat, map); 2446 2447 m = txm->txm_m; 2448 if (m != NULL) { 2449 txr->txr_opackets++; 2450 txr->txr_obytes += m->m_pkthdr.len; 2451 if (ISSET(m->m_flags, M_MCAST)) 2452 txr->txr_omcasts++; 2453 m_freem(m); 2454 } 2455 2456 txm->txm_m = NULL; 2457 txm->txm_eop = -1; 2458 2459 cons = last + 1; 2460 cons &= mask; 2461 done = 1; 2462 } while (cons != prod); 2463 2464 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2465 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2466 2467 txr->txr_cons = cons; 2468 2469 if (done) { 2470 softint_schedule(txr->txr_si); 2471 if (txr->txr_qid == 0) { 2472 CLR(ifp->if_flags, IFF_OACTIVE); 2473 if_schedule_deferred_start(ifp); 2474 } 2475 } 2476 2477 return more; 2478 } 2479 2480 static void 2481 ixl_start(struct ifnet *ifp) 2482 { 2483 struct ixl_softc *sc; 2484 struct ixl_tx_ring *txr; 2485 2486 sc = ifp->if_softc; 2487 txr = sc->sc_qps[0].qp_txr; 2488 2489 mutex_enter(&txr->txr_lock); 2490 ixl_tx_common_locked(ifp, txr, false); 2491 mutex_exit(&txr->txr_lock); 2492 } 2493 2494 static inline unsigned int 2495 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2496 { 2497 u_int cpuid; 2498 2499 cpuid = cpu_index(curcpu()); 2500 2501 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2502 } 2503 2504 static int 2505 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2506 { 2507 struct ixl_softc *sc; 2508 struct ixl_tx_ring *txr; 2509 unsigned int qid; 2510 2511 sc = ifp->if_softc; 2512 qid = ixl_select_txqueue(sc, m); 2513 2514 txr = sc->sc_qps[qid].qp_txr; 2515 2516 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2517 mutex_enter(&txr->txr_lock); 2518 txr->txr_pcqdrop.ev_count++; 2519 mutex_exit(&txr->txr_lock); 2520 2521 m_freem(m); 2522 return ENOBUFS; 2523 } 2524 2525 if (mutex_tryenter(&txr->txr_lock)) { 2526 ixl_tx_common_locked(ifp, txr, true); 2527 mutex_exit(&txr->txr_lock); 2528 } else { 2529 softint_schedule(txr->txr_si); 2530 } 2531 2532 return 0; 2533 } 2534 2535 static void 2536 ixl_deferred_transmit(void *xtxr) 2537 { 2538 struct ixl_tx_ring *txr = xtxr; 2539 struct ixl_softc *sc = txr->txr_sc; 2540 struct ifnet *ifp = &sc->sc_ec.ec_if; 2541 2542 mutex_enter(&txr->txr_lock); 2543 txr->txr_transmitdef.ev_count++; 2544 if (pcq_peek(txr->txr_intrq) != NULL) 2545 ixl_tx_common_locked(ifp, txr, true); 2546 mutex_exit(&txr->txr_lock); 2547 } 2548 2549 static struct ixl_rx_ring * 2550 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2551 { 2552 struct ixl_rx_ring *rxr = NULL; 2553 struct ixl_rx_map *maps = NULL, *rxm; 2554 unsigned int i; 2555 2556 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2557 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2558 KM_SLEEP); 2559 2560 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2561 sizeof(struct ixl_rx_rd_desc_16) * sc->sc_rx_ring_ndescs, 2562 IXL_RX_QUEUE_ALIGN) != 0) 2563 goto free; 2564 2565 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2566 rxm = &maps[i]; 2567 2568 if (bus_dmamap_create(sc->sc_dmat, 2569 IXL_HARDMTU, 1, IXL_HARDMTU, 0, 2570 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2571 goto uncreate; 2572 2573 rxm->rxm_m = NULL; 2574 } 2575 2576 rxr->rxr_cons = rxr->rxr_prod = 0; 2577 rxr->rxr_m_head = NULL; 2578 rxr->rxr_m_tail = &rxr->rxr_m_head; 2579 rxr->rxr_maps = maps; 2580 2581 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2582 rxr->rxr_qid = qid; 2583 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2584 2585 return rxr; 2586 2587 uncreate: 2588 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2589 rxm = &maps[i]; 2590 2591 if (rxm->rxm_map == NULL) 2592 continue; 2593 2594 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2595 } 2596 2597 ixl_dmamem_free(sc, &rxr->rxr_mem); 2598 free: 2599 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2600 kmem_free(rxr, sizeof(*rxr)); 2601 2602 return NULL; 2603 } 2604 2605 static void 2606 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2607 { 2608 struct ixl_rx_map *maps, *rxm; 2609 bus_dmamap_t map; 2610 unsigned int i; 2611 2612 maps = rxr->rxr_maps; 2613 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2614 rxm = &maps[i]; 2615 2616 if (rxm->rxm_m == NULL) 2617 continue; 2618 2619 map = rxm->rxm_map; 2620 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2621 BUS_DMASYNC_POSTWRITE); 2622 bus_dmamap_unload(sc->sc_dmat, map); 2623 2624 m_freem(rxm->rxm_m); 2625 rxm->rxm_m = NULL; 2626 } 2627 2628 m_freem(rxr->rxr_m_head); 2629 rxr->rxr_m_head = NULL; 2630 rxr->rxr_m_tail = &rxr->rxr_m_head; 2631 2632 rxr->rxr_prod = rxr->rxr_cons = 0; 2633 } 2634 2635 static int 2636 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2637 { 2638 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 2639 uint32_t reg; 2640 int i; 2641 2642 for (i = 0; i < 10; i++) { 2643 reg = ixl_rd(sc, ena); 2644 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 2645 return 0; 2646 2647 delaymsec(10); 2648 } 2649 2650 return ETIMEDOUT; 2651 } 2652 2653 static int 2654 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2655 { 2656 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 2657 uint32_t reg; 2658 int i; 2659 2660 KASSERT(mutex_owned(&rxr->rxr_lock)); 2661 2662 for (i = 0; i < 20; i++) { 2663 reg = ixl_rd(sc, ena); 2664 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 2665 return 0; 2666 2667 delaymsec(10); 2668 } 2669 2670 return ETIMEDOUT; 2671 } 2672 2673 static void 2674 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2675 { 2676 struct ixl_hmc_rxq rxq; 2677 void *hmc; 2678 2679 memset(&rxq, 0, sizeof(rxq)); 2680 2681 rxq.head = htole16(rxr->rxr_cons); 2682 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 2683 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 2684 rxq.dbuff = htole16(MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 2685 rxq.hbuff = 0; 2686 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 2687 rxq.dsize = IXL_HMC_RXQ_DSIZE_16; 2688 rxq.crcstrip = 1; 2689 rxq.l2sel = 0; 2690 rxq.showiv = 0; 2691 rxq.rxmax = htole16(IXL_HARDMTU); 2692 rxq.tphrdesc_ena = 0; 2693 rxq.tphwdesc_ena = 0; 2694 rxq.tphdata_ena = 0; 2695 rxq.tphhead_ena = 0; 2696 rxq.lrxqthresh = 0; 2697 rxq.prefena = 1; 2698 2699 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 2700 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 2701 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 2702 __arraycount(ixl_hmc_pack_rxq)); 2703 } 2704 2705 static void 2706 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2707 { 2708 void *hmc; 2709 2710 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 2711 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 2712 } 2713 2714 static void 2715 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2716 { 2717 struct ixl_rx_map *maps, *rxm; 2718 unsigned int i; 2719 2720 maps = rxr->rxr_maps; 2721 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2722 rxm = &maps[i]; 2723 2724 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2725 } 2726 2727 ixl_dmamem_free(sc, &rxr->rxr_mem); 2728 mutex_destroy(&rxr->rxr_lock); 2729 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2730 kmem_free(rxr, sizeof(*rxr)); 2731 } 2732 2733 static int 2734 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 2735 { 2736 struct ifnet *ifp = &sc->sc_ec.ec_if; 2737 struct ixl_rx_wb_desc_16 *ring, *rxd; 2738 struct ixl_rx_map *rxm; 2739 bus_dmamap_t map; 2740 unsigned int cons, prod; 2741 struct mbuf *m; 2742 uint64_t word; 2743 unsigned int len; 2744 unsigned int mask; 2745 int done = 0, more = 0; 2746 2747 KASSERT(mutex_owned(&rxr->rxr_lock)); 2748 2749 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2750 return 0; 2751 2752 prod = rxr->rxr_prod; 2753 cons = rxr->rxr_cons; 2754 2755 if (cons == prod) 2756 return 0; 2757 2758 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 2759 0, IXL_DMA_LEN(&rxr->rxr_mem), 2760 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 2761 2762 ring = IXL_DMA_KVA(&rxr->rxr_mem); 2763 mask = sc->sc_rx_ring_ndescs - 1; 2764 2765 do { 2766 if (rxlimit-- <= 0) { 2767 more = 1; 2768 break; 2769 } 2770 2771 rxd = &ring[cons]; 2772 2773 word = le64toh(rxd->qword1); 2774 2775 if (!ISSET(word, IXL_RX_DESC_DD)) 2776 break; 2777 2778 rxm = &rxr->rxr_maps[cons]; 2779 2780 map = rxm->rxm_map; 2781 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2782 BUS_DMASYNC_POSTREAD); 2783 bus_dmamap_unload(sc->sc_dmat, map); 2784 2785 m = rxm->rxm_m; 2786 rxm->rxm_m = NULL; 2787 2788 KASSERT(m != NULL); 2789 2790 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 2791 m->m_len = len; 2792 m->m_pkthdr.len = 0; 2793 2794 m->m_next = NULL; 2795 *rxr->rxr_m_tail = m; 2796 rxr->rxr_m_tail = &m->m_next; 2797 2798 m = rxr->rxr_m_head; 2799 m->m_pkthdr.len += len; 2800 2801 if (ISSET(word, IXL_RX_DESC_EOP)) { 2802 if (!ISSET(word, 2803 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 2804 m_set_rcvif(m, ifp); 2805 rxr->rxr_ipackets++; 2806 rxr->rxr_ibytes += m->m_pkthdr.len; 2807 if_percpuq_enqueue(ifp->if_percpuq, m); 2808 } else { 2809 rxr->rxr_ierrors++; 2810 m_freem(m); 2811 } 2812 2813 rxr->rxr_m_head = NULL; 2814 rxr->rxr_m_tail = &rxr->rxr_m_head; 2815 } 2816 2817 cons++; 2818 cons &= mask; 2819 2820 done = 1; 2821 } while (cons != prod); 2822 2823 if (done) { 2824 rxr->rxr_cons = cons; 2825 if (ixl_rxfill(sc, rxr) == -1) 2826 rxr->rxr_iqdrops++; 2827 } 2828 2829 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 2830 0, IXL_DMA_LEN(&rxr->rxr_mem), 2831 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 2832 2833 return more; 2834 } 2835 2836 static int 2837 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2838 { 2839 struct ixl_rx_rd_desc_16 *ring, *rxd; 2840 struct ixl_rx_map *rxm; 2841 bus_dmamap_t map; 2842 struct mbuf *m; 2843 unsigned int prod; 2844 unsigned int slots; 2845 unsigned int mask; 2846 int post = 0, error = 0; 2847 2848 KASSERT(mutex_owned(&rxr->rxr_lock)); 2849 2850 prod = rxr->rxr_prod; 2851 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 2852 sc->sc_rx_ring_ndescs); 2853 2854 ring = IXL_DMA_KVA(&rxr->rxr_mem); 2855 mask = sc->sc_rx_ring_ndescs - 1; 2856 2857 if (__predict_false(slots <= 0)) 2858 return -1; 2859 2860 do { 2861 rxm = &rxr->rxr_maps[prod]; 2862 2863 MGETHDR(m, M_DONTWAIT, MT_DATA); 2864 if (m == NULL) { 2865 rxr->rxr_mgethdr_failed.ev_count++; 2866 error = -1; 2867 break; 2868 } 2869 2870 MCLGET(m, M_DONTWAIT); 2871 if (!ISSET(m->m_flags, M_EXT)) { 2872 rxr->rxr_mgetcl_failed.ev_count++; 2873 error = -1; 2874 m_freem(m); 2875 break; 2876 } 2877 2878 m->m_len = m->m_pkthdr.len = MCLBYTES + ETHER_ALIGN; 2879 m_adj(m, ETHER_ALIGN); 2880 2881 map = rxm->rxm_map; 2882 2883 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 2884 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 2885 rxr->rxr_mbuf_load_failed.ev_count++; 2886 error = -1; 2887 m_freem(m); 2888 break; 2889 } 2890 2891 rxm->rxm_m = m; 2892 2893 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2894 BUS_DMASYNC_PREREAD); 2895 2896 rxd = &ring[prod]; 2897 2898 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 2899 rxd->haddr = htole64(0); 2900 2901 prod++; 2902 prod &= mask; 2903 2904 post = 1; 2905 2906 } while (--slots); 2907 2908 if (post) { 2909 rxr->rxr_prod = prod; 2910 ixl_wr(sc, rxr->rxr_tail, prod); 2911 } 2912 2913 return error; 2914 } 2915 2916 static inline int 2917 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 2918 u_int txlimit, struct evcnt *txevcnt, 2919 u_int rxlimit, struct evcnt *rxevcnt) 2920 { 2921 struct ixl_tx_ring *txr = qp->qp_txr; 2922 struct ixl_rx_ring *rxr = qp->qp_rxr; 2923 int txmore, rxmore; 2924 int rv; 2925 2926 KASSERT(!mutex_owned(&txr->txr_lock)); 2927 KASSERT(!mutex_owned(&rxr->rxr_lock)); 2928 2929 mutex_enter(&txr->txr_lock); 2930 txevcnt->ev_count++; 2931 txmore = ixl_txeof(sc, txr, txlimit); 2932 mutex_exit(&txr->txr_lock); 2933 2934 mutex_enter(&rxr->rxr_lock); 2935 rxevcnt->ev_count++; 2936 rxmore = ixl_rxeof(sc, rxr, rxlimit); 2937 mutex_exit(&rxr->rxr_lock); 2938 2939 rv = txmore | (rxmore << 1); 2940 2941 return rv; 2942 } 2943 2944 static void 2945 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 2946 { 2947 2948 if (qp->qp_workqueue) 2949 ixl_work_add(sc->sc_workq_txrx, &qp->qp_task); 2950 else 2951 softint_schedule(qp->qp_si); 2952 } 2953 2954 static int 2955 ixl_intr(void *xsc) 2956 { 2957 struct ixl_softc *sc = xsc; 2958 struct ixl_tx_ring *txr; 2959 struct ixl_rx_ring *rxr; 2960 uint32_t icr, rxintr, txintr; 2961 int rv = 0; 2962 unsigned int i; 2963 2964 KASSERT(sc != NULL); 2965 2966 ixl_enable_other_intr(sc); 2967 icr = ixl_rd(sc, I40E_PFINT_ICR0); 2968 2969 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 2970 atomic_inc_64(&sc->sc_event_atq.ev_count); 2971 ixl_atq_done(sc); 2972 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 2973 rv = 1; 2974 } 2975 2976 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 2977 atomic_inc_64(&sc->sc_event_link.ev_count); 2978 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 2979 rv = 1; 2980 } 2981 2982 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 2983 txintr = icr & I40E_INTR_NOTX_TX_MASK; 2984 2985 if (txintr || rxintr) { 2986 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2987 txr = sc->sc_qps[i].qp_txr; 2988 rxr = sc->sc_qps[i].qp_rxr; 2989 2990 ixl_handle_queue_common(sc, &sc->sc_qps[i], 2991 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 2992 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 2993 } 2994 rv = 1; 2995 } 2996 2997 return rv; 2998 } 2999 3000 static int 3001 ixl_queue_intr(void *xqp) 3002 { 3003 struct ixl_queue_pair *qp = xqp; 3004 struct ixl_tx_ring *txr = qp->qp_txr; 3005 struct ixl_rx_ring *rxr = qp->qp_rxr; 3006 struct ixl_softc *sc = qp->qp_sc; 3007 u_int txlimit, rxlimit; 3008 int more; 3009 3010 txlimit = sc->sc_tx_intr_process_limit; 3011 rxlimit = sc->sc_rx_intr_process_limit; 3012 qp->qp_workqueue = sc->sc_txrx_workqueue; 3013 3014 more = ixl_handle_queue_common(sc, qp, 3015 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3016 3017 if (more != 0) { 3018 ixl_sched_handle_queue(sc, qp); 3019 } else { 3020 /* for ALTQ */ 3021 if (txr->txr_qid == 0) 3022 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3023 softint_schedule(txr->txr_si); 3024 3025 ixl_enable_queue_intr(sc, qp); 3026 } 3027 3028 return 1; 3029 } 3030 3031 static void 3032 ixl_handle_queue(void *xqp) 3033 { 3034 struct ixl_queue_pair *qp = xqp; 3035 struct ixl_softc *sc = qp->qp_sc; 3036 struct ixl_tx_ring *txr = qp->qp_txr; 3037 struct ixl_rx_ring *rxr = qp->qp_rxr; 3038 u_int txlimit, rxlimit; 3039 int more; 3040 3041 txlimit = sc->sc_tx_process_limit; 3042 rxlimit = sc->sc_rx_process_limit; 3043 3044 more = ixl_handle_queue_common(sc, qp, 3045 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3046 3047 if (more != 0) 3048 ixl_sched_handle_queue(sc, qp); 3049 else 3050 ixl_enable_queue_intr(sc, qp); 3051 } 3052 3053 static inline void 3054 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3055 { 3056 uint32_t hmc_idx, hmc_isvf; 3057 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3058 3059 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3060 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3061 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3062 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3063 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3064 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3065 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3066 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3067 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3068 3069 device_printf(sc->sc_dev, 3070 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3071 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3072 } 3073 3074 static int 3075 ixl_other_intr(void *xsc) 3076 { 3077 struct ixl_softc *sc = xsc; 3078 uint32_t icr, mask, reg; 3079 int rv; 3080 3081 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3082 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3083 3084 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3085 atomic_inc_64(&sc->sc_event_atq.ev_count); 3086 ixl_atq_done(sc); 3087 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3088 rv = 1; 3089 } 3090 3091 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3092 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3093 device_printf(sc->sc_dev, "link stat changed\n"); 3094 3095 atomic_inc_64(&sc->sc_event_link.ev_count); 3096 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3097 rv = 1; 3098 } 3099 3100 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3101 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3102 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3103 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3104 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3105 3106 device_printf(sc->sc_dev, "GRST: %s\n", 3107 reg == I40E_RESET_CORER ? "CORER" : 3108 reg == I40E_RESET_GLOBR ? "GLOBR" : 3109 reg == I40E_RESET_EMPR ? "EMPR" : 3110 "POR"); 3111 } 3112 3113 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3114 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3115 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3116 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3117 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3118 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3119 3120 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3121 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3122 device_printf(sc->sc_dev, "critical error\n"); 3123 } 3124 3125 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3126 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3127 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3128 ixl_print_hmc_error(sc, reg); 3129 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3130 } 3131 3132 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3133 ixl_flush(sc); 3134 ixl_enable_other_intr(sc); 3135 return rv; 3136 } 3137 3138 static void 3139 ixl_get_link_status_done(struct ixl_softc *sc, 3140 const struct ixl_aq_desc *iaq) 3141 { 3142 3143 ixl_link_state_update(sc, iaq); 3144 } 3145 3146 static void 3147 ixl_get_link_status(void *xsc) 3148 { 3149 struct ixl_softc *sc = xsc; 3150 struct ixl_aq_desc *iaq; 3151 struct ixl_aq_link_param *param; 3152 3153 memset(&sc->sc_link_state_atq, 0, sizeof(sc->sc_link_state_atq)); 3154 iaq = &sc->sc_link_state_atq.iatq_desc; 3155 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3156 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3157 param->notify = IXL_AQ_LINK_NOTIFY; 3158 3159 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 3160 (void)ixl_atq_post(sc, &sc->sc_link_state_atq); 3161 } 3162 3163 static void 3164 ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3165 { 3166 struct ifnet *ifp = &sc->sc_ec.ec_if; 3167 int link_state; 3168 3169 link_state = ixl_set_link_status(sc, iaq); 3170 3171 if (ifp->if_link_state != link_state) 3172 if_link_state_change(ifp, link_state); 3173 3174 if (link_state != LINK_STATE_DOWN) { 3175 if_schedule_deferred_start(ifp); 3176 } 3177 } 3178 3179 static void 3180 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3181 const char *msg) 3182 { 3183 char buf[512]; 3184 size_t len; 3185 3186 len = sizeof(buf); 3187 buf[--len] = '\0'; 3188 3189 device_printf(sc->sc_dev, "%s\n", msg); 3190 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3191 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3192 buf, le16toh(iaq->iaq_opcode)); 3193 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3194 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3195 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3196 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3197 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3198 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3199 } 3200 3201 static void 3202 ixl_arq(void *xsc) 3203 { 3204 struct ixl_softc *sc = xsc; 3205 struct ixl_aq_desc *arq, *iaq; 3206 struct ixl_aq_buf *aqb; 3207 unsigned int cons = sc->sc_arq_cons; 3208 unsigned int prod; 3209 int done = 0; 3210 3211 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3212 sc->sc_aq_regs->arq_head_mask; 3213 3214 if (cons == prod) 3215 goto done; 3216 3217 arq = IXL_DMA_KVA(&sc->sc_arq); 3218 3219 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3220 0, IXL_DMA_LEN(&sc->sc_arq), 3221 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3222 3223 do { 3224 iaq = &arq[cons]; 3225 aqb = sc->sc_arq_live[cons]; 3226 3227 KASSERT(aqb != NULL); 3228 3229 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3230 BUS_DMASYNC_POSTREAD); 3231 3232 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3233 ixl_aq_dump(sc, iaq, "arq event"); 3234 3235 switch (iaq->iaq_opcode) { 3236 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3237 ixl_link_state_update(sc, iaq); 3238 break; 3239 } 3240 3241 memset(iaq, 0, sizeof(*iaq)); 3242 sc->sc_arq_live[cons] = NULL; 3243 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3244 3245 cons++; 3246 cons &= IXL_AQ_MASK; 3247 3248 done = 1; 3249 } while (cons != prod); 3250 3251 if (done) { 3252 sc->sc_arq_cons = cons; 3253 ixl_arq_fill(sc); 3254 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3255 0, IXL_DMA_LEN(&sc->sc_arq), 3256 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3257 } 3258 3259 done: 3260 ixl_enable_other_intr(sc); 3261 } 3262 3263 static void 3264 ixl_atq_set(struct ixl_atq *iatq, 3265 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3266 { 3267 3268 iatq->iatq_fn = fn; 3269 } 3270 3271 static int 3272 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3273 { 3274 struct ixl_aq_desc *atq, *slot; 3275 unsigned int prod, cons, prod_next; 3276 3277 /* assert locked */ 3278 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3279 3280 atq = IXL_DMA_KVA(&sc->sc_atq); 3281 prod = sc->sc_atq_prod; 3282 cons = sc->sc_atq_cons; 3283 prod_next = (prod +1) & IXL_AQ_MASK; 3284 3285 if (cons == prod_next) 3286 return ENOMEM; 3287 3288 slot = &atq[prod]; 3289 3290 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3291 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3292 3293 *slot = iatq->iatq_desc; 3294 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3295 3296 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3297 ixl_aq_dump(sc, slot, "atq command"); 3298 3299 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3300 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3301 3302 sc->sc_atq_prod = prod_next; 3303 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3304 3305 return 0; 3306 } 3307 3308 static int 3309 ixl_atq_post(struct ixl_softc *sc, struct ixl_atq *iatq) 3310 { 3311 int rv; 3312 3313 mutex_enter(&sc->sc_atq_lock); 3314 rv = ixl_atq_post_locked(sc, iatq); 3315 mutex_exit(&sc->sc_atq_lock); 3316 3317 return rv; 3318 } 3319 3320 static void 3321 ixl_atq_done_locked(struct ixl_softc *sc) 3322 { 3323 struct ixl_aq_desc *atq, *slot; 3324 struct ixl_atq *iatq; 3325 unsigned int cons; 3326 unsigned int prod; 3327 3328 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3329 3330 prod = sc->sc_atq_prod; 3331 cons = sc->sc_atq_cons; 3332 3333 if (prod == cons) 3334 return; 3335 3336 atq = IXL_DMA_KVA(&sc->sc_atq); 3337 3338 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3339 0, IXL_DMA_LEN(&sc->sc_atq), 3340 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3341 3342 do { 3343 slot = &atq[cons]; 3344 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3345 break; 3346 3347 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3348 iatq->iatq_desc = *slot; 3349 3350 memset(slot, 0, sizeof(*slot)); 3351 3352 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3353 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3354 3355 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3356 3357 cons++; 3358 cons &= IXL_AQ_MASK; 3359 } while (cons != prod); 3360 3361 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3362 0, IXL_DMA_LEN(&sc->sc_atq), 3363 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3364 3365 sc->sc_atq_cons = cons; 3366 } 3367 3368 static void 3369 ixl_atq_done(struct ixl_softc *sc) 3370 { 3371 3372 mutex_enter(&sc->sc_atq_lock); 3373 ixl_atq_done_locked(sc); 3374 mutex_exit(&sc->sc_atq_lock); 3375 } 3376 3377 static void 3378 ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3379 { 3380 3381 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3382 3383 cv_signal(&sc->sc_atq_cv); 3384 } 3385 3386 static int 3387 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3388 { 3389 int error; 3390 3391 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3392 3393 ixl_atq_set(iatq, ixl_wakeup); 3394 3395 mutex_enter(&sc->sc_atq_lock); 3396 error = ixl_atq_post_locked(sc, iatq); 3397 if (error) { 3398 mutex_exit(&sc->sc_atq_lock); 3399 return error; 3400 } 3401 3402 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3403 IXL_ATQ_EXEC_TIMEOUT); 3404 mutex_exit(&sc->sc_atq_lock); 3405 3406 return error; 3407 } 3408 3409 static int 3410 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3411 { 3412 struct ixl_aq_desc *atq, *slot; 3413 unsigned int prod; 3414 unsigned int t = 0; 3415 3416 mutex_enter(&sc->sc_atq_lock); 3417 3418 atq = IXL_DMA_KVA(&sc->sc_atq); 3419 prod = sc->sc_atq_prod; 3420 slot = atq + prod; 3421 3422 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3423 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3424 3425 *slot = *iaq; 3426 slot->iaq_flags |= htole16(IXL_AQ_SI); 3427 3428 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3429 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3430 3431 prod++; 3432 prod &= IXL_AQ_MASK; 3433 sc->sc_atq_prod = prod; 3434 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3435 3436 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3437 delaymsec(1); 3438 3439 if (t++ > tm) { 3440 mutex_exit(&sc->sc_atq_lock); 3441 return ETIMEDOUT; 3442 } 3443 } 3444 3445 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3446 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3447 *iaq = *slot; 3448 memset(slot, 0, sizeof(*slot)); 3449 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3450 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3451 3452 sc->sc_atq_cons = prod; 3453 3454 mutex_exit(&sc->sc_atq_lock); 3455 3456 return 0; 3457 } 3458 3459 static int 3460 ixl_get_version(struct ixl_softc *sc) 3461 { 3462 struct ixl_aq_desc iaq; 3463 uint32_t fwbuild, fwver, apiver; 3464 uint16_t api_maj_ver, api_min_ver; 3465 3466 memset(&iaq, 0, sizeof(iaq)); 3467 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3468 3469 iaq.iaq_retval = le16toh(23); 3470 3471 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3472 return ETIMEDOUT; 3473 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3474 return EIO; 3475 3476 fwbuild = le32toh(iaq.iaq_param[1]); 3477 fwver = le32toh(iaq.iaq_param[2]); 3478 apiver = le32toh(iaq.iaq_param[3]); 3479 3480 api_maj_ver = (uint16_t)apiver; 3481 api_min_ver = (uint16_t)(apiver >> 16); 3482 3483 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 3484 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 3485 3486 sc->sc_rxctl_atq = true; 3487 if (sc->sc_mac_type == I40E_MAC_X722) { 3488 if (api_maj_ver == 1 && api_min_ver < 5) { 3489 sc->sc_rxctl_atq = false; 3490 } 3491 } 3492 3493 return 0; 3494 } 3495 3496 static int 3497 ixl_pxe_clear(struct ixl_softc *sc) 3498 { 3499 struct ixl_aq_desc iaq; 3500 int rv; 3501 3502 memset(&iaq, 0, sizeof(iaq)); 3503 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 3504 iaq.iaq_param[0] = htole32(0x2); 3505 3506 rv = ixl_atq_poll(sc, &iaq, 250); 3507 3508 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 3509 3510 if (rv != 0) 3511 return ETIMEDOUT; 3512 3513 switch (iaq.iaq_retval) { 3514 case htole16(IXL_AQ_RC_OK): 3515 case htole16(IXL_AQ_RC_EEXIST): 3516 break; 3517 default: 3518 return EIO; 3519 } 3520 3521 return 0; 3522 } 3523 3524 static int 3525 ixl_lldp_shut(struct ixl_softc *sc) 3526 { 3527 struct ixl_aq_desc iaq; 3528 3529 memset(&iaq, 0, sizeof(iaq)); 3530 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 3531 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 3532 3533 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 3534 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 3535 return -1; 3536 } 3537 3538 switch (iaq.iaq_retval) { 3539 case htole16(IXL_AQ_RC_EMODE): 3540 case htole16(IXL_AQ_RC_EPERM): 3541 /* ignore silently */ 3542 default: 3543 break; 3544 } 3545 3546 return 0; 3547 } 3548 3549 static void 3550 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 3551 { 3552 uint16_t id; 3553 uint32_t number, logical_id; 3554 3555 id = le16toh(cap->cap_id); 3556 number = le32toh(cap->number); 3557 logical_id = le32toh(cap->logical_id); 3558 3559 switch (id) { 3560 case IXL_AQ_CAP_RSS: 3561 sc->sc_rss_table_size = number; 3562 sc->sc_rss_table_entry_width = logical_id; 3563 break; 3564 case IXL_AQ_CAP_RXQ: 3565 case IXL_AQ_CAP_TXQ: 3566 sc->sc_nqueue_pairs_device = MIN(number, 3567 sc->sc_nqueue_pairs_device); 3568 break; 3569 } 3570 } 3571 3572 static int 3573 ixl_get_hw_capabilities(struct ixl_softc *sc) 3574 { 3575 struct ixl_dmamem idm; 3576 struct ixl_aq_desc iaq; 3577 struct ixl_aq_capability *caps; 3578 size_t i, ncaps; 3579 bus_size_t caps_size; 3580 uint16_t status; 3581 int rv; 3582 3583 caps_size = sizeof(caps[0]) * 40; 3584 memset(&iaq, 0, sizeof(iaq)); 3585 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 3586 3587 do { 3588 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 3589 return -1; 3590 } 3591 3592 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3593 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3594 iaq.iaq_datalen = htole16(caps_size); 3595 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3596 3597 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 3598 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 3599 3600 rv = ixl_atq_poll(sc, &iaq, 250); 3601 3602 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 3603 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 3604 3605 if (rv != 0) { 3606 aprint_error(", HW capabilities timeout\n"); 3607 goto done; 3608 } 3609 3610 status = le16toh(iaq.iaq_retval); 3611 3612 if (status == IXL_AQ_RC_ENOMEM) { 3613 caps_size = le16toh(iaq.iaq_datalen); 3614 ixl_dmamem_free(sc, &idm); 3615 } 3616 } while (status == IXL_AQ_RC_ENOMEM); 3617 3618 if (status != IXL_AQ_RC_OK) { 3619 aprint_error(", HW capabilities error\n"); 3620 goto done; 3621 } 3622 3623 caps = IXL_DMA_KVA(&idm); 3624 ncaps = le16toh(iaq.iaq_param[1]); 3625 3626 for (i = 0; i < ncaps; i++) { 3627 ixl_parse_hw_capability(sc, &caps[i]); 3628 } 3629 3630 done: 3631 ixl_dmamem_free(sc, &idm); 3632 return rv; 3633 } 3634 3635 static int 3636 ixl_get_mac(struct ixl_softc *sc) 3637 { 3638 struct ixl_dmamem idm; 3639 struct ixl_aq_desc iaq; 3640 struct ixl_aq_mac_addresses *addrs; 3641 int rv; 3642 3643 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 3644 aprint_error(", unable to allocate mac addresses\n"); 3645 return -1; 3646 } 3647 3648 memset(&iaq, 0, sizeof(iaq)); 3649 iaq.iaq_flags = htole16(IXL_AQ_BUF); 3650 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 3651 iaq.iaq_datalen = htole16(sizeof(*addrs)); 3652 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3653 3654 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3655 BUS_DMASYNC_PREREAD); 3656 3657 rv = ixl_atq_poll(sc, &iaq, 250); 3658 3659 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3660 BUS_DMASYNC_POSTREAD); 3661 3662 if (rv != 0) { 3663 aprint_error(", MAC ADDRESS READ timeout\n"); 3664 rv = -1; 3665 goto done; 3666 } 3667 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3668 aprint_error(", MAC ADDRESS READ error\n"); 3669 rv = -1; 3670 goto done; 3671 } 3672 3673 addrs = IXL_DMA_KVA(&idm); 3674 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 3675 printf(", port address is not valid\n"); 3676 goto done; 3677 } 3678 3679 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 3680 rv = 0; 3681 3682 done: 3683 ixl_dmamem_free(sc, &idm); 3684 return rv; 3685 } 3686 3687 static int 3688 ixl_get_switch_config(struct ixl_softc *sc) 3689 { 3690 struct ixl_dmamem idm; 3691 struct ixl_aq_desc iaq; 3692 struct ixl_aq_switch_config *hdr; 3693 struct ixl_aq_switch_config_element *elms, *elm; 3694 unsigned int nelm, i; 3695 int rv; 3696 3697 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 3698 aprint_error_dev(sc->sc_dev, 3699 "unable to allocate switch config buffer\n"); 3700 return -1; 3701 } 3702 3703 memset(&iaq, 0, sizeof(iaq)); 3704 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3705 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3706 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 3707 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 3708 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3709 3710 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3711 BUS_DMASYNC_PREREAD); 3712 3713 rv = ixl_atq_poll(sc, &iaq, 250); 3714 3715 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3716 BUS_DMASYNC_POSTREAD); 3717 3718 if (rv != 0) { 3719 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 3720 rv = -1; 3721 goto done; 3722 } 3723 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3724 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 3725 rv = -1; 3726 goto done; 3727 } 3728 3729 hdr = IXL_DMA_KVA(&idm); 3730 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 3731 3732 nelm = le16toh(hdr->num_reported); 3733 if (nelm < 1) { 3734 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 3735 rv = -1; 3736 goto done; 3737 } 3738 3739 for (i = 0; i < nelm; i++) { 3740 elm = &elms[i]; 3741 3742 aprint_debug_dev(sc->sc_dev, 3743 "type %x revision %u seid %04x\n", 3744 elm->type, elm->revision, le16toh(elm->seid)); 3745 aprint_debug_dev(sc->sc_dev, 3746 "uplink %04x downlink %04x\n", 3747 le16toh(elm->uplink_seid), 3748 le16toh(elm->downlink_seid)); 3749 aprint_debug_dev(sc->sc_dev, 3750 "conntype %x scheduler %04x extra %04x\n", 3751 elm->connection_type, 3752 le16toh(elm->scheduler_id), 3753 le16toh(elm->element_info)); 3754 } 3755 3756 elm = &elms[0]; 3757 3758 sc->sc_uplink_seid = elm->uplink_seid; 3759 sc->sc_downlink_seid = elm->downlink_seid; 3760 sc->sc_seid = elm->seid; 3761 3762 if ((sc->sc_uplink_seid == htole16(0)) != 3763 (sc->sc_downlink_seid == htole16(0))) { 3764 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 3765 rv = -1; 3766 goto done; 3767 } 3768 3769 done: 3770 ixl_dmamem_free(sc, &idm); 3771 return rv; 3772 } 3773 3774 static int 3775 ixl_phy_mask_ints(struct ixl_softc *sc) 3776 { 3777 struct ixl_aq_desc iaq; 3778 3779 memset(&iaq, 0, sizeof(iaq)); 3780 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 3781 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 3782 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 3783 IXL_AQ_PHY_EV_MEDIA_NA)); 3784 3785 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 3786 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 3787 return -1; 3788 } 3789 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3790 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 3791 return -1; 3792 } 3793 3794 return 0; 3795 } 3796 3797 static int 3798 ixl_get_phy_abilities(struct ixl_softc *sc,struct ixl_dmamem *idm) 3799 { 3800 struct ixl_aq_desc iaq; 3801 int rv; 3802 3803 memset(&iaq, 0, sizeof(iaq)); 3804 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3805 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3806 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 3807 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 3808 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 3809 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 3810 3811 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 3812 BUS_DMASYNC_PREREAD); 3813 3814 rv = ixl_atq_poll(sc, &iaq, 250); 3815 3816 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 3817 BUS_DMASYNC_POSTREAD); 3818 3819 if (rv != 0) 3820 return -1; 3821 3822 return le16toh(iaq.iaq_retval); 3823 } 3824 3825 static int 3826 ixl_get_phy_types(struct ixl_softc *sc, uint64_t *phy_types_ptr) 3827 { 3828 struct ixl_dmamem idm; 3829 struct ixl_aq_phy_abilities *phy; 3830 uint64_t phy_types; 3831 int rv; 3832 3833 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 3834 aprint_error_dev(sc->sc_dev, 3835 "unable to allocate switch config buffer\n"); 3836 return -1; 3837 } 3838 3839 rv = ixl_get_phy_abilities(sc, &idm); 3840 switch (rv) { 3841 case -1: 3842 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 3843 goto done; 3844 case IXL_AQ_RC_OK: 3845 break; 3846 case IXL_AQ_RC_EIO: 3847 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 3848 break; 3849 default: 3850 aprint_error_dev(sc->sc_dev, 3851 "GET PHY ABILITIIES error %u\n", rv); 3852 goto done; 3853 } 3854 3855 phy = IXL_DMA_KVA(&idm); 3856 3857 phy_types = le32toh(phy->phy_type); 3858 phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 3859 3860 *phy_types_ptr = phy_types; 3861 3862 rv = 0; 3863 3864 done: 3865 ixl_dmamem_free(sc, &idm); 3866 return rv; 3867 } 3868 3869 static int 3870 ixl_get_link_status_poll(struct ixl_softc *sc) 3871 { 3872 struct ixl_aq_desc iaq; 3873 struct ixl_aq_link_param *param; 3874 int link; 3875 3876 memset(&iaq, 0, sizeof(iaq)); 3877 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3878 param = (struct ixl_aq_link_param *)iaq.iaq_param; 3879 param->notify = IXL_AQ_LINK_NOTIFY; 3880 3881 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 3882 return ETIMEDOUT; 3883 } 3884 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3885 return EIO; 3886 } 3887 3888 link = ixl_set_link_status(sc, &iaq); 3889 sc->sc_ec.ec_if.if_link_state = link; 3890 3891 return 0; 3892 } 3893 3894 static int 3895 ixl_get_vsi(struct ixl_softc *sc) 3896 { 3897 struct ixl_dmamem *vsi = &sc->sc_scratch; 3898 struct ixl_aq_desc iaq; 3899 struct ixl_aq_vsi_param *param; 3900 struct ixl_aq_vsi_reply *reply; 3901 int rv; 3902 3903 /* grumble, vsi info isn't "known" at compile time */ 3904 3905 memset(&iaq, 0, sizeof(iaq)); 3906 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3907 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3908 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 3909 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 3910 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 3911 3912 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 3913 param->uplink_seid = sc->sc_seid; 3914 3915 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 3916 BUS_DMASYNC_PREREAD); 3917 3918 rv = ixl_atq_poll(sc, &iaq, 250); 3919 3920 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 3921 BUS_DMASYNC_POSTREAD); 3922 3923 if (rv != 0) { 3924 aprint_error_dev(sc->sc_dev, "GET VSI timeout\n"); 3925 return -1; 3926 } 3927 3928 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3929 aprint_error_dev(sc->sc_dev, "GET VSI error %u\n", 3930 le16toh(iaq.iaq_retval)); 3931 return -1; 3932 } 3933 3934 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 3935 sc->sc_vsi_number = reply->vsi_number; 3936 3937 return 0; 3938 } 3939 3940 static int 3941 ixl_set_vsi(struct ixl_softc *sc) 3942 { 3943 struct ixl_dmamem *vsi = &sc->sc_scratch; 3944 struct ixl_aq_desc iaq; 3945 struct ixl_aq_vsi_param *param; 3946 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 3947 unsigned int qnum; 3948 int rv; 3949 3950 qnum = sc->sc_nqueue_pairs - 1; 3951 3952 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 3953 IXL_AQ_VSI_VALID_VLAN); 3954 3955 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 3956 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 3957 data->queue_mapping[0] = htole16(0); 3958 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 3959 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 3960 3961 CLR(data->port_vlan_flags, 3962 htole16(IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK)); 3963 SET(data->port_vlan_flags, 3964 htole16(IXL_AQ_VSI_PVLAN_MODE_ALL | IXL_AQ_VSI_PVLAN_EMOD_NOTHING)); 3965 3966 /* grumble, vsi info isn't "known" at compile time */ 3967 3968 memset(&iaq, 0, sizeof(iaq)); 3969 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 3970 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3971 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 3972 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 3973 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 3974 3975 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 3976 param->uplink_seid = sc->sc_seid; 3977 3978 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 3979 BUS_DMASYNC_PREWRITE); 3980 3981 rv = ixl_atq_poll(sc, &iaq, 250); 3982 3983 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 3984 BUS_DMASYNC_POSTWRITE); 3985 3986 if (rv != 0) { 3987 aprint_error_dev(sc->sc_dev, "UPDATE VSI timeout\n"); 3988 return -1; 3989 } 3990 3991 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3992 aprint_error_dev(sc->sc_dev, "UPDATE VSI error %u\n", 3993 le16toh(iaq.iaq_retval)); 3994 return -1; 3995 } 3996 3997 return 0; 3998 } 3999 4000 static void 4001 ixl_set_filter_control(struct ixl_softc *sc) 4002 { 4003 uint32_t reg; 4004 4005 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4006 4007 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4008 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4009 4010 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4011 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4012 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4013 4014 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4015 } 4016 4017 static inline void 4018 ixl_get_default_rss_key(uint32_t *buf, size_t len) 4019 { 4020 size_t cplen; 4021 uint8_t rss_seed[RSS_KEYSIZE]; 4022 4023 rss_getkey(rss_seed); 4024 memset(buf, 0, len); 4025 4026 cplen = MIN(len, sizeof(rss_seed)); 4027 memcpy(buf, rss_seed, cplen); 4028 } 4029 4030 static void 4031 ixl_set_rss_key(struct ixl_softc *sc) 4032 { 4033 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4034 size_t i; 4035 4036 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4037 4038 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4039 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4040 } 4041 } 4042 4043 static void 4044 ixl_set_rss_pctype(struct ixl_softc *sc) 4045 { 4046 uint64_t set_hena = 0; 4047 uint32_t hena0, hena1; 4048 4049 if (sc->sc_mac_type == I40E_MAC_X722) 4050 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4051 else 4052 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4053 4054 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4055 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4056 4057 SET(hena0, set_hena); 4058 SET(hena1, set_hena >> 32); 4059 4060 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4061 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4062 } 4063 4064 static void 4065 ixl_set_rss_hlut(struct ixl_softc *sc) 4066 { 4067 unsigned int qid; 4068 uint8_t hlut_buf[512], lut_mask; 4069 uint32_t *hluts; 4070 size_t i, hluts_num; 4071 4072 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4073 4074 for (i = 0; i < sc->sc_rss_table_size; i++) { 4075 qid = i % sc->sc_nqueue_pairs; 4076 hlut_buf[i] = qid & lut_mask; 4077 } 4078 4079 hluts = (uint32_t *)hlut_buf; 4080 hluts_num = sc->sc_rss_table_size >> 2; 4081 for (i = 0; i < hluts_num; i++) { 4082 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4083 } 4084 ixl_flush(sc); 4085 } 4086 4087 static void 4088 ixl_config_rss(struct ixl_softc *sc) 4089 { 4090 4091 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4092 4093 ixl_set_rss_key(sc); 4094 ixl_set_rss_pctype(sc); 4095 ixl_set_rss_hlut(sc); 4096 } 4097 4098 static const struct ixl_phy_type * 4099 ixl_search_phy_type(uint8_t phy_type) 4100 { 4101 const struct ixl_phy_type *itype; 4102 uint64_t mask; 4103 unsigned int i; 4104 4105 if (phy_type >= 64) 4106 return NULL; 4107 4108 mask = 1ULL << phy_type; 4109 4110 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4111 itype = &ixl_phy_type_map[i]; 4112 4113 if (ISSET(itype->phy_type, mask)) 4114 return itype; 4115 } 4116 4117 return NULL; 4118 } 4119 4120 static uint64_t 4121 ixl_search_link_speed(uint8_t link_speed) 4122 { 4123 const struct ixl_speed_type *type; 4124 unsigned int i; 4125 4126 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4127 type = &ixl_speed_type_map[i]; 4128 4129 if (ISSET(type->dev_speed, link_speed)) 4130 return type->net_speed; 4131 } 4132 4133 return 0; 4134 } 4135 4136 static int 4137 ixl_restart_an(struct ixl_softc *sc) 4138 { 4139 struct ixl_aq_desc iaq; 4140 4141 memset(&iaq, 0, sizeof(iaq)); 4142 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4143 iaq.iaq_param[0] = 4144 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4145 4146 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4147 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4148 return -1; 4149 } 4150 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4151 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4152 return -1; 4153 } 4154 4155 return 0; 4156 } 4157 4158 static int 4159 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4160 uint16_t vlan, uint16_t flags) 4161 { 4162 struct ixl_aq_desc iaq; 4163 struct ixl_aq_add_macvlan *param; 4164 struct ixl_aq_add_macvlan_elem *elem; 4165 4166 memset(&iaq, 0, sizeof(iaq)); 4167 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4168 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4169 iaq.iaq_datalen = htole16(sizeof(*elem)); 4170 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4171 4172 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4173 param->num_addrs = htole16(1); 4174 param->seid0 = htole16(0x8000) | sc->sc_seid; 4175 param->seid1 = 0; 4176 param->seid2 = 0; 4177 4178 elem = IXL_DMA_KVA(&sc->sc_scratch); 4179 memset(elem, 0, sizeof(*elem)); 4180 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4181 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4182 elem->vlan = htole16(vlan); 4183 4184 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4185 return IXL_AQ_RC_EINVAL; 4186 } 4187 4188 switch (le16toh(iaq.iaq_retval)) { 4189 case IXL_AQ_RC_OK: 4190 break; 4191 case IXL_AQ_RC_ENOSPC: 4192 return ENOSPC; 4193 case IXL_AQ_RC_ENOENT: 4194 return ENOENT; 4195 case IXL_AQ_RC_EACCES: 4196 return EACCES; 4197 case IXL_AQ_RC_EEXIST: 4198 return EEXIST; 4199 case IXL_AQ_RC_EINVAL: 4200 return EINVAL; 4201 default: 4202 return EIO; 4203 } 4204 4205 return 0; 4206 } 4207 4208 static int 4209 ixl_remove_macvlan(struct ixl_softc *sc, uint8_t *macaddr, 4210 uint16_t vlan, uint16_t flags) 4211 { 4212 struct ixl_aq_desc iaq; 4213 struct ixl_aq_remove_macvlan *param; 4214 struct ixl_aq_remove_macvlan_elem *elem; 4215 4216 memset(&iaq, 0, sizeof(iaq)); 4217 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4218 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 4219 iaq.iaq_datalen = htole16(sizeof(*elem)); 4220 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4221 4222 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 4223 param->num_addrs = htole16(1); 4224 param->seid0 = htole16(0x8000) | sc->sc_seid; 4225 param->seid1 = 0; 4226 param->seid2 = 0; 4227 4228 elem = IXL_DMA_KVA(&sc->sc_scratch); 4229 memset(elem, 0, sizeof(*elem)); 4230 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4231 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 4232 elem->vlan = htole16(vlan); 4233 4234 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4235 return EINVAL; 4236 } 4237 4238 switch (le16toh(iaq.iaq_retval)) { 4239 case IXL_AQ_RC_OK: 4240 break; 4241 case IXL_AQ_RC_ENOENT: 4242 return ENOENT; 4243 case IXL_AQ_RC_EACCES: 4244 return EACCES; 4245 case IXL_AQ_RC_EINVAL: 4246 return EINVAL; 4247 default: 4248 return EIO; 4249 } 4250 4251 return 0; 4252 } 4253 4254 static int 4255 ixl_hmc(struct ixl_softc *sc) 4256 { 4257 struct { 4258 uint32_t count; 4259 uint32_t minsize; 4260 bus_size_t objsiz; 4261 bus_size_t setoff; 4262 bus_size_t setcnt; 4263 } regs[] = { 4264 { 4265 0, 4266 IXL_HMC_TXQ_MINSIZE, 4267 I40E_GLHMC_LANTXOBJSZ, 4268 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 4269 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 4270 }, 4271 { 4272 0, 4273 IXL_HMC_RXQ_MINSIZE, 4274 I40E_GLHMC_LANRXOBJSZ, 4275 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 4276 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 4277 }, 4278 { 4279 0, 4280 0, 4281 I40E_GLHMC_FCOEDDPOBJSZ, 4282 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 4283 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 4284 }, 4285 { 4286 0, 4287 0, 4288 I40E_GLHMC_FCOEFOBJSZ, 4289 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 4290 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 4291 }, 4292 }; 4293 struct ixl_hmc_entry *e; 4294 uint64_t size, dva; 4295 uint8_t *kva; 4296 uint64_t *sdpage; 4297 unsigned int i; 4298 int npages, tables; 4299 uint32_t reg; 4300 4301 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 4302 4303 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 4304 ixl_rd(sc, I40E_GLHMC_LANQMAX); 4305 4306 size = 0; 4307 for (i = 0; i < __arraycount(regs); i++) { 4308 e = &sc->sc_hmc_entries[i]; 4309 4310 e->hmc_count = regs[i].count; 4311 reg = ixl_rd(sc, regs[i].objsiz); 4312 e->hmc_size = BIT_ULL(0x3F & reg); 4313 e->hmc_base = size; 4314 4315 if ((e->hmc_size * 8) < regs[i].minsize) { 4316 aprint_error_dev(sc->sc_dev, 4317 "kernel hmc entry is too big\n"); 4318 return -1; 4319 } 4320 4321 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 4322 } 4323 size = roundup(size, IXL_HMC_PGSIZE); 4324 npages = size / IXL_HMC_PGSIZE; 4325 4326 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 4327 4328 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 4329 aprint_error_dev(sc->sc_dev, 4330 "unable to allocate hmc pd memory\n"); 4331 return -1; 4332 } 4333 4334 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 4335 IXL_HMC_PGSIZE) != 0) { 4336 aprint_error_dev(sc->sc_dev, 4337 "unable to allocate hmc sd memory\n"); 4338 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 4339 return -1; 4340 } 4341 4342 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 4343 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 4344 4345 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 4346 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 4347 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4348 4349 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 4350 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 4351 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 4352 4353 for (i = 0; (int)i < npages; i++) { 4354 *sdpage = htole64(dva | IXL_HMC_PDVALID); 4355 sdpage++; 4356 4357 dva += IXL_HMC_PGSIZE; 4358 } 4359 4360 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 4361 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 4362 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4363 4364 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 4365 for (i = 0; (int)i < tables; i++) { 4366 uint32_t count; 4367 4368 KASSERT(npages >= 0); 4369 4370 count = ((unsigned int)npages > IXL_HMC_PGS) ? 4371 IXL_HMC_PGS : (unsigned int)npages; 4372 4373 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 4374 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 4375 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 4376 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 4377 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 4378 ixl_wr(sc, I40E_PFHMC_SDCMD, 4379 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 4380 4381 npages -= IXL_HMC_PGS; 4382 dva += IXL_HMC_PGSIZE; 4383 } 4384 4385 for (i = 0; i < __arraycount(regs); i++) { 4386 e = &sc->sc_hmc_entries[i]; 4387 4388 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 4389 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 4390 } 4391 4392 return 0; 4393 } 4394 4395 static void 4396 ixl_hmc_free(struct ixl_softc *sc) 4397 { 4398 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 4399 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 4400 } 4401 4402 static void 4403 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 4404 unsigned int npacking) 4405 { 4406 uint8_t *dst = d; 4407 const uint8_t *src = s; 4408 unsigned int i; 4409 4410 for (i = 0; i < npacking; i++) { 4411 const struct ixl_hmc_pack *pack = &packing[i]; 4412 unsigned int offset = pack->lsb / 8; 4413 unsigned int align = pack->lsb % 8; 4414 const uint8_t *in = src + pack->offset; 4415 uint8_t *out = dst + offset; 4416 int width = pack->width; 4417 unsigned int inbits = 0; 4418 4419 if (align) { 4420 inbits = (*in++) << align; 4421 *out++ |= (inbits & 0xff); 4422 inbits >>= 8; 4423 4424 width -= 8 - align; 4425 } 4426 4427 while (width >= 8) { 4428 inbits |= (*in++) << align; 4429 *out++ = (inbits & 0xff); 4430 inbits >>= 8; 4431 4432 width -= 8; 4433 } 4434 4435 if (width > 0) { 4436 inbits |= (*in) << align; 4437 *out |= (inbits & ((1 << width) - 1)); 4438 } 4439 } 4440 } 4441 4442 static struct ixl_aq_buf * 4443 ixl_aqb_alloc(struct ixl_softc *sc) 4444 { 4445 struct ixl_aq_buf *aqb; 4446 4447 aqb = malloc(sizeof(*aqb), M_DEVBUF, M_WAITOK); 4448 if (aqb == NULL) 4449 return NULL; 4450 4451 aqb->aqb_size = IXL_AQ_BUFLEN; 4452 4453 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 4454 aqb->aqb_size, 0, 4455 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 4456 goto free; 4457 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 4458 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 4459 BUS_DMA_WAITOK) != 0) 4460 goto destroy; 4461 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 4462 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 4463 goto dma_free; 4464 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 4465 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 4466 goto unmap; 4467 4468 return aqb; 4469 unmap: 4470 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 4471 dma_free: 4472 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 4473 destroy: 4474 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 4475 free: 4476 free(aqb, M_DEVBUF); 4477 4478 return NULL; 4479 } 4480 4481 static void 4482 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 4483 { 4484 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 4485 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 4486 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 4487 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 4488 free(aqb, M_DEVBUF); 4489 } 4490 4491 static int 4492 ixl_arq_fill(struct ixl_softc *sc) 4493 { 4494 struct ixl_aq_buf *aqb; 4495 struct ixl_aq_desc *arq, *iaq; 4496 unsigned int prod = sc->sc_arq_prod; 4497 unsigned int n; 4498 int post = 0; 4499 4500 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 4501 IXL_AQ_NUM); 4502 arq = IXL_DMA_KVA(&sc->sc_arq); 4503 4504 if (__predict_false(n <= 0)) 4505 return 0; 4506 4507 do { 4508 aqb = sc->sc_arq_live[prod]; 4509 iaq = &arq[prod]; 4510 4511 if (aqb == NULL) { 4512 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 4513 if (aqb != NULL) { 4514 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 4515 ixl_aq_buf, aqb_entry); 4516 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 4517 break; 4518 } 4519 4520 sc->sc_arq_live[prod] = aqb; 4521 memset(aqb->aqb_data, 0, aqb->aqb_size); 4522 4523 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 4524 aqb->aqb_size, BUS_DMASYNC_PREREAD); 4525 4526 iaq->iaq_flags = htole16(IXL_AQ_BUF | 4527 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 4528 IXL_AQ_LB : 0)); 4529 iaq->iaq_opcode = 0; 4530 iaq->iaq_datalen = htole16(aqb->aqb_size); 4531 iaq->iaq_retval = 0; 4532 iaq->iaq_cookie = 0; 4533 iaq->iaq_param[0] = 0; 4534 iaq->iaq_param[1] = 0; 4535 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 4536 } 4537 4538 prod++; 4539 prod &= IXL_AQ_MASK; 4540 4541 post = 1; 4542 4543 } while (--n); 4544 4545 if (post) { 4546 sc->sc_arq_prod = prod; 4547 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 4548 } 4549 4550 return post; 4551 } 4552 4553 static void 4554 ixl_arq_unfill(struct ixl_softc *sc) 4555 { 4556 struct ixl_aq_buf *aqb; 4557 unsigned int i; 4558 4559 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 4560 aqb = sc->sc_arq_live[i]; 4561 if (aqb == NULL) 4562 continue; 4563 4564 sc->sc_arq_live[i] = NULL; 4565 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 4566 BUS_DMASYNC_POSTREAD); 4567 ixl_aqb_free(sc, aqb); 4568 } 4569 4570 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 4571 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 4572 ixl_aq_buf, aqb_entry); 4573 ixl_aqb_free(sc, aqb); 4574 } 4575 } 4576 4577 static void 4578 ixl_clear_hw(struct ixl_softc *sc) 4579 { 4580 uint32_t num_queues, base_queue; 4581 uint32_t num_pf_int; 4582 uint32_t num_vf_int; 4583 uint32_t num_vfs; 4584 uint32_t i, j; 4585 uint32_t val; 4586 uint32_t eol = 0x7ff; 4587 4588 /* get number of interrupts, queues, and vfs */ 4589 val = ixl_rd(sc, I40E_GLPCI_CNF2); 4590 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 4591 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 4592 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 4593 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 4594 4595 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 4596 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 4597 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 4598 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 4599 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 4600 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 4601 num_queues = (j - base_queue) + 1; 4602 else 4603 num_queues = 0; 4604 4605 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 4606 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 4607 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 4608 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 4609 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 4610 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 4611 num_vfs = (j - i) + 1; 4612 else 4613 num_vfs = 0; 4614 4615 /* stop all the interrupts */ 4616 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 4617 ixl_flush(sc); 4618 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 4619 for (i = 0; i < num_pf_int - 2; i++) 4620 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 4621 ixl_flush(sc); 4622 4623 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 4624 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4625 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 4626 for (i = 0; i < num_pf_int - 2; i++) 4627 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 4628 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4629 for (i = 0; i < num_vfs; i++) 4630 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 4631 for (i = 0; i < num_vf_int - 2; i++) 4632 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 4633 4634 /* warn the HW of the coming Tx disables */ 4635 for (i = 0; i < num_queues; i++) { 4636 uint32_t abs_queue_idx = base_queue + i; 4637 uint32_t reg_block = 0; 4638 4639 if (abs_queue_idx >= 128) { 4640 reg_block = abs_queue_idx / 128; 4641 abs_queue_idx %= 128; 4642 } 4643 4644 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 4645 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 4646 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 4647 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 4648 4649 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 4650 } 4651 delaymsec(400); 4652 4653 /* stop all the queues */ 4654 for (i = 0; i < num_queues; i++) { 4655 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 4656 ixl_wr(sc, I40E_QTX_ENA(i), 0); 4657 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 4658 ixl_wr(sc, I40E_QRX_ENA(i), 0); 4659 } 4660 4661 /* short wait for all queue disables to settle */ 4662 delaymsec(50); 4663 } 4664 4665 static int 4666 ixl_pf_reset(struct ixl_softc *sc) 4667 { 4668 uint32_t cnt = 0; 4669 uint32_t cnt1 = 0; 4670 uint32_t reg = 0, reg0 = 0; 4671 uint32_t grst_del; 4672 4673 /* 4674 * Poll for Global Reset steady state in case of recent GRST. 4675 * The grst delay value is in 100ms units, and we'll wait a 4676 * couple counts longer to be sure we don't just miss the end. 4677 */ 4678 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 4679 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 4680 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 4681 4682 grst_del = grst_del * 20; 4683 4684 for (cnt = 0; cnt < grst_del; cnt++) { 4685 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 4686 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 4687 break; 4688 delaymsec(100); 4689 } 4690 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 4691 aprint_error(", Global reset polling failed to complete\n"); 4692 return -1; 4693 } 4694 4695 /* Now Wait for the FW to be ready */ 4696 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 4697 reg = ixl_rd(sc, I40E_GLNVM_ULD); 4698 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4699 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 4700 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4701 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 4702 break; 4703 4704 delaymsec(10); 4705 } 4706 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4707 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 4708 aprint_error(", wait for FW Reset complete timed out " 4709 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 4710 return -1; 4711 } 4712 4713 /* 4714 * If there was a Global Reset in progress when we got here, 4715 * we don't need to do the PF Reset 4716 */ 4717 if (cnt == 0) { 4718 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 4719 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 4720 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 4721 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 4722 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 4723 break; 4724 delaymsec(1); 4725 4726 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 4727 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 4728 aprint_error(", Core reset upcoming." 4729 " Skipping PF reset reset request\n"); 4730 return -1; 4731 } 4732 } 4733 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 4734 aprint_error(", PF reset polling failed to complete" 4735 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 4736 return -1; 4737 } 4738 } 4739 4740 return 0; 4741 } 4742 4743 static int 4744 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 4745 bus_size_t size, bus_size_t align) 4746 { 4747 ixm->ixm_size = size; 4748 4749 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 4750 ixm->ixm_size, 0, 4751 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 4752 &ixm->ixm_map) != 0) 4753 return 1; 4754 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 4755 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 4756 BUS_DMA_WAITOK) != 0) 4757 goto destroy; 4758 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 4759 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 4760 goto free; 4761 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 4762 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 4763 goto unmap; 4764 4765 memset(ixm->ixm_kva, 0, ixm->ixm_size); 4766 4767 return 0; 4768 unmap: 4769 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 4770 free: 4771 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 4772 destroy: 4773 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 4774 return 1; 4775 } 4776 4777 static void 4778 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 4779 { 4780 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 4781 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 4782 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 4783 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 4784 } 4785 4786 static int 4787 ixl_set_macvlan(struct ixl_softc *sc) 4788 { 4789 int error, rv = 0; 4790 4791 /* remove default mac filter and replace it so we can see vlans */ 4792 4793 error = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 4794 if (error != 0 && error != ENOENT) { 4795 aprint_debug_dev(sc->sc_dev, "unable to remove macvlan\n"); 4796 rv = -1; 4797 } 4798 4799 error = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 4800 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 4801 if (error != 0 && error != ENOENT) { 4802 aprint_debug_dev(sc->sc_dev, 4803 "unable to remove macvlan(IGNORE_VLAN)\n"); 4804 rv = -1; 4805 } 4806 4807 error = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 4808 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 4809 if (error != 0) { 4810 aprint_debug_dev(sc->sc_dev, "unable to add mac address\n"); 4811 rv = -1; 4812 } 4813 4814 error = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 4815 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 4816 if (error != 0) { 4817 aprint_debug_dev(sc->sc_dev, 4818 "unable to add broadcast mac address\n"); 4819 rv = -1; 4820 } 4821 4822 return rv; 4823 } 4824 4825 static int 4826 ixl_ifflags_cb(struct ethercom *ec) 4827 { 4828 struct ifnet *ifp = &ec->ec_if; 4829 struct ixl_softc *sc = ifp->if_softc; 4830 int rv; 4831 4832 mutex_enter(&sc->sc_cfg_lock); 4833 rv = ixl_iff(sc); 4834 mutex_exit(&sc->sc_cfg_lock); 4835 4836 return rv; 4837 } 4838 4839 static int 4840 ixl_set_link_status(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 4841 { 4842 const struct ixl_aq_link_status *status; 4843 const struct ixl_phy_type *itype; 4844 4845 uint64_t ifm_active = IFM_ETHER; 4846 uint64_t ifm_status = IFM_AVALID; 4847 int link_state = LINK_STATE_DOWN; 4848 uint64_t baudrate = 0; 4849 4850 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 4851 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) 4852 goto done; 4853 4854 ifm_active |= IFM_FDX; 4855 ifm_status |= IFM_ACTIVE; 4856 link_state = LINK_STATE_UP; 4857 4858 itype = ixl_search_phy_type(status->phy_type); 4859 if (itype != NULL) 4860 ifm_active |= itype->ifm_type; 4861 4862 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 4863 ifm_active |= IFM_ETH_TXPAUSE; 4864 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 4865 ifm_active |= IFM_ETH_RXPAUSE; 4866 4867 baudrate = ixl_search_link_speed(status->link_speed); 4868 4869 done: 4870 /* NET_ASSERT_LOCKED() except during attach */ 4871 sc->sc_media_active = ifm_active; 4872 sc->sc_media_status = ifm_status; 4873 4874 sc->sc_ec.ec_if.if_baudrate = baudrate; 4875 4876 return link_state; 4877 } 4878 4879 static int 4880 ixl_establish_intx(struct ixl_softc *sc) 4881 { 4882 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 4883 pci_intr_handle_t *intr; 4884 char xnamebuf[32]; 4885 char intrbuf[PCI_INTRSTR_LEN]; 4886 char const *intrstr; 4887 4888 KASSERT(sc->sc_nintrs == 1); 4889 4890 intr = &sc->sc_ihp[0]; 4891 4892 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 4893 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 4894 device_xname(sc->sc_dev)); 4895 4896 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 4897 sc, xnamebuf); 4898 4899 if (sc->sc_ihs[0] == NULL) { 4900 aprint_error_dev(sc->sc_dev, 4901 "unable to establish interrupt at %s\n", intrstr); 4902 return -1; 4903 } 4904 4905 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 4906 return 0; 4907 } 4908 4909 static int 4910 ixl_establish_msix(struct ixl_softc *sc) 4911 { 4912 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 4913 unsigned int vector = 0; 4914 unsigned int i; 4915 char xnamebuf[32]; 4916 char intrbuf[PCI_INTRSTR_LEN]; 4917 char const *intrstr; 4918 4919 /* the "other" intr is mapped to vector 0 */ 4920 vector = 0; 4921 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 4922 intrbuf, sizeof(intrbuf)); 4923 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 4924 device_xname(sc->sc_dev)); 4925 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 4926 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 4927 sc, xnamebuf); 4928 if (sc->sc_ihs[vector] == NULL) { 4929 aprint_error_dev(sc->sc_dev, 4930 "unable to establish interrupt at %s\n", intrstr); 4931 goto fail; 4932 } 4933 vector++; 4934 aprint_normal_dev(sc->sc_dev, "interrupt at %s\n", intrstr); 4935 4936 sc->sc_msix_vector_queue = vector; 4937 4938 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 4939 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 4940 intrbuf, sizeof(intrbuf)); 4941 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 4942 device_xname(sc->sc_dev), i); 4943 4944 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 4945 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 4946 (void *)&sc->sc_qps[i], xnamebuf); 4947 4948 if (sc->sc_ihs[vector] == NULL) { 4949 aprint_error_dev(sc->sc_dev, 4950 "unable to establish interrupt at %s\n", intrstr); 4951 goto fail; 4952 } 4953 vector++; 4954 aprint_normal_dev(sc->sc_dev, 4955 "interrupt at %s\n", intrstr); 4956 } 4957 4958 return 0; 4959 fail: 4960 for (i = 0; i < vector; i++) { 4961 pci_intr_disestablish(pc, sc->sc_ihs[i]); 4962 } 4963 4964 sc->sc_msix_vector_queue = 0; 4965 sc->sc_msix_vector_queue = 0; 4966 4967 return -1; 4968 } 4969 4970 static void 4971 ixl_set_affinity_msix(struct ixl_softc *sc) 4972 { 4973 kcpuset_t *affinity; 4974 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 4975 int affinity_to, r; 4976 unsigned int i, vector; 4977 char intrbuf[PCI_INTRSTR_LEN]; 4978 char const *intrstr; 4979 4980 affinity_to = 0; 4981 kcpuset_create(&affinity, false); 4982 4983 vector = sc->sc_msix_vector_queue; 4984 4985 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 4986 affinity_to = i % ncpu; 4987 4988 kcpuset_zero(affinity); 4989 kcpuset_set(affinity, affinity_to); 4990 4991 intrstr = pci_intr_string(pc, sc->sc_ihp[vector + i], 4992 intrbuf, sizeof(intrbuf)); 4993 r = interrupt_distribute(sc->sc_ihs[vector + i], 4994 affinity, NULL); 4995 if (r == 0) { 4996 aprint_normal_dev(sc->sc_dev, 4997 "for TXRX%u interrupting at %s affinity to %u\n", 4998 i, intrstr, affinity_to); 4999 } else { 5000 aprint_normal_dev(sc->sc_dev, 5001 "for TXRX%u interrupting at %s\n", 5002 i, intrstr); 5003 } 5004 } 5005 5006 vector = 0; /* vector 0 means "other" interrupt */ 5007 affinity_to = (affinity_to + 1) % ncpu; 5008 kcpuset_zero(affinity); 5009 kcpuset_set(affinity, affinity_to); 5010 5011 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5012 intrbuf, sizeof(intrbuf)); 5013 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5014 if (r == 0) { 5015 aprint_normal_dev(sc->sc_dev, 5016 "for other interrupting at %s affinity to %u\n", 5017 intrstr, affinity_to); 5018 } else { 5019 aprint_normal_dev(sc->sc_dev, 5020 "for other interrupting at %s", intrstr); 5021 } 5022 5023 kcpuset_destroy(affinity); 5024 } 5025 5026 static void 5027 ixl_config_queue_intr(struct ixl_softc *sc) 5028 { 5029 unsigned int i, vector; 5030 5031 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5032 vector = sc->sc_msix_vector_queue; 5033 } else { 5034 vector = I40E_INTR_NOTX_INTR; 5035 5036 ixl_wr(sc, I40E_PFINT_LNKLST0, 5037 (I40E_INTR_NOTX_QUEUE << 5038 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5039 (I40E_QUEUE_TYPE_RX << 5040 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5041 } 5042 5043 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5044 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5045 ixl_flush(sc); 5046 5047 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5048 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5049 (I40E_QUEUE_TYPE_RX << 5050 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5051 5052 ixl_wr(sc, I40E_QINT_RQCTL(i), 5053 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5054 (I40E_ITR_INDEX_RX << 5055 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5056 (I40E_INTR_NOTX_RX_QUEUE << 5057 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5058 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5059 (I40E_QUEUE_TYPE_TX << 5060 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5061 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5062 5063 ixl_wr(sc, I40E_QINT_TQCTL(i), 5064 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5065 (I40E_ITR_INDEX_TX << 5066 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5067 (I40E_INTR_NOTX_TX_QUEUE << 5068 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5069 (I40E_QUEUE_TYPE_EOL << 5070 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5071 (I40E_QUEUE_TYPE_RX << 5072 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5073 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5074 5075 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) 5076 vector++; 5077 } 5078 ixl_flush(sc); 5079 5080 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), 0x7a); 5081 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), 0x7a); 5082 ixl_flush(sc); 5083 } 5084 5085 static void 5086 ixl_config_other_intr(struct ixl_softc *sc) 5087 { 5088 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5089 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5090 5091 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5092 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5093 I40E_PFINT_ICR0_ENA_GRST_MASK | 5094 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5095 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5096 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5097 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5098 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5099 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5100 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5101 5102 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5103 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5104 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5105 (I40E_ITR_INDEX_OTHER << 5106 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5107 ixl_flush(sc); 5108 } 5109 5110 static int 5111 ixl_setup_interrupts(struct ixl_softc *sc) 5112 { 5113 struct pci_attach_args *pa = &sc->sc_pa; 5114 pci_intr_type_t max_type, intr_type; 5115 int counts[PCI_INTR_TYPE_SIZE]; 5116 int error; 5117 unsigned int i; 5118 bool retry, nomsix = IXL_NOMSIX; 5119 5120 memset(counts, 0, sizeof(counts)); 5121 max_type = PCI_INTR_TYPE_MSIX; 5122 /* QPs + other interrupt */ 5123 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5124 counts[PCI_INTR_TYPE_INTX] = 1; 5125 5126 if (nomsix) 5127 counts[PCI_INTR_TYPE_MSIX] = 0; 5128 5129 do { 5130 retry = false; 5131 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5132 if (error != 0) { 5133 aprint_error_dev(sc->sc_dev, 5134 "couldn't map interrupt\n"); 5135 break; 5136 } 5137 for (i = 0; i < sc->sc_nintrs; i++) { 5138 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5139 PCI_INTR_MPSAFE, true); 5140 } 5141 5142 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5143 sc->sc_nintrs = counts[intr_type]; 5144 KASSERT(sc->sc_nintrs > 0); 5145 5146 sc->sc_ihs = kmem_alloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5147 KM_SLEEP); 5148 5149 if (intr_type == PCI_INTR_TYPE_MSIX) { 5150 error = ixl_establish_msix(sc); 5151 if (error) { 5152 counts[PCI_INTR_TYPE_MSIX] = 0; 5153 retry = true; 5154 } else { 5155 ixl_set_affinity_msix(sc); 5156 } 5157 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5158 error = ixl_establish_intx(sc); 5159 } else { 5160 error = -1; 5161 } 5162 5163 if (error) { 5164 kmem_free(sc->sc_ihs, 5165 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5166 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5167 } else { 5168 sc->sc_intrtype = intr_type; 5169 } 5170 } while (retry); 5171 5172 return error; 5173 } 5174 5175 static void 5176 ixl_teardown_interrupts(struct ixl_softc *sc) 5177 { 5178 struct pci_attach_args *pa = &sc->sc_pa; 5179 unsigned int i; 5180 5181 for (i = 0; i < sc->sc_nintrs; i++) { 5182 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 5183 } 5184 5185 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5186 5187 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5188 sc->sc_ihs = NULL; 5189 sc->sc_nintrs = 0; 5190 } 5191 5192 static int 5193 ixl_setup_stats(struct ixl_softc *sc) 5194 { 5195 struct ixl_queue_pair *qp; 5196 struct ixl_tx_ring *txr; 5197 struct ixl_rx_ring *rxr; 5198 unsigned int i; 5199 5200 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5201 qp = &sc->sc_qps[i]; 5202 txr = qp->qp_txr; 5203 rxr = qp->qp_rxr; 5204 5205 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 5206 NULL, qp->qp_name, "m_defrag successed"); 5207 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 5208 NULL, qp->qp_name, "m_defrag_failed"); 5209 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 5210 NULL, qp->qp_name, "Dropped in pcq"); 5211 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 5212 NULL, qp->qp_name, "Deferred transmit"); 5213 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 5214 NULL, qp->qp_name, "Interrupt on queue"); 5215 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 5216 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5217 5218 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 5219 NULL, qp->qp_name, "MGETHDR failed"); 5220 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 5221 NULL, qp->qp_name, "MCLGET failed"); 5222 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 5223 EVCNT_TYPE_MISC, NULL, qp->qp_name, 5224 "bus_dmamap_load_mbuf failed"); 5225 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 5226 NULL, qp->qp_name, "Interrupt on queue"); 5227 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 5228 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5229 } 5230 5231 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 5232 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 5233 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 5234 NULL, device_xname(sc->sc_dev), "Link status event"); 5235 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 5236 NULL, device_xname(sc->sc_dev), "ECC error"); 5237 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 5238 NULL, device_xname(sc->sc_dev), "PCI exception"); 5239 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 5240 NULL, device_xname(sc->sc_dev), "Critical error"); 5241 5242 return 0; 5243 } 5244 5245 static void 5246 ixl_teardown_stats(struct ixl_softc *sc) 5247 { 5248 struct ixl_tx_ring *txr; 5249 struct ixl_rx_ring *rxr; 5250 unsigned int i; 5251 5252 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5253 txr = sc->sc_qps[i].qp_txr; 5254 rxr = sc->sc_qps[i].qp_rxr; 5255 5256 evcnt_detach(&txr->txr_defragged); 5257 evcnt_detach(&txr->txr_defrag_failed); 5258 evcnt_detach(&txr->txr_pcqdrop); 5259 evcnt_detach(&txr->txr_transmitdef); 5260 evcnt_detach(&txr->txr_intr); 5261 evcnt_detach(&txr->txr_defer); 5262 5263 evcnt_detach(&rxr->rxr_mgethdr_failed); 5264 evcnt_detach(&rxr->rxr_mgetcl_failed); 5265 evcnt_detach(&rxr->rxr_mbuf_load_failed); 5266 evcnt_detach(&rxr->rxr_intr); 5267 evcnt_detach(&rxr->rxr_defer); 5268 } 5269 5270 evcnt_detach(&sc->sc_event_atq); 5271 evcnt_detach(&sc->sc_event_link); 5272 evcnt_detach(&sc->sc_event_ecc_err); 5273 evcnt_detach(&sc->sc_event_pci_exception); 5274 evcnt_detach(&sc->sc_event_crit_err); 5275 } 5276 5277 static int 5278 ixl_setup_sysctls(struct ixl_softc *sc) 5279 { 5280 const char *devname; 5281 struct sysctllog **log; 5282 const struct sysctlnode *rnode, *rxnode, *txnode; 5283 int error; 5284 5285 log = &sc->sc_sysctllog; 5286 devname = device_xname(sc->sc_dev); 5287 5288 error = sysctl_createv(log, 0, NULL, &rnode, 5289 0, CTLTYPE_NODE, devname, 5290 SYSCTL_DESCR("ixl information and settings"), 5291 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 5292 if (error) 5293 goto out; 5294 5295 error = sysctl_createv(log, 0, &rnode, NULL, 5296 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 5297 SYSCTL_DESCR("Use workqueue for packet processing"), 5298 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 5299 if (error) 5300 goto out; 5301 5302 error = sysctl_createv(log, 0, &rnode, &rxnode, 5303 0, CTLTYPE_NODE, "rx", 5304 SYSCTL_DESCR("ixl information and settings for Rx"), 5305 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 5306 if (error) 5307 goto out; 5308 5309 error = sysctl_createv(log, 0, &rxnode, NULL, 5310 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 5311 SYSCTL_DESCR("max number of Rx packets" 5312 " to process for interrupt processing"), 5313 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 5314 if (error) 5315 goto out; 5316 5317 error = sysctl_createv(log, 0, &rxnode, NULL, 5318 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 5319 SYSCTL_DESCR("max number of Rx packets" 5320 " to process for deferred processing"), 5321 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 5322 if (error) 5323 goto out; 5324 5325 error = sysctl_createv(log, 0, &rnode, &txnode, 5326 0, CTLTYPE_NODE, "tx", 5327 SYSCTL_DESCR("ixl information and settings for Tx"), 5328 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 5329 if (error) 5330 goto out; 5331 5332 error = sysctl_createv(log, 0, &txnode, NULL, 5333 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 5334 SYSCTL_DESCR("max number of Tx packets" 5335 " to process for interrupt processing"), 5336 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 5337 if (error) 5338 goto out; 5339 5340 error = sysctl_createv(log, 0, &txnode, NULL, 5341 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 5342 SYSCTL_DESCR("max number of Tx packets" 5343 " to process for deferred processing"), 5344 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 5345 if (error) 5346 goto out; 5347 5348 out: 5349 if (error) { 5350 aprint_error_dev(sc->sc_dev, 5351 "unable to create sysctl node\n"); 5352 sysctl_teardown(log); 5353 } 5354 5355 return error; 5356 } 5357 5358 static void 5359 ixl_teardown_sysctls(struct ixl_softc *sc) 5360 { 5361 5362 sysctl_teardown(&sc->sc_sysctllog); 5363 } 5364 5365 static struct workqueue * 5366 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 5367 { 5368 struct workqueue *wq; 5369 int error; 5370 5371 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 5372 prio, ipl, flags); 5373 5374 if (error) 5375 return NULL; 5376 5377 return wq; 5378 } 5379 5380 static void 5381 ixl_workq_destroy(struct workqueue *wq) 5382 { 5383 5384 workqueue_destroy(wq); 5385 } 5386 5387 static void 5388 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 5389 { 5390 5391 memset(work, 0, sizeof(*work)); 5392 work->ixw_func = func; 5393 work->ixw_arg = arg; 5394 } 5395 5396 static void 5397 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 5398 { 5399 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 5400 return; 5401 5402 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 5403 } 5404 5405 static void 5406 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 5407 { 5408 5409 workqueue_wait(wq, &work->ixw_cookie); 5410 } 5411 5412 static void 5413 ixl_workq_work(struct work *wk, void *context) 5414 { 5415 struct ixl_work *work; 5416 5417 work = container_of(wk, struct ixl_work, ixw_cookie); 5418 5419 atomic_swap_uint(&work->ixw_added, 0); 5420 kpreempt_disable(); 5421 work->ixw_func(work->ixw_arg); 5422 kpreempt_enable(); 5423 } 5424 5425 static int 5426 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 5427 { 5428 struct ixl_aq_desc iaq; 5429 5430 memset(&iaq, 0, sizeof(iaq)); 5431 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 5432 iaq.iaq_param[1] = htole32(reg); 5433 5434 if (ixl_atq_poll(sc, &iaq, 250) != 0) 5435 return ETIMEDOUT; 5436 5437 switch (htole16(iaq.iaq_retval)) { 5438 case IXL_AQ_RC_OK: 5439 /* success */ 5440 break; 5441 case IXL_AQ_RC_EACCES: 5442 return EPERM; 5443 case IXL_AQ_RC_EAGAIN: 5444 return EAGAIN; 5445 default: 5446 return EIO; 5447 } 5448 5449 *rv = htole32(iaq.iaq_param[3]); 5450 return 0; 5451 } 5452 5453 5454 5455 static uint32_t 5456 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 5457 { 5458 uint32_t val; 5459 int rv, retry, retry_limit; 5460 5461 retry_limit = sc->sc_rxctl_atq ? 5 : 0; 5462 5463 for (retry = 0; retry < retry_limit; retry++) { 5464 rv = ixl_rx_ctl_read(sc, reg, &val); 5465 if (rv == 0) 5466 return val; 5467 else if (rv == EAGAIN) 5468 delaymsec(1); 5469 else 5470 break; 5471 } 5472 5473 val = ixl_rd(sc, reg); 5474 5475 return val; 5476 } 5477 5478 static int 5479 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 5480 { 5481 struct ixl_aq_desc iaq; 5482 5483 memset(&iaq, 0, sizeof(iaq)); 5484 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 5485 iaq.iaq_param[1] = htole32(reg); 5486 iaq.iaq_param[3] = htole32(value); 5487 5488 if (ixl_atq_poll(sc, &iaq, 250) != 0) 5489 return ETIMEDOUT; 5490 5491 switch (htole16(iaq.iaq_retval)) { 5492 case IXL_AQ_RC_OK: 5493 /* success */ 5494 break; 5495 case IXL_AQ_RC_EACCES: 5496 return EPERM; 5497 case IXL_AQ_RC_EAGAIN: 5498 return EAGAIN; 5499 default: 5500 return EIO; 5501 } 5502 5503 return 0; 5504 } 5505 5506 static void 5507 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 5508 { 5509 int rv, retry, retry_limit; 5510 5511 retry_limit = sc->sc_rxctl_atq ? 5 : 0; 5512 5513 for (retry = 0; retry < retry_limit; retry++) { 5514 rv = ixl_rx_ctl_write(sc, reg, value); 5515 if (rv == 0) 5516 return; 5517 else if (rv == EAGAIN) 5518 delaymsec(1); 5519 else 5520 break; 5521 } 5522 5523 ixl_wr(sc, reg, value); 5524 } 5525 5526 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 5527 5528 #ifdef _MODULE 5529 #include "ioconf.c" 5530 #endif 5531 5532 static int 5533 if_ixl_modcmd(modcmd_t cmd, void *opaque) 5534 { 5535 int error = 0; 5536 5537 #ifdef _MODULE 5538 switch (cmd) { 5539 case MODULE_CMD_INIT: 5540 error = config_init_component(cfdriver_ioconf_if_ixl, 5541 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 5542 break; 5543 case MODULE_CMD_FINI: 5544 error = config_fini_component(cfdriver_ioconf_if_ixl, 5545 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 5546 break; 5547 default: 5548 error = ENOTTY; 5549 break; 5550 } 5551 #endif 5552 5553 return error; 5554 } 5555
Indexes created Sat Sep 20 22:09:52 GMT 2025