if_ixl.c revision 1.3
1 /* $NetBSD: if_ixl.c,v 1.3 2019/12/11 05:50:03 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 78 #ifdef _KERNEL_OPT 79 #include "opt_net_mpsafe.h" 80 #endif 81 82 #include <sys/param.h> 83 #include <sys/types.h> 84 85 #include <sys/cpu.h> 86 #include <sys/device.h> 87 #include <sys/evcnt.h> 88 #include <sys/interrupt.h> 89 #include <sys/kmem.h> 90 #include <sys/malloc.h> 91 #include <sys/module.h> 92 #include <sys/mutex.h> 93 #include <sys/pcq.h> 94 #include <sys/syslog.h> 95 #include <sys/workqueue.h> 96 97 #include <sys/bus.h> 98 99 #include <net/bpf.h> 100 #include <net/if.h> 101 #include <net/if_dl.h> 102 #include <net/if_media.h> 103 #include <net/if_ether.h> 104 #include <net/rss_config.h> 105 106 #include <dev/pci/pcivar.h> 107 #include <dev/pci/pcidevs.h> 108 109 #include <dev/pci/if_ixlreg.h> 110 #include <dev/pci/if_ixlvar.h> 111 112 struct ixl_softc; /* defined */ 113 114 #define I40E_PF_RESET_WAIT_COUNT 200 115 #define I40E_AQ_LARGE_BUF 512 116 117 /* bitfields for Tx queue mapping in QTX_CTL */ 118 #define I40E_QTX_CTL_VF_QUEUE 0x0 119 #define I40E_QTX_CTL_VM_QUEUE 0x1 120 #define I40E_QTX_CTL_PF_QUEUE 0x2 121 122 #define I40E_QUEUE_TYPE_EOL 0x7ff 123 #define I40E_INTR_NOTX_QUEUE 0 124 125 #define I40E_QUEUE_TYPE_RX 0x0 126 #define I40E_QUEUE_TYPE_TX 0x1 127 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 128 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 129 130 #define I40E_ITR_INDEX_RX 0x0 131 #define I40E_ITR_INDEX_TX 0x1 132 #define I40E_ITR_INDEX_OTHER 0x2 133 #define I40E_ITR_INDEX_NONE 0x3 134 135 #define I40E_INTR_NOTX_QUEUE 0 136 #define I40E_INTR_NOTX_INTR 0 137 #define I40E_INTR_NOTX_RX_QUEUE 0 138 #define I40E_INTR_NOTX_TX_QUEUE 1 139 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 140 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 141 142 #define BIT_ULL(a) (1ULL << (a)) 143 #define IXL_RSS_HENA_DEFAULT_BASE \ 144 (BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 145 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 146 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 147 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 148 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 149 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 150 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 151 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 152 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 153 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 154 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 155 #define IXL_RSS_HENA_DEFAULT_XL710 IXL_RSS_HENA_DEFAULT_BASE 156 #define IXL_RSS_HENA_DEFAULT_X722 (IXL_RSS_HENA_DEFAULT_XL710 | \ 157 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 158 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 159 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 160 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 161 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 162 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 163 #define I40E_HASH_LUT_SIZE_128 0 164 #define IXL_RSS_KEY_SIZE_REG 13 165 166 #define IXL_ICR0_CRIT_ERR_MASK \ 167 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 168 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 169 I40E_PFINT_ICR0_PE_CRITERR_MASK) 170 171 #define IXL_TX_PKT_DESCS 8 172 #define IXL_TX_QUEUE_ALIGN 128 173 #define IXL_RX_QUEUE_ALIGN 128 174 175 #define IXL_HARDMTU 9712 /* 9726 - ETHER_HDR_LEN */ 176 177 #define IXL_PCIREG PCI_MAPREG_START 178 179 #define IXL_ITR0 0x0 180 #define IXL_ITR1 0x1 181 #define IXL_ITR2 0x2 182 #define IXL_NOITR 0x3 183 184 #define IXL_AQ_NUM 256 185 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 186 #define IXL_AQ_ALIGN 64 /* lol */ 187 #define IXL_AQ_BUFLEN 4096 188 189 #define IXL_HMC_ROUNDUP 512 190 #define IXL_HMC_PGSIZE 4096 191 #define IXL_HMC_DVASZ sizeof(uint64_t) 192 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 193 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 194 #define IXL_HMC_PDVALID 1ULL 195 196 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 197 198 struct ixl_aq_regs { 199 bus_size_t atq_tail; 200 bus_size_t atq_head; 201 bus_size_t atq_len; 202 bus_size_t atq_bal; 203 bus_size_t atq_bah; 204 205 bus_size_t arq_tail; 206 bus_size_t arq_head; 207 bus_size_t arq_len; 208 bus_size_t arq_bal; 209 bus_size_t arq_bah; 210 211 uint32_t atq_len_enable; 212 uint32_t atq_tail_mask; 213 uint32_t atq_head_mask; 214 215 uint32_t arq_len_enable; 216 uint32_t arq_tail_mask; 217 uint32_t arq_head_mask; 218 }; 219 220 struct ixl_phy_type { 221 uint64_t phy_type; 222 uint64_t ifm_type; 223 }; 224 225 struct ixl_speed_type { 226 uint8_t dev_speed; 227 uint64_t net_speed; 228 }; 229 230 struct ixl_aq_buf { 231 SIMPLEQ_ENTRY(ixl_aq_buf) 232 aqb_entry; 233 void *aqb_data; 234 bus_dmamap_t aqb_map; 235 bus_dma_segment_t aqb_seg; 236 size_t aqb_size; 237 int aqb_nsegs; 238 }; 239 SIMPLEQ_HEAD(ixl_aq_bufs, ixl_aq_buf); 240 241 struct ixl_dmamem { 242 bus_dmamap_t ixm_map; 243 bus_dma_segment_t ixm_seg; 244 int ixm_nsegs; 245 size_t ixm_size; 246 void *ixm_kva; 247 }; 248 249 #define IXL_DMA_MAP(_ixm) ((_ixm)->ixm_map) 250 #define IXL_DMA_DVA(_ixm) ((_ixm)->ixm_map->dm_segs[0].ds_addr) 251 #define IXL_DMA_KVA(_ixm) ((void *)(_ixm)->ixm_kva) 252 #define IXL_DMA_LEN(_ixm) ((_ixm)->ixm_size) 253 254 struct ixl_hmc_entry { 255 uint64_t hmc_base; 256 uint32_t hmc_count; 257 uint64_t hmc_size; 258 }; 259 260 enum ixl_hmc_types { 261 IXL_HMC_LAN_TX = 0, 262 IXL_HMC_LAN_RX, 263 IXL_HMC_FCOE_CTX, 264 IXL_HMC_FCOE_FILTER, 265 IXL_HMC_COUNT 266 }; 267 268 struct ixl_hmc_pack { 269 uint16_t offset; 270 uint16_t width; 271 uint16_t lsb; 272 }; 273 274 /* 275 * these hmc objects have weird sizes and alignments, so these are abstract 276 * representations of them that are nice for c to populate. 277 * 278 * the packing code relies on little-endian values being stored in the fields, 279 * no high bits in the fields being set, and the fields must be packed in the 280 * same order as they are in the ctx structure. 281 */ 282 283 struct ixl_hmc_rxq { 284 uint16_t head; 285 uint8_t cpuid; 286 uint64_t base; 287 #define IXL_HMC_RXQ_BASE_UNIT 128 288 uint16_t qlen; 289 uint16_t dbuff; 290 #define IXL_HMC_RXQ_DBUFF_UNIT 128 291 uint8_t hbuff; 292 #define IXL_HMC_RXQ_HBUFF_UNIT 64 293 uint8_t dtype; 294 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 295 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 296 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 297 uint8_t dsize; 298 #define IXL_HMC_RXQ_DSIZE_16 0 299 #define IXL_HMC_RXQ_DSIZE_32 1 300 uint8_t crcstrip; 301 uint8_t fc_ena; 302 uint8_t l2sel; 303 uint8_t hsplit_0; 304 uint8_t hsplit_1; 305 uint8_t showiv; 306 uint16_t rxmax; 307 uint8_t tphrdesc_ena; 308 uint8_t tphwdesc_ena; 309 uint8_t tphdata_ena; 310 uint8_t tphhead_ena; 311 uint8_t lrxqthresh; 312 uint8_t prefena; 313 }; 314 315 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 316 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 317 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 318 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 319 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 320 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 321 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 322 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 323 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 324 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 325 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 326 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 327 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 328 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 329 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 330 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 331 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 332 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 333 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 334 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 335 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 336 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 337 }; 338 339 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 340 341 struct ixl_hmc_txq { 342 uint16_t head; 343 uint8_t new_context; 344 uint64_t base; 345 #define IXL_HMC_TXQ_BASE_UNIT 128 346 uint8_t fc_ena; 347 uint8_t timesync_ena; 348 uint8_t fd_ena; 349 uint8_t alt_vlan_ena; 350 uint16_t thead_wb; 351 uint8_t cpuid; 352 uint8_t head_wb_ena; 353 #define IXL_HMC_TXQ_DESC_WB 0 354 #define IXL_HMC_TXQ_HEAD_WB 1 355 uint16_t qlen; 356 uint8_t tphrdesc_ena; 357 uint8_t tphrpacket_ena; 358 uint8_t tphwdesc_ena; 359 uint64_t head_wb_addr; 360 uint32_t crc; 361 uint16_t rdylist; 362 uint8_t rdylist_act; 363 }; 364 365 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 366 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 367 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 368 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 369 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 370 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 371 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 372 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 373 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 374 /* line 1 */ 375 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 376 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 377 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 378 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 379 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 380 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 381 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 382 /* line 7 */ 383 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 384 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 385 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 386 }; 387 388 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 389 390 struct ixl_work { 391 struct work ixw_cookie; 392 void (*ixw_func)(void *); 393 void *ixw_arg; 394 unsigned int ixw_added; 395 }; 396 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 397 398 struct ixl_tx_map { 399 struct mbuf *txm_m; 400 bus_dmamap_t txm_map; 401 unsigned int txm_eop; 402 }; 403 404 struct ixl_tx_ring { 405 kmutex_t txr_lock; 406 struct ixl_softc *txr_sc; 407 408 unsigned int txr_prod; 409 unsigned int txr_cons; 410 411 struct ixl_tx_map *txr_maps; 412 struct ixl_dmamem txr_mem; 413 414 bus_size_t txr_tail; 415 unsigned int txr_qid; 416 pcq_t *txr_intrq; 417 void *txr_si; 418 419 uint64_t txr_oerrors; /* if_oerrors */ 420 uint64_t txr_opackets; /* if_opackets */ 421 uint64_t txr_obytes; /* if_obytes */ 422 uint64_t txr_omcasts; /* if_omcasts */ 423 424 struct evcnt txr_defragged; 425 struct evcnt txr_defrag_failed; 426 struct evcnt txr_pcqdrop; 427 struct evcnt txr_transmitdef; 428 struct evcnt txr_intr; 429 struct evcnt txr_defer; 430 }; 431 432 struct ixl_rx_map { 433 struct mbuf *rxm_m; 434 bus_dmamap_t rxm_map; 435 }; 436 437 struct ixl_rx_ring { 438 kmutex_t rxr_lock; 439 440 unsigned int rxr_prod; 441 unsigned int rxr_cons; 442 443 struct ixl_rx_map *rxr_maps; 444 struct ixl_dmamem rxr_mem; 445 446 struct mbuf *rxr_m_head; 447 struct mbuf **rxr_m_tail; 448 449 bus_size_t rxr_tail; 450 unsigned int rxr_qid; 451 452 uint64_t rxr_ipackets; /* if_ipackets */ 453 uint64_t rxr_ibytes; /* if_ibytes */ 454 uint64_t rxr_iqdrops; /* iqdrops */ 455 uint64_t rxr_ierrors; /* if_ierrors */ 456 457 struct evcnt rxr_mgethdr_failed; 458 struct evcnt rxr_mgetcl_failed; 459 struct evcnt rxr_mbuf_load_failed; 460 struct evcnt rxr_intr; 461 struct evcnt rxr_defer; 462 }; 463 464 struct ixl_queue_pair { 465 struct ixl_softc *qp_sc; 466 struct ixl_tx_ring *qp_txr; 467 struct ixl_rx_ring *qp_rxr; 468 469 char qp_name[16]; 470 471 void *qp_si; 472 struct ixl_work qp_task; 473 bool qp_workqueue; 474 }; 475 476 struct ixl_atq { 477 struct ixl_aq_desc iatq_desc; 478 void (*iatq_fn)(struct ixl_softc *); 479 }; 480 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 481 482 struct ixl_product { 483 unsigned int vendor_id; 484 unsigned int product_id; 485 }; 486 487 /* 488 * Locking notes: 489 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 490 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 491 * - more than one lock of them cannot be held at once. 492 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 493 * (a spin mutex). 494 * - the lock cannot held with txr_lock or rxr_lock. 495 * + a field named sc_arq_* is not protected by any lock. 496 * - operations for sc_arq_* is done in one context related to 497 * sc_arq_task. 498 * + other fields in ixl_softc is protected by sc_cfg_lock 499 * (an adaptive mutex) 500 * - It must be held before another lock is held, and It can be 501 * released after the other lock is released. 502 * */ 503 504 struct ixl_softc { 505 device_t sc_dev; 506 struct ethercom sc_ec; 507 bool sc_attached; 508 bool sc_dead; 509 bool sc_rxctl_atq; 510 struct sysctllog *sc_sysctllog; 511 struct workqueue *sc_workq; 512 struct workqueue *sc_workq_txrx; 513 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 514 struct ifmedia sc_media; 515 uint64_t sc_media_status; 516 uint64_t sc_media_active; 517 kmutex_t sc_cfg_lock; 518 enum i40e_mac_type sc_mac_type; 519 uint32_t sc_rss_table_size; 520 uint32_t sc_rss_table_entry_width; 521 bool sc_txrx_workqueue; 522 u_int sc_tx_process_limit; 523 u_int sc_rx_process_limit; 524 u_int sc_tx_intr_process_limit; 525 u_int sc_rx_intr_process_limit; 526 527 struct pci_attach_args sc_pa; 528 pci_intr_handle_t *sc_ihp; 529 void **sc_ihs; 530 unsigned int sc_nintrs; 531 532 bus_dma_tag_t sc_dmat; 533 bus_space_tag_t sc_memt; 534 bus_space_handle_t sc_memh; 535 bus_size_t sc_mems; 536 537 uint8_t sc_pf_id; 538 uint16_t sc_uplink_seid; /* le */ 539 uint16_t sc_downlink_seid; /* le */ 540 uint16_t sc_vsi_number; /* le */ 541 uint16_t sc_seid; 542 unsigned int sc_base_queue; 543 544 pci_intr_type_t sc_intrtype; 545 unsigned int sc_msix_vector_queue; 546 547 struct ixl_dmamem sc_scratch; 548 549 const struct ixl_aq_regs * 550 sc_aq_regs; 551 552 kmutex_t sc_atq_lock; 553 kcondvar_t sc_atq_cv; 554 struct ixl_dmamem sc_atq; 555 unsigned int sc_atq_prod; 556 unsigned int sc_atq_cons; 557 558 struct ixl_dmamem sc_arq; 559 struct ixl_work sc_arq_task; 560 struct ixl_aq_bufs sc_arq_idle; 561 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 562 unsigned int sc_arq_prod; 563 unsigned int sc_arq_cons; 564 565 struct ixl_work sc_link_state_task; 566 struct ixl_atq sc_link_state_atq; 567 568 struct ixl_dmamem sc_hmc_sd; 569 struct ixl_dmamem sc_hmc_pd; 570 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 571 572 unsigned int sc_tx_ring_ndescs; 573 unsigned int sc_rx_ring_ndescs; 574 unsigned int sc_nqueue_pairs; 575 unsigned int sc_nqueue_pairs_max; 576 unsigned int sc_nqueue_pairs_device; 577 struct ixl_queue_pair *sc_qps; 578 579 struct evcnt sc_event_atq; 580 struct evcnt sc_event_link; 581 struct evcnt sc_event_ecc_err; 582 struct evcnt sc_event_pci_exception; 583 struct evcnt sc_event_crit_err; 584 }; 585 586 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 587 #define IXL_TX_PROCESS_LIMIT 256 588 #define IXL_RX_PROCESS_LIMIT 256 589 #define IXL_TX_INTR_PROCESS_LIMIT 256 590 #define IXL_RX_INTR_PROCESS_LIMIT 0U 591 592 #define delaymsec(_x) DELAY(1000 * (_x)) 593 #ifdef IXL_DEBUG 594 #define DDPRINTF(sc, fmt, args...) \ 595 do { \ 596 if (sc != NULL) \ 597 device_printf(sc->sc_dev, ""); \ 598 printf("%s:\t" fmt, __func__, ##args); \ 599 } while (0) 600 #else 601 #define DDPRINTF(sc, fmt, args...) __nothing 602 #endif 603 #define IXL_NOMSIX false 604 605 static enum i40e_mac_type 606 ixl_mactype(pci_product_id_t); 607 static void ixl_clear_hw(struct ixl_softc *); 608 static int ixl_pf_reset(struct ixl_softc *); 609 610 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 611 bus_size_t, bus_size_t); 612 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 613 614 static int ixl_arq_fill(struct ixl_softc *); 615 static void ixl_arq_unfill(struct ixl_softc *); 616 617 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 618 unsigned int); 619 static void ixl_atq_set(struct ixl_atq *, void (*)(struct ixl_softc *)); 620 static int ixl_atq_post(struct ixl_softc *, struct ixl_atq *); 621 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 622 static void ixl_atq_done(struct ixl_softc *); 623 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 624 static int ixl_get_version(struct ixl_softc *); 625 static int ixl_get_hw_capabilities(struct ixl_softc *); 626 static int ixl_pxe_clear(struct ixl_softc *); 627 static int ixl_lldp_shut(struct ixl_softc *); 628 static int ixl_get_mac(struct ixl_softc *); 629 static int ixl_get_switch_config(struct ixl_softc *); 630 static int ixl_phy_mask_ints(struct ixl_softc *); 631 static int ixl_get_phy_types(struct ixl_softc *, uint64_t *); 632 static int ixl_restart_an(struct ixl_softc *); 633 static int ixl_hmc(struct ixl_softc *); 634 static void ixl_hmc_free(struct ixl_softc *); 635 static int ixl_get_vsi(struct ixl_softc *); 636 static int ixl_set_vsi(struct ixl_softc *); 637 static void ixl_set_filter_control(struct ixl_softc *); 638 static int ixl_get_link_status(struct ixl_softc *); 639 static int ixl_set_link_status(struct ixl_softc *, 640 const struct ixl_aq_desc *); 641 static void ixl_config_rss(struct ixl_softc *); 642 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 643 uint16_t, uint16_t); 644 static int ixl_remove_macvlan(struct ixl_softc *, uint8_t *, uint16_t, 645 uint16_t); 646 static void ixl_arq(void *); 647 static void ixl_link_state_update(void *); 648 static void ixl_hmc_pack(void *, const void *, 649 const struct ixl_hmc_pack *, unsigned int); 650 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 651 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 652 653 static int ixl_match(device_t, cfdata_t, void *); 654 static void ixl_attach(device_t, device_t, void *); 655 static int ixl_detach(device_t, int); 656 657 static void ixl_media_add(struct ixl_softc *, uint64_t); 658 static int ixl_media_change(struct ifnet *); 659 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 660 static void ixl_watchdog(struct ifnet *); 661 static int ixl_ioctl(struct ifnet *, u_long, void *); 662 static void ixl_start(struct ifnet *); 663 static int ixl_transmit(struct ifnet *, struct mbuf *); 664 static void ixl_deferred_transmit(void *); 665 static int ixl_intr(void *); 666 static int ixl_queue_intr(void *); 667 static int ixl_other_intr(void *); 668 static void ixl_handle_queue(void *); 669 static void ixl_sched_handle_queue(struct ixl_softc *, 670 struct ixl_queue_pair *); 671 static int ixl_init(struct ifnet *); 672 static int ixl_init_locked(struct ixl_softc *); 673 static void ixl_stop(struct ifnet *, int); 674 static void ixl_stop_locked(struct ixl_softc *); 675 static int ixl_iff(struct ixl_softc *); 676 static int ixl_ifflags_cb(struct ethercom *); 677 static int ixl_setup_interrupts(struct ixl_softc *); 678 static int ixl_establish_intx(struct ixl_softc *); 679 static int ixl_establish_msix(struct ixl_softc *); 680 static void ixl_set_affinity_msix(struct ixl_softc *); 681 static void ixl_enable_queue_intr(struct ixl_softc *, 682 struct ixl_queue_pair *); 683 static void ixl_disable_queue_intr(struct ixl_softc *, 684 struct ixl_queue_pair *); 685 static void ixl_enable_other_intr(struct ixl_softc *); 686 static void ixl_disable_other_intr(struct ixl_softc *); 687 static void ixl_config_queue_intr(struct ixl_softc *); 688 static void ixl_config_other_intr(struct ixl_softc *); 689 690 static struct ixl_tx_ring * 691 ixl_txr_alloc(struct ixl_softc *, unsigned int); 692 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 693 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 694 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 695 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 696 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 697 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 698 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 699 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 700 701 static struct ixl_rx_ring * 702 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 703 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 704 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 705 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 706 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 707 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 708 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 709 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 710 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 711 712 static struct workqueue * 713 ixl_workq_create(const char *, pri_t, int, int); 714 static void ixl_workq_destroy(struct workqueue *); 715 static int ixl_workqs_teardown(device_t); 716 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 717 static void ixl_work_add(struct workqueue *, struct ixl_work *); 718 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 719 static void ixl_workq_work(struct work *, void *); 720 static const struct ixl_product * 721 ixl_lookup(const struct pci_attach_args *pa); 722 static void ixl_link_status(struct ixl_softc *); 723 static int ixl_set_macvlan(struct ixl_softc *); 724 static int ixl_setup_interrupts(struct ixl_softc *);; 725 static void ixl_teardown_interrupts(struct ixl_softc *); 726 static int ixl_setup_stats(struct ixl_softc *); 727 static void ixl_teardown_stats(struct ixl_softc *); 728 static int ixl_setup_sysctls(struct ixl_softc *); 729 static void ixl_teardown_sysctls(struct ixl_softc *); 730 static int ixl_queue_pairs_alloc(struct ixl_softc *); 731 static void ixl_queue_pairs_free(struct ixl_softc *); 732 733 static const struct ixl_phy_type ixl_phy_type_map[] = { 734 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 735 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 736 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 737 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 738 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 739 { 1ULL << IXL_PHY_TYPE_XAUI | 740 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 741 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 742 { 1ULL << IXL_PHY_TYPE_XLAUI | 743 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 744 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 745 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 746 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 747 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 748 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 749 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 750 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 751 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 752 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 753 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 754 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 755 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 756 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 757 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 758 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 759 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 760 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 761 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 762 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 763 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 764 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 765 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 766 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 767 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_CR }, 768 }; 769 770 static const struct ixl_speed_type ixl_speed_type_map[] = { 771 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 772 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 773 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 774 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 775 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 776 }; 777 778 static const struct ixl_aq_regs ixl_pf_aq_regs = { 779 .atq_tail = I40E_PF_ATQT, 780 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 781 .atq_head = I40E_PF_ATQH, 782 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 783 .atq_len = I40E_PF_ATQLEN, 784 .atq_bal = I40E_PF_ATQBAL, 785 .atq_bah = I40E_PF_ATQBAH, 786 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 787 788 .arq_tail = I40E_PF_ARQT, 789 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 790 .arq_head = I40E_PF_ARQH, 791 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 792 .arq_len = I40E_PF_ARQLEN, 793 .arq_bal = I40E_PF_ARQBAL, 794 .arq_bah = I40E_PF_ARQBAH, 795 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 796 }; 797 798 #define ixl_rd(_s, _r) \ 799 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 800 #define ixl_wr(_s, _r, _v) \ 801 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 802 #define ixl_barrier(_s, _r, _l, _o) \ 803 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 804 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 805 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 806 807 static inline uint32_t 808 ixl_dmamem_hi(struct ixl_dmamem *ixm) 809 { 810 uint32_t retval; 811 uint64_t val; 812 813 if (sizeof(bus_addr_t) > 4) { 814 val = (intptr_t)IXL_DMA_DVA(ixm); 815 retval = (uint32_t)(val >> 32); 816 } else { 817 retval = 0; 818 } 819 820 return retval; 821 } 822 823 static inline uint32_t 824 ixl_dmamem_lo(struct ixl_dmamem *ixm) 825 { 826 827 return (uint32_t)IXL_DMA_DVA(ixm); 828 } 829 830 static inline void 831 ixl_aq_dva(struct ixl_aq_desc *iaq, bus_addr_t addr) 832 { 833 uint64_t val; 834 835 if (sizeof(bus_addr_t) > 4) { 836 val = (intptr_t)addr; 837 iaq->iaq_param[2] = htole32(val >> 32); 838 } else { 839 iaq->iaq_param[2] = htole32(0); 840 } 841 842 iaq->iaq_param[3] = htole32(addr); 843 } 844 845 static inline unsigned int 846 ixl_rxr_unrefreshed(unsigned int prod, unsigned int cons, unsigned int ndescs) 847 { 848 unsigned int num; 849 850 if (prod < cons) 851 num = cons - prod; 852 else 853 num = (ndescs - prod) + cons; 854 855 if (__predict_true(num > 0)) { 856 /* device cannot receive packets if all descripter is filled */ 857 num -= 1; 858 } 859 860 return num; 861 } 862 863 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 864 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 865 DVF_DETACH_SHUTDOWN); 866 867 static const struct ixl_product ixl_products[] = { 868 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 869 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 870 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 871 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 872 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 873 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 874 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T }, 875 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 876 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 877 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 878 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 879 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 880 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 881 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 882 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 883 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 884 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 885 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 886 /* required last entry */ 887 {0, 0} 888 }; 889 890 static const struct ixl_product * 891 ixl_lookup(const struct pci_attach_args *pa) 892 { 893 const struct ixl_product *ixlp; 894 895 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 896 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 897 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 898 return ixlp; 899 } 900 901 return NULL; 902 } 903 904 static int 905 ixl_match(device_t parent, cfdata_t match, void *aux) 906 { 907 const struct pci_attach_args *pa = aux; 908 909 return (ixl_lookup(pa) != NULL) ? 1 : 0; 910 } 911 912 static void 913 ixl_attach(device_t parent, device_t self, void *aux) 914 { 915 struct ixl_softc *sc; 916 struct pci_attach_args *pa = aux; 917 struct ifnet *ifp; 918 pcireg_t memtype, reg; 919 uint32_t firstq, port, ari, func; 920 uint64_t phy_types = 0; 921 char xnamebuf[32]; 922 int tries, rv; 923 924 sc = device_private(self); 925 sc->sc_dev = self; 926 ifp = &sc->sc_ec.ec_if; 927 928 sc->sc_pa = *pa; 929 sc->sc_dmat = (pci_dma64_available(pa)) ? 930 pa->pa_dmat64 : pa->pa_dmat; 931 sc->sc_aq_regs = &ixl_pf_aq_regs; 932 933 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 934 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(reg)); 935 936 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 937 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 938 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 939 aprint_error(": unable to map registers\n"); 940 return; 941 } 942 943 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 944 945 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 946 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 947 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 948 sc->sc_base_queue = firstq; 949 950 ixl_clear_hw(sc); 951 if (ixl_pf_reset(sc) == -1) { 952 /* error printed by ixl pf_reset */ 953 goto unmap; 954 } 955 956 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 957 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 958 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 959 aprint_normal(": port %u", port); 960 961 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 962 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 963 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 964 965 func = ixl_rd(sc, I40E_PF_FUNC_RID); 966 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 967 968 /* initialise the adminq */ 969 970 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 971 972 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 973 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 974 aprint_error("\n" "%s: unable to allocate atq\n", 975 device_xname(self)); 976 goto unmap; 977 } 978 979 SIMPLEQ_INIT(&sc->sc_arq_idle); 980 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 981 sc->sc_arq_cons = 0; 982 sc->sc_arq_prod = 0; 983 984 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 985 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 986 aprint_error("\n" "%s: unable to allocate arq\n", 987 device_xname(self)); 988 goto free_atq; 989 } 990 991 if (!ixl_arq_fill(sc)) { 992 aprint_error("\n" "%s: unable to fill arq descriptors\n", 993 device_xname(self)); 994 goto free_arq; 995 } 996 997 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 998 0, IXL_DMA_LEN(&sc->sc_atq), 999 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1000 1001 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1002 0, IXL_DMA_LEN(&sc->sc_arq), 1003 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1004 1005 for (tries = 0; tries < 10; tries++) { 1006 sc->sc_atq_cons = 0; 1007 sc->sc_atq_prod = 0; 1008 1009 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1010 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1011 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1012 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1013 1014 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1015 1016 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1017 ixl_dmamem_lo(&sc->sc_atq)); 1018 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1019 ixl_dmamem_hi(&sc->sc_atq)); 1020 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1021 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1022 1023 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1024 ixl_dmamem_lo(&sc->sc_arq)); 1025 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1026 ixl_dmamem_hi(&sc->sc_arq)); 1027 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1028 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1029 1030 rv = ixl_get_version(sc); 1031 if (rv == 0) 1032 break; 1033 if (rv != ETIMEDOUT) { 1034 aprint_error(", unable to get firmware version\n"); 1035 goto shutdown; 1036 } 1037 1038 delaymsec(100); 1039 } 1040 1041 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1042 1043 if (sc->sc_mac_type == I40E_MAC_X722) 1044 sc->sc_nqueue_pairs_device = 128; 1045 else 1046 sc->sc_nqueue_pairs_device = 64; 1047 1048 rv = ixl_get_hw_capabilities(sc); 1049 if (rv != 0) { 1050 aprint_error(", GET HW CAPABILITIES %s\n", 1051 rv == ETIMEDOUT ? "timeout" : "error"); 1052 goto shutdown; 1053 } 1054 1055 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max = 1056 MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1057 sc->sc_tx_ring_ndescs = 1024; 1058 sc->sc_rx_ring_ndescs = 1024; 1059 1060 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1061 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1062 1063 if (ixl_get_mac(sc) != 0) { 1064 /* error printed by ixl_get_mac */ 1065 goto shutdown; 1066 } 1067 1068 aprint_normal("\n"); 1069 aprint_naive("\n"); 1070 1071 aprint_normal_dev(self, "Ethernet address %s\n", 1072 ether_sprintf(sc->sc_enaddr)); 1073 1074 rv = ixl_pxe_clear(sc); 1075 if (rv != 0) { 1076 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1077 rv == ETIMEDOUT ? "timeout" : "error"); 1078 } 1079 1080 ixl_set_filter_control(sc); 1081 1082 if (ixl_hmc(sc) != 0) { 1083 /* error printed by ixl_hmc */ 1084 goto shutdown; 1085 } 1086 1087 if (ixl_lldp_shut(sc) != 0) { 1088 /* error printed by ixl_lldp_shut */ 1089 goto free_hmc; 1090 } 1091 1092 if (ixl_phy_mask_ints(sc) != 0) { 1093 /* error printed by ixl_phy_mask_ints */ 1094 goto free_hmc; 1095 } 1096 1097 if (ixl_restart_an(sc) != 0) { 1098 /* error printed by ixl_restart_an */ 1099 goto free_hmc; 1100 } 1101 1102 if (ixl_get_switch_config(sc) != 0) { 1103 /* error printed by ixl_get_switch_config */ 1104 goto free_hmc; 1105 } 1106 1107 if (ixl_get_phy_types(sc, &phy_types) != 0) { 1108 /* error printed by ixl_get_phy_abilities */ 1109 goto free_hmc; 1110 } 1111 1112 rv = ixl_get_link_status(sc); 1113 if (rv != 0) { 1114 aprint_error_dev(self, "GET LINK STATUS %s\n", 1115 rv == ETIMEDOUT ? "timeout" : "error"); 1116 goto free_hmc; 1117 } 1118 1119 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1120 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1121 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1122 goto free_hmc; 1123 } 1124 1125 if (ixl_get_vsi(sc) != 0) { 1126 /* error printed by ixl_get_vsi */ 1127 goto free_scratch; 1128 } 1129 1130 if (ixl_set_vsi(sc) != 0) { 1131 /* error printed by ixl_set_vsi */ 1132 goto free_scratch; 1133 } 1134 1135 if (ixl_queue_pairs_alloc(sc) != 0) { 1136 /* error printed by ixl_queue_pairs_alloc */ 1137 goto free_scratch; 1138 } 1139 1140 if (ixl_setup_interrupts(sc) != 0) { 1141 /* error printed by ixl_setup_interrupts */ 1142 goto free_queue_pairs; 1143 } 1144 1145 if (ixl_setup_stats(sc) != 0) { 1146 aprint_error_dev(self, "failed to setup event counters\n"); 1147 goto teardown_intrs; 1148 } 1149 1150 if (ixl_setup_sysctls(sc) != 0) { 1151 /* error printed by ixl_setup_sysctls */ 1152 goto teardown_stats; 1153 } 1154 1155 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1156 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1157 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1158 if (sc->sc_workq == NULL) 1159 goto teardown_sysctls; 1160 1161 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1162 sc->sc_workq_txrx = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1163 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1164 if (sc->sc_workq_txrx == NULL) 1165 goto teardown_wqs; 1166 1167 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1168 cv_init(&sc->sc_atq_cv, xnamebuf); 1169 1170 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1171 1172 ifp->if_softc = sc; 1173 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1174 ifp->if_extflags = IFEF_MPSAFE; 1175 ifp->if_ioctl = ixl_ioctl; 1176 ifp->if_start = ixl_start; 1177 ifp->if_transmit = ixl_transmit; 1178 ifp->if_watchdog = ixl_watchdog; 1179 ifp->if_init = ixl_init; 1180 ifp->if_stop = ixl_stop; 1181 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1182 IFQ_SET_READY(&ifp->if_snd); 1183 #if 0 1184 ifp->if_capabilities |= IFCAP_CSUM_IPv4_Rx | 1185 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1186 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx; 1187 #endif 1188 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_MTU; 1189 #if 0 1190 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1191 #endif 1192 1193 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1194 ifmedia_init(&sc->sc_media, IFM_IMASK, ixl_media_change, 1195 ixl_media_status); 1196 1197 ixl_media_add(sc, phy_types); 1198 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1199 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1200 1201 if_attach(ifp); 1202 if_deferred_start_init(ifp, NULL); 1203 ether_ifattach(ifp, sc->sc_enaddr); 1204 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1205 (void)ixl_get_link_status(sc); 1206 1207 ixl_work_set(&sc->sc_link_state_task, ixl_link_state_update, sc); 1208 1209 ixl_config_other_intr(sc); 1210 1211 ixl_set_macvlan(sc); 1212 1213 ixl_enable_other_intr(sc); 1214 1215 sc->sc_txrx_workqueue = true; 1216 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1217 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1218 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1219 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1220 1221 if (pmf_device_register(self, NULL, NULL) != true) 1222 aprint_debug_dev(self, "couldn't establish power handler\n"); 1223 sc->sc_attached = true; 1224 return; 1225 1226 teardown_wqs: 1227 config_finalize_register(self, ixl_workqs_teardown); 1228 teardown_sysctls: 1229 ixl_teardown_sysctls(sc); 1230 teardown_stats: 1231 ixl_teardown_stats(sc); 1232 teardown_intrs: 1233 ixl_teardown_interrupts(sc); 1234 free_queue_pairs: 1235 ixl_queue_pairs_free(sc); 1236 free_scratch: 1237 ixl_dmamem_free(sc, &sc->sc_scratch); 1238 free_hmc: 1239 ixl_hmc_free(sc); 1240 shutdown: 1241 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1242 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1243 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1244 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1245 1246 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1247 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1248 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1249 1250 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1251 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1252 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1253 1254 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1255 0, IXL_DMA_LEN(&sc->sc_arq), 1256 BUS_DMASYNC_POSTREAD); 1257 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1258 0, IXL_DMA_LEN(&sc->sc_atq), 1259 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1260 1261 ixl_arq_unfill(sc); 1262 free_arq: 1263 ixl_dmamem_free(sc, &sc->sc_arq); 1264 free_atq: 1265 ixl_dmamem_free(sc, &sc->sc_atq); 1266 unmap: 1267 mutex_destroy(&sc->sc_atq_lock); 1268 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1269 mutex_destroy(&sc->sc_cfg_lock); 1270 sc->sc_mems = 0; 1271 1272 sc->sc_attached = false; 1273 } 1274 1275 static int 1276 ixl_detach(device_t self, int flags) 1277 { 1278 struct ixl_softc *sc = device_private(self); 1279 struct ifnet *ifp = &sc->sc_ec.ec_if; 1280 1281 if (!sc->sc_attached) 1282 return 0; 1283 1284 ixl_stop(ifp, 1); 1285 1286 if (sc->sc_workq != NULL) { 1287 ixl_workq_destroy(sc->sc_workq); 1288 sc->sc_workq = NULL; 1289 } 1290 1291 if (sc->sc_workq_txrx != NULL) { 1292 ixl_workq_destroy(sc->sc_workq_txrx); 1293 sc->sc_workq_txrx = NULL; 1294 } 1295 1296 ifmedia_delete_instance(&sc->sc_media, IFM_INST_ANY); 1297 ether_ifdetach(ifp); 1298 if_detach(ifp); 1299 1300 ixl_teardown_interrupts(sc); 1301 ixl_teardown_stats(sc); 1302 1303 ixl_queue_pairs_free(sc); 1304 1305 ixl_dmamem_free(sc, &sc->sc_scratch); 1306 ixl_hmc_free(sc); 1307 1308 /* shutdown */ 1309 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1310 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1311 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1312 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1313 1314 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1315 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1316 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1317 1318 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1319 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1320 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1321 1322 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1323 0, IXL_DMA_LEN(&sc->sc_arq), 1324 BUS_DMASYNC_POSTREAD); 1325 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1326 0, IXL_DMA_LEN(&sc->sc_atq), 1327 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1328 1329 ixl_arq_unfill(sc); 1330 1331 ixl_dmamem_free(sc, &sc->sc_arq); 1332 ixl_dmamem_free(sc, &sc->sc_atq); 1333 1334 cv_destroy(&sc->sc_atq_cv); 1335 mutex_destroy(&sc->sc_atq_lock); 1336 1337 if (sc->sc_mems != 0) { 1338 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1339 sc->sc_mems = 0; 1340 } 1341 1342 mutex_destroy(&sc->sc_cfg_lock); 1343 1344 return 0; 1345 } 1346 1347 static int 1348 ixl_workqs_teardown(device_t self) 1349 { 1350 struct ixl_softc *sc = device_private(self); 1351 1352 if (sc->sc_workq != NULL) { 1353 ixl_workq_destroy(sc->sc_workq); 1354 sc->sc_workq = NULL; 1355 } 1356 1357 if (sc->sc_workq_txrx != NULL) { 1358 ixl_workq_destroy(sc->sc_workq_txrx); 1359 sc->sc_workq_txrx = NULL; 1360 } 1361 1362 return 0; 1363 } 1364 1365 static void 1366 ixl_media_add(struct ixl_softc *sc, uint64_t phy_types) 1367 { 1368 struct ifmedia *ifm = &sc->sc_media; 1369 const struct ixl_phy_type *itype; 1370 unsigned int i; 1371 1372 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1373 itype = &ixl_phy_type_map[i]; 1374 1375 if (ISSET(phy_types, itype->phy_type)) { 1376 ifmedia_add(ifm, 1377 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1378 1379 if (itype->ifm_type == IFM_100_TX) { 1380 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1381 0, NULL); 1382 } 1383 } 1384 } 1385 } 1386 1387 static void 1388 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1389 { 1390 struct ixl_softc *sc = ifp->if_softc; 1391 1392 ifmr->ifm_status = sc->sc_media_status; 1393 ifmr->ifm_active = sc->sc_media_active; 1394 1395 mutex_enter(&sc->sc_cfg_lock); 1396 if (ifp->if_link_state == LINK_STATE_UP) 1397 SET(ifmr->ifm_status, IFM_ACTIVE); 1398 mutex_exit(&sc->sc_cfg_lock); 1399 } 1400 1401 static int 1402 ixl_media_change(struct ifnet *ifp) 1403 { 1404 1405 return 0; 1406 } 1407 1408 static void 1409 ixl_watchdog(struct ifnet *ifp) 1410 { 1411 1412 } 1413 1414 static void 1415 ixl_del_all_multiaddr(struct ixl_softc *sc) 1416 { 1417 struct ethercom *ec = &sc->sc_ec; 1418 struct ether_multi *enm; 1419 struct ether_multistep step; 1420 1421 ETHER_LOCK(ec); 1422 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1423 ETHER_NEXT_MULTI(step, enm)) { 1424 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1425 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1426 } 1427 ETHER_UNLOCK(ec); 1428 } 1429 1430 static int 1431 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1432 { 1433 struct ifnet *ifp = &sc->sc_ec.ec_if; 1434 int rv; 1435 1436 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1437 return 0; 1438 1439 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1440 ixl_del_all_multiaddr(sc); 1441 SET(ifp->if_flags, IFF_ALLMULTI); 1442 return 0; 1443 } 1444 1445 rv = ixl_add_macvlan(sc, addrlo, 0, 1446 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1447 1448 if (rv == IXL_AQ_RC_ENOSPC) { 1449 ixl_del_all_multiaddr(sc); 1450 SET(ifp->if_flags, IFF_ALLMULTI); 1451 return 0; 1452 } 1453 1454 if (rv != IXL_AQ_RC_OK) 1455 return EIO; 1456 1457 return 0; 1458 } 1459 1460 static void 1461 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1462 { 1463 struct ifnet *ifp = &sc->sc_ec.ec_if; 1464 struct ethercom *ec = &sc->sc_ec; 1465 struct ether_multi *enm, *enm_last; 1466 struct ether_multistep step; 1467 int rv; 1468 1469 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1470 ixl_remove_macvlan(sc, addrlo, 0, 1471 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1472 return; 1473 } 1474 1475 ETHER_LOCK(ec); 1476 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1477 ETHER_NEXT_MULTI(step, enm)) { 1478 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1479 ETHER_ADDR_LEN) != 0) { 1480 ETHER_UNLOCK(ec); 1481 return; 1482 } 1483 } 1484 1485 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1486 ETHER_NEXT_MULTI(step, enm)) { 1487 rv = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1488 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1489 if (rv != IXL_AQ_RC_OK) 1490 break; 1491 } 1492 1493 if (enm != NULL) { 1494 enm_last = enm; 1495 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1496 ETHER_NEXT_MULTI(step, enm)) { 1497 if (enm == enm_last) 1498 break; 1499 1500 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1501 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1502 } 1503 } else { 1504 CLR(ifp->if_flags, IFF_ALLMULTI); 1505 } 1506 1507 ETHER_UNLOCK(ec); 1508 } 1509 1510 static int 1511 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1512 { 1513 struct ifreq *ifr = (struct ifreq *)data; 1514 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1515 struct ixl_tx_ring *txr; 1516 struct ixl_rx_ring *rxr; 1517 const struct sockaddr *sa; 1518 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1519 int s, error = 0; 1520 unsigned int i; 1521 1522 switch (cmd) { 1523 case SIOCADDMULTI: 1524 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1525 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1526 error = ether_multiaddr(sa, addrlo, addrhi); 1527 if (error != 0) 1528 return error; 1529 1530 if (ixl_add_multi(sc, addrlo, addrhi) != 0) { 1531 ether_delmulti(sa, &sc->sc_ec); 1532 error = EIO; 1533 } 1534 } 1535 break; 1536 1537 case SIOCDELMULTI: 1538 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1539 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1540 error = ether_multiaddr(sa, addrlo, addrhi); 1541 if (error != 0) 1542 return error; 1543 1544 ixl_del_multi(sc, addrlo, addrhi); 1545 } 1546 break; 1547 1548 case SIOCGIFDATA: 1549 case SIOCZIFDATA: 1550 ifp->if_ipackets = 0; 1551 ifp->if_ibytes = 0; 1552 ifp->if_iqdrops = 0; 1553 ifp->if_ierrors = 0; 1554 ifp->if_opackets = 0; 1555 ifp->if_obytes = 0; 1556 ifp->if_omcasts = 0; 1557 1558 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 1559 txr = sc->sc_qps[i].qp_txr; 1560 rxr = sc->sc_qps[i].qp_rxr; 1561 1562 mutex_enter(&rxr->rxr_lock); 1563 ifp->if_ipackets += rxr->rxr_ipackets; 1564 ifp->if_ibytes += rxr->rxr_ibytes; 1565 ifp->if_iqdrops += rxr->rxr_iqdrops; 1566 ifp->if_ierrors += rxr->rxr_ierrors; 1567 if (cmd == SIOCZIFDATA) { 1568 rxr->rxr_ipackets = 0; 1569 rxr->rxr_ibytes = 0; 1570 rxr->rxr_iqdrops = 0; 1571 rxr->rxr_ierrors = 0; 1572 } 1573 mutex_exit(&rxr->rxr_lock); 1574 1575 mutex_enter(&txr->txr_lock); 1576 ifp->if_opackets += txr->txr_opackets; 1577 ifp->if_obytes += txr->txr_opackets; 1578 ifp->if_omcasts += txr->txr_omcasts; 1579 if (cmd == SIOCZIFDATA) { 1580 txr->txr_opackets = 0; 1581 txr->txr_opackets = 0; 1582 txr->txr_omcasts = 0; 1583 } 1584 mutex_exit(&txr->txr_lock); 1585 } 1586 /* FALLTHROUGH */ 1587 default: 1588 s = splnet(); 1589 error = ether_ioctl(ifp, cmd, data); 1590 splx(s); 1591 } 1592 1593 if (error == ENETRESET) 1594 error = ixl_iff(sc); 1595 1596 return error; 1597 } 1598 1599 static enum i40e_mac_type 1600 ixl_mactype(pci_product_id_t id) 1601 { 1602 1603 switch (id) { 1604 case PCI_PRODUCT_INTEL_XL710_SFP: 1605 case PCI_PRODUCT_INTEL_XL710_KX_B: 1606 case PCI_PRODUCT_INTEL_XL710_KX_C: 1607 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1608 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1609 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1610 case PCI_PRODUCT_INTEL_X710_10G_T: 1611 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1612 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1613 case PCI_PRODUCT_INTEL_X710_T4_10G: 1614 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1615 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1616 return I40E_MAC_XL710; 1617 1618 case PCI_PRODUCT_INTEL_X722_KX: 1619 case PCI_PRODUCT_INTEL_X722_QSFP: 1620 case PCI_PRODUCT_INTEL_X722_SFP: 1621 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1622 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1623 case PCI_PRODUCT_INTEL_X722_I_SFP: 1624 return I40E_MAC_X722; 1625 } 1626 1627 return I40E_MAC_GENERIC; 1628 } 1629 1630 static inline void * 1631 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1632 { 1633 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1634 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1635 1636 if (i >= e->hmc_count) 1637 return NULL; 1638 1639 kva += e->hmc_base; 1640 kva += i * e->hmc_size; 1641 1642 return kva; 1643 } 1644 1645 static inline size_t 1646 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1647 { 1648 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1649 1650 return e->hmc_size; 1651 } 1652 1653 static void 1654 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1655 { 1656 struct ixl_rx_ring *rxr = qp->qp_rxr; 1657 1658 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1659 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1660 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1661 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1662 ixl_flush(sc); 1663 } 1664 1665 static void 1666 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1667 { 1668 struct ixl_rx_ring *rxr = qp->qp_rxr; 1669 1670 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1671 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1672 ixl_flush(sc); 1673 } 1674 1675 static void 1676 ixl_enable_other_intr(struct ixl_softc *sc) 1677 { 1678 1679 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1680 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1681 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1682 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1683 ixl_flush(sc); 1684 } 1685 1686 static void 1687 ixl_disable_other_intr(struct ixl_softc *sc) 1688 { 1689 1690 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1691 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1692 ixl_flush(sc); 1693 } 1694 1695 static int 1696 ixl_reinit(struct ixl_softc *sc) 1697 { 1698 struct ixl_rx_ring *rxr; 1699 struct ixl_tx_ring *txr; 1700 unsigned int i; 1701 uint32_t reg; 1702 1703 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1704 1705 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1706 txr = sc->sc_qps[i].qp_txr; 1707 rxr = sc->sc_qps[i].qp_rxr; 1708 1709 txr->txr_cons = txr->txr_prod = 0; 1710 rxr->rxr_cons = rxr->rxr_prod = 0; 1711 1712 ixl_txr_config(sc, txr); 1713 ixl_rxr_config(sc, rxr); 1714 } 1715 1716 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1717 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 1718 1719 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1720 txr = sc->sc_qps[i].qp_txr; 1721 rxr = sc->sc_qps[i].qp_rxr; 1722 1723 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 1724 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 1725 ixl_flush(sc); 1726 1727 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 1728 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 1729 1730 1731 /* ixl_rxfill() needs lock held */ 1732 mutex_enter(&rxr->rxr_lock); 1733 ixl_rxfill(sc, rxr); 1734 mutex_exit(&rxr->rxr_lock); 1735 1736 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 1737 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 1738 ixl_wr(sc, I40E_QRX_ENA(i), reg); 1739 if (ixl_rxr_enabled(sc, rxr) != 0) 1740 goto stop; 1741 1742 ixl_txr_qdis(sc, txr, 1); 1743 1744 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 1745 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 1746 ixl_wr(sc, I40E_QTX_ENA(i), reg); 1747 1748 if (ixl_txr_enabled(sc, txr) != 0) 1749 goto stop; 1750 } 1751 1752 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1753 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 1754 1755 return 0; 1756 1757 stop: 1758 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1759 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 1760 1761 return ETIMEDOUT; 1762 } 1763 1764 static int 1765 ixl_init_locked(struct ixl_softc *sc) 1766 { 1767 struct ifnet *ifp = &sc->sc_ec.ec_if; 1768 unsigned int i; 1769 int error; 1770 1771 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1772 1773 if (sc->sc_dead) { 1774 return ENXIO; 1775 } 1776 1777 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 1778 sc->sc_nqueue_pairs = 1; 1779 else 1780 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1781 1782 error = ixl_reinit(sc); 1783 if (error) { 1784 ixl_stop_locked(sc); 1785 return error; 1786 } 1787 1788 SET(ifp->if_flags, IFF_RUNNING); 1789 CLR(ifp->if_flags, IFF_OACTIVE); 1790 ixl_link_status(sc); 1791 1792 ixl_config_rss(sc); 1793 ixl_config_queue_intr(sc); 1794 1795 ixl_enable_other_intr(sc); 1796 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1797 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 1798 } 1799 1800 error = ixl_iff(sc); 1801 if (error) { 1802 ixl_stop_locked(sc); 1803 return error; 1804 } 1805 1806 return 0; 1807 } 1808 1809 static int 1810 ixl_init(struct ifnet *ifp) 1811 { 1812 struct ixl_softc *sc = ifp->if_softc; 1813 int error; 1814 1815 mutex_enter(&sc->sc_cfg_lock); 1816 error = ixl_init_locked(sc); 1817 mutex_exit(&sc->sc_cfg_lock); 1818 1819 return error; 1820 } 1821 1822 static int 1823 ixl_iff(struct ixl_softc *sc) 1824 { 1825 struct ifnet *ifp = &sc->sc_ec.ec_if; 1826 struct ixl_atq iatq; 1827 struct ixl_aq_desc *iaq; 1828 struct ixl_aq_vsi_promisc_param *param; 1829 int error; 1830 1831 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 1832 return 0; 1833 1834 memset(&iatq, 0, sizeof(iatq)); 1835 1836 iaq = &iatq.iatq_desc; 1837 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 1838 1839 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 1840 param->flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 1841 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 1842 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 1843 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 1844 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 1845 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1846 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 1847 } 1848 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 1849 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 1850 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 1851 param->seid = sc->sc_seid; 1852 1853 error = ixl_atq_exec(sc, &iatq); 1854 if (error) 1855 return error; 1856 1857 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 1858 return EIO; 1859 1860 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 1861 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1862 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1863 1864 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 1865 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 1866 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1867 } 1868 return 0; 1869 } 1870 1871 static void 1872 ixl_stop_rendezvous(struct ixl_softc *sc) 1873 { 1874 struct ixl_tx_ring *txr; 1875 struct ixl_rx_ring *rxr; 1876 unsigned int i; 1877 1878 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1879 txr = sc->sc_qps[i].qp_txr; 1880 rxr = sc->sc_qps[i].qp_rxr; 1881 1882 mutex_enter(&txr->txr_lock); 1883 mutex_exit(&txr->txr_lock); 1884 1885 mutex_enter(&rxr->rxr_lock); 1886 mutex_exit(&rxr->rxr_lock); 1887 1888 ixl_work_wait(sc->sc_workq_txrx, 1889 &sc->sc_qps[i].qp_task); 1890 } 1891 1892 mutex_enter(&sc->sc_atq_lock); 1893 mutex_exit(&sc->sc_atq_lock); 1894 1895 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1896 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1897 1898 } 1899 1900 static void 1901 ixl_stop_locked(struct ixl_softc *sc) 1902 { 1903 struct ifnet *ifp = &sc->sc_ec.ec_if; 1904 struct ixl_rx_ring *rxr; 1905 struct ixl_tx_ring *txr; 1906 unsigned int i; 1907 uint32_t reg; 1908 1909 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1910 1911 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 1912 1913 ixl_disable_other_intr(sc); 1914 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1915 txr = sc->sc_qps[i].qp_txr; 1916 rxr = sc->sc_qps[i].qp_rxr; 1917 1918 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 1919 1920 mutex_enter(&txr->txr_lock); 1921 ixl_txr_qdis(sc, txr, 0); 1922 /* XXX wait at least 400 usec for all tx queues in one go */ 1923 ixl_flush(sc); 1924 DELAY(500); 1925 1926 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 1927 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 1928 ixl_wr(sc, I40E_QTX_ENA(i), reg); 1929 /* XXX wait 50ms from completaion of the TX queue disable*/ 1930 ixl_flush(sc); 1931 DELAY(50); 1932 1933 if (ixl_txr_disabled(sc, txr) != 0) { 1934 mutex_exit(&txr->txr_lock); 1935 goto die; 1936 } 1937 mutex_exit(&txr->txr_lock); 1938 1939 mutex_enter(&rxr->rxr_lock); 1940 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 1941 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 1942 ixl_wr(sc, I40E_QRX_ENA(i), reg); 1943 /* XXX wait 50ms from completion of the RX queue disable */ 1944 ixl_flush(sc); 1945 DELAY(50); 1946 1947 if (ixl_rxr_disabled(sc, rxr) != 0) { 1948 mutex_exit(&rxr->rxr_lock); 1949 goto die; 1950 } 1951 mutex_exit(&rxr->rxr_lock); 1952 } 1953 1954 ixl_stop_rendezvous(sc); 1955 1956 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1957 txr = sc->sc_qps[i].qp_txr; 1958 rxr = sc->sc_qps[i].qp_rxr; 1959 1960 ixl_txr_unconfig(sc, txr); 1961 ixl_rxr_unconfig(sc, rxr); 1962 1963 ixl_txr_clean(sc, txr); 1964 ixl_rxr_clean(sc, rxr); 1965 } 1966 1967 return; 1968 die: 1969 sc->sc_dead = true; 1970 log(LOG_CRIT, "%s: failed to shut down rings", 1971 device_xname(sc->sc_dev)); 1972 return; 1973 } 1974 1975 static void 1976 ixl_stop(struct ifnet *ifp, int disable) 1977 { 1978 struct ixl_softc *sc = ifp->if_softc; 1979 1980 mutex_enter(&sc->sc_cfg_lock); 1981 ixl_stop_locked(sc); 1982 mutex_exit(&sc->sc_cfg_lock); 1983 } 1984 1985 static int 1986 ixl_queue_pairs_alloc(struct ixl_softc *sc) 1987 { 1988 struct ixl_queue_pair *qp; 1989 unsigned int i; 1990 size_t sz; 1991 1992 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 1993 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 1994 1995 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 1996 qp = &sc->sc_qps[i]; 1997 1998 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1999 ixl_handle_queue, qp); 2000 2001 qp->qp_txr = ixl_txr_alloc(sc, i); 2002 if (qp->qp_txr == NULL) 2003 goto free; 2004 2005 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2006 if (qp->qp_rxr == NULL) 2007 goto free; 2008 2009 qp->qp_sc = sc; 2010 ixl_work_set(&qp->qp_task, ixl_handle_queue, qp); 2011 snprintf(qp->qp_name, sizeof(qp->qp_name), 2012 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2013 } 2014 2015 return 0; 2016 free: 2017 if (sc->sc_qps != NULL) { 2018 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2019 qp = &sc->sc_qps[i]; 2020 2021 if (qp->qp_txr != NULL) 2022 ixl_txr_free(sc, qp->qp_txr); 2023 if (qp->qp_rxr != NULL) 2024 ixl_rxr_free(sc, qp->qp_rxr); 2025 } 2026 2027 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2028 kmem_free(sc->sc_qps, sz); 2029 sc->sc_qps = NULL; 2030 } 2031 2032 return -1; 2033 } 2034 2035 static void 2036 ixl_queue_pairs_free(struct ixl_softc *sc) 2037 { 2038 struct ixl_queue_pair *qp; 2039 unsigned int i; 2040 size_t sz; 2041 2042 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2043 qp = &sc->sc_qps[i]; 2044 ixl_txr_free(sc, qp->qp_txr); 2045 ixl_rxr_free(sc, qp->qp_rxr); 2046 } 2047 2048 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2049 kmem_free(sc->sc_qps, sz); 2050 sc->sc_qps = NULL; 2051 } 2052 2053 static struct ixl_tx_ring * 2054 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2055 { 2056 struct ixl_tx_ring *txr = NULL; 2057 struct ixl_tx_map *maps = NULL, *txm; 2058 unsigned int i; 2059 2060 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2061 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2062 KM_SLEEP); 2063 2064 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2065 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2066 IXL_TX_QUEUE_ALIGN) != 0) 2067 goto free; 2068 2069 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2070 txm = &maps[i]; 2071 2072 if (bus_dmamap_create(sc->sc_dmat, 2073 IXL_HARDMTU, IXL_TX_PKT_DESCS, IXL_HARDMTU, 0, 2074 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2075 goto uncreate; 2076 2077 txm->txm_eop = -1; 2078 txm->txm_m = NULL; 2079 } 2080 2081 txr->txr_cons = txr->txr_prod = 0; 2082 txr->txr_maps = maps; 2083 2084 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2085 if (txr->txr_intrq == NULL) 2086 goto uncreate; 2087 2088 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2089 ixl_deferred_transmit, txr); 2090 if (txr->txr_si == NULL) 2091 goto destroy_pcq; 2092 2093 txr->txr_tail = I40E_QTX_TAIL(qid); 2094 txr->txr_qid = qid; 2095 txr->txr_sc = sc; 2096 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2097 2098 return txr; 2099 2100 destroy_pcq: 2101 pcq_destroy(txr->txr_intrq); 2102 uncreate: 2103 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2104 txm = &maps[i]; 2105 2106 if (txm->txm_map == NULL) 2107 continue; 2108 2109 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2110 } 2111 2112 ixl_dmamem_free(sc, &txr->txr_mem); 2113 free: 2114 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2115 kmem_free(txr, sizeof(*txr)); 2116 2117 return NULL; 2118 } 2119 2120 static void 2121 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2122 { 2123 unsigned int qid; 2124 bus_size_t reg; 2125 uint32_t r; 2126 2127 qid = txr->txr_qid + sc->sc_base_queue; 2128 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2129 qid %= 128; 2130 2131 r = ixl_rd(sc, reg); 2132 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2133 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2134 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2135 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2136 ixl_wr(sc, reg, r); 2137 } 2138 2139 static void 2140 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2141 { 2142 struct ixl_hmc_txq txq; 2143 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2144 void *hmc; 2145 2146 memset(&txq, 0, sizeof(txq)); 2147 txq.head = htole16(txr->txr_cons); 2148 txq.new_context = 1; 2149 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2150 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2151 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2152 txq.tphrdesc_ena = 0; 2153 txq.tphrpacket_ena = 0; 2154 txq.tphwdesc_ena = 0; 2155 txq.rdylist = data->qs_handle[0]; 2156 2157 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2158 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2159 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2160 __arraycount(ixl_hmc_pack_txq)); 2161 } 2162 2163 static void 2164 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2165 { 2166 void *hmc; 2167 2168 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2169 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2170 } 2171 2172 static void 2173 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2174 { 2175 struct ixl_tx_map *maps, *txm; 2176 bus_dmamap_t map; 2177 unsigned int i; 2178 2179 maps = txr->txr_maps; 2180 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2181 txm = &maps[i]; 2182 2183 if (txm->txm_m == NULL) 2184 continue; 2185 2186 map = txm->txm_map; 2187 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2188 BUS_DMASYNC_POSTWRITE); 2189 bus_dmamap_unload(sc->sc_dmat, map); 2190 2191 m_freem(txm->txm_m); 2192 txm->txm_m = NULL; 2193 } 2194 } 2195 2196 static int 2197 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2198 { 2199 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2200 uint32_t reg; 2201 int i; 2202 2203 for (i = 0; i < 10; i++) { 2204 reg = ixl_rd(sc, ena); 2205 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2206 return 0; 2207 2208 delaymsec(10); 2209 } 2210 2211 return ETIMEDOUT; 2212 } 2213 2214 static int 2215 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2216 { 2217 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2218 uint32_t reg; 2219 int i; 2220 2221 KASSERT(mutex_owned(&txr->txr_lock)); 2222 2223 for (i = 0; i < 20; i++) { 2224 reg = ixl_rd(sc, ena); 2225 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2226 return 0; 2227 2228 delaymsec(10); 2229 } 2230 2231 return ETIMEDOUT; 2232 } 2233 2234 static void 2235 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2236 { 2237 struct ixl_tx_map *maps, *txm; 2238 struct mbuf *m; 2239 unsigned int i; 2240 2241 softint_disestablish(txr->txr_si); 2242 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2243 m_freem(m); 2244 pcq_destroy(txr->txr_intrq); 2245 2246 maps = txr->txr_maps; 2247 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2248 txm = &maps[i]; 2249 2250 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2251 } 2252 2253 ixl_dmamem_free(sc, &txr->txr_mem); 2254 mutex_destroy(&txr->txr_lock); 2255 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2256 kmem_free(txr, sizeof(*txr)); 2257 } 2258 2259 static inline int 2260 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2261 struct ixl_tx_ring *txr) 2262 { 2263 struct mbuf *m; 2264 int error; 2265 2266 KASSERT(mutex_owned(&txr->txr_lock)); 2267 2268 m = *m0; 2269 2270 error = bus_dmamap_load_mbuf(dmat, map, m, 2271 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2272 if (error != EFBIG) 2273 return error; 2274 2275 m = m_defrag(m, M_DONTWAIT); 2276 if (m != NULL) { 2277 *m0 = m; 2278 txr->txr_defragged.ev_count++; 2279 2280 error = bus_dmamap_load_mbuf(dmat, map, m, 2281 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2282 } else { 2283 txr->txr_defrag_failed.ev_count++; 2284 error = ENOBUFS; 2285 } 2286 2287 return error; 2288 } 2289 2290 static void 2291 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2292 bool is_transmit) 2293 { 2294 struct ixl_softc *sc = ifp->if_softc; 2295 struct ixl_tx_desc *ring, *txd; 2296 struct ixl_tx_map *txm; 2297 bus_dmamap_t map; 2298 struct mbuf *m; 2299 uint64_t cmd; 2300 unsigned int prod, free, last, i; 2301 unsigned int mask; 2302 int post = 0; 2303 2304 KASSERT(mutex_owned(&txr->txr_lock)); 2305 2306 if (ifp->if_link_state != LINK_STATE_UP 2307 || !ISSET(ifp->if_flags, IFF_RUNNING) 2308 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2309 if (!is_transmit) 2310 IFQ_PURGE(&ifp->if_snd); 2311 return; 2312 } 2313 2314 prod = txr->txr_prod; 2315 free = txr->txr_cons; 2316 if (free <= prod) 2317 free += sc->sc_tx_ring_ndescs; 2318 free -= prod; 2319 2320 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2321 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2322 2323 ring = IXL_DMA_KVA(&txr->txr_mem); 2324 mask = sc->sc_tx_ring_ndescs - 1; 2325 last = prod; 2326 cmd = 0; 2327 txd = NULL; 2328 2329 for (;;) { 2330 if (free <= IXL_TX_PKT_DESCS) { 2331 if (!is_transmit) 2332 SET(ifp->if_flags, IFF_OACTIVE); 2333 break; 2334 } 2335 2336 if (is_transmit) 2337 m = pcq_get(txr->txr_intrq); 2338 else 2339 IFQ_DEQUEUE(&ifp->if_snd, m); 2340 2341 if (m == NULL) 2342 break; 2343 2344 txm = &txr->txr_maps[prod]; 2345 map = txm->txm_map; 2346 2347 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2348 txr->txr_oerrors++; 2349 m_freem(m); 2350 continue; 2351 } 2352 2353 bus_dmamap_sync(sc->sc_dmat, map, 0, 2354 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2355 2356 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2357 txd = &ring[prod]; 2358 2359 cmd = (uint64_t)map->dm_segs[i].ds_len << 2360 IXL_TX_DESC_BSIZE_SHIFT; 2361 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2362 2363 txd->addr = htole64(map->dm_segs[i].ds_addr); 2364 txd->cmd = htole64(cmd); 2365 2366 last = prod; 2367 2368 prod++; 2369 prod &= mask; 2370 } 2371 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2372 txd->cmd = htole64(cmd); 2373 2374 txm->txm_m = m; 2375 txm->txm_eop = last; 2376 2377 bpf_mtap(ifp, m, BPF_D_OUT); 2378 2379 free -= i; 2380 post = 1; 2381 } 2382 2383 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2384 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2385 2386 if (post) { 2387 txr->txr_prod = prod; 2388 ixl_wr(sc, txr->txr_tail, prod); 2389 } 2390 } 2391 2392 static int 2393 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2394 { 2395 struct ifnet *ifp = &sc->sc_ec.ec_if; 2396 struct ixl_tx_desc *ring, *txd; 2397 struct ixl_tx_map *txm; 2398 struct mbuf *m; 2399 bus_dmamap_t map; 2400 unsigned int cons, prod, last; 2401 unsigned int mask; 2402 uint64_t dtype; 2403 int done = 0, more = 0; 2404 2405 KASSERT(mutex_owned(&txr->txr_lock)); 2406 2407 prod = txr->txr_prod; 2408 cons = txr->txr_cons; 2409 2410 if (cons == prod) 2411 return 0; 2412 2413 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2414 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2415 2416 ring = IXL_DMA_KVA(&txr->txr_mem); 2417 mask = sc->sc_tx_ring_ndescs - 1; 2418 2419 do { 2420 if (txlimit-- <= 0) { 2421 more = 1; 2422 break; 2423 } 2424 2425 txm = &txr->txr_maps[cons]; 2426 last = txm->txm_eop; 2427 txd = &ring[last]; 2428 2429 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2430 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2431 break; 2432 2433 map = txm->txm_map; 2434 2435 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2436 BUS_DMASYNC_POSTWRITE); 2437 bus_dmamap_unload(sc->sc_dmat, map); 2438 2439 m = txm->txm_m; 2440 if (m != NULL) { 2441 txr->txr_opackets++; 2442 txr->txr_obytes += m->m_pkthdr.len; 2443 if (ISSET(m->m_flags, M_MCAST)) 2444 txr->txr_omcasts++; 2445 m_freem(m); 2446 } 2447 2448 txm->txm_m = NULL; 2449 txm->txm_eop = -1; 2450 2451 cons = last + 1; 2452 cons &= mask; 2453 done = 1; 2454 } while (cons != prod); 2455 2456 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2457 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2458 2459 txr->txr_cons = cons; 2460 2461 if (done) { 2462 softint_schedule(txr->txr_si); 2463 if (txr->txr_qid == 0) { 2464 CLR(ifp->if_flags, IFF_OACTIVE); 2465 if_schedule_deferred_start(ifp); 2466 } 2467 } 2468 2469 return more; 2470 } 2471 2472 static void 2473 ixl_start(struct ifnet *ifp) 2474 { 2475 struct ixl_softc *sc; 2476 struct ixl_tx_ring *txr; 2477 2478 sc = ifp->if_softc; 2479 txr = sc->sc_qps[0].qp_txr; 2480 2481 mutex_enter(&txr->txr_lock); 2482 ixl_tx_common_locked(ifp, txr, false); 2483 mutex_exit(&txr->txr_lock); 2484 } 2485 2486 static inline unsigned int 2487 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2488 { 2489 u_int cpuid; 2490 2491 cpuid = cpu_index(curcpu()); 2492 2493 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2494 } 2495 2496 static int 2497 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2498 { 2499 struct ixl_softc *sc; 2500 struct ixl_tx_ring *txr; 2501 unsigned int qid; 2502 2503 sc = ifp->if_softc; 2504 qid = ixl_select_txqueue(sc, m); 2505 2506 txr = sc->sc_qps[qid].qp_txr; 2507 2508 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2509 mutex_enter(&txr->txr_lock); 2510 txr->txr_pcqdrop.ev_count++; 2511 mutex_exit(&txr->txr_lock); 2512 2513 m_freem(m); 2514 return ENOBUFS; 2515 } 2516 2517 if (mutex_tryenter(&txr->txr_lock)) { 2518 ixl_tx_common_locked(ifp, txr, true); 2519 mutex_exit(&txr->txr_lock); 2520 } else { 2521 softint_schedule(txr->txr_si); 2522 } 2523 2524 return 0; 2525 } 2526 2527 static void 2528 ixl_deferred_transmit(void *xtxr) 2529 { 2530 struct ixl_tx_ring *txr = xtxr; 2531 struct ixl_softc *sc = txr->txr_sc; 2532 struct ifnet *ifp = &sc->sc_ec.ec_if; 2533 2534 mutex_enter(&txr->txr_lock); 2535 txr->txr_transmitdef.ev_count++; 2536 if (pcq_peek(txr->txr_intrq) != NULL) 2537 ixl_tx_common_locked(ifp, txr, true); 2538 mutex_exit(&txr->txr_lock); 2539 } 2540 2541 static struct ixl_rx_ring * 2542 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2543 { 2544 struct ixl_rx_ring *rxr = NULL; 2545 struct ixl_rx_map *maps = NULL, *rxm; 2546 unsigned int i; 2547 2548 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2549 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2550 KM_SLEEP); 2551 2552 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2553 sizeof(struct ixl_rx_rd_desc_16) * sc->sc_rx_ring_ndescs, 2554 IXL_RX_QUEUE_ALIGN) != 0) 2555 goto free; 2556 2557 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2558 rxm = &maps[i]; 2559 2560 if (bus_dmamap_create(sc->sc_dmat, 2561 IXL_HARDMTU, 1, IXL_HARDMTU, 0, 2562 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2563 goto uncreate; 2564 2565 rxm->rxm_m = NULL; 2566 } 2567 2568 rxr->rxr_cons = rxr->rxr_prod = 0; 2569 rxr->rxr_m_head = NULL; 2570 rxr->rxr_m_tail = &rxr->rxr_m_head; 2571 rxr->rxr_maps = maps; 2572 2573 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2574 rxr->rxr_qid = qid; 2575 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2576 2577 return rxr; 2578 2579 uncreate: 2580 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2581 rxm = &maps[i]; 2582 2583 if (rxm->rxm_map == NULL) 2584 continue; 2585 2586 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2587 } 2588 2589 ixl_dmamem_free(sc, &rxr->rxr_mem); 2590 free: 2591 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2592 kmem_free(rxr, sizeof(*rxr)); 2593 2594 return NULL; 2595 } 2596 2597 static void 2598 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2599 { 2600 struct ixl_rx_map *maps, *rxm; 2601 bus_dmamap_t map; 2602 unsigned int i; 2603 2604 maps = rxr->rxr_maps; 2605 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2606 rxm = &maps[i]; 2607 2608 if (rxm->rxm_m == NULL) 2609 continue; 2610 2611 map = rxm->rxm_map; 2612 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2613 BUS_DMASYNC_POSTWRITE); 2614 bus_dmamap_unload(sc->sc_dmat, map); 2615 2616 m_freem(rxm->rxm_m); 2617 rxm->rxm_m = NULL; 2618 } 2619 2620 m_freem(rxr->rxr_m_head); 2621 rxr->rxr_m_head = NULL; 2622 rxr->rxr_m_tail = &rxr->rxr_m_head; 2623 2624 rxr->rxr_prod = rxr->rxr_cons = 0; 2625 } 2626 2627 static int 2628 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2629 { 2630 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 2631 uint32_t reg; 2632 int i; 2633 2634 for (i = 0; i < 10; i++) { 2635 reg = ixl_rd(sc, ena); 2636 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 2637 return 0; 2638 2639 delaymsec(10); 2640 } 2641 2642 return ETIMEDOUT; 2643 } 2644 2645 static int 2646 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2647 { 2648 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 2649 uint32_t reg; 2650 int i; 2651 2652 KASSERT(mutex_owned(&rxr->rxr_lock)); 2653 2654 for (i = 0; i < 20; i++) { 2655 reg = ixl_rd(sc, ena); 2656 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 2657 return 0; 2658 2659 delaymsec(10); 2660 } 2661 2662 return ETIMEDOUT; 2663 } 2664 2665 static void 2666 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2667 { 2668 struct ixl_hmc_rxq rxq; 2669 void *hmc; 2670 2671 memset(&rxq, 0, sizeof(rxq)); 2672 2673 rxq.head = htole16(rxr->rxr_cons); 2674 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 2675 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 2676 rxq.dbuff = htole16(MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 2677 rxq.hbuff = 0; 2678 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 2679 rxq.dsize = IXL_HMC_RXQ_DSIZE_16; 2680 rxq.crcstrip = 1; 2681 rxq.l2sel = 0; 2682 rxq.showiv = 0; 2683 rxq.rxmax = htole16(IXL_HARDMTU); 2684 rxq.tphrdesc_ena = 0; 2685 rxq.tphwdesc_ena = 0; 2686 rxq.tphdata_ena = 0; 2687 rxq.tphhead_ena = 0; 2688 rxq.lrxqthresh = 0; 2689 rxq.prefena = 1; 2690 2691 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 2692 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 2693 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 2694 __arraycount(ixl_hmc_pack_rxq)); 2695 } 2696 2697 static void 2698 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2699 { 2700 void *hmc; 2701 2702 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 2703 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 2704 } 2705 2706 static void 2707 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2708 { 2709 struct ixl_rx_map *maps, *rxm; 2710 unsigned int i; 2711 2712 maps = rxr->rxr_maps; 2713 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2714 rxm = &maps[i]; 2715 2716 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2717 } 2718 2719 ixl_dmamem_free(sc, &rxr->rxr_mem); 2720 mutex_destroy(&rxr->rxr_lock); 2721 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2722 kmem_free(rxr, sizeof(*rxr)); 2723 } 2724 2725 static int 2726 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 2727 { 2728 struct ifnet *ifp = &sc->sc_ec.ec_if; 2729 struct ixl_rx_wb_desc_16 *ring, *rxd; 2730 struct ixl_rx_map *rxm; 2731 bus_dmamap_t map; 2732 unsigned int cons, prod; 2733 struct mbuf *m; 2734 uint64_t word; 2735 unsigned int len; 2736 unsigned int mask; 2737 int done = 0, more = 0; 2738 2739 KASSERT(mutex_owned(&rxr->rxr_lock)); 2740 2741 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2742 return 0; 2743 2744 prod = rxr->rxr_prod; 2745 cons = rxr->rxr_cons; 2746 2747 if (cons == prod) 2748 return 0; 2749 2750 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 2751 0, IXL_DMA_LEN(&rxr->rxr_mem), 2752 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 2753 2754 ring = IXL_DMA_KVA(&rxr->rxr_mem); 2755 mask = sc->sc_rx_ring_ndescs - 1; 2756 2757 do { 2758 if (rxlimit-- <= 0) { 2759 more = 1; 2760 break; 2761 } 2762 2763 rxd = &ring[cons]; 2764 2765 word = le64toh(rxd->qword1); 2766 2767 if (!ISSET(word, IXL_RX_DESC_DD)) 2768 break; 2769 2770 rxm = &rxr->rxr_maps[cons]; 2771 2772 map = rxm->rxm_map; 2773 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2774 BUS_DMASYNC_POSTREAD); 2775 bus_dmamap_unload(sc->sc_dmat, map); 2776 2777 m = rxm->rxm_m; 2778 rxm->rxm_m = NULL; 2779 2780 KASSERT(m != NULL); 2781 2782 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 2783 m->m_len = len; 2784 m->m_pkthdr.len = 0; 2785 2786 m->m_next = NULL; 2787 *rxr->rxr_m_tail = m; 2788 rxr->rxr_m_tail = &m->m_next; 2789 2790 m = rxr->rxr_m_head; 2791 m->m_pkthdr.len += len; 2792 2793 if (ISSET(word, IXL_RX_DESC_EOP)) { 2794 if (!ISSET(word, 2795 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 2796 m_set_rcvif(m, ifp); 2797 rxr->rxr_ipackets++; 2798 rxr->rxr_ibytes += m->m_pkthdr.len; 2799 if_percpuq_enqueue(ifp->if_percpuq, m); 2800 } else { 2801 rxr->rxr_ierrors++; 2802 m_freem(m); 2803 } 2804 2805 rxr->rxr_m_head = NULL; 2806 rxr->rxr_m_tail = &rxr->rxr_m_head; 2807 } 2808 2809 cons++; 2810 cons &= mask; 2811 2812 done = 1; 2813 } while (cons != prod); 2814 2815 if (done) { 2816 rxr->rxr_cons = cons; 2817 if (ixl_rxfill(sc, rxr) == -1) 2818 rxr->rxr_iqdrops++; 2819 } 2820 2821 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 2822 0, IXL_DMA_LEN(&rxr->rxr_mem), 2823 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 2824 2825 return more; 2826 } 2827 2828 static int 2829 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2830 { 2831 struct ixl_rx_rd_desc_16 *ring, *rxd; 2832 struct ixl_rx_map *rxm; 2833 bus_dmamap_t map; 2834 struct mbuf *m; 2835 unsigned int prod; 2836 unsigned int slots; 2837 unsigned int mask; 2838 int post = 0, error = 0; 2839 2840 KASSERT(mutex_owned(&rxr->rxr_lock)); 2841 2842 prod = rxr->rxr_prod; 2843 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 2844 sc->sc_rx_ring_ndescs); 2845 2846 ring = IXL_DMA_KVA(&rxr->rxr_mem); 2847 mask = sc->sc_rx_ring_ndescs - 1; 2848 2849 if (__predict_false(slots <= 0)) 2850 return -1; 2851 2852 do { 2853 rxm = &rxr->rxr_maps[prod]; 2854 2855 MGETHDR(m, M_DONTWAIT, MT_DATA); 2856 if (m == NULL) { 2857 rxr->rxr_mgethdr_failed.ev_count++; 2858 error = -1; 2859 break; 2860 } 2861 2862 MCLGET(m, M_DONTWAIT); 2863 if (!ISSET(m->m_flags, M_EXT)) { 2864 rxr->rxr_mgetcl_failed.ev_count++; 2865 error = -1; 2866 m_freem(m); 2867 break; 2868 } 2869 2870 m->m_len = m->m_pkthdr.len = MCLBYTES + ETHER_ALIGN; 2871 m_adj(m, ETHER_ALIGN); 2872 2873 map = rxm->rxm_map; 2874 2875 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 2876 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 2877 rxr->rxr_mbuf_load_failed.ev_count++; 2878 error = -1; 2879 m_freem(m); 2880 break; 2881 } 2882 2883 rxm->rxm_m = m; 2884 2885 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2886 BUS_DMASYNC_PREREAD); 2887 2888 rxd = &ring[prod]; 2889 2890 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 2891 rxd->haddr = htole64(0); 2892 2893 prod++; 2894 prod &= mask; 2895 2896 post = 1; 2897 2898 } while (--slots); 2899 2900 if (post) { 2901 rxr->rxr_prod = prod; 2902 ixl_wr(sc, rxr->rxr_tail, prod); 2903 } 2904 2905 return error; 2906 } 2907 2908 static inline int 2909 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 2910 u_int txlimit, struct evcnt *txevcnt, 2911 u_int rxlimit, struct evcnt *rxevcnt) 2912 { 2913 struct ixl_tx_ring *txr = qp->qp_txr; 2914 struct ixl_rx_ring *rxr = qp->qp_rxr; 2915 int txmore, rxmore; 2916 int rv; 2917 2918 KASSERT(!mutex_owned(&txr->txr_lock)); 2919 KASSERT(!mutex_owned(&rxr->rxr_lock)); 2920 2921 mutex_enter(&txr->txr_lock); 2922 txevcnt->ev_count++; 2923 txmore = ixl_txeof(sc, txr, txlimit); 2924 mutex_exit(&txr->txr_lock); 2925 2926 mutex_enter(&rxr->rxr_lock); 2927 rxevcnt->ev_count++; 2928 rxmore = ixl_rxeof(sc, rxr, rxlimit); 2929 mutex_exit(&rxr->rxr_lock); 2930 2931 rv = txmore | (rxmore << 1); 2932 2933 return rv; 2934 } 2935 2936 static void 2937 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 2938 { 2939 2940 if (qp->qp_workqueue) 2941 ixl_work_add(sc->sc_workq_txrx, &qp->qp_task); 2942 else 2943 softint_schedule(qp->qp_si); 2944 } 2945 2946 static int 2947 ixl_intr(void *xsc) 2948 { 2949 struct ixl_softc *sc = xsc; 2950 struct ixl_tx_ring *txr; 2951 struct ixl_rx_ring *rxr; 2952 uint32_t icr, rxintr, txintr; 2953 int rv = 0; 2954 unsigned int i; 2955 2956 KASSERT(sc != NULL); 2957 2958 ixl_enable_other_intr(sc); 2959 icr = ixl_rd(sc, I40E_PFINT_ICR0); 2960 2961 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 2962 atomic_inc_64(&sc->sc_event_atq.ev_count); 2963 ixl_atq_done(sc); 2964 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 2965 rv = 1; 2966 } 2967 2968 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 2969 atomic_inc_64(&sc->sc_event_link.ev_count); 2970 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 2971 rv = 1; 2972 } 2973 2974 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 2975 txintr = icr & I40E_INTR_NOTX_TX_MASK; 2976 2977 if (txintr || rxintr) { 2978 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2979 txr = sc->sc_qps[i].qp_txr; 2980 rxr = sc->sc_qps[i].qp_rxr; 2981 2982 ixl_handle_queue_common(sc, &sc->sc_qps[i], 2983 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 2984 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 2985 } 2986 rv = 1; 2987 } 2988 2989 return rv; 2990 } 2991 2992 static int 2993 ixl_queue_intr(void *xqp) 2994 { 2995 struct ixl_queue_pair *qp = xqp; 2996 struct ixl_tx_ring *txr = qp->qp_txr; 2997 struct ixl_rx_ring *rxr = qp->qp_rxr; 2998 struct ixl_softc *sc = qp->qp_sc; 2999 u_int txlimit, rxlimit; 3000 int more; 3001 3002 txlimit = sc->sc_tx_intr_process_limit; 3003 rxlimit = sc->sc_rx_intr_process_limit; 3004 qp->qp_workqueue = sc->sc_txrx_workqueue; 3005 3006 more = ixl_handle_queue_common(sc, qp, 3007 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3008 3009 if (more != 0) { 3010 ixl_sched_handle_queue(sc, qp); 3011 } else { 3012 /* for ALTQ */ 3013 if (txr->txr_qid == 0) 3014 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3015 softint_schedule(txr->txr_si); 3016 3017 ixl_enable_queue_intr(sc, qp); 3018 } 3019 3020 return 1; 3021 } 3022 3023 static void 3024 ixl_handle_queue(void *xqp) 3025 { 3026 struct ixl_queue_pair *qp = xqp; 3027 struct ixl_softc *sc = qp->qp_sc; 3028 struct ixl_tx_ring *txr = qp->qp_txr; 3029 struct ixl_rx_ring *rxr = qp->qp_rxr; 3030 u_int txlimit, rxlimit; 3031 int more; 3032 3033 txlimit = sc->sc_tx_process_limit; 3034 rxlimit = sc->sc_rx_process_limit; 3035 3036 more = ixl_handle_queue_common(sc, qp, 3037 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3038 3039 if (more != 0) 3040 ixl_sched_handle_queue(sc, qp); 3041 else 3042 ixl_enable_queue_intr(sc, qp); 3043 } 3044 3045 static inline void 3046 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3047 { 3048 uint32_t hmc_idx, hmc_isvf; 3049 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3050 3051 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3052 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3053 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3054 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3055 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3056 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3057 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3058 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3059 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3060 3061 device_printf(sc->sc_dev, 3062 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3063 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3064 } 3065 3066 static int 3067 ixl_other_intr(void *xsc) 3068 { 3069 struct ixl_softc *sc = xsc; 3070 uint32_t icr, mask, reg; 3071 int rv; 3072 3073 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3074 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3075 3076 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3077 atomic_inc_64(&sc->sc_event_atq.ev_count); 3078 ixl_atq_done(sc); 3079 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3080 rv = 1; 3081 } 3082 3083 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3084 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3085 device_printf(sc->sc_dev, "link stat changed\n"); 3086 3087 atomic_inc_64(&sc->sc_event_link.ev_count); 3088 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3089 rv = 1; 3090 } 3091 3092 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3093 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3094 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3095 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3096 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3097 3098 device_printf(sc->sc_dev, "GRST: %s\n", 3099 reg == I40E_RESET_CORER ? "CORER" : 3100 reg == I40E_RESET_GLOBR ? "GLOBR" : 3101 reg == I40E_RESET_EMPR ? "EMPR" : 3102 "POR"); 3103 } 3104 3105 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3106 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3107 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3108 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3109 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3110 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3111 3112 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3113 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3114 device_printf(sc->sc_dev, "critical error\n"); 3115 } 3116 3117 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3118 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3119 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3120 ixl_print_hmc_error(sc, reg); 3121 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3122 } 3123 3124 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3125 ixl_flush(sc); 3126 ixl_enable_other_intr(sc); 3127 return rv; 3128 } 3129 3130 static void 3131 ixl_link_state_update_done(struct ixl_softc *sc) 3132 { 3133 3134 /* IXL_AQ_OP_PHY_LINK_STATUS already posted to admin reply queue */ 3135 } 3136 3137 static void 3138 ixl_link_state_update(void *xsc) 3139 { 3140 struct ixl_softc *sc = xsc; 3141 struct ixl_aq_desc *iaq; 3142 struct ixl_aq_link_param *param; 3143 3144 memset(&sc->sc_link_state_atq, 0, sizeof(sc->sc_link_state_atq)); 3145 iaq = &sc->sc_link_state_atq.iatq_desc; 3146 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3147 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3148 param->notify = IXL_AQ_LINK_NOTIFY; 3149 3150 ixl_atq_set(&sc->sc_link_state_atq, ixl_link_state_update_done); 3151 (void)ixl_atq_post(sc, &sc->sc_link_state_atq); 3152 } 3153 3154 static void 3155 ixl_arq_link_status(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3156 { 3157 struct ifnet *ifp = &sc->sc_ec.ec_if; 3158 int link_state; 3159 3160 link_state = ixl_set_link_status(sc, iaq); 3161 3162 if (ifp->if_link_state != link_state) 3163 if_link_state_change(ifp, link_state); 3164 3165 if (link_state != LINK_STATE_DOWN) { 3166 if_schedule_deferred_start(ifp); 3167 } 3168 } 3169 3170 static void 3171 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3172 { 3173 char buf[512]; 3174 size_t len; 3175 3176 len = sizeof(buf); 3177 buf[--len] = '\0'; 3178 3179 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3180 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3181 buf, le16toh(iaq->iaq_opcode)); 3182 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3183 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3184 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3185 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3186 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3187 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3188 } 3189 3190 static void 3191 ixl_arq(void *xsc) 3192 { 3193 struct ixl_softc *sc = xsc; 3194 struct ixl_aq_desc *arq, *iaq; 3195 struct ixl_aq_buf *aqb; 3196 unsigned int cons = sc->sc_arq_cons; 3197 unsigned int prod; 3198 int done = 0; 3199 3200 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3201 sc->sc_aq_regs->arq_head_mask; 3202 3203 if (cons == prod) 3204 goto done; 3205 3206 arq = IXL_DMA_KVA(&sc->sc_arq); 3207 3208 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3209 0, IXL_DMA_LEN(&sc->sc_arq), 3210 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3211 3212 do { 3213 iaq = &arq[cons]; 3214 aqb = sc->sc_arq_live[cons]; 3215 3216 KASSERT(aqb != NULL); 3217 3218 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3219 BUS_DMASYNC_POSTREAD); 3220 3221 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3222 ixl_aq_dump(sc, iaq); 3223 3224 switch (iaq->iaq_opcode) { 3225 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3226 ixl_arq_link_status(sc, iaq); 3227 break; 3228 } 3229 3230 memset(iaq, 0, sizeof(*iaq)); 3231 sc->sc_arq_live[cons] = NULL; 3232 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3233 3234 cons++; 3235 cons &= IXL_AQ_MASK; 3236 3237 done = 1; 3238 } while (cons != prod); 3239 3240 if (done) { 3241 sc->sc_arq_cons = cons; 3242 ixl_arq_fill(sc); 3243 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3244 0, IXL_DMA_LEN(&sc->sc_arq), 3245 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3246 } 3247 3248 done: 3249 ixl_enable_other_intr(sc); 3250 } 3251 3252 static void 3253 ixl_atq_set(struct ixl_atq *iatq, void (*fn)(struct ixl_softc *)) 3254 { 3255 iatq->iatq_fn = fn; 3256 } 3257 3258 static int 3259 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3260 { 3261 struct ixl_aq_desc *atq, *slot; 3262 unsigned int prod, cons, prod_next; 3263 3264 /* assert locked */ 3265 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3266 3267 atq = IXL_DMA_KVA(&sc->sc_atq); 3268 prod = sc->sc_atq_prod; 3269 cons = sc->sc_atq_cons; 3270 prod_next = (prod +1) & IXL_AQ_MASK; 3271 3272 if (cons == prod_next) 3273 return ENOMEM; 3274 3275 slot = &atq[prod]; 3276 3277 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3278 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3279 3280 *slot = iatq->iatq_desc; 3281 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3282 3283 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3284 ixl_aq_dump(sc, slot); 3285 3286 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3287 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3288 3289 sc->sc_atq_prod = prod_next; 3290 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3291 3292 return 0; 3293 } 3294 3295 static int 3296 ixl_atq_post(struct ixl_softc *sc, struct ixl_atq *iatq) 3297 { 3298 int rv; 3299 3300 mutex_enter(&sc->sc_atq_lock); 3301 rv = ixl_atq_post_locked(sc, iatq); 3302 mutex_exit(&sc->sc_atq_lock); 3303 3304 return rv; 3305 } 3306 3307 static void 3308 ixl_atq_done_locked(struct ixl_softc *sc) 3309 { 3310 struct ixl_aq_desc *atq, *slot; 3311 struct ixl_atq *iatq; 3312 unsigned int cons; 3313 unsigned int prod; 3314 3315 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3316 3317 prod = sc->sc_atq_prod; 3318 cons = sc->sc_atq_cons; 3319 3320 if (prod == cons) 3321 return; 3322 3323 atq = IXL_DMA_KVA(&sc->sc_atq); 3324 3325 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3326 0, IXL_DMA_LEN(&sc->sc_atq), 3327 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3328 3329 do { 3330 slot = &atq[cons]; 3331 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3332 break; 3333 3334 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3335 iatq->iatq_desc = *slot; 3336 3337 memset(slot, 0, sizeof(*slot)); 3338 3339 (*iatq->iatq_fn)(sc); 3340 3341 cons++; 3342 cons &= IXL_AQ_MASK; 3343 } while (cons != prod); 3344 3345 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3346 0, IXL_DMA_LEN(&sc->sc_atq), 3347 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3348 3349 sc->sc_atq_cons = cons; 3350 } 3351 3352 static void 3353 ixl_atq_done(struct ixl_softc *sc) 3354 { 3355 3356 mutex_enter(&sc->sc_atq_lock); 3357 ixl_atq_done_locked(sc); 3358 mutex_exit(&sc->sc_atq_lock); 3359 } 3360 3361 static void 3362 ixl_wakeup(struct ixl_softc *sc) 3363 { 3364 3365 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3366 3367 cv_signal(&sc->sc_atq_cv); 3368 } 3369 3370 static int 3371 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3372 { 3373 int error; 3374 3375 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3376 3377 ixl_atq_set(iatq, ixl_wakeup); 3378 3379 mutex_enter(&sc->sc_atq_lock); 3380 error = ixl_atq_post_locked(sc, iatq); 3381 if (error) { 3382 mutex_exit(&sc->sc_atq_lock); 3383 return error; 3384 } 3385 3386 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3387 IXL_ATQ_EXEC_TIMEOUT); 3388 mutex_exit(&sc->sc_atq_lock); 3389 3390 return error; 3391 } 3392 3393 static int 3394 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3395 { 3396 struct ixl_aq_desc *atq, *slot; 3397 unsigned int prod; 3398 unsigned int t = 0; 3399 3400 atq = IXL_DMA_KVA(&sc->sc_atq); 3401 prod = sc->sc_atq_prod; 3402 slot = atq + prod; 3403 3404 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3405 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3406 3407 *slot = *iaq; 3408 slot->iaq_flags |= htole16(IXL_AQ_SI); 3409 3410 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3411 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3412 3413 prod++; 3414 prod &= IXL_AQ_MASK; 3415 sc->sc_atq_prod = prod; 3416 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3417 3418 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3419 delaymsec(1); 3420 3421 if (t++ > tm) 3422 return ETIMEDOUT; 3423 } 3424 3425 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3426 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3427 *iaq = *slot; 3428 memset(slot, 0, sizeof(*slot)); 3429 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3430 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3431 3432 sc->sc_atq_cons = prod; 3433 3434 return 0; 3435 } 3436 3437 static int 3438 ixl_get_version(struct ixl_softc *sc) 3439 { 3440 struct ixl_aq_desc iaq; 3441 uint32_t fwbuild, fwver, apiver; 3442 uint16_t api_maj_ver, api_min_ver; 3443 3444 memset(&iaq, 0, sizeof(iaq)); 3445 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3446 3447 iaq.iaq_retval = le16toh(23); 3448 3449 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3450 return ETIMEDOUT; 3451 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3452 return EIO; 3453 3454 fwbuild = le32toh(iaq.iaq_param[1]); 3455 fwver = le32toh(iaq.iaq_param[2]); 3456 apiver = le32toh(iaq.iaq_param[3]); 3457 3458 api_maj_ver = (uint16_t)apiver; 3459 api_min_ver = (uint16_t)(apiver >> 16); 3460 3461 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 3462 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 3463 3464 sc->sc_rxctl_atq = true; 3465 if (sc->sc_mac_type == I40E_MAC_X722) { 3466 if (api_maj_ver == 1 && api_min_ver < 5) { 3467 sc->sc_rxctl_atq = false; 3468 } 3469 } 3470 3471 return 0; 3472 } 3473 3474 static int 3475 ixl_pxe_clear(struct ixl_softc *sc) 3476 { 3477 struct ixl_aq_desc iaq; 3478 int rv; 3479 3480 memset(&iaq, 0, sizeof(iaq)); 3481 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 3482 iaq.iaq_param[0] = htole32(0x2); 3483 3484 rv = ixl_atq_poll(sc, &iaq, 250); 3485 3486 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 3487 3488 if (rv != 0) 3489 return ETIMEDOUT; 3490 3491 switch (iaq.iaq_retval) { 3492 case htole16(IXL_AQ_RC_OK): 3493 case htole16(IXL_AQ_RC_EEXIST): 3494 break; 3495 default: 3496 return EIO; 3497 } 3498 3499 return 0; 3500 } 3501 3502 static int 3503 ixl_lldp_shut(struct ixl_softc *sc) 3504 { 3505 struct ixl_aq_desc iaq; 3506 3507 memset(&iaq, 0, sizeof(iaq)); 3508 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 3509 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 3510 3511 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 3512 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 3513 return -1; 3514 } 3515 3516 switch (iaq.iaq_retval) { 3517 case htole16(IXL_AQ_RC_EMODE): 3518 case htole16(IXL_AQ_RC_EPERM): 3519 /* ignore silently */ 3520 default: 3521 break; 3522 } 3523 3524 return 0; 3525 } 3526 3527 static void 3528 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 3529 { 3530 uint16_t id; 3531 uint32_t number, logical_id; 3532 3533 id = le16toh(cap->cap_id); 3534 number = le32toh(cap->number); 3535 logical_id = le32toh(cap->logical_id); 3536 3537 switch (id) { 3538 case IXL_AQ_CAP_RSS: 3539 sc->sc_rss_table_size = number; 3540 sc->sc_rss_table_entry_width = logical_id; 3541 break; 3542 case IXL_AQ_CAP_RXQ: 3543 case IXL_AQ_CAP_TXQ: 3544 sc->sc_nqueue_pairs_device = MIN(number, 3545 sc->sc_nqueue_pairs_device); 3546 break; 3547 } 3548 } 3549 3550 static int 3551 ixl_get_hw_capabilities(struct ixl_softc *sc) 3552 { 3553 struct ixl_dmamem idm; 3554 struct ixl_aq_desc iaq; 3555 struct ixl_aq_capability *caps; 3556 size_t i, ncaps; 3557 bus_size_t caps_size; 3558 uint16_t status; 3559 int rv; 3560 3561 caps_size = sizeof(caps[0]) * 40; 3562 memset(&iaq, 0, sizeof(iaq)); 3563 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 3564 3565 do { 3566 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 3567 return -1; 3568 } 3569 3570 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3571 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3572 iaq.iaq_datalen = htole16(caps_size); 3573 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3574 3575 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 3576 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 3577 3578 rv = ixl_atq_poll(sc, &iaq, 250); 3579 3580 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 3581 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 3582 3583 if (rv != 0) { 3584 aprint_error(", HW capabilities timeout\n"); 3585 goto done; 3586 } 3587 3588 status = le16toh(iaq.iaq_retval); 3589 3590 if (status == IXL_AQ_RC_ENOMEM) { 3591 caps_size = le16toh(iaq.iaq_datalen); 3592 ixl_dmamem_free(sc, &idm); 3593 } 3594 } while (status == IXL_AQ_RC_ENOMEM); 3595 3596 if (status != IXL_AQ_RC_OK) { 3597 aprint_error(", HW capabilities error\n"); 3598 goto done; 3599 } 3600 3601 caps = IXL_DMA_KVA(&idm); 3602 ncaps = le16toh(iaq.iaq_param[1]); 3603 3604 for (i = 0; i < ncaps; i++) { 3605 ixl_parse_hw_capability(sc, &caps[i]); 3606 } 3607 3608 done: 3609 ixl_dmamem_free(sc, &idm); 3610 return rv; 3611 } 3612 3613 static int 3614 ixl_get_mac(struct ixl_softc *sc) 3615 { 3616 struct ixl_dmamem idm; 3617 struct ixl_aq_desc iaq; 3618 struct ixl_aq_mac_addresses *addrs; 3619 int rv; 3620 3621 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 3622 aprint_error(", unable to allocate mac addresses\n"); 3623 return -1; 3624 } 3625 3626 memset(&iaq, 0, sizeof(iaq)); 3627 iaq.iaq_flags = htole16(IXL_AQ_BUF); 3628 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 3629 iaq.iaq_datalen = htole16(sizeof(*addrs)); 3630 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3631 3632 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3633 BUS_DMASYNC_PREREAD); 3634 3635 rv = ixl_atq_poll(sc, &iaq, 250); 3636 3637 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3638 BUS_DMASYNC_POSTREAD); 3639 3640 if (rv != 0) { 3641 aprint_error(", MAC ADDRESS READ timeout\n"); 3642 rv = -1; 3643 goto done; 3644 } 3645 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3646 aprint_error(", MAC ADDRESS READ error\n"); 3647 rv = -1; 3648 goto done; 3649 } 3650 3651 addrs = IXL_DMA_KVA(&idm); 3652 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 3653 printf(", port address is not valid\n"); 3654 goto done; 3655 } 3656 3657 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 3658 rv = 0; 3659 3660 done: 3661 ixl_dmamem_free(sc, &idm); 3662 return rv; 3663 } 3664 3665 static int 3666 ixl_get_switch_config(struct ixl_softc *sc) 3667 { 3668 struct ixl_dmamem idm; 3669 struct ixl_aq_desc iaq; 3670 struct ixl_aq_switch_config *hdr; 3671 struct ixl_aq_switch_config_element *elms, *elm; 3672 unsigned int nelm, i; 3673 int rv; 3674 3675 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 3676 aprint_error_dev(sc->sc_dev, 3677 "unable to allocate switch config buffer\n"); 3678 return -1; 3679 } 3680 3681 memset(&iaq, 0, sizeof(iaq)); 3682 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3683 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3684 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 3685 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 3686 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3687 3688 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3689 BUS_DMASYNC_PREREAD); 3690 3691 rv = ixl_atq_poll(sc, &iaq, 250); 3692 3693 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3694 BUS_DMASYNC_POSTREAD); 3695 3696 if (rv != 0) { 3697 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 3698 rv = -1; 3699 goto done; 3700 } 3701 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3702 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 3703 rv = -1; 3704 goto done; 3705 } 3706 3707 hdr = IXL_DMA_KVA(&idm); 3708 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 3709 3710 nelm = le16toh(hdr->num_reported); 3711 if (nelm < 1) { 3712 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 3713 rv = -1; 3714 goto done; 3715 } 3716 3717 for (i = 0; i < nelm; i++) { 3718 elm = &elms[i]; 3719 3720 aprint_debug_dev(sc->sc_dev, 3721 "type %x revision %u seid %04x\n", 3722 elm->type, elm->revision, le16toh(elm->seid)); 3723 aprint_debug_dev(sc->sc_dev, 3724 "uplink %04x downlink %04x\n", 3725 le16toh(elm->uplink_seid), 3726 le16toh(elm->downlink_seid)); 3727 aprint_debug_dev(sc->sc_dev, 3728 "conntype %x scheduler %04x extra %04x\n", 3729 elm->connection_type, 3730 le16toh(elm->scheduler_id), 3731 le16toh(elm->element_info)); 3732 } 3733 3734 elm = &elms[0]; 3735 3736 sc->sc_uplink_seid = elm->uplink_seid; 3737 sc->sc_downlink_seid = elm->downlink_seid; 3738 sc->sc_seid = elm->seid; 3739 3740 if ((sc->sc_uplink_seid == htole16(0)) != 3741 (sc->sc_downlink_seid == htole16(0))) { 3742 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 3743 rv = -1; 3744 goto done; 3745 } 3746 3747 done: 3748 ixl_dmamem_free(sc, &idm); 3749 return rv; 3750 } 3751 3752 static int 3753 ixl_phy_mask_ints(struct ixl_softc *sc) 3754 { 3755 struct ixl_aq_desc iaq; 3756 3757 memset(&iaq, 0, sizeof(iaq)); 3758 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 3759 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 3760 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 3761 IXL_AQ_PHY_EV_MEDIA_NA)); 3762 3763 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 3764 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 3765 return -1; 3766 } 3767 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3768 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 3769 return -1; 3770 } 3771 3772 return 0; 3773 } 3774 3775 static int 3776 ixl_get_phy_abilities(struct ixl_softc *sc,struct ixl_dmamem *idm) 3777 { 3778 struct ixl_aq_desc iaq; 3779 int rv; 3780 3781 memset(&iaq, 0, sizeof(iaq)); 3782 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3783 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3784 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 3785 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 3786 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 3787 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 3788 3789 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 3790 BUS_DMASYNC_PREREAD); 3791 3792 rv = ixl_atq_poll(sc, &iaq, 250); 3793 3794 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 3795 BUS_DMASYNC_POSTREAD); 3796 3797 if (rv != 0) 3798 return -1; 3799 3800 return le16toh(iaq.iaq_retval); 3801 } 3802 3803 static int 3804 ixl_get_phy_types(struct ixl_softc *sc, uint64_t *phy_types_ptr) 3805 { 3806 struct ixl_dmamem idm; 3807 struct ixl_aq_phy_abilities *phy; 3808 uint64_t phy_types; 3809 int rv; 3810 3811 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 3812 aprint_error_dev(sc->sc_dev, 3813 "unable to allocate switch config buffer\n"); 3814 return -1; 3815 } 3816 3817 rv = ixl_get_phy_abilities(sc, &idm); 3818 switch (rv) { 3819 case -1: 3820 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 3821 goto done; 3822 case IXL_AQ_RC_OK: 3823 break; 3824 case IXL_AQ_RC_EIO: 3825 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 3826 break; 3827 default: 3828 aprint_error_dev(sc->sc_dev, 3829 "GET PHY ABILITIIES error %u\n", rv); 3830 goto done; 3831 } 3832 3833 phy = IXL_DMA_KVA(&idm); 3834 3835 phy_types = le32toh(phy->phy_type); 3836 phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 3837 3838 *phy_types_ptr = phy_types; 3839 3840 rv = 0; 3841 3842 done: 3843 ixl_dmamem_free(sc, &idm); 3844 return rv; 3845 } 3846 3847 static int 3848 ixl_get_link_status(struct ixl_softc *sc) 3849 { 3850 struct ixl_aq_desc iaq; 3851 struct ixl_aq_link_param *param; 3852 int link; 3853 3854 memset(&iaq, 0, sizeof(iaq)); 3855 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3856 param = (struct ixl_aq_link_param *)iaq.iaq_param; 3857 param->notify = IXL_AQ_LINK_NOTIFY; 3858 3859 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 3860 return ETIMEDOUT; 3861 } 3862 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3863 return EIO; 3864 } 3865 3866 link = ixl_set_link_status(sc, &iaq); 3867 sc->sc_ec.ec_if.if_link_state = link; 3868 3869 return 0; 3870 } 3871 3872 static int 3873 ixl_get_vsi(struct ixl_softc *sc) 3874 { 3875 struct ixl_dmamem *vsi = &sc->sc_scratch; 3876 struct ixl_aq_desc iaq; 3877 struct ixl_aq_vsi_param *param; 3878 struct ixl_aq_vsi_reply *reply; 3879 int rv; 3880 3881 /* grumble, vsi info isn't "known" at compile time */ 3882 3883 memset(&iaq, 0, sizeof(iaq)); 3884 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3885 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3886 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 3887 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 3888 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 3889 3890 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 3891 param->uplink_seid = sc->sc_seid; 3892 3893 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 3894 BUS_DMASYNC_PREREAD); 3895 3896 rv = ixl_atq_poll(sc, &iaq, 250); 3897 3898 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 3899 BUS_DMASYNC_POSTREAD); 3900 3901 if (rv != 0) { 3902 aprint_error_dev(sc->sc_dev, "GET VSI timeout\n"); 3903 return -1; 3904 } 3905 3906 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3907 aprint_error_dev(sc->sc_dev, "GET VSI error %u\n", 3908 le16toh(iaq.iaq_retval)); 3909 return -1; 3910 } 3911 3912 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 3913 sc->sc_vsi_number = reply->vsi_number; 3914 3915 return 0; 3916 } 3917 3918 static int 3919 ixl_set_vsi(struct ixl_softc *sc) 3920 { 3921 struct ixl_dmamem *vsi = &sc->sc_scratch; 3922 struct ixl_aq_desc iaq; 3923 struct ixl_aq_vsi_param *param; 3924 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 3925 unsigned int qnum; 3926 int rv; 3927 3928 qnum = sc->sc_nqueue_pairs - 1; 3929 3930 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 3931 IXL_AQ_VSI_VALID_VLAN); 3932 3933 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 3934 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 3935 data->queue_mapping[0] = htole16(0); 3936 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 3937 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 3938 3939 CLR(data->port_vlan_flags, 3940 htole16(IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK)); 3941 SET(data->port_vlan_flags, 3942 htole16(IXL_AQ_VSI_PVLAN_MODE_ALL | IXL_AQ_VSI_PVLAN_EMOD_NOTHING)); 3943 3944 /* grumble, vsi info isn't "known" at compile time */ 3945 3946 memset(&iaq, 0, sizeof(iaq)); 3947 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 3948 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3949 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 3950 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 3951 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 3952 3953 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 3954 param->uplink_seid = sc->sc_seid; 3955 3956 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 3957 BUS_DMASYNC_PREWRITE); 3958 3959 rv = ixl_atq_poll(sc, &iaq, 250); 3960 3961 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 3962 BUS_DMASYNC_POSTWRITE); 3963 3964 if (rv != 0) { 3965 aprint_error_dev(sc->sc_dev, "UPDATE VSI timeout\n"); 3966 return -1; 3967 } 3968 3969 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3970 aprint_error_dev(sc->sc_dev, "UPDATE VSI error %u\n", 3971 le16toh(iaq.iaq_retval)); 3972 return -1; 3973 } 3974 3975 return 0; 3976 } 3977 3978 static void 3979 ixl_set_filter_control(struct ixl_softc *sc) 3980 { 3981 uint32_t reg; 3982 3983 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 3984 3985 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 3986 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 3987 3988 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 3989 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 3990 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 3991 3992 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 3993 } 3994 3995 static inline void 3996 ixl_get_default_rss_key(uint32_t *buf, size_t len) 3997 { 3998 size_t cplen; 3999 uint8_t rss_seed[RSS_KEYSIZE]; 4000 4001 rss_getkey(rss_seed); 4002 memset(buf, 0, len); 4003 4004 cplen = MIN(len, sizeof(rss_seed)); 4005 memcpy(buf, rss_seed, cplen); 4006 } 4007 4008 static void 4009 ixl_set_rss_key(struct ixl_softc *sc) 4010 { 4011 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4012 size_t i; 4013 4014 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4015 4016 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4017 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4018 } 4019 } 4020 4021 static void 4022 ixl_set_rss_pctype(struct ixl_softc *sc) 4023 { 4024 uint64_t set_hena = 0; 4025 uint32_t hena0, hena1; 4026 4027 if (sc->sc_mac_type == I40E_MAC_X722) 4028 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4029 else 4030 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4031 4032 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4033 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4034 4035 SET(hena0, set_hena); 4036 SET(hena1, set_hena >> 32); 4037 4038 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4039 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4040 } 4041 4042 static void 4043 ixl_set_rss_hlut(struct ixl_softc *sc) 4044 { 4045 unsigned int qid; 4046 uint8_t hlut_buf[512], lut_mask; 4047 uint32_t *hluts; 4048 size_t i, hluts_num; 4049 4050 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4051 4052 for (i = 0; i < sc->sc_rss_table_size; i++) { 4053 qid = i % sc->sc_nqueue_pairs; 4054 hlut_buf[i] = qid & lut_mask; 4055 } 4056 4057 hluts = (uint32_t *)hlut_buf; 4058 hluts_num = sc->sc_rss_table_size >> 2; 4059 for (i = 0; i < hluts_num; i++) { 4060 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4061 } 4062 ixl_flush(sc); 4063 } 4064 4065 static void 4066 ixl_config_rss(struct ixl_softc *sc) 4067 { 4068 4069 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4070 4071 ixl_set_rss_key(sc); 4072 ixl_set_rss_pctype(sc); 4073 ixl_set_rss_hlut(sc); 4074 } 4075 4076 static const struct ixl_phy_type * 4077 ixl_search_phy_type(uint8_t phy_type) 4078 { 4079 const struct ixl_phy_type *itype; 4080 uint64_t mask; 4081 unsigned int i; 4082 4083 if (phy_type >= 64) 4084 return NULL; 4085 4086 mask = 1ULL << phy_type; 4087 4088 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4089 itype = &ixl_phy_type_map[i]; 4090 4091 if (ISSET(itype->phy_type, mask)) 4092 return itype; 4093 } 4094 4095 return NULL; 4096 } 4097 4098 static uint64_t 4099 ixl_search_link_speed(uint8_t link_speed) 4100 { 4101 const struct ixl_speed_type *type; 4102 unsigned int i; 4103 4104 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4105 type = &ixl_speed_type_map[i]; 4106 4107 if (ISSET(type->dev_speed, link_speed)) 4108 return type->net_speed; 4109 } 4110 4111 return 0; 4112 } 4113 4114 static int 4115 ixl_restart_an(struct ixl_softc *sc) 4116 { 4117 struct ixl_aq_desc iaq; 4118 4119 memset(&iaq, 0, sizeof(iaq)); 4120 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4121 iaq.iaq_param[0] = 4122 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4123 4124 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4125 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4126 return -1; 4127 } 4128 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4129 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4130 return -1; 4131 } 4132 4133 return 0; 4134 } 4135 4136 static int 4137 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4138 uint16_t vlan, uint16_t flags) 4139 { 4140 struct ixl_aq_desc iaq; 4141 struct ixl_aq_add_macvlan *param; 4142 struct ixl_aq_add_macvlan_elem *elem; 4143 4144 memset(&iaq, 0, sizeof(iaq)); 4145 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4146 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4147 iaq.iaq_datalen = htole16(sizeof(*elem)); 4148 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4149 4150 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4151 param->num_addrs = htole16(1); 4152 param->seid0 = htole16(0x8000) | sc->sc_seid; 4153 param->seid1 = 0; 4154 param->seid2 = 0; 4155 4156 elem = IXL_DMA_KVA(&sc->sc_scratch); 4157 memset(elem, 0, sizeof(*elem)); 4158 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4159 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4160 elem->vlan = htole16(vlan); 4161 4162 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4163 return IXL_AQ_RC_EINVAL; 4164 } 4165 4166 return le16toh(iaq.iaq_retval); 4167 } 4168 4169 static int 4170 ixl_remove_macvlan(struct ixl_softc *sc, uint8_t *macaddr, 4171 uint16_t vlan, uint16_t flags) 4172 { 4173 struct ixl_aq_desc iaq; 4174 struct ixl_aq_remove_macvlan *param; 4175 struct ixl_aq_remove_macvlan_elem *elem; 4176 4177 memset(&iaq, 0, sizeof(iaq)); 4178 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4179 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 4180 iaq.iaq_datalen = htole16(sizeof(*elem)); 4181 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4182 4183 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 4184 param->num_addrs = htole16(1); 4185 param->seid0 = htole16(0x8000) | sc->sc_seid; 4186 param->seid1 = 0; 4187 param->seid2 = 0; 4188 4189 elem = IXL_DMA_KVA(&sc->sc_scratch); 4190 memset(elem, 0, sizeof(*elem)); 4191 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4192 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 4193 elem->vlan = htole16(vlan); 4194 4195 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4196 return IXL_AQ_RC_EINVAL; 4197 } 4198 4199 return le16toh(iaq.iaq_retval); 4200 } 4201 4202 static int 4203 ixl_hmc(struct ixl_softc *sc) 4204 { 4205 struct { 4206 uint32_t count; 4207 uint32_t minsize; 4208 bus_size_t objsiz; 4209 bus_size_t setoff; 4210 bus_size_t setcnt; 4211 } regs[] = { 4212 { 4213 0, 4214 IXL_HMC_TXQ_MINSIZE, 4215 I40E_GLHMC_LANTXOBJSZ, 4216 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 4217 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 4218 }, 4219 { 4220 0, 4221 IXL_HMC_RXQ_MINSIZE, 4222 I40E_GLHMC_LANRXOBJSZ, 4223 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 4224 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 4225 }, 4226 { 4227 0, 4228 0, 4229 I40E_GLHMC_FCOEDDPOBJSZ, 4230 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 4231 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 4232 }, 4233 { 4234 0, 4235 0, 4236 I40E_GLHMC_FCOEFOBJSZ, 4237 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 4238 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 4239 }, 4240 }; 4241 struct ixl_hmc_entry *e; 4242 uint64_t size, dva; 4243 uint8_t *kva; 4244 uint64_t *sdpage; 4245 unsigned int i; 4246 int npages, tables; 4247 uint32_t reg; 4248 4249 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 4250 4251 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 4252 ixl_rd(sc, I40E_GLHMC_LANQMAX); 4253 4254 size = 0; 4255 for (i = 0; i < __arraycount(regs); i++) { 4256 e = &sc->sc_hmc_entries[i]; 4257 4258 e->hmc_count = regs[i].count; 4259 reg = ixl_rd(sc, regs[i].objsiz); 4260 e->hmc_size = BIT_ULL(0x3F & reg); 4261 e->hmc_base = size; 4262 4263 if ((e->hmc_size * 8) < regs[i].minsize) { 4264 aprint_error_dev(sc->sc_dev, 4265 "kernel hmc entry is too big\n"); 4266 return -1; 4267 } 4268 4269 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 4270 } 4271 size = roundup(size, IXL_HMC_PGSIZE); 4272 npages = size / IXL_HMC_PGSIZE; 4273 4274 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 4275 4276 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 4277 aprint_error_dev(sc->sc_dev, 4278 "unable to allocate hmc pd memory\n"); 4279 return -1; 4280 } 4281 4282 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 4283 IXL_HMC_PGSIZE) != 0) { 4284 aprint_error_dev(sc->sc_dev, 4285 "unable to allocate hmc sd memory\n"); 4286 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 4287 return -1; 4288 } 4289 4290 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 4291 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 4292 4293 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 4294 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 4295 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4296 4297 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 4298 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 4299 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 4300 4301 for (i = 0; (int)i < npages; i++) { 4302 *sdpage = htole64(dva | IXL_HMC_PDVALID); 4303 sdpage++; 4304 4305 dva += IXL_HMC_PGSIZE; 4306 } 4307 4308 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 4309 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 4310 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4311 4312 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 4313 for (i = 0; (int)i < tables; i++) { 4314 uint32_t count; 4315 4316 KASSERT(npages >= 0); 4317 4318 count = ((unsigned int)npages > IXL_HMC_PGS) ? 4319 IXL_HMC_PGS : (unsigned int)npages; 4320 4321 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 4322 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 4323 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 4324 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 4325 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 4326 ixl_wr(sc, I40E_PFHMC_SDCMD, 4327 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 4328 4329 npages -= IXL_HMC_PGS; 4330 dva += IXL_HMC_PGSIZE; 4331 } 4332 4333 for (i = 0; i < __arraycount(regs); i++) { 4334 e = &sc->sc_hmc_entries[i]; 4335 4336 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 4337 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 4338 } 4339 4340 return 0; 4341 } 4342 4343 static void 4344 ixl_hmc_free(struct ixl_softc *sc) 4345 { 4346 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 4347 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 4348 } 4349 4350 static void 4351 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 4352 unsigned int npacking) 4353 { 4354 uint8_t *dst = d; 4355 const uint8_t *src = s; 4356 unsigned int i; 4357 4358 for (i = 0; i < npacking; i++) { 4359 const struct ixl_hmc_pack *pack = &packing[i]; 4360 unsigned int offset = pack->lsb / 8; 4361 unsigned int align = pack->lsb % 8; 4362 const uint8_t *in = src + pack->offset; 4363 uint8_t *out = dst + offset; 4364 int width = pack->width; 4365 unsigned int inbits = 0; 4366 4367 if (align) { 4368 inbits = (*in++) << align; 4369 *out++ |= (inbits & 0xff); 4370 inbits >>= 8; 4371 4372 width -= 8 - align; 4373 } 4374 4375 while (width >= 8) { 4376 inbits |= (*in++) << align; 4377 *out++ = (inbits & 0xff); 4378 inbits >>= 8; 4379 4380 width -= 8; 4381 } 4382 4383 if (width > 0) { 4384 inbits |= (*in) << align; 4385 *out |= (inbits & ((1 << width) - 1)); 4386 } 4387 } 4388 } 4389 4390 static struct ixl_aq_buf * 4391 ixl_aqb_alloc(struct ixl_softc *sc) 4392 { 4393 struct ixl_aq_buf *aqb; 4394 4395 aqb = malloc(sizeof(*aqb), M_DEVBUF, M_WAITOK); 4396 if (aqb == NULL) 4397 return NULL; 4398 4399 aqb->aqb_size = IXL_AQ_BUFLEN; 4400 4401 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 4402 aqb->aqb_size, 0, 4403 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 4404 goto free; 4405 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 4406 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 4407 BUS_DMA_WAITOK) != 0) 4408 goto destroy; 4409 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 4410 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 4411 goto dma_free; 4412 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 4413 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 4414 goto unmap; 4415 4416 return aqb; 4417 unmap: 4418 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 4419 dma_free: 4420 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 4421 destroy: 4422 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 4423 free: 4424 free(aqb, M_DEVBUF); 4425 4426 return NULL; 4427 } 4428 4429 static void 4430 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 4431 { 4432 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 4433 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 4434 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 4435 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 4436 free(aqb, M_DEVBUF); 4437 } 4438 4439 static int 4440 ixl_arq_fill(struct ixl_softc *sc) 4441 { 4442 struct ixl_aq_buf *aqb; 4443 struct ixl_aq_desc *arq, *iaq; 4444 unsigned int prod = sc->sc_arq_prod; 4445 unsigned int n; 4446 int post = 0; 4447 4448 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 4449 IXL_AQ_NUM); 4450 arq = IXL_DMA_KVA(&sc->sc_arq); 4451 4452 if (__predict_false(n <= 0)) 4453 return 0; 4454 4455 do { 4456 aqb = sc->sc_arq_live[prod]; 4457 iaq = &arq[prod]; 4458 4459 if (aqb == NULL) { 4460 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 4461 if (aqb != NULL) { 4462 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 4463 ixl_aq_buf, aqb_entry); 4464 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 4465 break; 4466 } 4467 4468 sc->sc_arq_live[prod] = aqb; 4469 memset(aqb->aqb_data, 0, aqb->aqb_size); 4470 4471 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 4472 aqb->aqb_size, BUS_DMASYNC_PREREAD); 4473 4474 iaq->iaq_flags = htole16(IXL_AQ_BUF | 4475 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 4476 IXL_AQ_LB : 0)); 4477 iaq->iaq_opcode = 0; 4478 iaq->iaq_datalen = htole16(aqb->aqb_size); 4479 iaq->iaq_retval = 0; 4480 iaq->iaq_cookie = 0; 4481 iaq->iaq_param[0] = 0; 4482 iaq->iaq_param[1] = 0; 4483 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 4484 } 4485 4486 prod++; 4487 prod &= IXL_AQ_MASK; 4488 4489 post = 1; 4490 4491 } while (--n); 4492 4493 if (post) { 4494 sc->sc_arq_prod = prod; 4495 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 4496 } 4497 4498 return post; 4499 } 4500 4501 static void 4502 ixl_arq_unfill(struct ixl_softc *sc) 4503 { 4504 struct ixl_aq_buf *aqb; 4505 unsigned int i; 4506 4507 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 4508 aqb = sc->sc_arq_live[i]; 4509 if (aqb == NULL) 4510 continue; 4511 4512 sc->sc_arq_live[i] = NULL; 4513 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 4514 BUS_DMASYNC_POSTREAD); 4515 ixl_aqb_free(sc, aqb); 4516 } 4517 4518 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 4519 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 4520 ixl_aq_buf, aqb_entry); 4521 ixl_aqb_free(sc, aqb); 4522 } 4523 } 4524 4525 static void 4526 ixl_clear_hw(struct ixl_softc *sc) 4527 { 4528 uint32_t num_queues, base_queue; 4529 uint32_t num_pf_int; 4530 uint32_t num_vf_int; 4531 uint32_t num_vfs; 4532 uint32_t i, j; 4533 uint32_t val; 4534 uint32_t eol = 0x7ff; 4535 4536 /* get number of interrupts, queues, and vfs */ 4537 val = ixl_rd(sc, I40E_GLPCI_CNF2); 4538 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 4539 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 4540 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 4541 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 4542 4543 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 4544 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 4545 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 4546 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 4547 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 4548 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 4549 num_queues = (j - base_queue) + 1; 4550 else 4551 num_queues = 0; 4552 4553 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 4554 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 4555 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 4556 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 4557 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 4558 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 4559 num_vfs = (j - i) + 1; 4560 else 4561 num_vfs = 0; 4562 4563 /* stop all the interrupts */ 4564 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 4565 ixl_flush(sc); 4566 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 4567 for (i = 0; i < num_pf_int - 2; i++) 4568 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 4569 ixl_flush(sc); 4570 4571 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 4572 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4573 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 4574 for (i = 0; i < num_pf_int - 2; i++) 4575 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 4576 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4577 for (i = 0; i < num_vfs; i++) 4578 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 4579 for (i = 0; i < num_vf_int - 2; i++) 4580 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 4581 4582 /* warn the HW of the coming Tx disables */ 4583 for (i = 0; i < num_queues; i++) { 4584 uint32_t abs_queue_idx = base_queue + i; 4585 uint32_t reg_block = 0; 4586 4587 if (abs_queue_idx >= 128) { 4588 reg_block = abs_queue_idx / 128; 4589 abs_queue_idx %= 128; 4590 } 4591 4592 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 4593 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 4594 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 4595 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 4596 4597 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 4598 } 4599 delaymsec(400); 4600 4601 /* stop all the queues */ 4602 for (i = 0; i < num_queues; i++) { 4603 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 4604 ixl_wr(sc, I40E_QTX_ENA(i), 0); 4605 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 4606 ixl_wr(sc, I40E_QRX_ENA(i), 0); 4607 } 4608 4609 /* short wait for all queue disables to settle */ 4610 delaymsec(50); 4611 } 4612 4613 static int 4614 ixl_pf_reset(struct ixl_softc *sc) 4615 { 4616 uint32_t cnt = 0; 4617 uint32_t cnt1 = 0; 4618 uint32_t reg = 0, reg0 = 0; 4619 uint32_t grst_del; 4620 4621 /* 4622 * Poll for Global Reset steady state in case of recent GRST. 4623 * The grst delay value is in 100ms units, and we'll wait a 4624 * couple counts longer to be sure we don't just miss the end. 4625 */ 4626 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 4627 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 4628 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 4629 4630 grst_del = grst_del * 20; 4631 4632 for (cnt = 0; cnt < grst_del; cnt++) { 4633 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 4634 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 4635 break; 4636 delaymsec(100); 4637 } 4638 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 4639 aprint_error(", Global reset polling failed to complete\n"); 4640 return -1; 4641 } 4642 4643 /* Now Wait for the FW to be ready */ 4644 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 4645 reg = ixl_rd(sc, I40E_GLNVM_ULD); 4646 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4647 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 4648 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4649 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 4650 break; 4651 4652 delaymsec(10); 4653 } 4654 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4655 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 4656 aprint_error(", wait for FW Reset complete timed out " 4657 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 4658 return -1; 4659 } 4660 4661 /* 4662 * If there was a Global Reset in progress when we got here, 4663 * we don't need to do the PF Reset 4664 */ 4665 if (cnt == 0) { 4666 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 4667 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 4668 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 4669 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 4670 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 4671 break; 4672 delaymsec(1); 4673 4674 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 4675 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 4676 aprint_error(", Core reset upcoming." 4677 " Skipping PF reset reset request\n"); 4678 return -1; 4679 } 4680 } 4681 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 4682 aprint_error(", PF reset polling failed to complete" 4683 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 4684 return -1; 4685 } 4686 } 4687 4688 return 0; 4689 } 4690 4691 static int 4692 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 4693 bus_size_t size, bus_size_t align) 4694 { 4695 ixm->ixm_size = size; 4696 4697 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 4698 ixm->ixm_size, 0, 4699 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 4700 &ixm->ixm_map) != 0) 4701 return 1; 4702 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 4703 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 4704 BUS_DMA_WAITOK) != 0) 4705 goto destroy; 4706 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 4707 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 4708 goto free; 4709 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 4710 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 4711 goto unmap; 4712 4713 memset(ixm->ixm_kva, 0, ixm->ixm_size); 4714 4715 return 0; 4716 unmap: 4717 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 4718 free: 4719 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 4720 destroy: 4721 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 4722 return 1; 4723 } 4724 4725 static void 4726 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 4727 { 4728 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 4729 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 4730 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 4731 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 4732 } 4733 4734 static int 4735 ixl_set_macvlan(struct ixl_softc *sc) 4736 { 4737 int error, rv = 0; 4738 4739 /* remove default mac filter and replace it so we can see vlans */ 4740 4741 error = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 4742 if (error != IXL_AQ_RC_OK) { 4743 aprint_debug_dev(sc->sc_dev, "unable to remove macvlan\n"); 4744 rv = -1; 4745 } 4746 4747 error = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 4748 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 4749 if (error != IXL_AQ_RC_OK && error != IXL_AQ_RC_ENOENT) { 4750 aprint_debug_dev(sc->sc_dev, 4751 "unable to remove macvlan(IGNORE_VLAN)\n"); 4752 rv = -1; 4753 } 4754 4755 error = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 4756 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 4757 if (error != IXL_AQ_RC_OK) { 4758 aprint_debug_dev(sc->sc_dev, "unable to add mac address\n"); 4759 rv = -1; 4760 } 4761 4762 error = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 4763 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 4764 if (error != IXL_AQ_RC_OK) { 4765 aprint_debug_dev(sc->sc_dev, 4766 "unable to add broadcast mac address\n"); 4767 rv = -1; 4768 } 4769 4770 return rv; 4771 } 4772 4773 static void 4774 ixl_link_status(struct ixl_softc *sc) 4775 { 4776 4777 (void)ixl_get_link_status(sc); 4778 } 4779 4780 static int 4781 ixl_ifflags_cb(struct ethercom *ec) 4782 { 4783 4784 return 0; 4785 } 4786 4787 static int 4788 ixl_set_link_status(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 4789 { 4790 const struct ixl_aq_link_status *status; 4791 const struct ixl_phy_type *itype; 4792 4793 uint64_t ifm_active = IFM_ETHER; 4794 uint64_t ifm_status = IFM_AVALID; 4795 int link_state = LINK_STATE_DOWN; 4796 uint64_t baudrate = 0; 4797 4798 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 4799 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) 4800 goto done; 4801 4802 ifm_active |= IFM_FDX; 4803 ifm_status |= IFM_ACTIVE; 4804 link_state = LINK_STATE_UP; 4805 4806 itype = ixl_search_phy_type(status->phy_type); 4807 if (itype != NULL) 4808 ifm_active |= itype->ifm_type; 4809 4810 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 4811 ifm_active |= IFM_ETH_TXPAUSE; 4812 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 4813 ifm_active |= IFM_ETH_RXPAUSE; 4814 4815 baudrate = ixl_search_link_speed(status->link_speed); 4816 4817 done: 4818 /* NET_ASSERT_LOCKED() except during attach */ 4819 sc->sc_media_active = ifm_active; 4820 sc->sc_media_status = ifm_status; 4821 4822 sc->sc_ec.ec_if.if_baudrate = baudrate; 4823 4824 return link_state; 4825 } 4826 4827 static int 4828 ixl_establish_intx(struct ixl_softc *sc) 4829 { 4830 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 4831 pci_intr_handle_t *intr; 4832 char xnamebuf[32]; 4833 char intrbuf[PCI_INTRSTR_LEN]; 4834 char const *intrstr; 4835 4836 KASSERT(sc->sc_nintrs == 1); 4837 4838 intr = &sc->sc_ihp[0]; 4839 4840 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 4841 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 4842 device_xname(sc->sc_dev)); 4843 4844 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 4845 sc, xnamebuf); 4846 4847 if (sc->sc_ihs[0] == NULL) { 4848 aprint_error_dev(sc->sc_dev, 4849 "unable to establish interrupt at %s\n", intrstr); 4850 return -1; 4851 } 4852 4853 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 4854 return 0; 4855 } 4856 4857 static int 4858 ixl_establish_msix(struct ixl_softc *sc) 4859 { 4860 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 4861 unsigned int vector = 0; 4862 unsigned int i; 4863 char xnamebuf[32]; 4864 char intrbuf[PCI_INTRSTR_LEN]; 4865 char const *intrstr; 4866 4867 /* the "other" intr is mapped to vector 0 */ 4868 vector = 0; 4869 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 4870 intrbuf, sizeof(intrbuf)); 4871 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 4872 device_xname(sc->sc_dev)); 4873 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 4874 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 4875 sc, xnamebuf); 4876 if (sc->sc_ihs[vector] == NULL) { 4877 aprint_error_dev(sc->sc_dev, 4878 "unable to establish interrupt at %s\n", intrstr); 4879 goto fail; 4880 } 4881 vector++; 4882 aprint_normal_dev(sc->sc_dev, "interrupt at %s\n", intrstr); 4883 4884 sc->sc_msix_vector_queue = vector; 4885 4886 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 4887 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 4888 intrbuf, sizeof(intrbuf)); 4889 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 4890 device_xname(sc->sc_dev), i); 4891 4892 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 4893 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 4894 (void *)&sc->sc_qps[i], xnamebuf); 4895 4896 if (sc->sc_ihs[vector] == NULL) { 4897 aprint_error_dev(sc->sc_dev, 4898 "unable to establish interrupt at %s\n", intrstr); 4899 goto fail; 4900 } 4901 vector++; 4902 aprint_normal_dev(sc->sc_dev, 4903 "interrupt at %s\n", intrstr); 4904 } 4905 4906 return 0; 4907 fail: 4908 for (i = 0; i < vector; i++) { 4909 pci_intr_disestablish(pc, sc->sc_ihs[i]); 4910 } 4911 4912 sc->sc_msix_vector_queue = 0; 4913 sc->sc_msix_vector_queue = 0; 4914 4915 return -1; 4916 } 4917 4918 static void 4919 ixl_set_affinity_msix(struct ixl_softc *sc) 4920 { 4921 kcpuset_t *affinity; 4922 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 4923 int affinity_to, r; 4924 unsigned int i, vector; 4925 char intrbuf[PCI_INTRSTR_LEN]; 4926 char const *intrstr; 4927 4928 affinity_to = 0; 4929 kcpuset_create(&affinity, false); 4930 4931 vector = sc->sc_msix_vector_queue; 4932 4933 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 4934 affinity_to = i % ncpu; 4935 4936 kcpuset_zero(affinity); 4937 kcpuset_set(affinity, affinity_to); 4938 4939 intrstr = pci_intr_string(pc, sc->sc_ihp[vector + i], 4940 intrbuf, sizeof(intrbuf)); 4941 r = interrupt_distribute(sc->sc_ihs[vector + i], 4942 affinity, NULL); 4943 if (r == 0) { 4944 aprint_normal_dev(sc->sc_dev, 4945 "for TXRX%u interrupting at %s affinity to %u\n", 4946 i, intrstr, affinity_to); 4947 } else { 4948 aprint_normal_dev(sc->sc_dev, 4949 "for TXRX%u interrupting at %s\n", 4950 i, intrstr); 4951 } 4952 } 4953 4954 vector = 0; /* vector 0 means "other" interrupt */ 4955 affinity_to = (affinity_to + 1) % ncpu; 4956 kcpuset_zero(affinity); 4957 kcpuset_set(affinity, affinity_to); 4958 4959 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 4960 intrbuf, sizeof(intrbuf)); 4961 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 4962 if (r == 0) { 4963 aprint_normal_dev(sc->sc_dev, 4964 "for other interrupting at %s affinity to %u\n", 4965 intrstr, affinity_to); 4966 } else { 4967 aprint_normal_dev(sc->sc_dev, 4968 "for other interrupting at %s", intrstr); 4969 } 4970 4971 kcpuset_destroy(affinity); 4972 } 4973 4974 static void 4975 ixl_config_queue_intr(struct ixl_softc *sc) 4976 { 4977 unsigned int i, vector; 4978 4979 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 4980 vector = sc->sc_msix_vector_queue; 4981 } else { 4982 vector = I40E_INTR_NOTX_INTR; 4983 4984 ixl_wr(sc, I40E_PFINT_LNKLST0, 4985 (I40E_INTR_NOTX_QUEUE << 4986 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 4987 (I40E_QUEUE_TYPE_RX << 4988 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 4989 } 4990 4991 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 4992 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 4993 ixl_flush(sc); 4994 4995 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 4996 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 4997 (I40E_QUEUE_TYPE_RX << 4998 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 4999 5000 ixl_wr(sc, I40E_QINT_RQCTL(i), 5001 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5002 (I40E_ITR_INDEX_RX << 5003 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5004 (I40E_INTR_NOTX_RX_QUEUE << 5005 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5006 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5007 (I40E_QUEUE_TYPE_TX << 5008 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5009 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5010 5011 ixl_wr(sc, I40E_QINT_TQCTL(i), 5012 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5013 (I40E_ITR_INDEX_TX << 5014 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5015 (I40E_INTR_NOTX_TX_QUEUE << 5016 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5017 (I40E_QUEUE_TYPE_EOL << 5018 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5019 (I40E_QUEUE_TYPE_RX << 5020 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5021 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5022 5023 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) 5024 vector++; 5025 } 5026 ixl_flush(sc); 5027 5028 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), 0x7a); 5029 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), 0x7a); 5030 ixl_flush(sc); 5031 } 5032 5033 static void 5034 ixl_config_other_intr(struct ixl_softc *sc) 5035 { 5036 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5037 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5038 5039 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5040 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5041 I40E_PFINT_ICR0_ENA_GRST_MASK | 5042 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5043 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5044 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5045 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5046 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5047 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5048 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5049 5050 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5051 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5052 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5053 (I40E_ITR_INDEX_OTHER << 5054 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5055 ixl_flush(sc); 5056 } 5057 5058 static int 5059 ixl_setup_interrupts(struct ixl_softc *sc) 5060 { 5061 struct pci_attach_args *pa = &sc->sc_pa; 5062 pci_intr_type_t max_type, intr_type; 5063 int counts[PCI_INTR_TYPE_SIZE]; 5064 int error; 5065 unsigned int i; 5066 bool retry, nomsix = IXL_NOMSIX; 5067 5068 memset(counts, 0, sizeof(counts)); 5069 max_type = PCI_INTR_TYPE_MSIX; 5070 /* QPs + other interrupt */ 5071 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5072 counts[PCI_INTR_TYPE_INTX] = 1; 5073 5074 if (nomsix) 5075 counts[PCI_INTR_TYPE_MSIX] = 0; 5076 5077 do { 5078 retry = false; 5079 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5080 if (error != 0) { 5081 aprint_error_dev(sc->sc_dev, 5082 "couldn't map interrupt\n"); 5083 break; 5084 } 5085 for (i = 0; i < sc->sc_nintrs; i++) { 5086 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5087 PCI_INTR_MPSAFE, true); 5088 } 5089 5090 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5091 sc->sc_nintrs = counts[intr_type]; 5092 KASSERT(sc->sc_nintrs > 0); 5093 5094 sc->sc_ihs = kmem_alloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5095 KM_SLEEP); 5096 5097 if (intr_type == PCI_INTR_TYPE_MSIX) { 5098 error = ixl_establish_msix(sc); 5099 if (error) { 5100 counts[PCI_INTR_TYPE_MSIX] = 0; 5101 retry = true; 5102 } else { 5103 ixl_set_affinity_msix(sc); 5104 } 5105 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5106 error = ixl_establish_intx(sc); 5107 } else { 5108 error = -1; 5109 } 5110 5111 if (error) { 5112 kmem_free(sc->sc_ihs, 5113 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5114 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5115 } else { 5116 sc->sc_intrtype = intr_type; 5117 } 5118 } while (retry); 5119 5120 return error; 5121 } 5122 5123 static void 5124 ixl_teardown_interrupts(struct ixl_softc *sc) 5125 { 5126 struct pci_attach_args *pa = &sc->sc_pa; 5127 unsigned int i; 5128 5129 for (i = 0; i < sc->sc_nintrs; i++) { 5130 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 5131 } 5132 5133 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5134 5135 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5136 sc->sc_ihs = NULL; 5137 sc->sc_nintrs = 0; 5138 } 5139 5140 static int 5141 ixl_setup_stats(struct ixl_softc *sc) 5142 { 5143 struct ixl_queue_pair *qp; 5144 struct ixl_tx_ring *txr; 5145 struct ixl_rx_ring *rxr; 5146 unsigned int i; 5147 5148 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5149 qp = &sc->sc_qps[i]; 5150 txr = qp->qp_txr; 5151 rxr = qp->qp_rxr; 5152 5153 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 5154 NULL, qp->qp_name, "m_defrag successed"); 5155 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 5156 NULL, qp->qp_name, "m_defrag_failed"); 5157 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 5158 NULL, qp->qp_name, "Dropped in pcq"); 5159 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 5160 NULL, qp->qp_name, "Deferred transmit"); 5161 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 5162 NULL, qp->qp_name, "Interrupt on queue"); 5163 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 5164 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5165 5166 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 5167 NULL, qp->qp_name, "MGETHDR failed"); 5168 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 5169 NULL, qp->qp_name, "MCLGET failed"); 5170 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 5171 EVCNT_TYPE_MISC, NULL, qp->qp_name, 5172 "bus_dmamap_load_mbuf failed"); 5173 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 5174 NULL, qp->qp_name, "Interrupt on queue"); 5175 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 5176 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5177 } 5178 5179 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 5180 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 5181 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 5182 NULL, device_xname(sc->sc_dev), "Link status event"); 5183 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 5184 NULL, device_xname(sc->sc_dev), "ECC error"); 5185 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 5186 NULL, device_xname(sc->sc_dev), "PCI exception"); 5187 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 5188 NULL, device_xname(sc->sc_dev), "Critical error"); 5189 5190 return 0; 5191 } 5192 5193 static void 5194 ixl_teardown_stats(struct ixl_softc *sc) 5195 { 5196 struct ixl_tx_ring *txr; 5197 struct ixl_rx_ring *rxr; 5198 unsigned int i; 5199 5200 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5201 txr = sc->sc_qps[i].qp_txr; 5202 rxr = sc->sc_qps[i].qp_rxr; 5203 5204 evcnt_detach(&txr->txr_defragged); 5205 evcnt_detach(&txr->txr_defrag_failed); 5206 evcnt_detach(&txr->txr_pcqdrop); 5207 evcnt_detach(&txr->txr_transmitdef); 5208 evcnt_detach(&txr->txr_intr); 5209 evcnt_detach(&txr->txr_defer); 5210 5211 evcnt_detach(&rxr->rxr_mgethdr_failed); 5212 evcnt_detach(&rxr->rxr_mgetcl_failed); 5213 evcnt_detach(&rxr->rxr_mbuf_load_failed); 5214 evcnt_detach(&rxr->rxr_intr); 5215 evcnt_detach(&rxr->rxr_defer); 5216 } 5217 5218 evcnt_detach(&sc->sc_event_atq); 5219 evcnt_detach(&sc->sc_event_link); 5220 evcnt_detach(&sc->sc_event_ecc_err); 5221 evcnt_detach(&sc->sc_event_pci_exception); 5222 evcnt_detach(&sc->sc_event_crit_err); 5223 } 5224 5225 static int 5226 ixl_setup_sysctls(struct ixl_softc *sc) 5227 { 5228 const char *devname; 5229 struct sysctllog **log; 5230 const struct sysctlnode *rnode, *rxnode, *txnode; 5231 int error; 5232 5233 log = &sc->sc_sysctllog; 5234 devname = device_xname(sc->sc_dev); 5235 5236 error = sysctl_createv(log, 0, NULL, &rnode, 5237 0, CTLTYPE_NODE, devname, 5238 SYSCTL_DESCR("ixl information and settings"), 5239 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 5240 if (error) 5241 goto out; 5242 5243 error = sysctl_createv(log, 0, &rnode, NULL, 5244 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 5245 SYSCTL_DESCR("Use workqueue for packet processing"), 5246 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 5247 if (error) 5248 goto out; 5249 5250 error = sysctl_createv(log, 0, &rnode, &rxnode, 5251 0, CTLTYPE_NODE, "rx", 5252 SYSCTL_DESCR("ixl information and settings for Rx"), 5253 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 5254 if (error) 5255 goto out; 5256 5257 error = sysctl_createv(log, 0, &rxnode, NULL, 5258 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 5259 SYSCTL_DESCR("max number of Rx packets" 5260 " to process for interrupt processing"), 5261 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 5262 if (error) 5263 goto out; 5264 5265 error = sysctl_createv(log, 0, &rxnode, NULL, 5266 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 5267 SYSCTL_DESCR("max number of Rx packets" 5268 " to process for deferred processing"), 5269 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 5270 if (error) 5271 goto out; 5272 5273 error = sysctl_createv(log, 0, &rnode, &txnode, 5274 0, CTLTYPE_NODE, "tx", 5275 SYSCTL_DESCR("ixl information and settings for Tx"), 5276 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 5277 if (error) 5278 goto out; 5279 5280 error = sysctl_createv(log, 0, &txnode, NULL, 5281 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 5282 SYSCTL_DESCR("max number of Tx packets" 5283 " to process for interrupt processing"), 5284 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 5285 if (error) 5286 goto out; 5287 5288 error = sysctl_createv(log, 0, &txnode, NULL, 5289 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 5290 SYSCTL_DESCR("max number of Tx packets" 5291 " to process for deferred processing"), 5292 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 5293 if (error) 5294 goto out; 5295 5296 out: 5297 if (error) { 5298 aprint_error_dev(sc->sc_dev, 5299 "unable to create sysctl node\n"); 5300 sysctl_teardown(log); 5301 } 5302 5303 return error; 5304 } 5305 5306 static void 5307 ixl_teardown_sysctls(struct ixl_softc *sc) 5308 { 5309 5310 sysctl_teardown(&sc->sc_sysctllog); 5311 } 5312 5313 static struct workqueue * 5314 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 5315 { 5316 struct workqueue *wq; 5317 int error; 5318 5319 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 5320 prio, ipl, flags); 5321 5322 if (error) 5323 return NULL; 5324 5325 return wq; 5326 } 5327 5328 static void 5329 ixl_workq_destroy(struct workqueue *wq) 5330 { 5331 5332 workqueue_destroy(wq); 5333 } 5334 5335 static void 5336 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 5337 { 5338 5339 memset(work, 0, sizeof(*work)); 5340 work->ixw_func = func; 5341 work->ixw_arg = arg; 5342 } 5343 5344 static void 5345 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 5346 { 5347 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 5348 return; 5349 5350 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 5351 } 5352 5353 static void 5354 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 5355 { 5356 5357 workqueue_wait(wq, &work->ixw_cookie); 5358 } 5359 5360 static void 5361 ixl_workq_work(struct work *wk, void *context) 5362 { 5363 struct ixl_work *work; 5364 5365 work = container_of(wk, struct ixl_work, ixw_cookie); 5366 5367 atomic_swap_uint(&work->ixw_added, 0); 5368 kpreempt_disable(); 5369 work->ixw_func(work->ixw_arg); 5370 kpreempt_enable(); 5371 } 5372 5373 static int 5374 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 5375 { 5376 struct ixl_aq_desc iaq; 5377 5378 memset(&iaq, 0, sizeof(iaq)); 5379 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 5380 iaq.iaq_param[1] = htole32(reg); 5381 5382 if (ixl_atq_poll(sc, &iaq, 250) != 0) 5383 return ETIMEDOUT; 5384 5385 switch (htole16(iaq.iaq_retval)) { 5386 case IXL_AQ_RC_OK: 5387 /* success */ 5388 break; 5389 case IXL_AQ_RC_EACCES: 5390 return EPERM; 5391 case IXL_AQ_RC_EAGAIN: 5392 return EAGAIN; 5393 default: 5394 return EIO; 5395 } 5396 5397 *rv = htole32(iaq.iaq_param[3]); 5398 return 0; 5399 } 5400 5401 5402 5403 static uint32_t 5404 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 5405 { 5406 uint32_t val; 5407 int rv, retry, retry_limit; 5408 5409 retry_limit = sc->sc_rxctl_atq ? 5 : 0; 5410 5411 for (retry = 0; retry < retry_limit; retry++) { 5412 rv = ixl_rx_ctl_read(sc, reg, &val); 5413 if (rv == 0) 5414 return val; 5415 else if (rv == EAGAIN) 5416 delaymsec(1); 5417 else 5418 break; 5419 } 5420 5421 val = ixl_rd(sc, reg); 5422 5423 return val; 5424 } 5425 5426 static int 5427 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 5428 { 5429 struct ixl_aq_desc iaq; 5430 5431 memset(&iaq, 0, sizeof(iaq)); 5432 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 5433 iaq.iaq_param[1] = htole32(reg); 5434 iaq.iaq_param[3] = htole32(value); 5435 5436 if (ixl_atq_poll(sc, &iaq, 250) != 0) 5437 return ETIMEDOUT; 5438 5439 switch (htole16(iaq.iaq_retval)) { 5440 case IXL_AQ_RC_OK: 5441 /* success */ 5442 break; 5443 case IXL_AQ_RC_EACCES: 5444 return EPERM; 5445 case IXL_AQ_RC_EAGAIN: 5446 return EAGAIN; 5447 default: 5448 return EIO; 5449 } 5450 5451 return 0; 5452 } 5453 5454 static void 5455 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 5456 { 5457 int rv, retry, retry_limit; 5458 5459 retry_limit = sc->sc_rxctl_atq ? 5 : 0; 5460 5461 for (retry = 0; retry < retry_limit; retry++) { 5462 rv = ixl_rx_ctl_write(sc, reg, value); 5463 if (rv == 0) 5464 return; 5465 else if (rv == EAGAIN) 5466 delaymsec(1); 5467 else 5468 break; 5469 } 5470 5471 ixl_wr(sc, reg, value); 5472 } 5473 5474 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 5475 5476 #ifdef _MODULE 5477 #include "ioconf.c" 5478 #endif 5479 5480 static int 5481 if_ixl_modcmd(modcmd_t cmd, void *opaque) 5482 { 5483 int error = 0; 5484 5485 #ifdef _MODULE 5486 switch (cmd) { 5487 case MODULE_CMD_INIT: 5488 error = config_init_component(cfdriver_ioconf_if_ixl, 5489 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 5490 break; 5491 case MODULE_CMD_FINI: 5492 error = config_fini_component(cfdriver_ioconf_if_ixl, 5493 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 5494 break; 5495 default: 5496 error = ENOTTY; 5497 break; 5498 } 5499 #endif 5500 5501 return error; 5502 } 5503
Indexes created Sun Sep 21 14:09:52 GMT 2025