if_ixl.c revision 1.2
1 /* $NetBSD: if_ixl.c,v 1.2 2019/12/11 01:51:22 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 78 #ifdef _KERNEL_OPT 79 #include "opt_net_mpsafe.h" 80 #endif 81 82 #include <sys/param.h> 83 #include <sys/types.h> 84 85 #include <sys/cpu.h> 86 #include <sys/device.h> 87 #include <sys/evcnt.h> 88 #include <sys/interrupt.h> 89 #include <sys/kmem.h> 90 #include <sys/malloc.h> 91 #include <sys/module.h> 92 #include <sys/mutex.h> 93 #include <sys/pcq.h> 94 #include <sys/syslog.h> 95 #include <sys/workqueue.h> 96 97 #include <sys/bus.h> 98 99 #include <net/bpf.h> 100 #include <net/if.h> 101 #include <net/if_dl.h> 102 #include <net/if_media.h> 103 #include <net/if_ether.h> 104 #include <net/rss_config.h> 105 106 #include <dev/pci/pcivar.h> 107 #include <dev/pci/pcidevs.h> 108 109 #include <dev/pci/if_ixlreg.h> 110 #include <dev/pci/if_ixlvar.h> 111 112 struct ixl_softc; /* defined */ 113 114 #define I40E_PF_RESET_WAIT_COUNT 200 115 #define I40E_AQ_LARGE_BUF 512 116 117 /* bitfields for Tx queue mapping in QTX_CTL */ 118 #define I40E_QTX_CTL_VF_QUEUE 0x0 119 #define I40E_QTX_CTL_VM_QUEUE 0x1 120 #define I40E_QTX_CTL_PF_QUEUE 0x2 121 122 #define I40E_QUEUE_TYPE_EOL 0x7ff 123 #define I40E_INTR_NOTX_QUEUE 0 124 125 #define I40E_QUEUE_TYPE_RX 0x0 126 #define I40E_QUEUE_TYPE_TX 0x1 127 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 128 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 129 130 #define I40E_ITR_INDEX_RX 0x0 131 #define I40E_ITR_INDEX_TX 0x1 132 #define I40E_ITR_INDEX_OTHER 0x2 133 #define I40E_ITR_INDEX_NONE 0x3 134 135 #define I40E_INTR_NOTX_QUEUE 0 136 #define I40E_INTR_NOTX_INTR 0 137 #define I40E_INTR_NOTX_RX_QUEUE 0 138 #define I40E_INTR_NOTX_TX_QUEUE 1 139 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 140 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 141 142 #define BIT_ULL(a) (1ULL << (a)) 143 #define IXL_RSS_HENA_DEFAULT_BASE \ 144 (BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 145 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 146 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 147 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 148 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 149 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 150 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 151 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 152 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 153 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 154 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 155 #define IXL_RSS_HENA_DEFAULT_XL710 IXL_RSS_HENA_DEFAULT_BASE 156 #define IXL_RSS_HENA_DEFAULT_X722 (IXL_RSS_HENA_DEFAULT_XL710 | \ 157 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 158 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 159 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 160 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 161 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 162 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 163 #define I40E_HASH_LUT_SIZE_128 0 164 #define IXL_RSS_KEY_SIZE_REG 13 165 166 #define IXL_ICR0_CRIT_ERR_MASK \ 167 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 168 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 169 I40E_PFINT_ICR0_PE_CRITERR_MASK) 170 171 #define IXL_TX_PKT_DESCS 8 172 #define IXL_TX_QUEUE_ALIGN 128 173 #define IXL_RX_QUEUE_ALIGN 128 174 175 #define IXL_HARDMTU 9712 /* 9726 - ETHER_HDR_LEN */ 176 177 #define IXL_PCIREG PCI_MAPREG_START 178 179 #define IXL_ITR0 0x0 180 #define IXL_ITR1 0x1 181 #define IXL_ITR2 0x2 182 #define IXL_NOITR 0x3 183 184 #define IXL_AQ_NUM 256 185 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 186 #define IXL_AQ_ALIGN 64 /* lol */ 187 #define IXL_AQ_BUFLEN 4096 188 189 #define IXL_HMC_ROUNDUP 512 190 #define IXL_HMC_PGSIZE 4096 191 #define IXL_HMC_DVASZ sizeof(uint64_t) 192 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 193 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 194 #define IXL_HMC_PDVALID 1ULL 195 196 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 197 198 struct ixl_aq_regs { 199 bus_size_t atq_tail; 200 bus_size_t atq_head; 201 bus_size_t atq_len; 202 bus_size_t atq_bal; 203 bus_size_t atq_bah; 204 205 bus_size_t arq_tail; 206 bus_size_t arq_head; 207 bus_size_t arq_len; 208 bus_size_t arq_bal; 209 bus_size_t arq_bah; 210 211 uint32_t atq_len_enable; 212 uint32_t atq_tail_mask; 213 uint32_t atq_head_mask; 214 215 uint32_t arq_len_enable; 216 uint32_t arq_tail_mask; 217 uint32_t arq_head_mask; 218 }; 219 220 struct ixl_phy_type { 221 uint64_t phy_type; 222 uint64_t ifm_type; 223 }; 224 225 struct ixl_speed_type { 226 uint8_t dev_speed; 227 uint64_t net_speed; 228 }; 229 230 struct ixl_aq_buf { 231 SIMPLEQ_ENTRY(ixl_aq_buf) 232 aqb_entry; 233 void *aqb_data; 234 bus_dmamap_t aqb_map; 235 bus_dma_segment_t aqb_seg; 236 size_t aqb_size; 237 int aqb_nsegs; 238 }; 239 SIMPLEQ_HEAD(ixl_aq_bufs, ixl_aq_buf); 240 241 struct ixl_dmamem { 242 bus_dmamap_t ixm_map; 243 bus_dma_segment_t ixm_seg; 244 int ixm_nsegs; 245 size_t ixm_size; 246 void *ixm_kva; 247 }; 248 249 #define IXL_DMA_MAP(_ixm) ((_ixm)->ixm_map) 250 #define IXL_DMA_DVA(_ixm) ((_ixm)->ixm_map->dm_segs[0].ds_addr) 251 #define IXL_DMA_KVA(_ixm) ((void *)(_ixm)->ixm_kva) 252 #define IXL_DMA_LEN(_ixm) ((_ixm)->ixm_size) 253 254 struct ixl_hmc_entry { 255 uint64_t hmc_base; 256 uint32_t hmc_count; 257 uint64_t hmc_size; 258 }; 259 260 enum ixl_hmc_types { 261 IXL_HMC_LAN_TX = 0, 262 IXL_HMC_LAN_RX, 263 IXL_HMC_FCOE_CTX, 264 IXL_HMC_FCOE_FILTER, 265 IXL_HMC_COUNT 266 }; 267 268 struct ixl_hmc_pack { 269 uint16_t offset; 270 uint16_t width; 271 uint16_t lsb; 272 }; 273 274 /* 275 * these hmc objects have weird sizes and alignments, so these are abstract 276 * representations of them that are nice for c to populate. 277 * 278 * the packing code relies on little-endian values being stored in the fields, 279 * no high bits in the fields being set, and the fields must be packed in the 280 * same order as they are in the ctx structure. 281 */ 282 283 struct ixl_hmc_rxq { 284 uint16_t head; 285 uint8_t cpuid; 286 uint64_t base; 287 #define IXL_HMC_RXQ_BASE_UNIT 128 288 uint16_t qlen; 289 uint16_t dbuff; 290 #define IXL_HMC_RXQ_DBUFF_UNIT 128 291 uint8_t hbuff; 292 #define IXL_HMC_RXQ_HBUFF_UNIT 64 293 uint8_t dtype; 294 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 295 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 296 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 297 uint8_t dsize; 298 #define IXL_HMC_RXQ_DSIZE_16 0 299 #define IXL_HMC_RXQ_DSIZE_32 1 300 uint8_t crcstrip; 301 uint8_t fc_ena; 302 uint8_t l2sel; 303 uint8_t hsplit_0; 304 uint8_t hsplit_1; 305 uint8_t showiv; 306 uint16_t rxmax; 307 uint8_t tphrdesc_ena; 308 uint8_t tphwdesc_ena; 309 uint8_t tphdata_ena; 310 uint8_t tphhead_ena; 311 uint8_t lrxqthresh; 312 uint8_t prefena; 313 }; 314 315 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 316 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 317 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 318 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 319 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 320 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 321 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 322 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 323 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 324 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 325 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 326 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 327 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 328 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 329 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 330 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 331 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 332 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 333 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 334 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 335 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 336 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 337 }; 338 339 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 340 341 struct ixl_hmc_txq { 342 uint16_t head; 343 uint8_t new_context; 344 uint64_t base; 345 #define IXL_HMC_TXQ_BASE_UNIT 128 346 uint8_t fc_ena; 347 uint8_t timesync_ena; 348 uint8_t fd_ena; 349 uint8_t alt_vlan_ena; 350 uint16_t thead_wb; 351 uint8_t cpuid; 352 uint8_t head_wb_ena; 353 #define IXL_HMC_TXQ_DESC_WB 0 354 #define IXL_HMC_TXQ_HEAD_WB 1 355 uint16_t qlen; 356 uint8_t tphrdesc_ena; 357 uint8_t tphrpacket_ena; 358 uint8_t tphwdesc_ena; 359 uint64_t head_wb_addr; 360 uint32_t crc; 361 uint16_t rdylist; 362 uint8_t rdylist_act; 363 }; 364 365 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 366 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 367 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 368 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 369 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 370 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 371 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 372 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 373 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 374 /* line 1 */ 375 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 376 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 377 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 378 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 379 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 380 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 381 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 382 /* line 7 */ 383 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 384 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 385 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 386 }; 387 388 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 389 390 struct ixl_work { 391 struct work ixw_cookie; 392 void (*ixw_func)(void *); 393 void *ixw_arg; 394 unsigned int ixw_added; 395 }; 396 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 397 398 struct ixl_tx_map { 399 struct mbuf *txm_m; 400 bus_dmamap_t txm_map; 401 unsigned int txm_eop; 402 }; 403 404 struct ixl_tx_ring { 405 kmutex_t txr_lock; 406 struct ixl_softc *txr_sc; 407 408 unsigned int txr_prod; 409 unsigned int txr_cons; 410 411 struct ixl_tx_map *txr_maps; 412 struct ixl_dmamem txr_mem; 413 414 bus_size_t txr_tail; 415 unsigned int txr_qid; 416 pcq_t *txr_intrq; 417 void *txr_si; 418 419 uint64_t txr_oerrors; /* if_oerrors */ 420 uint64_t txr_opackets; /* if_opackets */ 421 uint64_t txr_obytes; /* if_obytes */ 422 uint64_t txr_omcasts; /* if_omcasts */ 423 424 struct evcnt txr_defragged; 425 struct evcnt txr_defrag_failed; 426 struct evcnt txr_pcqdrop; 427 struct evcnt txr_transmitdef; 428 struct evcnt txr_intr; 429 struct evcnt txr_defer; 430 }; 431 432 struct ixl_rx_map { 433 struct mbuf *rxm_m; 434 bus_dmamap_t rxm_map; 435 }; 436 437 struct ixl_rx_ring { 438 kmutex_t rxr_lock; 439 440 unsigned int rxr_prod; 441 unsigned int rxr_cons; 442 443 struct ixl_rx_map *rxr_maps; 444 struct ixl_dmamem rxr_mem; 445 446 struct mbuf *rxr_m_head; 447 struct mbuf **rxr_m_tail; 448 449 bus_size_t rxr_tail; 450 unsigned int rxr_qid; 451 452 uint64_t rxr_ipackets; /* if_ipackets */ 453 uint64_t rxr_ibytes; /* if_ibytes */ 454 uint64_t rxr_iqdrops; /* iqdrops */ 455 uint64_t rxr_ierrors; /* if_ierrors */ 456 457 struct evcnt rxr_mgethdr_failed; 458 struct evcnt rxr_mgetcl_failed; 459 struct evcnt rxr_mbuf_load_failed; 460 struct evcnt rxr_intr; 461 struct evcnt rxr_defer; 462 }; 463 464 struct ixl_queue_pair { 465 struct ixl_softc *qp_sc; 466 struct ixl_tx_ring *qp_txr; 467 struct ixl_rx_ring *qp_rxr; 468 469 char qp_name[16]; 470 471 void *qp_si; 472 struct ixl_work qp_task; 473 bool qp_workqueue; 474 }; 475 476 struct ixl_atq { 477 struct ixl_aq_desc iatq_desc; 478 void (*iatq_fn)(struct ixl_softc *); 479 }; 480 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 481 482 struct ixl_product { 483 unsigned int vendor_id; 484 unsigned int product_id; 485 }; 486 487 /* 488 * Locking notes: 489 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 490 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 491 * - more than one lock of them cannot be held at once. 492 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 493 * (a spin mutex). 494 * - the lock cannot held with txr_lock or rxr_lock. 495 * + a field named sc_arq_* is not protected by any lock. 496 * - operations for sc_arq_* is done in one context related to 497 * sc_arq_task. 498 * + other fields in ixl_softc is protected by sc_cfg_lock 499 * (an adaptive mutex) 500 * - It must be held before another lock is held, and It can be 501 * released after the other lock is released. 502 * */ 503 504 struct ixl_softc { 505 device_t sc_dev; 506 struct ethercom sc_ec; 507 bool sc_attached; 508 bool sc_dead; 509 bool sc_rxctl_atq; 510 struct sysctllog *sc_sysctllog; 511 struct workqueue *sc_workq; 512 struct workqueue *sc_workq_txrx; 513 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 514 struct ifmedia sc_media; 515 uint64_t sc_media_status; 516 uint64_t sc_media_active; 517 kmutex_t sc_cfg_lock; 518 enum i40e_mac_type sc_mac_type; 519 uint32_t sc_rss_table_size; 520 uint32_t sc_rss_table_entry_width; 521 bool sc_txrx_workqueue; 522 u_int sc_tx_process_limit; 523 u_int sc_rx_process_limit; 524 u_int sc_tx_intr_process_limit; 525 u_int sc_rx_intr_process_limit; 526 527 struct pci_attach_args sc_pa; 528 pci_intr_handle_t *sc_ihp; 529 void **sc_ihs; 530 unsigned int sc_nintrs; 531 532 bus_dma_tag_t sc_dmat; 533 bus_space_tag_t sc_memt; 534 bus_space_handle_t sc_memh; 535 bus_size_t sc_mems; 536 537 uint8_t sc_pf_id; 538 uint16_t sc_uplink_seid; /* le */ 539 uint16_t sc_downlink_seid; /* le */ 540 uint16_t sc_vsi_number; /* le */ 541 uint16_t sc_seid; 542 unsigned int sc_base_queue; 543 544 pci_intr_type_t sc_intrtype; 545 unsigned int sc_msix_vector_queue; 546 547 struct ixl_dmamem sc_scratch; 548 549 const struct ixl_aq_regs * 550 sc_aq_regs; 551 552 kmutex_t sc_atq_lock; 553 kcondvar_t sc_atq_cv; 554 struct ixl_dmamem sc_atq; 555 unsigned int sc_atq_prod; 556 unsigned int sc_atq_cons; 557 558 struct ixl_dmamem sc_arq; 559 struct ixl_work sc_arq_task; 560 struct ixl_aq_bufs sc_arq_idle; 561 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 562 unsigned int sc_arq_prod; 563 unsigned int sc_arq_cons; 564 565 struct ixl_work sc_link_state_task; 566 struct ixl_atq sc_link_state_atq; 567 568 struct ixl_dmamem sc_hmc_sd; 569 struct ixl_dmamem sc_hmc_pd; 570 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 571 572 unsigned int sc_tx_ring_ndescs; 573 unsigned int sc_rx_ring_ndescs; 574 unsigned int sc_nqueue_pairs; 575 unsigned int sc_nqueue_pairs_max; 576 unsigned int sc_nqueue_pairs_device; 577 struct ixl_queue_pair *sc_qps; 578 579 struct evcnt sc_event_atq; 580 struct evcnt sc_event_link; 581 struct evcnt sc_event_ecc_err; 582 struct evcnt sc_event_pci_exception; 583 struct evcnt sc_event_crit_err; 584 }; 585 586 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 587 #define IXL_TX_PROCESS_LIMIT 256 588 #define IXL_RX_PROCESS_LIMIT 256 589 #define IXL_TX_INTR_PROCESS_LIMIT 256 590 #define IXL_RX_INTR_PROCESS_LIMIT 0U 591 592 #define delaymsec(_x) DELAY(1000 * (_x)) 593 #ifdef IXL_DEBUG 594 #define DDPRINTF(sc, fmt, args...) \ 595 do { \ 596 if (sc != NULL) \ 597 device_printf(sc->sc_dev, ""); \ 598 printf("%s:\t" fmt, __func__, ##args); \ 599 } while (0) 600 #else 601 #define DDPRINTF(sc, fmt, args...) __nothing 602 #endif 603 #define IXL_NOMSIX false 604 605 static enum i40e_mac_type 606 ixl_mactype(pci_product_id_t); 607 static void ixl_clear_hw(struct ixl_softc *); 608 static int ixl_pf_reset(struct ixl_softc *); 609 610 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 611 bus_size_t, bus_size_t); 612 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 613 614 static int ixl_arq_fill(struct ixl_softc *); 615 static void ixl_arq_unfill(struct ixl_softc *); 616 617 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 618 unsigned int); 619 static void ixl_atq_set(struct ixl_atq *, void (*)(struct ixl_softc *)); 620 static int ixl_atq_post(struct ixl_softc *, struct ixl_atq *); 621 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 622 static void ixl_atq_done(struct ixl_softc *); 623 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 624 static int ixl_get_version(struct ixl_softc *); 625 static int ixl_get_hw_capabilities(struct ixl_softc *); 626 static int ixl_pxe_clear(struct ixl_softc *); 627 static int ixl_lldp_shut(struct ixl_softc *); 628 static int ixl_get_mac(struct ixl_softc *); 629 static int ixl_get_switch_config(struct ixl_softc *); 630 static int ixl_phy_mask_ints(struct ixl_softc *); 631 static int ixl_get_phy_types(struct ixl_softc *, uint64_t *); 632 static int ixl_restart_an(struct ixl_softc *); 633 static int ixl_hmc(struct ixl_softc *); 634 static void ixl_hmc_free(struct ixl_softc *); 635 static int ixl_get_vsi(struct ixl_softc *); 636 static int ixl_set_vsi(struct ixl_softc *); 637 static void ixl_set_filter_control(struct ixl_softc *); 638 static int ixl_get_link_status(struct ixl_softc *); 639 static int ixl_set_link_status(struct ixl_softc *, 640 const struct ixl_aq_desc *); 641 static void ixl_config_rss(struct ixl_softc *); 642 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 643 uint16_t, uint16_t); 644 static int ixl_remove_macvlan(struct ixl_softc *, uint8_t *, uint16_t, 645 uint16_t); 646 static void ixl_arq(void *); 647 static void ixl_link_state_update(void *); 648 static void ixl_hmc_pack(void *, const void *, 649 const struct ixl_hmc_pack *, unsigned int); 650 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 651 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 652 653 static int ixl_match(device_t, cfdata_t, void *); 654 static void ixl_attach(device_t, device_t, void *); 655 static int ixl_detach(device_t, int); 656 657 static void ixl_media_add(struct ixl_softc *, uint64_t); 658 static int ixl_media_change(struct ifnet *); 659 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 660 static void ixl_watchdog(struct ifnet *); 661 static int ixl_ioctl(struct ifnet *, u_long, void *); 662 static void ixl_start(struct ifnet *); 663 static int ixl_transmit(struct ifnet *, struct mbuf *); 664 static void ixl_deferred_transmit(void *); 665 static int ixl_intr(void *); 666 static int ixl_queue_intr(void *); 667 static int ixl_other_intr(void *); 668 static void ixl_handle_queue(void *); 669 static void ixl_sched_handle_queue(struct ixl_softc *, 670 struct ixl_queue_pair *); 671 static int ixl_init(struct ifnet *); 672 static int ixl_init_locked(struct ixl_softc *); 673 static void ixl_stop(struct ifnet *, int); 674 static void ixl_stop_locked(struct ixl_softc *); 675 static int ixl_iff(struct ixl_softc *); 676 static int ixl_ifflags_cb(struct ethercom *); 677 static int ixl_setup_interrupts(struct ixl_softc *); 678 static int ixl_establish_intx(struct ixl_softc *); 679 static int ixl_establish_msix(struct ixl_softc *); 680 static void ixl_set_affinity_msix(struct ixl_softc *); 681 static void ixl_enable_queue_intr(struct ixl_softc *, 682 struct ixl_queue_pair *); 683 static void ixl_disable_queue_intr(struct ixl_softc *, 684 struct ixl_queue_pair *); 685 static void ixl_enable_other_intr(struct ixl_softc *); 686 static void ixl_disable_other_intr(struct ixl_softc *); 687 static void ixl_config_queue_intr(struct ixl_softc *); 688 static void ixl_config_other_intr(struct ixl_softc *); 689 690 static struct ixl_tx_ring * 691 ixl_txr_alloc(struct ixl_softc *, unsigned int); 692 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 693 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 694 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 695 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 696 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 697 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 698 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 699 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 700 701 static struct ixl_rx_ring * 702 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 703 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 704 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 705 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 706 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 707 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 708 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 709 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 710 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 711 712 static struct workqueue * 713 ixl_workq_create(const char *, pri_t, int, int); 714 static void ixl_workq_destroy(struct workqueue *); 715 static int ixl_workqs_teardown(device_t); 716 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 717 static void ixl_work_add(struct workqueue *, struct ixl_work *); 718 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 719 static void ixl_workq_work(struct work *, void *); 720 static const struct ixl_product * 721 ixl_lookup(const struct pci_attach_args *pa); 722 static void ixl_link_status(struct ixl_softc *); 723 static int ixl_set_macvlan(struct ixl_softc *); 724 static int ixl_setup_interrupts(struct ixl_softc *);; 725 static void ixl_teardown_interrupts(struct ixl_softc *); 726 static int ixl_setup_stats(struct ixl_softc *); 727 static void ixl_teardown_stats(struct ixl_softc *); 728 static int ixl_setup_sysctls(struct ixl_softc *); 729 static void ixl_teardown_sysctls(struct ixl_softc *); 730 static int ixl_queue_pairs_alloc(struct ixl_softc *); 731 static void ixl_queue_pairs_free(struct ixl_softc *); 732 733 static const struct ixl_phy_type ixl_phy_type_map[] = { 734 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 735 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 736 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 737 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 738 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 739 { 1ULL << IXL_PHY_TYPE_XAUI | 740 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 741 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 742 { 1ULL << IXL_PHY_TYPE_XLAUI | 743 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 744 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 745 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 746 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 747 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 748 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 749 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 750 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 751 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 752 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 753 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 754 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 755 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 756 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 757 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 758 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 759 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 760 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 761 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 762 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 763 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 764 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 765 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 766 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 767 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_CR }, 768 }; 769 770 static const struct ixl_speed_type ixl_speed_type_map[] = { 771 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 772 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 773 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 774 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 775 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 776 }; 777 778 static const struct ixl_aq_regs ixl_pf_aq_regs = { 779 .atq_tail = I40E_PF_ATQT, 780 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 781 .atq_head = I40E_PF_ATQH, 782 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 783 .atq_len = I40E_PF_ATQLEN, 784 .atq_bal = I40E_PF_ATQBAL, 785 .atq_bah = I40E_PF_ATQBAH, 786 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 787 788 .arq_tail = I40E_PF_ARQT, 789 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 790 .arq_head = I40E_PF_ARQH, 791 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 792 .arq_len = I40E_PF_ARQLEN, 793 .arq_bal = I40E_PF_ARQBAL, 794 .arq_bah = I40E_PF_ARQBAH, 795 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 796 }; 797 798 #define ixl_rd(_s, _r) \ 799 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 800 #define ixl_wr(_s, _r, _v) \ 801 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 802 #define ixl_barrier(_s, _r, _l, _o) \ 803 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 804 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 805 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 806 807 static inline uint32_t 808 ixl_dmamem_hi(struct ixl_dmamem *ixm) 809 { 810 uint32_t val; 811 812 if (sizeof(IXL_DMA_DVA(ixm)) > 4) 813 val = (uint32_t)(IXL_DMA_DVA(ixm) >> 32); 814 else 815 val = 0; 816 817 return val; 818 } 819 820 static inline uint32_t 821 ixl_dmamem_lo(struct ixl_dmamem *ixm) 822 { 823 824 return (uint32_t)IXL_DMA_DVA(ixm); 825 } 826 827 static inline void 828 ixl_aq_dva(struct ixl_aq_desc *iaq, bus_addr_t addr) 829 { 830 831 if (sizeof(addr) > 4) 832 iaq->iaq_param[2] = htole32(addr >> 32); 833 else 834 iaq->iaq_param[2] = htole32(0); 835 836 iaq->iaq_param[3] = htole32(addr); 837 } 838 839 static inline unsigned int 840 ixl_rxr_unrefreshed(unsigned int prod, unsigned int cons, unsigned int ndescs) 841 { 842 unsigned int num; 843 844 if (prod < cons) 845 num = cons - prod; 846 else 847 num = (ndescs - prod) + cons; 848 849 if (__predict_true(num > 0)) { 850 /* device cannot receive packets if all descripter is filled */ 851 num -= 1; 852 } 853 854 return num; 855 } 856 857 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 858 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 859 DVF_DETACH_SHUTDOWN); 860 861 static const struct ixl_product ixl_products[] = { 862 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 863 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 864 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 865 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 866 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 867 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 868 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T }, 869 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 870 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 871 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 872 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 873 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 874 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 875 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 876 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 877 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 878 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 879 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 880 /* required last entry */ 881 {0, 0} 882 }; 883 884 static const struct ixl_product * 885 ixl_lookup(const struct pci_attach_args *pa) 886 { 887 const struct ixl_product *ixlp; 888 889 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 890 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 891 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 892 return ixlp; 893 } 894 895 return NULL; 896 } 897 898 static int 899 ixl_match(device_t parent, cfdata_t match, void *aux) 900 { 901 const struct pci_attach_args *pa = aux; 902 903 return (ixl_lookup(pa) != NULL) ? 1 : 0; 904 } 905 906 static void 907 ixl_attach(device_t parent, device_t self, void *aux) 908 { 909 struct ixl_softc *sc; 910 struct pci_attach_args *pa = aux; 911 struct ifnet *ifp; 912 pcireg_t memtype, reg; 913 uint32_t firstq, port, ari, func; 914 uint64_t phy_types = 0; 915 char xnamebuf[32]; 916 int tries, rv; 917 918 sc = device_private(self); 919 sc->sc_dev = self; 920 ifp = &sc->sc_ec.ec_if; 921 922 sc->sc_pa = *pa; 923 sc->sc_dmat = (pci_dma64_available(pa)) ? 924 pa->pa_dmat64 : pa->pa_dmat; 925 sc->sc_aq_regs = &ixl_pf_aq_regs; 926 927 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 928 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(reg)); 929 930 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 931 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 932 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 933 aprint_error(": unable to map registers\n"); 934 return; 935 } 936 937 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 938 939 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 940 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 941 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 942 sc->sc_base_queue = firstq; 943 944 ixl_clear_hw(sc); 945 if (ixl_pf_reset(sc) == -1) { 946 /* error printed by ixl pf_reset */ 947 goto unmap; 948 } 949 950 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 951 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 952 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 953 aprint_normal(": port %u", port); 954 955 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 956 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 957 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 958 959 func = ixl_rd(sc, I40E_PF_FUNC_RID); 960 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 961 962 /* initialise the adminq */ 963 964 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 965 966 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 967 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 968 aprint_error("\n" "%s: unable to allocate atq\n", 969 device_xname(self)); 970 goto unmap; 971 } 972 973 SIMPLEQ_INIT(&sc->sc_arq_idle); 974 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 975 sc->sc_arq_cons = 0; 976 sc->sc_arq_prod = 0; 977 978 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 979 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 980 aprint_error("\n" "%s: unable to allocate arq\n", 981 device_xname(self)); 982 goto free_atq; 983 } 984 985 if (!ixl_arq_fill(sc)) { 986 aprint_error("\n" "%s: unable to fill arq descriptors\n", 987 device_xname(self)); 988 goto free_arq; 989 } 990 991 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 992 0, IXL_DMA_LEN(&sc->sc_atq), 993 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 994 995 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 996 0, IXL_DMA_LEN(&sc->sc_arq), 997 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 998 999 for (tries = 0; tries < 10; tries++) { 1000 sc->sc_atq_cons = 0; 1001 sc->sc_atq_prod = 0; 1002 1003 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1004 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1005 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1006 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1007 1008 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1009 1010 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1011 ixl_dmamem_lo(&sc->sc_atq)); 1012 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1013 ixl_dmamem_hi(&sc->sc_atq)); 1014 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1015 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1016 1017 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1018 ixl_dmamem_lo(&sc->sc_arq)); 1019 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1020 ixl_dmamem_hi(&sc->sc_arq)); 1021 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1022 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1023 1024 rv = ixl_get_version(sc); 1025 if (rv == 0) 1026 break; 1027 if (rv != ETIMEDOUT) { 1028 aprint_error(", unable to get firmware version\n"); 1029 goto shutdown; 1030 } 1031 1032 delaymsec(100); 1033 } 1034 1035 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1036 1037 if (sc->sc_mac_type == I40E_MAC_X722) 1038 sc->sc_nqueue_pairs_device = 128; 1039 else 1040 sc->sc_nqueue_pairs_device = 64; 1041 1042 rv = ixl_get_hw_capabilities(sc); 1043 if (rv != 0) { 1044 aprint_error(", GET HW CAPABILITIES %s\n", 1045 rv == ETIMEDOUT ? "timeout" : "error"); 1046 goto shutdown; 1047 } 1048 1049 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max = 1050 MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1051 sc->sc_tx_ring_ndescs = 1024; 1052 sc->sc_rx_ring_ndescs = 1024; 1053 1054 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1055 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1056 1057 if (ixl_get_mac(sc) != 0) { 1058 /* error printed by ixl_get_mac */ 1059 goto shutdown; 1060 } 1061 1062 aprint_normal("\n"); 1063 aprint_naive("\n"); 1064 1065 aprint_normal_dev(self, "Ethernet address %s\n", 1066 ether_sprintf(sc->sc_enaddr)); 1067 1068 rv = ixl_pxe_clear(sc); 1069 if (rv != 0) { 1070 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1071 rv == ETIMEDOUT ? "timeout" : "error"); 1072 } 1073 1074 ixl_set_filter_control(sc); 1075 1076 if (ixl_hmc(sc) != 0) { 1077 /* error printed by ixl_hmc */ 1078 goto shutdown; 1079 } 1080 1081 if (ixl_lldp_shut(sc) != 0) { 1082 /* error printed by ixl_lldp_shut */ 1083 goto free_hmc; 1084 } 1085 1086 if (ixl_phy_mask_ints(sc) != 0) { 1087 /* error printed by ixl_phy_mask_ints */ 1088 goto free_hmc; 1089 } 1090 1091 if (ixl_restart_an(sc) != 0) { 1092 /* error printed by ixl_restart_an */ 1093 goto free_hmc; 1094 } 1095 1096 if (ixl_get_switch_config(sc) != 0) { 1097 /* error printed by ixl_get_switch_config */ 1098 goto free_hmc; 1099 } 1100 1101 if (ixl_get_phy_types(sc, &phy_types) != 0) { 1102 /* error printed by ixl_get_phy_abilities */ 1103 goto free_hmc; 1104 } 1105 1106 rv = ixl_get_link_status(sc); 1107 if (rv != 0) { 1108 aprint_error_dev(self, "GET LINK STATUS %s\n", 1109 rv == ETIMEDOUT ? "timeout" : "error"); 1110 goto free_hmc; 1111 } 1112 1113 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1114 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1115 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1116 goto free_hmc; 1117 } 1118 1119 if (ixl_get_vsi(sc) != 0) { 1120 /* error printed by ixl_get_vsi */ 1121 goto free_scratch; 1122 } 1123 1124 if (ixl_set_vsi(sc) != 0) { 1125 /* error printed by ixl_set_vsi */ 1126 goto free_scratch; 1127 } 1128 1129 if (ixl_queue_pairs_alloc(sc) != 0) { 1130 /* error printed by ixl_queue_pairs_alloc */ 1131 goto free_scratch; 1132 } 1133 1134 if (ixl_setup_interrupts(sc) != 0) { 1135 /* error printed by ixl_setup_interrupts */ 1136 goto free_queue_pairs; 1137 } 1138 1139 if (ixl_setup_stats(sc) != 0) { 1140 aprint_error_dev(self, "failed to setup event counters\n"); 1141 goto teardown_intrs; 1142 } 1143 1144 if (ixl_setup_sysctls(sc) != 0) { 1145 /* error printed by ixl_setup_sysctls */ 1146 goto teardown_stats; 1147 } 1148 1149 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1150 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1151 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1152 if (sc->sc_workq == NULL) 1153 goto teardown_sysctls; 1154 1155 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1156 sc->sc_workq_txrx = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1157 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1158 if (sc->sc_workq_txrx == NULL) 1159 goto teardown_wqs; 1160 1161 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1162 cv_init(&sc->sc_atq_cv, xnamebuf); 1163 1164 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1165 1166 ifp->if_softc = sc; 1167 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1168 ifp->if_extflags = IFEF_MPSAFE; 1169 ifp->if_ioctl = ixl_ioctl; 1170 ifp->if_start = ixl_start; 1171 ifp->if_transmit = ixl_transmit; 1172 ifp->if_watchdog = ixl_watchdog; 1173 ifp->if_init = ixl_init; 1174 ifp->if_stop = ixl_stop; 1175 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1176 IFQ_SET_READY(&ifp->if_snd); 1177 #if 0 1178 ifp->if_capabilities |= IFCAP_CSUM_IPv4_Rx | 1179 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1180 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx; 1181 #endif 1182 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_MTU; 1183 #if 0 1184 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1185 #endif 1186 1187 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1188 ifmedia_init(&sc->sc_media, IFM_IMASK, ixl_media_change, 1189 ixl_media_status); 1190 1191 ixl_media_add(sc, phy_types); 1192 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1193 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1194 1195 if_attach(ifp); 1196 if_deferred_start_init(ifp, NULL); 1197 ether_ifattach(ifp, sc->sc_enaddr); 1198 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1199 (void)ixl_get_link_status(sc); 1200 1201 ixl_work_set(&sc->sc_link_state_task, ixl_link_state_update, sc); 1202 1203 ixl_config_other_intr(sc); 1204 1205 ixl_set_macvlan(sc); 1206 1207 ixl_enable_other_intr(sc); 1208 1209 sc->sc_txrx_workqueue = true; 1210 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1211 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1212 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1213 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1214 1215 if (pmf_device_register(self, NULL, NULL) != true) 1216 aprint_debug_dev(self, "couldn't establish power handler\n"); 1217 sc->sc_attached = true; 1218 return; 1219 1220 teardown_wqs: 1221 config_finalize_register(self, ixl_workqs_teardown); 1222 teardown_sysctls: 1223 ixl_teardown_sysctls(sc); 1224 teardown_stats: 1225 ixl_teardown_stats(sc); 1226 teardown_intrs: 1227 ixl_teardown_interrupts(sc); 1228 free_queue_pairs: 1229 ixl_queue_pairs_free(sc); 1230 free_scratch: 1231 ixl_dmamem_free(sc, &sc->sc_scratch); 1232 free_hmc: 1233 ixl_hmc_free(sc); 1234 shutdown: 1235 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1236 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1237 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1238 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1239 1240 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1241 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1242 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1243 1244 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1245 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1246 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1247 1248 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1249 0, IXL_DMA_LEN(&sc->sc_arq), 1250 BUS_DMASYNC_POSTREAD); 1251 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1252 0, IXL_DMA_LEN(&sc->sc_atq), 1253 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1254 1255 ixl_arq_unfill(sc); 1256 free_arq: 1257 ixl_dmamem_free(sc, &sc->sc_arq); 1258 free_atq: 1259 ixl_dmamem_free(sc, &sc->sc_atq); 1260 unmap: 1261 mutex_destroy(&sc->sc_atq_lock); 1262 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1263 mutex_destroy(&sc->sc_cfg_lock); 1264 sc->sc_mems = 0; 1265 1266 sc->sc_attached = false; 1267 } 1268 1269 static int 1270 ixl_detach(device_t self, int flags) 1271 { 1272 struct ixl_softc *sc = device_private(self); 1273 struct ifnet *ifp = &sc->sc_ec.ec_if; 1274 1275 if (!sc->sc_attached) 1276 return 0; 1277 1278 ixl_stop(ifp, 1); 1279 1280 if (sc->sc_workq != NULL) { 1281 ixl_workq_destroy(sc->sc_workq); 1282 sc->sc_workq = NULL; 1283 } 1284 1285 if (sc->sc_workq_txrx != NULL) { 1286 ixl_workq_destroy(sc->sc_workq_txrx); 1287 sc->sc_workq_txrx = NULL; 1288 } 1289 1290 ifmedia_delete_instance(&sc->sc_media, IFM_INST_ANY); 1291 ether_ifdetach(ifp); 1292 if_detach(ifp); 1293 1294 ixl_teardown_interrupts(sc); 1295 ixl_teardown_stats(sc); 1296 1297 ixl_queue_pairs_free(sc); 1298 1299 ixl_dmamem_free(sc, &sc->sc_scratch); 1300 ixl_hmc_free(sc); 1301 1302 /* shutdown */ 1303 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1304 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1305 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1306 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1307 1308 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1309 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1310 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1311 1312 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1313 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1314 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1315 1316 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1317 0, IXL_DMA_LEN(&sc->sc_arq), 1318 BUS_DMASYNC_POSTREAD); 1319 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1320 0, IXL_DMA_LEN(&sc->sc_atq), 1321 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1322 1323 ixl_arq_unfill(sc); 1324 1325 ixl_dmamem_free(sc, &sc->sc_arq); 1326 ixl_dmamem_free(sc, &sc->sc_atq); 1327 1328 cv_destroy(&sc->sc_atq_cv); 1329 mutex_destroy(&sc->sc_atq_lock); 1330 1331 if (sc->sc_mems != 0) { 1332 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1333 sc->sc_mems = 0; 1334 } 1335 1336 mutex_destroy(&sc->sc_cfg_lock); 1337 1338 return 0; 1339 } 1340 1341 static int 1342 ixl_workqs_teardown(device_t self) 1343 { 1344 struct ixl_softc *sc = device_private(self); 1345 1346 if (sc->sc_workq != NULL) { 1347 ixl_workq_destroy(sc->sc_workq); 1348 sc->sc_workq = NULL; 1349 } 1350 1351 if (sc->sc_workq_txrx != NULL) { 1352 ixl_workq_destroy(sc->sc_workq_txrx); 1353 sc->sc_workq_txrx = NULL; 1354 } 1355 1356 return 0; 1357 } 1358 1359 static void 1360 ixl_media_add(struct ixl_softc *sc, uint64_t phy_types) 1361 { 1362 struct ifmedia *ifm = &sc->sc_media; 1363 const struct ixl_phy_type *itype; 1364 unsigned int i; 1365 1366 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1367 itype = &ixl_phy_type_map[i]; 1368 1369 if (ISSET(phy_types, itype->phy_type)) { 1370 ifmedia_add(ifm, 1371 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1372 1373 if (itype->ifm_type == IFM_100_TX) { 1374 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1375 0, NULL); 1376 } 1377 } 1378 } 1379 } 1380 1381 static void 1382 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1383 { 1384 struct ixl_softc *sc = ifp->if_softc; 1385 1386 ifmr->ifm_status = sc->sc_media_status; 1387 ifmr->ifm_active = sc->sc_media_active; 1388 1389 mutex_enter(&sc->sc_cfg_lock); 1390 if (ifp->if_link_state == LINK_STATE_UP) 1391 SET(ifmr->ifm_status, IFM_ACTIVE); 1392 mutex_exit(&sc->sc_cfg_lock); 1393 } 1394 1395 static int 1396 ixl_media_change(struct ifnet *ifp) 1397 { 1398 1399 return 0; 1400 } 1401 1402 static void 1403 ixl_watchdog(struct ifnet *ifp) 1404 { 1405 1406 } 1407 1408 static void 1409 ixl_del_all_multiaddr(struct ixl_softc *sc) 1410 { 1411 struct ethercom *ec = &sc->sc_ec; 1412 struct ether_multi *enm; 1413 struct ether_multistep step; 1414 1415 ETHER_LOCK(ec); 1416 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1417 ETHER_NEXT_MULTI(step, enm)) { 1418 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1419 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1420 } 1421 ETHER_UNLOCK(ec); 1422 } 1423 1424 static int 1425 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1426 { 1427 struct ifnet *ifp = &sc->sc_ec.ec_if; 1428 int rv; 1429 1430 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1431 return 0; 1432 1433 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1434 ixl_del_all_multiaddr(sc); 1435 SET(ifp->if_flags, IFF_ALLMULTI); 1436 return 0; 1437 } 1438 1439 rv = ixl_add_macvlan(sc, addrlo, 0, 1440 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1441 1442 if (rv == IXL_AQ_RC_ENOSPC) { 1443 ixl_del_all_multiaddr(sc); 1444 SET(ifp->if_flags, IFF_ALLMULTI); 1445 return 0; 1446 } 1447 1448 if (rv != IXL_AQ_RC_OK) 1449 return EIO; 1450 1451 return 0; 1452 } 1453 1454 static void 1455 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1456 { 1457 struct ifnet *ifp = &sc->sc_ec.ec_if; 1458 struct ethercom *ec = &sc->sc_ec; 1459 struct ether_multi *enm, *enm_last; 1460 struct ether_multistep step; 1461 int rv; 1462 1463 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1464 ixl_remove_macvlan(sc, addrlo, 0, 1465 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1466 return; 1467 } 1468 1469 ETHER_LOCK(ec); 1470 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1471 ETHER_NEXT_MULTI(step, enm)) { 1472 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1473 ETHER_ADDR_LEN) != 0) { 1474 ETHER_UNLOCK(ec); 1475 return; 1476 } 1477 } 1478 1479 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1480 ETHER_NEXT_MULTI(step, enm)) { 1481 rv = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1482 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1483 if (rv != IXL_AQ_RC_OK) 1484 break; 1485 } 1486 1487 if (enm != NULL) { 1488 enm_last = enm; 1489 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1490 ETHER_NEXT_MULTI(step, enm)) { 1491 if (enm == enm_last) 1492 break; 1493 1494 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1495 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1496 } 1497 } else { 1498 CLR(ifp->if_flags, IFF_ALLMULTI); 1499 } 1500 1501 ETHER_UNLOCK(ec); 1502 } 1503 1504 static int 1505 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1506 { 1507 struct ifreq *ifr = (struct ifreq *)data; 1508 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1509 struct ixl_tx_ring *txr; 1510 struct ixl_rx_ring *rxr; 1511 const struct sockaddr *sa; 1512 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1513 int s, error = 0; 1514 unsigned int i; 1515 1516 switch (cmd) { 1517 case SIOCADDMULTI: 1518 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1519 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1520 error = ether_multiaddr(sa, addrlo, addrhi); 1521 if (error != 0) 1522 return error; 1523 1524 if (ixl_add_multi(sc, addrlo, addrhi) != 0) { 1525 ether_delmulti(sa, &sc->sc_ec); 1526 error = EIO; 1527 } 1528 } 1529 break; 1530 1531 case SIOCDELMULTI: 1532 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1533 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1534 error = ether_multiaddr(sa, addrlo, addrhi); 1535 if (error != 0) 1536 return error; 1537 1538 ixl_del_multi(sc, addrlo, addrhi); 1539 } 1540 break; 1541 1542 case SIOCGIFDATA: 1543 case SIOCZIFDATA: 1544 ifp->if_ipackets = 0; 1545 ifp->if_ibytes = 0; 1546 ifp->if_iqdrops = 0; 1547 ifp->if_ierrors = 0; 1548 ifp->if_opackets = 0; 1549 ifp->if_obytes = 0; 1550 ifp->if_omcasts = 0; 1551 1552 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 1553 txr = sc->sc_qps[i].qp_txr; 1554 rxr = sc->sc_qps[i].qp_rxr; 1555 1556 mutex_enter(&rxr->rxr_lock); 1557 ifp->if_ipackets += rxr->rxr_ipackets; 1558 ifp->if_ibytes += rxr->rxr_ibytes; 1559 ifp->if_iqdrops += rxr->rxr_iqdrops; 1560 ifp->if_ierrors += rxr->rxr_ierrors; 1561 if (cmd == SIOCZIFDATA) { 1562 rxr->rxr_ipackets = 0; 1563 rxr->rxr_ibytes = 0; 1564 rxr->rxr_iqdrops = 0; 1565 rxr->rxr_ierrors = 0; 1566 } 1567 mutex_exit(&rxr->rxr_lock); 1568 1569 mutex_enter(&txr->txr_lock); 1570 ifp->if_opackets += txr->txr_opackets; 1571 ifp->if_obytes += txr->txr_opackets; 1572 ifp->if_omcasts += txr->txr_omcasts; 1573 if (cmd == SIOCZIFDATA) { 1574 txr->txr_opackets = 0; 1575 txr->txr_opackets = 0; 1576 txr->txr_omcasts = 0; 1577 } 1578 mutex_exit(&txr->txr_lock); 1579 } 1580 /* FALLTHROUGH */ 1581 default: 1582 s = splnet(); 1583 error = ether_ioctl(ifp, cmd, data); 1584 splx(s); 1585 } 1586 1587 if (error == ENETRESET) 1588 error = ixl_iff(sc); 1589 1590 return error; 1591 } 1592 1593 static enum i40e_mac_type 1594 ixl_mactype(pci_product_id_t id) 1595 { 1596 1597 switch (id) { 1598 case PCI_PRODUCT_INTEL_XL710_SFP: 1599 case PCI_PRODUCT_INTEL_XL710_KX_B: 1600 case PCI_PRODUCT_INTEL_XL710_KX_C: 1601 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1602 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1603 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1604 case PCI_PRODUCT_INTEL_X710_10G_T: 1605 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1606 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1607 case PCI_PRODUCT_INTEL_X710_T4_10G: 1608 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1609 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1610 return I40E_MAC_XL710; 1611 1612 case PCI_PRODUCT_INTEL_X722_KX: 1613 case PCI_PRODUCT_INTEL_X722_QSFP: 1614 case PCI_PRODUCT_INTEL_X722_SFP: 1615 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1616 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1617 case PCI_PRODUCT_INTEL_X722_I_SFP: 1618 return I40E_MAC_X722; 1619 } 1620 1621 return I40E_MAC_GENERIC; 1622 } 1623 1624 static inline void * 1625 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1626 { 1627 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1628 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1629 1630 if (i >= e->hmc_count) 1631 return NULL; 1632 1633 kva += e->hmc_base; 1634 kva += i * e->hmc_size; 1635 1636 return kva; 1637 } 1638 1639 static inline size_t 1640 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1641 { 1642 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1643 1644 return e->hmc_size; 1645 } 1646 1647 static void 1648 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1649 { 1650 struct ixl_rx_ring *rxr = qp->qp_rxr; 1651 1652 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1653 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1654 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1655 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1656 ixl_flush(sc); 1657 } 1658 1659 static void 1660 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1661 { 1662 struct ixl_rx_ring *rxr = qp->qp_rxr; 1663 1664 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1665 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1666 ixl_flush(sc); 1667 } 1668 1669 static void 1670 ixl_enable_other_intr(struct ixl_softc *sc) 1671 { 1672 1673 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1674 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1675 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1676 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1677 ixl_flush(sc); 1678 } 1679 1680 static void 1681 ixl_disable_other_intr(struct ixl_softc *sc) 1682 { 1683 1684 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1685 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1686 ixl_flush(sc); 1687 } 1688 1689 static int 1690 ixl_reinit(struct ixl_softc *sc) 1691 { 1692 struct ixl_rx_ring *rxr; 1693 struct ixl_tx_ring *txr; 1694 unsigned int i; 1695 uint32_t reg; 1696 1697 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1698 1699 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1700 txr = sc->sc_qps[i].qp_txr; 1701 rxr = sc->sc_qps[i].qp_rxr; 1702 1703 txr->txr_cons = txr->txr_prod = 0; 1704 rxr->rxr_cons = rxr->rxr_prod = 0; 1705 1706 ixl_txr_config(sc, txr); 1707 ixl_rxr_config(sc, rxr); 1708 } 1709 1710 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1711 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 1712 1713 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1714 txr = sc->sc_qps[i].qp_txr; 1715 rxr = sc->sc_qps[i].qp_rxr; 1716 1717 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 1718 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 1719 ixl_flush(sc); 1720 1721 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 1722 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 1723 1724 1725 /* ixl_rxfill() needs lock held */ 1726 mutex_enter(&rxr->rxr_lock); 1727 ixl_rxfill(sc, rxr); 1728 mutex_exit(&rxr->rxr_lock); 1729 1730 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 1731 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 1732 ixl_wr(sc, I40E_QRX_ENA(i), reg); 1733 if (ixl_rxr_enabled(sc, rxr) != 0) 1734 goto stop; 1735 1736 ixl_txr_qdis(sc, txr, 1); 1737 1738 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 1739 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 1740 ixl_wr(sc, I40E_QTX_ENA(i), reg); 1741 1742 if (ixl_txr_enabled(sc, txr) != 0) 1743 goto stop; 1744 } 1745 1746 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1747 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 1748 1749 return 0; 1750 1751 stop: 1752 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1753 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 1754 1755 return ETIMEDOUT; 1756 } 1757 1758 static int 1759 ixl_init_locked(struct ixl_softc *sc) 1760 { 1761 struct ifnet *ifp = &sc->sc_ec.ec_if; 1762 unsigned int i; 1763 int error; 1764 1765 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1766 1767 if (sc->sc_dead) { 1768 return ENXIO; 1769 } 1770 1771 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 1772 sc->sc_nqueue_pairs = 1; 1773 else 1774 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1775 1776 error = ixl_reinit(sc); 1777 if (error) { 1778 ixl_stop_locked(sc); 1779 return error; 1780 } 1781 1782 SET(ifp->if_flags, IFF_RUNNING); 1783 CLR(ifp->if_flags, IFF_OACTIVE); 1784 ixl_link_status(sc); 1785 1786 ixl_config_rss(sc); 1787 ixl_config_queue_intr(sc); 1788 1789 ixl_enable_other_intr(sc); 1790 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1791 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 1792 } 1793 1794 error = ixl_iff(sc); 1795 if (error) { 1796 ixl_stop_locked(sc); 1797 return error; 1798 } 1799 1800 return 0; 1801 } 1802 1803 static int 1804 ixl_init(struct ifnet *ifp) 1805 { 1806 struct ixl_softc *sc = ifp->if_softc; 1807 int error; 1808 1809 mutex_enter(&sc->sc_cfg_lock); 1810 error = ixl_init_locked(sc); 1811 mutex_exit(&sc->sc_cfg_lock); 1812 1813 return error; 1814 } 1815 1816 static int 1817 ixl_iff(struct ixl_softc *sc) 1818 { 1819 struct ifnet *ifp = &sc->sc_ec.ec_if; 1820 struct ixl_atq iatq; 1821 struct ixl_aq_desc *iaq; 1822 struct ixl_aq_vsi_promisc_param *param; 1823 int error; 1824 1825 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 1826 return 0; 1827 1828 memset(&iatq, 0, sizeof(iatq)); 1829 1830 iaq = &iatq.iatq_desc; 1831 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 1832 1833 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 1834 param->flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 1835 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 1836 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 1837 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 1838 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 1839 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1840 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 1841 } 1842 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 1843 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 1844 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 1845 param->seid = sc->sc_seid; 1846 1847 error = ixl_atq_exec(sc, &iatq); 1848 if (error) 1849 return error; 1850 1851 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 1852 return EIO; 1853 1854 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 1855 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1856 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1857 1858 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 1859 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 1860 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1861 } 1862 return 0; 1863 } 1864 1865 static void 1866 ixl_stop_rendezvous(struct ixl_softc *sc) 1867 { 1868 struct ixl_tx_ring *txr; 1869 struct ixl_rx_ring *rxr; 1870 unsigned int i; 1871 1872 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1873 txr = sc->sc_qps[i].qp_txr; 1874 rxr = sc->sc_qps[i].qp_rxr; 1875 1876 mutex_enter(&txr->txr_lock); 1877 mutex_exit(&txr->txr_lock); 1878 1879 mutex_enter(&rxr->rxr_lock); 1880 mutex_exit(&rxr->rxr_lock); 1881 1882 ixl_work_wait(sc->sc_workq_txrx, 1883 &sc->sc_qps[i].qp_task); 1884 } 1885 1886 mutex_enter(&sc->sc_atq_lock); 1887 mutex_exit(&sc->sc_atq_lock); 1888 1889 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1890 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1891 1892 } 1893 1894 static void 1895 ixl_stop_locked(struct ixl_softc *sc) 1896 { 1897 struct ifnet *ifp = &sc->sc_ec.ec_if; 1898 struct ixl_rx_ring *rxr; 1899 struct ixl_tx_ring *txr; 1900 unsigned int i; 1901 uint32_t reg; 1902 1903 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1904 1905 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 1906 1907 ixl_disable_other_intr(sc); 1908 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1909 txr = sc->sc_qps[i].qp_txr; 1910 rxr = sc->sc_qps[i].qp_rxr; 1911 1912 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 1913 1914 mutex_enter(&txr->txr_lock); 1915 ixl_txr_qdis(sc, txr, 0); 1916 /* XXX wait at least 400 usec for all tx queues in one go */ 1917 ixl_flush(sc); 1918 DELAY(500); 1919 1920 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 1921 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 1922 ixl_wr(sc, I40E_QTX_ENA(i), reg); 1923 /* XXX wait 50ms from completaion of the TX queue disable*/ 1924 ixl_flush(sc); 1925 DELAY(50); 1926 1927 if (ixl_txr_disabled(sc, txr) != 0) { 1928 mutex_exit(&txr->txr_lock); 1929 goto die; 1930 } 1931 mutex_exit(&txr->txr_lock); 1932 1933 mutex_enter(&rxr->rxr_lock); 1934 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 1935 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 1936 ixl_wr(sc, I40E_QRX_ENA(i), reg); 1937 /* XXX wait 50ms from completion of the RX queue disable */ 1938 ixl_flush(sc); 1939 DELAY(50); 1940 1941 if (ixl_rxr_disabled(sc, rxr) != 0) { 1942 mutex_exit(&rxr->rxr_lock); 1943 goto die; 1944 } 1945 mutex_exit(&rxr->rxr_lock); 1946 } 1947 1948 ixl_stop_rendezvous(sc); 1949 1950 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1951 txr = sc->sc_qps[i].qp_txr; 1952 rxr = sc->sc_qps[i].qp_rxr; 1953 1954 ixl_txr_unconfig(sc, txr); 1955 ixl_rxr_unconfig(sc, rxr); 1956 1957 ixl_txr_clean(sc, txr); 1958 ixl_rxr_clean(sc, rxr); 1959 } 1960 1961 return; 1962 die: 1963 sc->sc_dead = true; 1964 log(LOG_CRIT, "%s: failed to shut down rings", 1965 device_xname(sc->sc_dev)); 1966 return; 1967 } 1968 1969 static void 1970 ixl_stop(struct ifnet *ifp, int disable) 1971 { 1972 struct ixl_softc *sc = ifp->if_softc; 1973 1974 mutex_enter(&sc->sc_cfg_lock); 1975 ixl_stop_locked(sc); 1976 mutex_exit(&sc->sc_cfg_lock); 1977 } 1978 1979 static int 1980 ixl_queue_pairs_alloc(struct ixl_softc *sc) 1981 { 1982 struct ixl_queue_pair *qp; 1983 unsigned int i; 1984 size_t sz; 1985 1986 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 1987 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 1988 1989 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 1990 qp = &sc->sc_qps[i]; 1991 1992 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1993 ixl_handle_queue, qp); 1994 1995 qp->qp_txr = ixl_txr_alloc(sc, i); 1996 if (qp->qp_txr == NULL) 1997 goto free; 1998 1999 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2000 if (qp->qp_rxr == NULL) 2001 goto free; 2002 2003 qp->qp_sc = sc; 2004 ixl_work_set(&qp->qp_task, ixl_handle_queue, qp); 2005 snprintf(qp->qp_name, sizeof(qp->qp_name), 2006 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2007 } 2008 2009 return 0; 2010 free: 2011 if (sc->sc_qps != NULL) { 2012 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2013 qp = &sc->sc_qps[i]; 2014 2015 if (qp->qp_txr != NULL) 2016 ixl_txr_free(sc, qp->qp_txr); 2017 if (qp->qp_rxr != NULL) 2018 ixl_rxr_free(sc, qp->qp_rxr); 2019 } 2020 2021 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2022 kmem_free(sc->sc_qps, sz); 2023 sc->sc_qps = NULL; 2024 } 2025 2026 return -1; 2027 } 2028 2029 static void 2030 ixl_queue_pairs_free(struct ixl_softc *sc) 2031 { 2032 struct ixl_queue_pair *qp; 2033 unsigned int i; 2034 size_t sz; 2035 2036 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2037 qp = &sc->sc_qps[i]; 2038 ixl_txr_free(sc, qp->qp_txr); 2039 ixl_rxr_free(sc, qp->qp_rxr); 2040 } 2041 2042 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2043 kmem_free(sc->sc_qps, sz); 2044 sc->sc_qps = NULL; 2045 } 2046 2047 static struct ixl_tx_ring * 2048 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2049 { 2050 struct ixl_tx_ring *txr = NULL; 2051 struct ixl_tx_map *maps = NULL, *txm; 2052 unsigned int i; 2053 2054 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2055 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2056 KM_SLEEP); 2057 2058 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2059 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2060 IXL_TX_QUEUE_ALIGN) != 0) 2061 goto free; 2062 2063 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2064 txm = &maps[i]; 2065 2066 if (bus_dmamap_create(sc->sc_dmat, 2067 IXL_HARDMTU, IXL_TX_PKT_DESCS, IXL_HARDMTU, 0, 2068 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2069 goto uncreate; 2070 2071 txm->txm_eop = -1; 2072 txm->txm_m = NULL; 2073 } 2074 2075 txr->txr_cons = txr->txr_prod = 0; 2076 txr->txr_maps = maps; 2077 2078 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2079 if (txr->txr_intrq == NULL) 2080 goto uncreate; 2081 2082 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2083 ixl_deferred_transmit, txr); 2084 if (txr->txr_si == NULL) 2085 goto destroy_pcq; 2086 2087 txr->txr_tail = I40E_QTX_TAIL(qid); 2088 txr->txr_qid = qid; 2089 txr->txr_sc = sc; 2090 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2091 2092 return txr; 2093 2094 destroy_pcq: 2095 pcq_destroy(txr->txr_intrq); 2096 uncreate: 2097 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2098 txm = &maps[i]; 2099 2100 if (txm->txm_map == NULL) 2101 continue; 2102 2103 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2104 } 2105 2106 ixl_dmamem_free(sc, &txr->txr_mem); 2107 free: 2108 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2109 kmem_free(txr, sizeof(*txr)); 2110 2111 return NULL; 2112 } 2113 2114 static void 2115 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2116 { 2117 unsigned int qid; 2118 bus_size_t reg; 2119 uint32_t r; 2120 2121 qid = txr->txr_qid + sc->sc_base_queue; 2122 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2123 qid %= 128; 2124 2125 r = ixl_rd(sc, reg); 2126 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2127 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2128 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2129 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2130 ixl_wr(sc, reg, r); 2131 } 2132 2133 static void 2134 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2135 { 2136 struct ixl_hmc_txq txq; 2137 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2138 void *hmc; 2139 2140 memset(&txq, 0, sizeof(txq)); 2141 txq.head = htole16(txr->txr_cons); 2142 txq.new_context = 1; 2143 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2144 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2145 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2146 txq.tphrdesc_ena = 0; 2147 txq.tphrpacket_ena = 0; 2148 txq.tphwdesc_ena = 0; 2149 txq.rdylist = data->qs_handle[0]; 2150 2151 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2152 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2153 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2154 __arraycount(ixl_hmc_pack_txq)); 2155 } 2156 2157 static void 2158 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2159 { 2160 void *hmc; 2161 2162 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2163 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2164 } 2165 2166 static void 2167 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2168 { 2169 struct ixl_tx_map *maps, *txm; 2170 bus_dmamap_t map; 2171 unsigned int i; 2172 2173 maps = txr->txr_maps; 2174 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2175 txm = &maps[i]; 2176 2177 if (txm->txm_m == NULL) 2178 continue; 2179 2180 map = txm->txm_map; 2181 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2182 BUS_DMASYNC_POSTWRITE); 2183 bus_dmamap_unload(sc->sc_dmat, map); 2184 2185 m_freem(txm->txm_m); 2186 txm->txm_m = NULL; 2187 } 2188 } 2189 2190 static int 2191 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2192 { 2193 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2194 uint32_t reg; 2195 int i; 2196 2197 for (i = 0; i < 10; i++) { 2198 reg = ixl_rd(sc, ena); 2199 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2200 return 0; 2201 2202 delaymsec(10); 2203 } 2204 2205 return ETIMEDOUT; 2206 } 2207 2208 static int 2209 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2210 { 2211 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2212 uint32_t reg; 2213 int i; 2214 2215 KASSERT(mutex_owned(&txr->txr_lock)); 2216 2217 for (i = 0; i < 20; i++) { 2218 reg = ixl_rd(sc, ena); 2219 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2220 return 0; 2221 2222 delaymsec(10); 2223 } 2224 2225 return ETIMEDOUT; 2226 } 2227 2228 static void 2229 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2230 { 2231 struct ixl_tx_map *maps, *txm; 2232 struct mbuf *m; 2233 unsigned int i; 2234 2235 softint_disestablish(txr->txr_si); 2236 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2237 m_freem(m); 2238 pcq_destroy(txr->txr_intrq); 2239 2240 maps = txr->txr_maps; 2241 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2242 txm = &maps[i]; 2243 2244 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2245 } 2246 2247 ixl_dmamem_free(sc, &txr->txr_mem); 2248 mutex_destroy(&txr->txr_lock); 2249 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2250 kmem_free(txr, sizeof(*txr)); 2251 } 2252 2253 static inline int 2254 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2255 struct ixl_tx_ring *txr) 2256 { 2257 struct mbuf *m; 2258 int error; 2259 2260 KASSERT(mutex_owned(&txr->txr_lock)); 2261 2262 m = *m0; 2263 2264 error = bus_dmamap_load_mbuf(dmat, map, m, 2265 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2266 if (error != EFBIG) 2267 return error; 2268 2269 m = m_defrag(m, M_DONTWAIT); 2270 if (m != NULL) { 2271 *m0 = m; 2272 txr->txr_defragged.ev_count++; 2273 2274 error = bus_dmamap_load_mbuf(dmat, map, m, 2275 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2276 } else { 2277 txr->txr_defrag_failed.ev_count++; 2278 error = ENOBUFS; 2279 } 2280 2281 return error; 2282 } 2283 2284 static void 2285 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2286 bool is_transmit) 2287 { 2288 struct ixl_softc *sc = ifp->if_softc; 2289 struct ixl_tx_desc *ring, *txd; 2290 struct ixl_tx_map *txm; 2291 bus_dmamap_t map; 2292 struct mbuf *m; 2293 uint64_t cmd; 2294 unsigned int prod, free, last, i; 2295 unsigned int mask; 2296 int post = 0; 2297 2298 KASSERT(mutex_owned(&txr->txr_lock)); 2299 2300 if (ifp->if_link_state != LINK_STATE_UP 2301 || !ISSET(ifp->if_flags, IFF_RUNNING) 2302 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2303 if (!is_transmit) 2304 IFQ_PURGE(&ifp->if_snd); 2305 return; 2306 } 2307 2308 prod = txr->txr_prod; 2309 free = txr->txr_cons; 2310 if (free <= prod) 2311 free += sc->sc_tx_ring_ndescs; 2312 free -= prod; 2313 2314 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2315 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2316 2317 ring = IXL_DMA_KVA(&txr->txr_mem); 2318 mask = sc->sc_tx_ring_ndescs - 1; 2319 last = prod; 2320 cmd = 0; 2321 txd = NULL; 2322 2323 for (;;) { 2324 if (free <= IXL_TX_PKT_DESCS) { 2325 if (!is_transmit) 2326 SET(ifp->if_flags, IFF_OACTIVE); 2327 break; 2328 } 2329 2330 if (is_transmit) 2331 m = pcq_get(txr->txr_intrq); 2332 else 2333 IFQ_DEQUEUE(&ifp->if_snd, m); 2334 2335 if (m == NULL) 2336 break; 2337 2338 txm = &txr->txr_maps[prod]; 2339 map = txm->txm_map; 2340 2341 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2342 txr->txr_oerrors++; 2343 m_freem(m); 2344 continue; 2345 } 2346 2347 bus_dmamap_sync(sc->sc_dmat, map, 0, 2348 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2349 2350 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2351 txd = &ring[prod]; 2352 2353 cmd = (uint64_t)map->dm_segs[i].ds_len << 2354 IXL_TX_DESC_BSIZE_SHIFT; 2355 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2356 2357 txd->addr = htole64(map->dm_segs[i].ds_addr); 2358 txd->cmd = htole64(cmd); 2359 2360 last = prod; 2361 2362 prod++; 2363 prod &= mask; 2364 } 2365 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2366 txd->cmd = htole64(cmd); 2367 2368 txm->txm_m = m; 2369 txm->txm_eop = last; 2370 2371 bpf_mtap(ifp, m, BPF_D_OUT); 2372 2373 free -= i; 2374 post = 1; 2375 } 2376 2377 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2378 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2379 2380 if (post) { 2381 txr->txr_prod = prod; 2382 ixl_wr(sc, txr->txr_tail, prod); 2383 } 2384 } 2385 2386 static int 2387 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2388 { 2389 struct ifnet *ifp = &sc->sc_ec.ec_if; 2390 struct ixl_tx_desc *ring, *txd; 2391 struct ixl_tx_map *txm; 2392 struct mbuf *m; 2393 bus_dmamap_t map; 2394 unsigned int cons, prod, last; 2395 unsigned int mask; 2396 uint64_t dtype; 2397 int done = 0, more = 0; 2398 2399 KASSERT(mutex_owned(&txr->txr_lock)); 2400 2401 prod = txr->txr_prod; 2402 cons = txr->txr_cons; 2403 2404 if (cons == prod) 2405 return 0; 2406 2407 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2408 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2409 2410 ring = IXL_DMA_KVA(&txr->txr_mem); 2411 mask = sc->sc_tx_ring_ndescs - 1; 2412 2413 do { 2414 if (txlimit-- <= 0) { 2415 more = 1; 2416 break; 2417 } 2418 2419 txm = &txr->txr_maps[cons]; 2420 last = txm->txm_eop; 2421 txd = &ring[last]; 2422 2423 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2424 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2425 break; 2426 2427 map = txm->txm_map; 2428 2429 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2430 BUS_DMASYNC_POSTWRITE); 2431 bus_dmamap_unload(sc->sc_dmat, map); 2432 2433 m = txm->txm_m; 2434 if (m != NULL) { 2435 txr->txr_opackets++; 2436 txr->txr_obytes += m->m_pkthdr.len; 2437 if (ISSET(m->m_flags, M_MCAST)) 2438 txr->txr_omcasts++; 2439 m_freem(m); 2440 } 2441 2442 txm->txm_m = NULL; 2443 txm->txm_eop = -1; 2444 2445 cons = last + 1; 2446 cons &= mask; 2447 done = 1; 2448 } while (cons != prod); 2449 2450 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2451 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2452 2453 txr->txr_cons = cons; 2454 2455 if (done) { 2456 softint_schedule(txr->txr_si); 2457 if (txr->txr_qid == 0) { 2458 CLR(ifp->if_flags, IFF_OACTIVE); 2459 if_schedule_deferred_start(ifp); 2460 } 2461 } 2462 2463 return more; 2464 } 2465 2466 static void 2467 ixl_start(struct ifnet *ifp) 2468 { 2469 struct ixl_softc *sc; 2470 struct ixl_tx_ring *txr; 2471 2472 sc = ifp->if_softc; 2473 txr = sc->sc_qps[0].qp_txr; 2474 2475 mutex_enter(&txr->txr_lock); 2476 ixl_tx_common_locked(ifp, txr, false); 2477 mutex_exit(&txr->txr_lock); 2478 } 2479 2480 static inline unsigned int 2481 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2482 { 2483 u_int cpuid; 2484 2485 cpuid = cpu_index(curcpu()); 2486 2487 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2488 } 2489 2490 static int 2491 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2492 { 2493 struct ixl_softc *sc; 2494 struct ixl_tx_ring *txr; 2495 unsigned int qid; 2496 2497 sc = ifp->if_softc; 2498 qid = ixl_select_txqueue(sc, m); 2499 2500 txr = sc->sc_qps[qid].qp_txr; 2501 2502 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2503 mutex_enter(&txr->txr_lock); 2504 txr->txr_pcqdrop.ev_count++; 2505 mutex_exit(&txr->txr_lock); 2506 2507 m_freem(m); 2508 return ENOBUFS; 2509 } 2510 2511 if (mutex_tryenter(&txr->txr_lock)) { 2512 ixl_tx_common_locked(ifp, txr, true); 2513 mutex_exit(&txr->txr_lock); 2514 } else { 2515 softint_schedule(txr->txr_si); 2516 } 2517 2518 return 0; 2519 } 2520 2521 static void 2522 ixl_deferred_transmit(void *xtxr) 2523 { 2524 struct ixl_tx_ring *txr = xtxr; 2525 struct ixl_softc *sc = txr->txr_sc; 2526 struct ifnet *ifp = &sc->sc_ec.ec_if; 2527 2528 mutex_enter(&txr->txr_lock); 2529 txr->txr_transmitdef.ev_count++; 2530 if (pcq_peek(txr->txr_intrq) != NULL) 2531 ixl_tx_common_locked(ifp, txr, true); 2532 mutex_exit(&txr->txr_lock); 2533 } 2534 2535 static struct ixl_rx_ring * 2536 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2537 { 2538 struct ixl_rx_ring *rxr = NULL; 2539 struct ixl_rx_map *maps = NULL, *rxm; 2540 unsigned int i; 2541 2542 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2543 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2544 KM_SLEEP); 2545 2546 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2547 sizeof(struct ixl_rx_rd_desc_16) * sc->sc_rx_ring_ndescs, 2548 IXL_RX_QUEUE_ALIGN) != 0) 2549 goto free; 2550 2551 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2552 rxm = &maps[i]; 2553 2554 if (bus_dmamap_create(sc->sc_dmat, 2555 IXL_HARDMTU, 1, IXL_HARDMTU, 0, 2556 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2557 goto uncreate; 2558 2559 rxm->rxm_m = NULL; 2560 } 2561 2562 rxr->rxr_cons = rxr->rxr_prod = 0; 2563 rxr->rxr_m_head = NULL; 2564 rxr->rxr_m_tail = &rxr->rxr_m_head; 2565 rxr->rxr_maps = maps; 2566 2567 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2568 rxr->rxr_qid = qid; 2569 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2570 2571 return rxr; 2572 2573 uncreate: 2574 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2575 rxm = &maps[i]; 2576 2577 if (rxm->rxm_map == NULL) 2578 continue; 2579 2580 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2581 } 2582 2583 ixl_dmamem_free(sc, &rxr->rxr_mem); 2584 free: 2585 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2586 kmem_free(rxr, sizeof(*rxr)); 2587 2588 return NULL; 2589 } 2590 2591 static void 2592 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2593 { 2594 struct ixl_rx_map *maps, *rxm; 2595 bus_dmamap_t map; 2596 unsigned int i; 2597 2598 maps = rxr->rxr_maps; 2599 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2600 rxm = &maps[i]; 2601 2602 if (rxm->rxm_m == NULL) 2603 continue; 2604 2605 map = rxm->rxm_map; 2606 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2607 BUS_DMASYNC_POSTWRITE); 2608 bus_dmamap_unload(sc->sc_dmat, map); 2609 2610 m_freem(rxm->rxm_m); 2611 rxm->rxm_m = NULL; 2612 } 2613 2614 m_freem(rxr->rxr_m_head); 2615 rxr->rxr_m_head = NULL; 2616 rxr->rxr_m_tail = &rxr->rxr_m_head; 2617 2618 rxr->rxr_prod = rxr->rxr_cons = 0; 2619 } 2620 2621 static int 2622 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2623 { 2624 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 2625 uint32_t reg; 2626 int i; 2627 2628 for (i = 0; i < 10; i++) { 2629 reg = ixl_rd(sc, ena); 2630 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 2631 return 0; 2632 2633 delaymsec(10); 2634 } 2635 2636 return ETIMEDOUT; 2637 } 2638 2639 static int 2640 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2641 { 2642 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 2643 uint32_t reg; 2644 int i; 2645 2646 KASSERT(mutex_owned(&rxr->rxr_lock)); 2647 2648 for (i = 0; i < 20; i++) { 2649 reg = ixl_rd(sc, ena); 2650 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 2651 return 0; 2652 2653 delaymsec(10); 2654 } 2655 2656 return ETIMEDOUT; 2657 } 2658 2659 static void 2660 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2661 { 2662 struct ixl_hmc_rxq rxq; 2663 void *hmc; 2664 2665 memset(&rxq, 0, sizeof(rxq)); 2666 2667 rxq.head = htole16(rxr->rxr_cons); 2668 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 2669 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 2670 rxq.dbuff = htole16(MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 2671 rxq.hbuff = 0; 2672 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 2673 rxq.dsize = IXL_HMC_RXQ_DSIZE_16; 2674 rxq.crcstrip = 1; 2675 rxq.l2sel = 0; 2676 rxq.showiv = 0; 2677 rxq.rxmax = htole16(IXL_HARDMTU); 2678 rxq.tphrdesc_ena = 0; 2679 rxq.tphwdesc_ena = 0; 2680 rxq.tphdata_ena = 0; 2681 rxq.tphhead_ena = 0; 2682 rxq.lrxqthresh = 0; 2683 rxq.prefena = 1; 2684 2685 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 2686 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 2687 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 2688 __arraycount(ixl_hmc_pack_rxq)); 2689 } 2690 2691 static void 2692 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2693 { 2694 void *hmc; 2695 2696 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 2697 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 2698 } 2699 2700 static void 2701 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2702 { 2703 struct ixl_rx_map *maps, *rxm; 2704 unsigned int i; 2705 2706 maps = rxr->rxr_maps; 2707 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2708 rxm = &maps[i]; 2709 2710 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2711 } 2712 2713 ixl_dmamem_free(sc, &rxr->rxr_mem); 2714 mutex_destroy(&rxr->rxr_lock); 2715 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2716 kmem_free(rxr, sizeof(*rxr)); 2717 } 2718 2719 static int 2720 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 2721 { 2722 struct ifnet *ifp = &sc->sc_ec.ec_if; 2723 struct ixl_rx_wb_desc_16 *ring, *rxd; 2724 struct ixl_rx_map *rxm; 2725 bus_dmamap_t map; 2726 unsigned int cons, prod; 2727 struct mbuf *m; 2728 uint64_t word; 2729 unsigned int len; 2730 unsigned int mask; 2731 int done = 0, more = 0; 2732 2733 KASSERT(mutex_owned(&rxr->rxr_lock)); 2734 2735 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2736 return 0; 2737 2738 prod = rxr->rxr_prod; 2739 cons = rxr->rxr_cons; 2740 2741 if (cons == prod) 2742 return 0; 2743 2744 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 2745 0, IXL_DMA_LEN(&rxr->rxr_mem), 2746 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 2747 2748 ring = IXL_DMA_KVA(&rxr->rxr_mem); 2749 mask = sc->sc_rx_ring_ndescs - 1; 2750 2751 do { 2752 if (rxlimit-- <= 0) { 2753 more = 1; 2754 break; 2755 } 2756 2757 rxd = &ring[cons]; 2758 2759 word = le64toh(rxd->qword1); 2760 2761 if (!ISSET(word, IXL_RX_DESC_DD)) 2762 break; 2763 2764 rxm = &rxr->rxr_maps[cons]; 2765 2766 map = rxm->rxm_map; 2767 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2768 BUS_DMASYNC_POSTREAD); 2769 bus_dmamap_unload(sc->sc_dmat, map); 2770 2771 m = rxm->rxm_m; 2772 rxm->rxm_m = NULL; 2773 2774 KASSERT(m != NULL); 2775 2776 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 2777 m->m_len = len; 2778 m->m_pkthdr.len = 0; 2779 2780 m->m_next = NULL; 2781 *rxr->rxr_m_tail = m; 2782 rxr->rxr_m_tail = &m->m_next; 2783 2784 m = rxr->rxr_m_head; 2785 m->m_pkthdr.len += len; 2786 2787 if (ISSET(word, IXL_RX_DESC_EOP)) { 2788 if (!ISSET(word, 2789 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 2790 m_set_rcvif(m, ifp); 2791 rxr->rxr_ipackets++; 2792 rxr->rxr_ibytes += m->m_pkthdr.len; 2793 if_percpuq_enqueue(ifp->if_percpuq, m); 2794 } else { 2795 rxr->rxr_ierrors++; 2796 m_freem(m); 2797 } 2798 2799 rxr->rxr_m_head = NULL; 2800 rxr->rxr_m_tail = &rxr->rxr_m_head; 2801 } 2802 2803 cons++; 2804 cons &= mask; 2805 2806 done = 1; 2807 } while (cons != prod); 2808 2809 if (done) { 2810 rxr->rxr_cons = cons; 2811 if (ixl_rxfill(sc, rxr) == -1) 2812 rxr->rxr_iqdrops++; 2813 } 2814 2815 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 2816 0, IXL_DMA_LEN(&rxr->rxr_mem), 2817 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 2818 2819 return more; 2820 } 2821 2822 static int 2823 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2824 { 2825 struct ixl_rx_rd_desc_16 *ring, *rxd; 2826 struct ixl_rx_map *rxm; 2827 bus_dmamap_t map; 2828 struct mbuf *m; 2829 unsigned int prod; 2830 unsigned int slots; 2831 unsigned int mask; 2832 int post = 0, error = 0; 2833 2834 KASSERT(mutex_owned(&rxr->rxr_lock)); 2835 2836 prod = rxr->rxr_prod; 2837 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 2838 sc->sc_rx_ring_ndescs); 2839 2840 ring = IXL_DMA_KVA(&rxr->rxr_mem); 2841 mask = sc->sc_rx_ring_ndescs - 1; 2842 2843 if (__predict_false(slots <= 0)) 2844 return -1; 2845 2846 do { 2847 rxm = &rxr->rxr_maps[prod]; 2848 2849 MGETHDR(m, M_DONTWAIT, MT_DATA); 2850 if (m == NULL) { 2851 rxr->rxr_mgethdr_failed.ev_count++; 2852 error = -1; 2853 break; 2854 } 2855 2856 MCLGET(m, M_DONTWAIT); 2857 if (!ISSET(m->m_flags, M_EXT)) { 2858 rxr->rxr_mgetcl_failed.ev_count++; 2859 error = -1; 2860 m_freem(m); 2861 break; 2862 } 2863 2864 m->m_len = m->m_pkthdr.len = MCLBYTES + ETHER_ALIGN; 2865 m_adj(m, ETHER_ALIGN); 2866 2867 map = rxm->rxm_map; 2868 2869 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 2870 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 2871 rxr->rxr_mbuf_load_failed.ev_count++; 2872 error = -1; 2873 m_freem(m); 2874 break; 2875 } 2876 2877 rxm->rxm_m = m; 2878 2879 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2880 BUS_DMASYNC_PREREAD); 2881 2882 rxd = &ring[prod]; 2883 2884 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 2885 rxd->haddr = htole64(0); 2886 2887 prod++; 2888 prod &= mask; 2889 2890 post = 1; 2891 2892 } while (--slots); 2893 2894 if (post) { 2895 rxr->rxr_prod = prod; 2896 ixl_wr(sc, rxr->rxr_tail, prod); 2897 } 2898 2899 return error; 2900 } 2901 2902 static inline int 2903 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 2904 u_int txlimit, struct evcnt *txevcnt, 2905 u_int rxlimit, struct evcnt *rxevcnt) 2906 { 2907 struct ixl_tx_ring *txr = qp->qp_txr; 2908 struct ixl_rx_ring *rxr = qp->qp_rxr; 2909 int txmore, rxmore; 2910 int rv; 2911 2912 KASSERT(!mutex_owned(&txr->txr_lock)); 2913 KASSERT(!mutex_owned(&rxr->rxr_lock)); 2914 2915 mutex_enter(&txr->txr_lock); 2916 txevcnt->ev_count++; 2917 txmore = ixl_txeof(sc, txr, txlimit); 2918 mutex_exit(&txr->txr_lock); 2919 2920 mutex_enter(&rxr->rxr_lock); 2921 rxevcnt->ev_count++; 2922 rxmore = ixl_rxeof(sc, rxr, rxlimit); 2923 mutex_exit(&rxr->rxr_lock); 2924 2925 rv = txmore | (rxmore << 1); 2926 2927 return rv; 2928 } 2929 2930 static void 2931 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 2932 { 2933 2934 if (qp->qp_workqueue) 2935 ixl_work_add(sc->sc_workq_txrx, &qp->qp_task); 2936 else 2937 softint_schedule(qp->qp_si); 2938 } 2939 2940 static int 2941 ixl_intr(void *xsc) 2942 { 2943 struct ixl_softc *sc = xsc; 2944 struct ixl_tx_ring *txr; 2945 struct ixl_rx_ring *rxr; 2946 uint32_t icr, rxintr, txintr; 2947 int rv = 0; 2948 unsigned int i; 2949 2950 KASSERT(sc != NULL); 2951 2952 ixl_enable_other_intr(sc); 2953 icr = ixl_rd(sc, I40E_PFINT_ICR0); 2954 2955 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 2956 atomic_inc_64(&sc->sc_event_atq.ev_count); 2957 ixl_atq_done(sc); 2958 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 2959 rv = 1; 2960 } 2961 2962 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 2963 atomic_inc_64(&sc->sc_event_link.ev_count); 2964 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 2965 rv = 1; 2966 } 2967 2968 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 2969 txintr = icr & I40E_INTR_NOTX_TX_MASK; 2970 2971 if (txintr || rxintr) { 2972 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2973 txr = sc->sc_qps[i].qp_txr; 2974 rxr = sc->sc_qps[i].qp_rxr; 2975 2976 ixl_handle_queue_common(sc, &sc->sc_qps[i], 2977 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 2978 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 2979 } 2980 rv = 1; 2981 } 2982 2983 return rv; 2984 } 2985 2986 static int 2987 ixl_queue_intr(void *xqp) 2988 { 2989 struct ixl_queue_pair *qp = xqp; 2990 struct ixl_tx_ring *txr = qp->qp_txr; 2991 struct ixl_rx_ring *rxr = qp->qp_rxr; 2992 struct ixl_softc *sc = qp->qp_sc; 2993 u_int txlimit, rxlimit; 2994 int more; 2995 2996 txlimit = sc->sc_tx_intr_process_limit; 2997 rxlimit = sc->sc_rx_intr_process_limit; 2998 qp->qp_workqueue = sc->sc_txrx_workqueue; 2999 3000 more = ixl_handle_queue_common(sc, qp, 3001 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3002 3003 if (more != 0) { 3004 ixl_sched_handle_queue(sc, qp); 3005 } else { 3006 /* for ALTQ */ 3007 if (txr->txr_qid == 0) 3008 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3009 softint_schedule(txr->txr_si); 3010 3011 ixl_enable_queue_intr(sc, qp); 3012 } 3013 3014 return 1; 3015 } 3016 3017 static void 3018 ixl_handle_queue(void *xqp) 3019 { 3020 struct ixl_queue_pair *qp = xqp; 3021 struct ixl_softc *sc = qp->qp_sc; 3022 struct ixl_tx_ring *txr = qp->qp_txr; 3023 struct ixl_rx_ring *rxr = qp->qp_rxr; 3024 u_int txlimit, rxlimit; 3025 int more; 3026 3027 txlimit = sc->sc_tx_process_limit; 3028 rxlimit = sc->sc_rx_process_limit; 3029 3030 more = ixl_handle_queue_common(sc, qp, 3031 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3032 3033 if (more != 0) 3034 ixl_sched_handle_queue(sc, qp); 3035 else 3036 ixl_enable_queue_intr(sc, qp); 3037 } 3038 3039 static inline void 3040 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3041 { 3042 uint32_t hmc_idx, hmc_isvf; 3043 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3044 3045 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3046 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3047 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3048 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3049 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3050 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3051 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3052 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3053 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3054 3055 device_printf(sc->sc_dev, 3056 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3057 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3058 } 3059 3060 static int 3061 ixl_other_intr(void *xsc) 3062 { 3063 struct ixl_softc *sc = xsc; 3064 uint32_t icr, mask, reg; 3065 int rv; 3066 3067 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3068 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3069 3070 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3071 atomic_inc_64(&sc->sc_event_atq.ev_count); 3072 ixl_atq_done(sc); 3073 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3074 rv = 1; 3075 } 3076 3077 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3078 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3079 device_printf(sc->sc_dev, "link stat changed\n"); 3080 3081 atomic_inc_64(&sc->sc_event_link.ev_count); 3082 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3083 rv = 1; 3084 } 3085 3086 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3087 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3088 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3089 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3090 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3091 3092 device_printf(sc->sc_dev, "GRST: %s\n", 3093 reg == I40E_RESET_CORER ? "CORER" : 3094 reg == I40E_RESET_GLOBR ? "GLOBR" : 3095 reg == I40E_RESET_EMPR ? "EMPR" : 3096 "POR"); 3097 } 3098 3099 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3100 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3101 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3102 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3103 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3104 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3105 3106 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3107 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3108 device_printf(sc->sc_dev, "critical error\n"); 3109 } 3110 3111 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3112 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3113 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3114 ixl_print_hmc_error(sc, reg); 3115 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3116 } 3117 3118 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3119 ixl_flush(sc); 3120 ixl_enable_other_intr(sc); 3121 return rv; 3122 } 3123 3124 static void 3125 ixl_link_state_update_done(struct ixl_softc *sc) 3126 { 3127 3128 /* IXL_AQ_OP_PHY_LINK_STATUS already posted to admin reply queue */ 3129 } 3130 3131 static void 3132 ixl_link_state_update(void *xsc) 3133 { 3134 struct ixl_softc *sc = xsc; 3135 struct ixl_aq_desc *iaq; 3136 struct ixl_aq_link_param *param; 3137 3138 memset(&sc->sc_link_state_atq, 0, sizeof(sc->sc_link_state_atq)); 3139 iaq = &sc->sc_link_state_atq.iatq_desc; 3140 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3141 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3142 param->notify = IXL_AQ_LINK_NOTIFY; 3143 3144 ixl_atq_set(&sc->sc_link_state_atq, ixl_link_state_update_done); 3145 (void)ixl_atq_post(sc, &sc->sc_link_state_atq); 3146 } 3147 3148 static void 3149 ixl_arq_link_status(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3150 { 3151 struct ifnet *ifp = &sc->sc_ec.ec_if; 3152 int link_state; 3153 3154 link_state = ixl_set_link_status(sc, iaq); 3155 3156 if (ifp->if_link_state != link_state) 3157 if_link_state_change(ifp, link_state); 3158 3159 if (link_state != LINK_STATE_DOWN) { 3160 if_schedule_deferred_start(ifp); 3161 } 3162 } 3163 3164 static void 3165 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3166 { 3167 char buf[512]; 3168 size_t len; 3169 3170 len = sizeof(buf); 3171 buf[--len] = '\0'; 3172 3173 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3174 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3175 buf, le16toh(iaq->iaq_opcode)); 3176 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3177 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3178 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3179 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3180 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3181 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3182 } 3183 3184 static void 3185 ixl_arq(void *xsc) 3186 { 3187 struct ixl_softc *sc = xsc; 3188 struct ixl_aq_desc *arq, *iaq; 3189 struct ixl_aq_buf *aqb; 3190 unsigned int cons = sc->sc_arq_cons; 3191 unsigned int prod; 3192 int done = 0; 3193 3194 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3195 sc->sc_aq_regs->arq_head_mask; 3196 3197 if (cons == prod) 3198 goto done; 3199 3200 arq = IXL_DMA_KVA(&sc->sc_arq); 3201 3202 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3203 0, IXL_DMA_LEN(&sc->sc_arq), 3204 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3205 3206 do { 3207 iaq = &arq[cons]; 3208 aqb = sc->sc_arq_live[cons]; 3209 3210 KASSERT(aqb != NULL); 3211 3212 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3213 BUS_DMASYNC_POSTREAD); 3214 3215 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3216 ixl_aq_dump(sc, iaq); 3217 3218 switch (iaq->iaq_opcode) { 3219 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3220 ixl_arq_link_status(sc, iaq); 3221 break; 3222 } 3223 3224 memset(iaq, 0, sizeof(*iaq)); 3225 sc->sc_arq_live[cons] = NULL; 3226 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3227 3228 cons++; 3229 cons &= IXL_AQ_MASK; 3230 3231 done = 1; 3232 } while (cons != prod); 3233 3234 if (done) { 3235 sc->sc_arq_cons = cons; 3236 ixl_arq_fill(sc); 3237 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3238 0, IXL_DMA_LEN(&sc->sc_arq), 3239 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3240 } 3241 3242 done: 3243 ixl_enable_other_intr(sc); 3244 } 3245 3246 static void 3247 ixl_atq_set(struct ixl_atq *iatq, void (*fn)(struct ixl_softc *)) 3248 { 3249 iatq->iatq_fn = fn; 3250 } 3251 3252 static int 3253 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3254 { 3255 struct ixl_aq_desc *atq, *slot; 3256 unsigned int prod, cons, prod_next; 3257 3258 /* assert locked */ 3259 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3260 3261 atq = IXL_DMA_KVA(&sc->sc_atq); 3262 prod = sc->sc_atq_prod; 3263 cons = sc->sc_atq_cons; 3264 prod_next = (prod +1) & IXL_AQ_MASK; 3265 3266 if (cons == prod_next) 3267 return ENOMEM; 3268 3269 slot = &atq[prod]; 3270 3271 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3272 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3273 3274 *slot = iatq->iatq_desc; 3275 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3276 3277 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3278 ixl_aq_dump(sc, slot); 3279 3280 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3281 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3282 3283 sc->sc_atq_prod = prod_next; 3284 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3285 3286 return 0; 3287 } 3288 3289 static int 3290 ixl_atq_post(struct ixl_softc *sc, struct ixl_atq *iatq) 3291 { 3292 int rv; 3293 3294 mutex_enter(&sc->sc_atq_lock); 3295 rv = ixl_atq_post_locked(sc, iatq); 3296 mutex_exit(&sc->sc_atq_lock); 3297 3298 return rv; 3299 } 3300 3301 static void 3302 ixl_atq_done_locked(struct ixl_softc *sc) 3303 { 3304 struct ixl_aq_desc *atq, *slot; 3305 struct ixl_atq *iatq; 3306 unsigned int cons; 3307 unsigned int prod; 3308 3309 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3310 3311 prod = sc->sc_atq_prod; 3312 cons = sc->sc_atq_cons; 3313 3314 if (prod == cons) 3315 return; 3316 3317 atq = IXL_DMA_KVA(&sc->sc_atq); 3318 3319 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3320 0, IXL_DMA_LEN(&sc->sc_atq), 3321 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3322 3323 do { 3324 slot = &atq[cons]; 3325 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3326 break; 3327 3328 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3329 iatq->iatq_desc = *slot; 3330 3331 memset(slot, 0, sizeof(*slot)); 3332 3333 (*iatq->iatq_fn)(sc); 3334 3335 cons++; 3336 cons &= IXL_AQ_MASK; 3337 } while (cons != prod); 3338 3339 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3340 0, IXL_DMA_LEN(&sc->sc_atq), 3341 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3342 3343 sc->sc_atq_cons = cons; 3344 } 3345 3346 static void 3347 ixl_atq_done(struct ixl_softc *sc) 3348 { 3349 3350 mutex_enter(&sc->sc_atq_lock); 3351 ixl_atq_done_locked(sc); 3352 mutex_exit(&sc->sc_atq_lock); 3353 } 3354 3355 static void 3356 ixl_wakeup(struct ixl_softc *sc) 3357 { 3358 3359 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3360 3361 cv_signal(&sc->sc_atq_cv); 3362 } 3363 3364 static int 3365 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3366 { 3367 int error; 3368 3369 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3370 3371 ixl_atq_set(iatq, ixl_wakeup); 3372 3373 mutex_enter(&sc->sc_atq_lock); 3374 error = ixl_atq_post_locked(sc, iatq); 3375 if (error) { 3376 mutex_exit(&sc->sc_atq_lock); 3377 return error; 3378 } 3379 3380 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3381 IXL_ATQ_EXEC_TIMEOUT); 3382 mutex_exit(&sc->sc_atq_lock); 3383 3384 return error; 3385 } 3386 3387 static int 3388 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3389 { 3390 struct ixl_aq_desc *atq, *slot; 3391 unsigned int prod; 3392 unsigned int t = 0; 3393 3394 atq = IXL_DMA_KVA(&sc->sc_atq); 3395 prod = sc->sc_atq_prod; 3396 slot = atq + prod; 3397 3398 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3399 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3400 3401 *slot = *iaq; 3402 slot->iaq_flags |= htole16(IXL_AQ_SI); 3403 3404 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3405 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3406 3407 prod++; 3408 prod &= IXL_AQ_MASK; 3409 sc->sc_atq_prod = prod; 3410 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3411 3412 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3413 delaymsec(1); 3414 3415 if (t++ > tm) 3416 return ETIMEDOUT; 3417 } 3418 3419 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3420 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3421 *iaq = *slot; 3422 memset(slot, 0, sizeof(*slot)); 3423 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3424 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3425 3426 sc->sc_atq_cons = prod; 3427 3428 return 0; 3429 } 3430 3431 static int 3432 ixl_get_version(struct ixl_softc *sc) 3433 { 3434 struct ixl_aq_desc iaq; 3435 uint32_t fwbuild, fwver, apiver; 3436 uint16_t api_maj_ver, api_min_ver; 3437 3438 memset(&iaq, 0, sizeof(iaq)); 3439 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3440 3441 iaq.iaq_retval = le16toh(23); 3442 3443 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3444 return ETIMEDOUT; 3445 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3446 return EIO; 3447 3448 fwbuild = le32toh(iaq.iaq_param[1]); 3449 fwver = le32toh(iaq.iaq_param[2]); 3450 apiver = le32toh(iaq.iaq_param[3]); 3451 3452 api_maj_ver = (uint16_t)apiver; 3453 api_min_ver = (uint16_t)(apiver >> 16); 3454 3455 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 3456 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 3457 3458 sc->sc_rxctl_atq = true; 3459 if (sc->sc_mac_type == I40E_MAC_X722) { 3460 if (api_maj_ver == 1 && api_min_ver < 5) { 3461 sc->sc_rxctl_atq = false; 3462 } 3463 } 3464 3465 return 0; 3466 } 3467 3468 static int 3469 ixl_pxe_clear(struct ixl_softc *sc) 3470 { 3471 struct ixl_aq_desc iaq; 3472 int rv; 3473 3474 memset(&iaq, 0, sizeof(iaq)); 3475 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 3476 iaq.iaq_param[0] = htole32(0x2); 3477 3478 rv = ixl_atq_poll(sc, &iaq, 250); 3479 3480 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 3481 3482 if (rv != 0) 3483 return ETIMEDOUT; 3484 3485 switch (iaq.iaq_retval) { 3486 case htole16(IXL_AQ_RC_OK): 3487 case htole16(IXL_AQ_RC_EEXIST): 3488 break; 3489 default: 3490 return EIO; 3491 } 3492 3493 return 0; 3494 } 3495 3496 static int 3497 ixl_lldp_shut(struct ixl_softc *sc) 3498 { 3499 struct ixl_aq_desc iaq; 3500 3501 memset(&iaq, 0, sizeof(iaq)); 3502 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 3503 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 3504 3505 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 3506 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 3507 return -1; 3508 } 3509 3510 switch (iaq.iaq_retval) { 3511 case htole16(IXL_AQ_RC_EMODE): 3512 case htole16(IXL_AQ_RC_EPERM): 3513 /* ignore silently */ 3514 default: 3515 break; 3516 } 3517 3518 return 0; 3519 } 3520 3521 static void 3522 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 3523 { 3524 uint16_t id; 3525 uint32_t number, logical_id; 3526 3527 id = le16toh(cap->cap_id); 3528 number = le32toh(cap->number); 3529 logical_id = le32toh(cap->logical_id); 3530 3531 switch (id) { 3532 case IXL_AQ_CAP_RSS: 3533 sc->sc_rss_table_size = number; 3534 sc->sc_rss_table_entry_width = logical_id; 3535 break; 3536 case IXL_AQ_CAP_RXQ: 3537 case IXL_AQ_CAP_TXQ: 3538 sc->sc_nqueue_pairs_device = MIN(number, 3539 sc->sc_nqueue_pairs_device); 3540 break; 3541 } 3542 } 3543 3544 static int 3545 ixl_get_hw_capabilities(struct ixl_softc *sc) 3546 { 3547 struct ixl_dmamem idm; 3548 struct ixl_aq_desc iaq; 3549 struct ixl_aq_capability *caps; 3550 size_t i, ncaps; 3551 bus_size_t caps_size; 3552 uint16_t status; 3553 int rv; 3554 3555 caps_size = sizeof(caps[0]) * 40; 3556 memset(&iaq, 0, sizeof(iaq)); 3557 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 3558 3559 do { 3560 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 3561 return -1; 3562 } 3563 3564 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3565 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3566 iaq.iaq_datalen = htole16(caps_size); 3567 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3568 3569 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 3570 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 3571 3572 rv = ixl_atq_poll(sc, &iaq, 250); 3573 3574 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 3575 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 3576 3577 if (rv != 0) { 3578 aprint_error(", HW capabilities timeout\n"); 3579 goto done; 3580 } 3581 3582 status = le16toh(iaq.iaq_retval); 3583 3584 if (status == IXL_AQ_RC_ENOMEM) { 3585 caps_size = le16toh(iaq.iaq_datalen); 3586 ixl_dmamem_free(sc, &idm); 3587 } 3588 } while (status == IXL_AQ_RC_ENOMEM); 3589 3590 if (status != IXL_AQ_RC_OK) { 3591 aprint_error(", HW capabilities error\n"); 3592 goto done; 3593 } 3594 3595 caps = IXL_DMA_KVA(&idm); 3596 ncaps = le16toh(iaq.iaq_param[1]); 3597 3598 for (i = 0; i < ncaps; i++) { 3599 ixl_parse_hw_capability(sc, &caps[i]); 3600 } 3601 3602 done: 3603 ixl_dmamem_free(sc, &idm); 3604 return rv; 3605 } 3606 3607 static int 3608 ixl_get_mac(struct ixl_softc *sc) 3609 { 3610 struct ixl_dmamem idm; 3611 struct ixl_aq_desc iaq; 3612 struct ixl_aq_mac_addresses *addrs; 3613 int rv; 3614 3615 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 3616 aprint_error(", unable to allocate mac addresses\n"); 3617 return -1; 3618 } 3619 3620 memset(&iaq, 0, sizeof(iaq)); 3621 iaq.iaq_flags = htole16(IXL_AQ_BUF); 3622 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 3623 iaq.iaq_datalen = htole16(sizeof(*addrs)); 3624 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3625 3626 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3627 BUS_DMASYNC_PREREAD); 3628 3629 rv = ixl_atq_poll(sc, &iaq, 250); 3630 3631 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3632 BUS_DMASYNC_POSTREAD); 3633 3634 if (rv != 0) { 3635 aprint_error(", MAC ADDRESS READ timeout\n"); 3636 rv = -1; 3637 goto done; 3638 } 3639 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3640 aprint_error(", MAC ADDRESS READ error\n"); 3641 rv = -1; 3642 goto done; 3643 } 3644 3645 addrs = IXL_DMA_KVA(&idm); 3646 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 3647 printf(", port address is not valid\n"); 3648 goto done; 3649 } 3650 3651 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 3652 rv = 0; 3653 3654 done: 3655 ixl_dmamem_free(sc, &idm); 3656 return rv; 3657 } 3658 3659 static int 3660 ixl_get_switch_config(struct ixl_softc *sc) 3661 { 3662 struct ixl_dmamem idm; 3663 struct ixl_aq_desc iaq; 3664 struct ixl_aq_switch_config *hdr; 3665 struct ixl_aq_switch_config_element *elms, *elm; 3666 unsigned int nelm, i; 3667 int rv; 3668 3669 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 3670 aprint_error_dev(sc->sc_dev, 3671 "unable to allocate switch config buffer\n"); 3672 return -1; 3673 } 3674 3675 memset(&iaq, 0, sizeof(iaq)); 3676 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3677 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3678 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 3679 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 3680 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3681 3682 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3683 BUS_DMASYNC_PREREAD); 3684 3685 rv = ixl_atq_poll(sc, &iaq, 250); 3686 3687 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3688 BUS_DMASYNC_POSTREAD); 3689 3690 if (rv != 0) { 3691 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 3692 rv = -1; 3693 goto done; 3694 } 3695 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3696 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 3697 rv = -1; 3698 goto done; 3699 } 3700 3701 hdr = IXL_DMA_KVA(&idm); 3702 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 3703 3704 nelm = le16toh(hdr->num_reported); 3705 if (nelm < 1) { 3706 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 3707 rv = -1; 3708 goto done; 3709 } 3710 3711 for (i = 0; i < nelm; i++) { 3712 elm = &elms[i]; 3713 3714 aprint_debug_dev(sc->sc_dev, 3715 "type %x revision %u seid %04x\n", 3716 elm->type, elm->revision, le16toh(elm->seid)); 3717 aprint_debug_dev(sc->sc_dev, 3718 "uplink %04x downlink %04x\n", 3719 le16toh(elm->uplink_seid), 3720 le16toh(elm->downlink_seid)); 3721 aprint_debug_dev(sc->sc_dev, 3722 "conntype %x scheduler %04x extra %04x\n", 3723 elm->connection_type, 3724 le16toh(elm->scheduler_id), 3725 le16toh(elm->element_info)); 3726 } 3727 3728 elm = &elms[0]; 3729 3730 sc->sc_uplink_seid = elm->uplink_seid; 3731 sc->sc_downlink_seid = elm->downlink_seid; 3732 sc->sc_seid = elm->seid; 3733 3734 if ((sc->sc_uplink_seid == htole16(0)) != 3735 (sc->sc_downlink_seid == htole16(0))) { 3736 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 3737 rv = -1; 3738 goto done; 3739 } 3740 3741 done: 3742 ixl_dmamem_free(sc, &idm); 3743 return rv; 3744 } 3745 3746 static int 3747 ixl_phy_mask_ints(struct ixl_softc *sc) 3748 { 3749 struct ixl_aq_desc iaq; 3750 3751 memset(&iaq, 0, sizeof(iaq)); 3752 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 3753 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 3754 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 3755 IXL_AQ_PHY_EV_MEDIA_NA)); 3756 3757 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 3758 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 3759 return -1; 3760 } 3761 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3762 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 3763 return -1; 3764 } 3765 3766 return 0; 3767 } 3768 3769 static int 3770 ixl_get_phy_abilities(struct ixl_softc *sc,struct ixl_dmamem *idm) 3771 { 3772 struct ixl_aq_desc iaq; 3773 int rv; 3774 3775 memset(&iaq, 0, sizeof(iaq)); 3776 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3777 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3778 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 3779 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 3780 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 3781 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 3782 3783 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 3784 BUS_DMASYNC_PREREAD); 3785 3786 rv = ixl_atq_poll(sc, &iaq, 250); 3787 3788 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 3789 BUS_DMASYNC_POSTREAD); 3790 3791 if (rv != 0) 3792 return -1; 3793 3794 return le16toh(iaq.iaq_retval); 3795 } 3796 3797 static int 3798 ixl_get_phy_types(struct ixl_softc *sc, uint64_t *phy_types_ptr) 3799 { 3800 struct ixl_dmamem idm; 3801 struct ixl_aq_phy_abilities *phy; 3802 uint64_t phy_types; 3803 int rv; 3804 3805 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 3806 aprint_error_dev(sc->sc_dev, 3807 "unable to allocate switch config buffer\n"); 3808 return -1; 3809 } 3810 3811 rv = ixl_get_phy_abilities(sc, &idm); 3812 switch (rv) { 3813 case -1: 3814 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 3815 goto done; 3816 case IXL_AQ_RC_OK: 3817 break; 3818 case IXL_AQ_RC_EIO: 3819 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 3820 break; 3821 default: 3822 aprint_error_dev(sc->sc_dev, 3823 "GET PHY ABILITIIES error %u\n", rv); 3824 goto done; 3825 } 3826 3827 phy = IXL_DMA_KVA(&idm); 3828 3829 phy_types = le32toh(phy->phy_type); 3830 phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 3831 3832 *phy_types_ptr = phy_types; 3833 3834 rv = 0; 3835 3836 done: 3837 ixl_dmamem_free(sc, &idm); 3838 return rv; 3839 } 3840 3841 static int 3842 ixl_get_link_status(struct ixl_softc *sc) 3843 { 3844 struct ixl_aq_desc iaq; 3845 struct ixl_aq_link_param *param; 3846 int link; 3847 3848 memset(&iaq, 0, sizeof(iaq)); 3849 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3850 param = (struct ixl_aq_link_param *)iaq.iaq_param; 3851 param->notify = IXL_AQ_LINK_NOTIFY; 3852 3853 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 3854 return ETIMEDOUT; 3855 } 3856 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3857 return EIO; 3858 } 3859 3860 link = ixl_set_link_status(sc, &iaq); 3861 sc->sc_ec.ec_if.if_link_state = link; 3862 3863 return 0; 3864 } 3865 3866 static int 3867 ixl_get_vsi(struct ixl_softc *sc) 3868 { 3869 struct ixl_dmamem *vsi = &sc->sc_scratch; 3870 struct ixl_aq_desc iaq; 3871 struct ixl_aq_vsi_param *param; 3872 struct ixl_aq_vsi_reply *reply; 3873 int rv; 3874 3875 /* grumble, vsi info isn't "known" at compile time */ 3876 3877 memset(&iaq, 0, sizeof(iaq)); 3878 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3879 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3880 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 3881 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 3882 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 3883 3884 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 3885 param->uplink_seid = sc->sc_seid; 3886 3887 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 3888 BUS_DMASYNC_PREREAD); 3889 3890 rv = ixl_atq_poll(sc, &iaq, 250); 3891 3892 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 3893 BUS_DMASYNC_POSTREAD); 3894 3895 if (rv != 0) { 3896 aprint_error_dev(sc->sc_dev, "GET VSI timeout\n"); 3897 return -1; 3898 } 3899 3900 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3901 aprint_error_dev(sc->sc_dev, "GET VSI error %u\n", 3902 le16toh(iaq.iaq_retval)); 3903 return -1; 3904 } 3905 3906 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 3907 sc->sc_vsi_number = reply->vsi_number; 3908 3909 return 0; 3910 } 3911 3912 static int 3913 ixl_set_vsi(struct ixl_softc *sc) 3914 { 3915 struct ixl_dmamem *vsi = &sc->sc_scratch; 3916 struct ixl_aq_desc iaq; 3917 struct ixl_aq_vsi_param *param; 3918 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 3919 unsigned int qnum; 3920 int rv; 3921 3922 qnum = sc->sc_nqueue_pairs - 1; 3923 3924 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 3925 IXL_AQ_VSI_VALID_VLAN); 3926 3927 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 3928 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 3929 data->queue_mapping[0] = htole16(0); 3930 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 3931 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 3932 3933 CLR(data->port_vlan_flags, 3934 htole16(IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK)); 3935 SET(data->port_vlan_flags, 3936 htole16(IXL_AQ_VSI_PVLAN_MODE_ALL | IXL_AQ_VSI_PVLAN_EMOD_NOTHING)); 3937 3938 /* grumble, vsi info isn't "known" at compile time */ 3939 3940 memset(&iaq, 0, sizeof(iaq)); 3941 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 3942 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3943 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 3944 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 3945 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 3946 3947 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 3948 param->uplink_seid = sc->sc_seid; 3949 3950 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 3951 BUS_DMASYNC_PREWRITE); 3952 3953 rv = ixl_atq_poll(sc, &iaq, 250); 3954 3955 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 3956 BUS_DMASYNC_POSTWRITE); 3957 3958 if (rv != 0) { 3959 aprint_error_dev(sc->sc_dev, "UPDATE VSI timeout\n"); 3960 return -1; 3961 } 3962 3963 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3964 aprint_error_dev(sc->sc_dev, "UPDATE VSI error %u\n", 3965 le16toh(iaq.iaq_retval)); 3966 return -1; 3967 } 3968 3969 return 0; 3970 } 3971 3972 static void 3973 ixl_set_filter_control(struct ixl_softc *sc) 3974 { 3975 uint32_t reg; 3976 3977 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 3978 3979 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 3980 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 3981 3982 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 3983 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 3984 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 3985 3986 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 3987 } 3988 3989 static inline void 3990 ixl_get_default_rss_key(uint32_t *buf, size_t len) 3991 { 3992 size_t cplen; 3993 uint8_t rss_seed[RSS_KEYSIZE]; 3994 3995 rss_getkey(rss_seed); 3996 memset(buf, 0, len); 3997 3998 cplen = MIN(len, sizeof(rss_seed)); 3999 memcpy(buf, rss_seed, cplen); 4000 } 4001 4002 static void 4003 ixl_set_rss_key(struct ixl_softc *sc) 4004 { 4005 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4006 size_t i; 4007 4008 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4009 4010 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4011 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4012 } 4013 } 4014 4015 static void 4016 ixl_set_rss_pctype(struct ixl_softc *sc) 4017 { 4018 uint64_t set_hena = 0; 4019 uint32_t hena0, hena1; 4020 4021 if (sc->sc_mac_type == I40E_MAC_X722) 4022 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4023 else 4024 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4025 4026 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4027 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4028 4029 SET(hena0, set_hena); 4030 SET(hena1, set_hena >> 32); 4031 4032 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4033 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4034 } 4035 4036 static void 4037 ixl_set_rss_hlut(struct ixl_softc *sc) 4038 { 4039 unsigned int qid; 4040 uint8_t hlut_buf[512], lut_mask; 4041 uint32_t *hluts; 4042 size_t i, hluts_num; 4043 4044 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4045 4046 for (i = 0; i < sc->sc_rss_table_size; i++) { 4047 qid = i % sc->sc_nqueue_pairs; 4048 hlut_buf[i] = qid & lut_mask; 4049 } 4050 4051 hluts = (uint32_t *)hlut_buf; 4052 hluts_num = sc->sc_rss_table_size >> 2; 4053 for (i = 0; i < hluts_num; i++) { 4054 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4055 } 4056 ixl_flush(sc); 4057 } 4058 4059 static void 4060 ixl_config_rss(struct ixl_softc *sc) 4061 { 4062 4063 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4064 4065 ixl_set_rss_key(sc); 4066 ixl_set_rss_pctype(sc); 4067 ixl_set_rss_hlut(sc); 4068 } 4069 4070 static const struct ixl_phy_type * 4071 ixl_search_phy_type(uint8_t phy_type) 4072 { 4073 const struct ixl_phy_type *itype; 4074 uint64_t mask; 4075 unsigned int i; 4076 4077 if (phy_type >= 64) 4078 return NULL; 4079 4080 mask = 1ULL << phy_type; 4081 4082 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4083 itype = &ixl_phy_type_map[i]; 4084 4085 if (ISSET(itype->phy_type, mask)) 4086 return itype; 4087 } 4088 4089 return NULL; 4090 } 4091 4092 static uint64_t 4093 ixl_search_link_speed(uint8_t link_speed) 4094 { 4095 const struct ixl_speed_type *type; 4096 unsigned int i; 4097 4098 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4099 type = &ixl_speed_type_map[i]; 4100 4101 if (ISSET(type->dev_speed, link_speed)) 4102 return type->net_speed; 4103 } 4104 4105 return 0; 4106 } 4107 4108 static int 4109 ixl_restart_an(struct ixl_softc *sc) 4110 { 4111 struct ixl_aq_desc iaq; 4112 4113 memset(&iaq, 0, sizeof(iaq)); 4114 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4115 iaq.iaq_param[0] = 4116 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4117 4118 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4119 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4120 return -1; 4121 } 4122 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4123 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4124 return -1; 4125 } 4126 4127 return 0; 4128 } 4129 4130 static int 4131 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4132 uint16_t vlan, uint16_t flags) 4133 { 4134 struct ixl_aq_desc iaq; 4135 struct ixl_aq_add_macvlan *param; 4136 struct ixl_aq_add_macvlan_elem *elem; 4137 4138 memset(&iaq, 0, sizeof(iaq)); 4139 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4140 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4141 iaq.iaq_datalen = htole16(sizeof(*elem)); 4142 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4143 4144 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4145 param->num_addrs = htole16(1); 4146 param->seid0 = htole16(0x8000) | sc->sc_seid; 4147 param->seid1 = 0; 4148 param->seid2 = 0; 4149 4150 elem = IXL_DMA_KVA(&sc->sc_scratch); 4151 memset(elem, 0, sizeof(*elem)); 4152 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4153 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4154 elem->vlan = htole16(vlan); 4155 4156 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4157 return IXL_AQ_RC_EINVAL; 4158 } 4159 4160 return le16toh(iaq.iaq_retval); 4161 } 4162 4163 static int 4164 ixl_remove_macvlan(struct ixl_softc *sc, uint8_t *macaddr, 4165 uint16_t vlan, uint16_t flags) 4166 { 4167 struct ixl_aq_desc iaq; 4168 struct ixl_aq_remove_macvlan *param; 4169 struct ixl_aq_remove_macvlan_elem *elem; 4170 4171 memset(&iaq, 0, sizeof(iaq)); 4172 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4173 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 4174 iaq.iaq_datalen = htole16(sizeof(*elem)); 4175 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4176 4177 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 4178 param->num_addrs = htole16(1); 4179 param->seid0 = htole16(0x8000) | sc->sc_seid; 4180 param->seid1 = 0; 4181 param->seid2 = 0; 4182 4183 elem = IXL_DMA_KVA(&sc->sc_scratch); 4184 memset(elem, 0, sizeof(*elem)); 4185 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4186 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 4187 elem->vlan = htole16(vlan); 4188 4189 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4190 return IXL_AQ_RC_EINVAL; 4191 } 4192 4193 return le16toh(iaq.iaq_retval); 4194 } 4195 4196 static int 4197 ixl_hmc(struct ixl_softc *sc) 4198 { 4199 struct { 4200 uint32_t count; 4201 uint32_t minsize; 4202 bus_size_t objsiz; 4203 bus_size_t setoff; 4204 bus_size_t setcnt; 4205 } regs[] = { 4206 { 4207 0, 4208 IXL_HMC_TXQ_MINSIZE, 4209 I40E_GLHMC_LANTXOBJSZ, 4210 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 4211 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 4212 }, 4213 { 4214 0, 4215 IXL_HMC_RXQ_MINSIZE, 4216 I40E_GLHMC_LANRXOBJSZ, 4217 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 4218 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 4219 }, 4220 { 4221 0, 4222 0, 4223 I40E_GLHMC_FCOEDDPOBJSZ, 4224 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 4225 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 4226 }, 4227 { 4228 0, 4229 0, 4230 I40E_GLHMC_FCOEFOBJSZ, 4231 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 4232 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 4233 }, 4234 }; 4235 struct ixl_hmc_entry *e; 4236 uint64_t size, dva; 4237 uint8_t *kva; 4238 uint64_t *sdpage; 4239 unsigned int i; 4240 int npages, tables; 4241 uint32_t reg; 4242 4243 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 4244 4245 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 4246 ixl_rd(sc, I40E_GLHMC_LANQMAX); 4247 4248 size = 0; 4249 for (i = 0; i < __arraycount(regs); i++) { 4250 e = &sc->sc_hmc_entries[i]; 4251 4252 e->hmc_count = regs[i].count; 4253 reg = ixl_rd(sc, regs[i].objsiz); 4254 e->hmc_size = BIT_ULL(0x3F & reg); 4255 e->hmc_base = size; 4256 4257 if ((e->hmc_size * 8) < regs[i].minsize) { 4258 aprint_error_dev(sc->sc_dev, 4259 "kernel hmc entry is too big\n"); 4260 return -1; 4261 } 4262 4263 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 4264 } 4265 size = roundup(size, IXL_HMC_PGSIZE); 4266 npages = size / IXL_HMC_PGSIZE; 4267 4268 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 4269 4270 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 4271 aprint_error_dev(sc->sc_dev, 4272 "unable to allocate hmc pd memory\n"); 4273 return -1; 4274 } 4275 4276 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 4277 IXL_HMC_PGSIZE) != 0) { 4278 aprint_error_dev(sc->sc_dev, 4279 "unable to allocate hmc sd memory\n"); 4280 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 4281 return -1; 4282 } 4283 4284 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 4285 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 4286 4287 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 4288 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 4289 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4290 4291 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 4292 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 4293 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 4294 4295 for (i = 0; (int)i < npages; i++) { 4296 *sdpage = htole64(dva | IXL_HMC_PDVALID); 4297 sdpage++; 4298 4299 dva += IXL_HMC_PGSIZE; 4300 } 4301 4302 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 4303 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 4304 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4305 4306 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 4307 for (i = 0; (int)i < tables; i++) { 4308 uint32_t count; 4309 4310 KASSERT(npages >= 0); 4311 4312 count = ((unsigned int)npages > IXL_HMC_PGS) ? 4313 IXL_HMC_PGS : (unsigned int)npages; 4314 4315 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 4316 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 4317 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 4318 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 4319 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 4320 ixl_wr(sc, I40E_PFHMC_SDCMD, 4321 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 4322 4323 npages -= IXL_HMC_PGS; 4324 dva += IXL_HMC_PGSIZE; 4325 } 4326 4327 for (i = 0; i < __arraycount(regs); i++) { 4328 e = &sc->sc_hmc_entries[i]; 4329 4330 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 4331 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 4332 } 4333 4334 return 0; 4335 } 4336 4337 static void 4338 ixl_hmc_free(struct ixl_softc *sc) 4339 { 4340 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 4341 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 4342 } 4343 4344 static void 4345 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 4346 unsigned int npacking) 4347 { 4348 uint8_t *dst = d; 4349 const uint8_t *src = s; 4350 unsigned int i; 4351 4352 for (i = 0; i < npacking; i++) { 4353 const struct ixl_hmc_pack *pack = &packing[i]; 4354 unsigned int offset = pack->lsb / 8; 4355 unsigned int align = pack->lsb % 8; 4356 const uint8_t *in = src + pack->offset; 4357 uint8_t *out = dst + offset; 4358 int width = pack->width; 4359 unsigned int inbits = 0; 4360 4361 if (align) { 4362 inbits = (*in++) << align; 4363 *out++ |= (inbits & 0xff); 4364 inbits >>= 8; 4365 4366 width -= 8 - align; 4367 } 4368 4369 while (width >= 8) { 4370 inbits |= (*in++) << align; 4371 *out++ = (inbits & 0xff); 4372 inbits >>= 8; 4373 4374 width -= 8; 4375 } 4376 4377 if (width > 0) { 4378 inbits |= (*in) << align; 4379 *out |= (inbits & ((1 << width) - 1)); 4380 } 4381 } 4382 } 4383 4384 static struct ixl_aq_buf * 4385 ixl_aqb_alloc(struct ixl_softc *sc) 4386 { 4387 struct ixl_aq_buf *aqb; 4388 4389 aqb = malloc(sizeof(*aqb), M_DEVBUF, M_WAITOK); 4390 if (aqb == NULL) 4391 return NULL; 4392 4393 aqb->aqb_size = IXL_AQ_BUFLEN; 4394 4395 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 4396 aqb->aqb_size, 0, 4397 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 4398 goto free; 4399 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 4400 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 4401 BUS_DMA_WAITOK) != 0) 4402 goto destroy; 4403 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 4404 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 4405 goto dma_free; 4406 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 4407 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 4408 goto unmap; 4409 4410 return aqb; 4411 unmap: 4412 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 4413 dma_free: 4414 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 4415 destroy: 4416 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 4417 free: 4418 free(aqb, M_DEVBUF); 4419 4420 return NULL; 4421 } 4422 4423 static void 4424 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 4425 { 4426 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 4427 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 4428 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 4429 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 4430 free(aqb, M_DEVBUF); 4431 } 4432 4433 static int 4434 ixl_arq_fill(struct ixl_softc *sc) 4435 { 4436 struct ixl_aq_buf *aqb; 4437 struct ixl_aq_desc *arq, *iaq; 4438 unsigned int prod = sc->sc_arq_prod; 4439 unsigned int n; 4440 int post = 0; 4441 4442 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 4443 IXL_AQ_NUM); 4444 arq = IXL_DMA_KVA(&sc->sc_arq); 4445 4446 if (__predict_false(n <= 0)) 4447 return 0; 4448 4449 do { 4450 aqb = sc->sc_arq_live[prod]; 4451 iaq = &arq[prod]; 4452 4453 if (aqb == NULL) { 4454 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 4455 if (aqb != NULL) { 4456 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 4457 ixl_aq_buf, aqb_entry); 4458 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 4459 break; 4460 } 4461 4462 sc->sc_arq_live[prod] = aqb; 4463 memset(aqb->aqb_data, 0, aqb->aqb_size); 4464 4465 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 4466 aqb->aqb_size, BUS_DMASYNC_PREREAD); 4467 4468 iaq->iaq_flags = htole16(IXL_AQ_BUF | 4469 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 4470 IXL_AQ_LB : 0)); 4471 iaq->iaq_opcode = 0; 4472 iaq->iaq_datalen = htole16(aqb->aqb_size); 4473 iaq->iaq_retval = 0; 4474 iaq->iaq_cookie = 0; 4475 iaq->iaq_param[0] = 0; 4476 iaq->iaq_param[1] = 0; 4477 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 4478 } 4479 4480 prod++; 4481 prod &= IXL_AQ_MASK; 4482 4483 post = 1; 4484 4485 } while (--n); 4486 4487 if (post) { 4488 sc->sc_arq_prod = prod; 4489 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 4490 } 4491 4492 return post; 4493 } 4494 4495 static void 4496 ixl_arq_unfill(struct ixl_softc *sc) 4497 { 4498 struct ixl_aq_buf *aqb; 4499 unsigned int i; 4500 4501 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 4502 aqb = sc->sc_arq_live[i]; 4503 if (aqb == NULL) 4504 continue; 4505 4506 sc->sc_arq_live[i] = NULL; 4507 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 4508 BUS_DMASYNC_POSTREAD); 4509 ixl_aqb_free(sc, aqb); 4510 } 4511 4512 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 4513 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 4514 ixl_aq_buf, aqb_entry); 4515 ixl_aqb_free(sc, aqb); 4516 } 4517 } 4518 4519 static void 4520 ixl_clear_hw(struct ixl_softc *sc) 4521 { 4522 uint32_t num_queues, base_queue; 4523 uint32_t num_pf_int; 4524 uint32_t num_vf_int; 4525 uint32_t num_vfs; 4526 uint32_t i, j; 4527 uint32_t val; 4528 uint32_t eol = 0x7ff; 4529 4530 /* get number of interrupts, queues, and vfs */ 4531 val = ixl_rd(sc, I40E_GLPCI_CNF2); 4532 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 4533 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 4534 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 4535 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 4536 4537 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 4538 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 4539 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 4540 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 4541 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 4542 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 4543 num_queues = (j - base_queue) + 1; 4544 else 4545 num_queues = 0; 4546 4547 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 4548 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 4549 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 4550 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 4551 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 4552 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 4553 num_vfs = (j - i) + 1; 4554 else 4555 num_vfs = 0; 4556 4557 /* stop all the interrupts */ 4558 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 4559 ixl_flush(sc); 4560 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 4561 for (i = 0; i < num_pf_int - 2; i++) 4562 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 4563 ixl_flush(sc); 4564 4565 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 4566 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4567 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 4568 for (i = 0; i < num_pf_int - 2; i++) 4569 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 4570 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4571 for (i = 0; i < num_vfs; i++) 4572 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 4573 for (i = 0; i < num_vf_int - 2; i++) 4574 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 4575 4576 /* warn the HW of the coming Tx disables */ 4577 for (i = 0; i < num_queues; i++) { 4578 uint32_t abs_queue_idx = base_queue + i; 4579 uint32_t reg_block = 0; 4580 4581 if (abs_queue_idx >= 128) { 4582 reg_block = abs_queue_idx / 128; 4583 abs_queue_idx %= 128; 4584 } 4585 4586 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 4587 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 4588 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 4589 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 4590 4591 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 4592 } 4593 delaymsec(400); 4594 4595 /* stop all the queues */ 4596 for (i = 0; i < num_queues; i++) { 4597 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 4598 ixl_wr(sc, I40E_QTX_ENA(i), 0); 4599 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 4600 ixl_wr(sc, I40E_QRX_ENA(i), 0); 4601 } 4602 4603 /* short wait for all queue disables to settle */ 4604 delaymsec(50); 4605 } 4606 4607 static int 4608 ixl_pf_reset(struct ixl_softc *sc) 4609 { 4610 uint32_t cnt = 0; 4611 uint32_t cnt1 = 0; 4612 uint32_t reg = 0, reg0 = 0; 4613 uint32_t grst_del; 4614 4615 /* 4616 * Poll for Global Reset steady state in case of recent GRST. 4617 * The grst delay value is in 100ms units, and we'll wait a 4618 * couple counts longer to be sure we don't just miss the end. 4619 */ 4620 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 4621 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 4622 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 4623 4624 grst_del = grst_del * 20; 4625 4626 for (cnt = 0; cnt < grst_del; cnt++) { 4627 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 4628 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 4629 break; 4630 delaymsec(100); 4631 } 4632 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 4633 aprint_error(", Global reset polling failed to complete\n"); 4634 return -1; 4635 } 4636 4637 /* Now Wait for the FW to be ready */ 4638 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 4639 reg = ixl_rd(sc, I40E_GLNVM_ULD); 4640 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4641 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 4642 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4643 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 4644 break; 4645 4646 delaymsec(10); 4647 } 4648 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4649 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 4650 aprint_error(", wait for FW Reset complete timed out " 4651 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 4652 return -1; 4653 } 4654 4655 /* 4656 * If there was a Global Reset in progress when we got here, 4657 * we don't need to do the PF Reset 4658 */ 4659 if (cnt == 0) { 4660 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 4661 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 4662 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 4663 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 4664 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 4665 break; 4666 delaymsec(1); 4667 4668 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 4669 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 4670 aprint_error(", Core reset upcoming." 4671 " Skipping PF reset reset request\n"); 4672 return -1; 4673 } 4674 } 4675 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 4676 aprint_error(", PF reset polling failed to complete" 4677 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 4678 return -1; 4679 } 4680 } 4681 4682 return 0; 4683 } 4684 4685 static int 4686 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 4687 bus_size_t size, bus_size_t align) 4688 { 4689 ixm->ixm_size = size; 4690 4691 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 4692 ixm->ixm_size, 0, 4693 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 4694 &ixm->ixm_map) != 0) 4695 return 1; 4696 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 4697 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 4698 BUS_DMA_WAITOK) != 0) 4699 goto destroy; 4700 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 4701 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 4702 goto free; 4703 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 4704 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 4705 goto unmap; 4706 4707 memset(ixm->ixm_kva, 0, ixm->ixm_size); 4708 4709 return 0; 4710 unmap: 4711 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 4712 free: 4713 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 4714 destroy: 4715 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 4716 return 1; 4717 } 4718 4719 static void 4720 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 4721 { 4722 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 4723 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 4724 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 4725 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 4726 } 4727 4728 static int 4729 ixl_set_macvlan(struct ixl_softc *sc) 4730 { 4731 int error, rv = 0; 4732 4733 /* remove default mac filter and replace it so we can see vlans */ 4734 4735 error = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 4736 if (error != IXL_AQ_RC_OK) { 4737 aprint_debug_dev(sc->sc_dev, "unable to remove macvlan\n"); 4738 rv = -1; 4739 } 4740 4741 error = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 4742 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 4743 if (error != IXL_AQ_RC_OK && error != IXL_AQ_RC_ENOENT) { 4744 aprint_debug_dev(sc->sc_dev, 4745 "unable to remove macvlan(IGNORE_VLAN)\n"); 4746 rv = -1; 4747 } 4748 4749 error = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 4750 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 4751 if (error != IXL_AQ_RC_OK) { 4752 aprint_debug_dev(sc->sc_dev, "unable to add mac address\n"); 4753 rv = -1; 4754 } 4755 4756 error = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 4757 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 4758 if (error != IXL_AQ_RC_OK) { 4759 aprint_debug_dev(sc->sc_dev, 4760 "unable to add broadcast mac address\n"); 4761 rv = -1; 4762 } 4763 4764 return rv; 4765 } 4766 4767 static void 4768 ixl_link_status(struct ixl_softc *sc) 4769 { 4770 4771 (void)ixl_get_link_status(sc); 4772 } 4773 4774 static int 4775 ixl_ifflags_cb(struct ethercom *ec) 4776 { 4777 4778 return 0; 4779 } 4780 4781 static int 4782 ixl_set_link_status(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 4783 { 4784 const struct ixl_aq_link_status *status; 4785 const struct ixl_phy_type *itype; 4786 4787 uint64_t ifm_active = IFM_ETHER; 4788 uint64_t ifm_status = IFM_AVALID; 4789 int link_state = LINK_STATE_DOWN; 4790 uint64_t baudrate = 0; 4791 4792 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 4793 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) 4794 goto done; 4795 4796 ifm_active |= IFM_FDX; 4797 ifm_status |= IFM_ACTIVE; 4798 link_state = LINK_STATE_UP; 4799 4800 itype = ixl_search_phy_type(status->phy_type); 4801 if (itype != NULL) 4802 ifm_active |= itype->ifm_type; 4803 4804 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 4805 ifm_active |= IFM_ETH_TXPAUSE; 4806 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 4807 ifm_active |= IFM_ETH_RXPAUSE; 4808 4809 baudrate = ixl_search_link_speed(status->link_speed); 4810 4811 done: 4812 /* NET_ASSERT_LOCKED() except during attach */ 4813 sc->sc_media_active = ifm_active; 4814 sc->sc_media_status = ifm_status; 4815 4816 sc->sc_ec.ec_if.if_baudrate = baudrate; 4817 4818 return link_state; 4819 } 4820 4821 static int 4822 ixl_establish_intx(struct ixl_softc *sc) 4823 { 4824 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 4825 pci_intr_handle_t *intr; 4826 char xnamebuf[32]; 4827 char intrbuf[PCI_INTRSTR_LEN]; 4828 char const *intrstr; 4829 4830 KASSERT(sc->sc_nintrs == 1); 4831 4832 intr = &sc->sc_ihp[0]; 4833 4834 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 4835 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 4836 device_xname(sc->sc_dev)); 4837 4838 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 4839 sc, xnamebuf); 4840 4841 if (sc->sc_ihs[0] == NULL) { 4842 aprint_error_dev(sc->sc_dev, 4843 "unable to establish interrupt at %s\n", intrstr); 4844 return -1; 4845 } 4846 4847 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 4848 return 0; 4849 } 4850 4851 static int 4852 ixl_establish_msix(struct ixl_softc *sc) 4853 { 4854 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 4855 unsigned int vector = 0; 4856 unsigned int i; 4857 char xnamebuf[32]; 4858 char intrbuf[PCI_INTRSTR_LEN]; 4859 char const *intrstr; 4860 4861 /* the "other" intr is mapped to vector 0 */ 4862 vector = 0; 4863 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 4864 intrbuf, sizeof(intrbuf)); 4865 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 4866 device_xname(sc->sc_dev)); 4867 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 4868 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 4869 sc, xnamebuf); 4870 if (sc->sc_ihs[vector] == NULL) { 4871 aprint_error_dev(sc->sc_dev, 4872 "unable to establish interrupt at %s\n", intrstr); 4873 goto fail; 4874 } 4875 vector++; 4876 aprint_normal_dev(sc->sc_dev, "interrupt at %s\n", intrstr); 4877 4878 sc->sc_msix_vector_queue = vector; 4879 4880 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 4881 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 4882 intrbuf, sizeof(intrbuf)); 4883 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 4884 device_xname(sc->sc_dev), i); 4885 4886 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 4887 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 4888 (void *)&sc->sc_qps[i], xnamebuf); 4889 4890 if (sc->sc_ihs[vector] == NULL) { 4891 aprint_error_dev(sc->sc_dev, 4892 "unable to establish interrupt at %s\n", intrstr); 4893 goto fail; 4894 } 4895 vector++; 4896 aprint_normal_dev(sc->sc_dev, 4897 "interrupt at %s\n", intrstr); 4898 } 4899 4900 return 0; 4901 fail: 4902 for (i = 0; i < vector; i++) { 4903 pci_intr_disestablish(pc, sc->sc_ihs[i]); 4904 } 4905 4906 sc->sc_msix_vector_queue = 0; 4907 sc->sc_msix_vector_queue = 0; 4908 4909 return -1; 4910 } 4911 4912 static void 4913 ixl_set_affinity_msix(struct ixl_softc *sc) 4914 { 4915 kcpuset_t *affinity; 4916 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 4917 int affinity_to, r; 4918 unsigned int i, vector; 4919 char intrbuf[PCI_INTRSTR_LEN]; 4920 char const *intrstr; 4921 4922 affinity_to = 0; 4923 kcpuset_create(&affinity, false); 4924 4925 vector = sc->sc_msix_vector_queue; 4926 4927 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 4928 affinity_to = i % ncpu; 4929 4930 kcpuset_zero(affinity); 4931 kcpuset_set(affinity, affinity_to); 4932 4933 intrstr = pci_intr_string(pc, sc->sc_ihp[vector + i], 4934 intrbuf, sizeof(intrbuf)); 4935 r = interrupt_distribute(sc->sc_ihs[vector + i], 4936 affinity, NULL); 4937 if (r == 0) { 4938 aprint_normal_dev(sc->sc_dev, 4939 "for TXRX%u interrupting at %s affinity to %u\n", 4940 i, intrstr, affinity_to); 4941 } else { 4942 aprint_normal_dev(sc->sc_dev, 4943 "for TXRX%u interrupting at %s\n", 4944 i, intrstr); 4945 } 4946 } 4947 4948 vector = 0; /* vector 0 means "other" interrupt */ 4949 affinity_to = (affinity_to + 1) % ncpu; 4950 kcpuset_zero(affinity); 4951 kcpuset_set(affinity, affinity_to); 4952 4953 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 4954 intrbuf, sizeof(intrbuf)); 4955 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 4956 if (r == 0) { 4957 aprint_normal_dev(sc->sc_dev, 4958 "for other interrupting at %s affinity to %u\n", 4959 intrstr, affinity_to); 4960 } else { 4961 aprint_normal_dev(sc->sc_dev, 4962 "for other interrupting at %s", intrstr); 4963 } 4964 4965 kcpuset_destroy(affinity); 4966 } 4967 4968 static void 4969 ixl_config_queue_intr(struct ixl_softc *sc) 4970 { 4971 unsigned int i, vector; 4972 4973 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 4974 vector = sc->sc_msix_vector_queue; 4975 } else { 4976 vector = I40E_INTR_NOTX_INTR; 4977 4978 ixl_wr(sc, I40E_PFINT_LNKLST0, 4979 (I40E_INTR_NOTX_QUEUE << 4980 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 4981 (I40E_QUEUE_TYPE_RX << 4982 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 4983 } 4984 4985 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 4986 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 4987 ixl_flush(sc); 4988 4989 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 4990 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 4991 (I40E_QUEUE_TYPE_RX << 4992 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 4993 4994 ixl_wr(sc, I40E_QINT_RQCTL(i), 4995 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 4996 (I40E_ITR_INDEX_RX << 4997 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 4998 (I40E_INTR_NOTX_RX_QUEUE << 4999 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5000 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5001 (I40E_QUEUE_TYPE_TX << 5002 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5003 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5004 5005 ixl_wr(sc, I40E_QINT_TQCTL(i), 5006 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5007 (I40E_ITR_INDEX_TX << 5008 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5009 (I40E_INTR_NOTX_TX_QUEUE << 5010 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5011 (I40E_QUEUE_TYPE_EOL << 5012 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5013 (I40E_QUEUE_TYPE_RX << 5014 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5015 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5016 5017 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) 5018 vector++; 5019 } 5020 ixl_flush(sc); 5021 5022 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), 0x7a); 5023 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), 0x7a); 5024 ixl_flush(sc); 5025 } 5026 5027 static void 5028 ixl_config_other_intr(struct ixl_softc *sc) 5029 { 5030 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5031 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5032 5033 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5034 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5035 I40E_PFINT_ICR0_ENA_GRST_MASK | 5036 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5037 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5038 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5039 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5040 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5041 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5042 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5043 5044 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5045 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5046 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5047 (I40E_ITR_INDEX_OTHER << 5048 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5049 ixl_flush(sc); 5050 } 5051 5052 static int 5053 ixl_setup_interrupts(struct ixl_softc *sc) 5054 { 5055 struct pci_attach_args *pa = &sc->sc_pa; 5056 pci_intr_type_t max_type, intr_type; 5057 int counts[PCI_INTR_TYPE_SIZE]; 5058 int error; 5059 unsigned int i; 5060 bool retry, nomsix = IXL_NOMSIX; 5061 5062 memset(counts, 0, sizeof(counts)); 5063 max_type = PCI_INTR_TYPE_MSIX; 5064 /* QPs + other interrupt */ 5065 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5066 counts[PCI_INTR_TYPE_INTX] = 1; 5067 5068 if (nomsix) 5069 counts[PCI_INTR_TYPE_MSIX] = 0; 5070 5071 do { 5072 retry = false; 5073 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5074 if (error != 0) { 5075 aprint_error_dev(sc->sc_dev, 5076 "couldn't map interrupt\n"); 5077 break; 5078 } 5079 for (i = 0; i < sc->sc_nintrs; i++) { 5080 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5081 PCI_INTR_MPSAFE, true); 5082 } 5083 5084 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5085 sc->sc_nintrs = counts[intr_type]; 5086 KASSERT(sc->sc_nintrs > 0); 5087 5088 sc->sc_ihs = kmem_alloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5089 KM_SLEEP); 5090 5091 if (intr_type == PCI_INTR_TYPE_MSIX) { 5092 error = ixl_establish_msix(sc); 5093 if (error) { 5094 counts[PCI_INTR_TYPE_MSIX] = 0; 5095 retry = true; 5096 } else { 5097 ixl_set_affinity_msix(sc); 5098 } 5099 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5100 error = ixl_establish_intx(sc); 5101 } else { 5102 error = -1; 5103 } 5104 5105 if (error) { 5106 kmem_free(sc->sc_ihs, 5107 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5108 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5109 } else { 5110 sc->sc_intrtype = intr_type; 5111 } 5112 } while (retry); 5113 5114 return error; 5115 } 5116 5117 static void 5118 ixl_teardown_interrupts(struct ixl_softc *sc) 5119 { 5120 struct pci_attach_args *pa = &sc->sc_pa; 5121 unsigned int i; 5122 5123 for (i = 0; i < sc->sc_nintrs; i++) { 5124 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 5125 } 5126 5127 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5128 5129 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5130 sc->sc_ihs = NULL; 5131 sc->sc_nintrs = 0; 5132 } 5133 5134 static int 5135 ixl_setup_stats(struct ixl_softc *sc) 5136 { 5137 struct ixl_queue_pair *qp; 5138 struct ixl_tx_ring *txr; 5139 struct ixl_rx_ring *rxr; 5140 unsigned int i; 5141 5142 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5143 qp = &sc->sc_qps[i]; 5144 txr = qp->qp_txr; 5145 rxr = qp->qp_rxr; 5146 5147 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 5148 NULL, qp->qp_name, "m_defrag successed"); 5149 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 5150 NULL, qp->qp_name, "m_defrag_failed"); 5151 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 5152 NULL, qp->qp_name, "Dropped in pcq"); 5153 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 5154 NULL, qp->qp_name, "Deferred transmit"); 5155 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 5156 NULL, qp->qp_name, "Interrupt on queue"); 5157 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 5158 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5159 5160 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 5161 NULL, qp->qp_name, "MGETHDR failed"); 5162 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 5163 NULL, qp->qp_name, "MCLGET failed"); 5164 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 5165 EVCNT_TYPE_MISC, NULL, qp->qp_name, 5166 "bus_dmamap_load_mbuf failed"); 5167 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 5168 NULL, qp->qp_name, "Interrupt on queue"); 5169 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 5170 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5171 } 5172 5173 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 5174 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 5175 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 5176 NULL, device_xname(sc->sc_dev), "Link status event"); 5177 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 5178 NULL, device_xname(sc->sc_dev), "ECC error"); 5179 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 5180 NULL, device_xname(sc->sc_dev), "PCI exception"); 5181 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 5182 NULL, device_xname(sc->sc_dev), "Critical error"); 5183 5184 return 0; 5185 } 5186 5187 static void 5188 ixl_teardown_stats(struct ixl_softc *sc) 5189 { 5190 struct ixl_tx_ring *txr; 5191 struct ixl_rx_ring *rxr; 5192 unsigned int i; 5193 5194 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5195 txr = sc->sc_qps[i].qp_txr; 5196 rxr = sc->sc_qps[i].qp_rxr; 5197 5198 evcnt_detach(&txr->txr_defragged); 5199 evcnt_detach(&txr->txr_defrag_failed); 5200 evcnt_detach(&txr->txr_pcqdrop); 5201 evcnt_detach(&txr->txr_transmitdef); 5202 evcnt_detach(&txr->txr_intr); 5203 evcnt_detach(&txr->txr_defer); 5204 5205 evcnt_detach(&rxr->rxr_mgethdr_failed); 5206 evcnt_detach(&rxr->rxr_mgetcl_failed); 5207 evcnt_detach(&rxr->rxr_mbuf_load_failed); 5208 evcnt_detach(&rxr->rxr_intr); 5209 evcnt_detach(&rxr->rxr_defer); 5210 } 5211 5212 evcnt_detach(&sc->sc_event_atq); 5213 evcnt_detach(&sc->sc_event_link); 5214 evcnt_detach(&sc->sc_event_ecc_err); 5215 evcnt_detach(&sc->sc_event_pci_exception); 5216 evcnt_detach(&sc->sc_event_crit_err); 5217 } 5218 5219 static int 5220 ixl_setup_sysctls(struct ixl_softc *sc) 5221 { 5222 const char *devname; 5223 struct sysctllog **log; 5224 const struct sysctlnode *rnode, *rxnode, *txnode; 5225 int error; 5226 5227 log = &sc->sc_sysctllog; 5228 devname = device_xname(sc->sc_dev); 5229 5230 error = sysctl_createv(log, 0, NULL, &rnode, 5231 0, CTLTYPE_NODE, devname, 5232 SYSCTL_DESCR("ixl information and settings"), 5233 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 5234 if (error) 5235 goto out; 5236 5237 error = sysctl_createv(log, 0, &rnode, NULL, 5238 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 5239 SYSCTL_DESCR("Use workqueue for packet processing"), 5240 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 5241 if (error) 5242 goto out; 5243 5244 error = sysctl_createv(log, 0, &rnode, &rxnode, 5245 0, CTLTYPE_NODE, "rx", 5246 SYSCTL_DESCR("ixl information and settings for Rx"), 5247 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 5248 if (error) 5249 goto out; 5250 5251 error = sysctl_createv(log, 0, &rxnode, NULL, 5252 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 5253 SYSCTL_DESCR("max number of Rx packets" 5254 " to process for interrupt processing"), 5255 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 5256 if (error) 5257 goto out; 5258 5259 error = sysctl_createv(log, 0, &rxnode, NULL, 5260 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 5261 SYSCTL_DESCR("max number of Rx packets" 5262 " to process for deferred processing"), 5263 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 5264 if (error) 5265 goto out; 5266 5267 error = sysctl_createv(log, 0, &rnode, &txnode, 5268 0, CTLTYPE_NODE, "tx", 5269 SYSCTL_DESCR("ixl information and settings for Tx"), 5270 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 5271 if (error) 5272 goto out; 5273 5274 error = sysctl_createv(log, 0, &txnode, NULL, 5275 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 5276 SYSCTL_DESCR("max number of Tx packets" 5277 " to process for interrupt processing"), 5278 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 5279 if (error) 5280 goto out; 5281 5282 error = sysctl_createv(log, 0, &txnode, NULL, 5283 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 5284 SYSCTL_DESCR("max number of Tx packets" 5285 " to process for deferred processing"), 5286 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 5287 if (error) 5288 goto out; 5289 5290 out: 5291 if (error) { 5292 aprint_error_dev(sc->sc_dev, 5293 "unable to create sysctl node\n"); 5294 sysctl_teardown(log); 5295 } 5296 5297 return error; 5298 } 5299 5300 static void 5301 ixl_teardown_sysctls(struct ixl_softc *sc) 5302 { 5303 5304 sysctl_teardown(&sc->sc_sysctllog); 5305 } 5306 5307 static struct workqueue * 5308 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 5309 { 5310 struct workqueue *wq; 5311 int error; 5312 5313 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 5314 prio, ipl, flags); 5315 5316 if (error) 5317 return NULL; 5318 5319 return wq; 5320 } 5321 5322 static void 5323 ixl_workq_destroy(struct workqueue *wq) 5324 { 5325 5326 workqueue_destroy(wq); 5327 } 5328 5329 static void 5330 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 5331 { 5332 5333 memset(work, 0, sizeof(*work)); 5334 work->ixw_func = func; 5335 work->ixw_arg = arg; 5336 } 5337 5338 static void 5339 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 5340 { 5341 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 5342 return; 5343 5344 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 5345 } 5346 5347 static void 5348 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 5349 { 5350 5351 workqueue_wait(wq, &work->ixw_cookie); 5352 } 5353 5354 static void 5355 ixl_workq_work(struct work *wk, void *context) 5356 { 5357 struct ixl_work *work; 5358 5359 work = container_of(wk, struct ixl_work, ixw_cookie); 5360 5361 atomic_swap_uint(&work->ixw_added, 0); 5362 kpreempt_disable(); 5363 work->ixw_func(work->ixw_arg); 5364 kpreempt_enable(); 5365 } 5366 5367 static int 5368 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 5369 { 5370 struct ixl_aq_desc iaq; 5371 5372 memset(&iaq, 0, sizeof(iaq)); 5373 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 5374 iaq.iaq_param[1] = htole32(reg); 5375 5376 if (ixl_atq_poll(sc, &iaq, 250) != 0) 5377 return ETIMEDOUT; 5378 5379 switch (htole16(iaq.iaq_retval)) { 5380 case IXL_AQ_RC_OK: 5381 /* success */ 5382 break; 5383 case IXL_AQ_RC_EACCES: 5384 return EPERM; 5385 case IXL_AQ_RC_EAGAIN: 5386 return EAGAIN; 5387 default: 5388 return EIO; 5389 } 5390 5391 *rv = htole32(iaq.iaq_param[3]); 5392 return 0; 5393 } 5394 5395 5396 5397 static uint32_t 5398 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 5399 { 5400 uint32_t val; 5401 int rv, retry, retry_limit; 5402 5403 retry_limit = sc->sc_rxctl_atq ? 5 : 0; 5404 5405 for (retry = 0; retry < retry_limit; retry++) { 5406 rv = ixl_rx_ctl_read(sc, reg, &val); 5407 if (rv == 0) 5408 return val; 5409 else if (rv == EAGAIN) 5410 delaymsec(1); 5411 else 5412 break; 5413 } 5414 5415 val = ixl_rd(sc, reg); 5416 5417 return val; 5418 } 5419 5420 static int 5421 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 5422 { 5423 struct ixl_aq_desc iaq; 5424 5425 memset(&iaq, 0, sizeof(iaq)); 5426 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 5427 iaq.iaq_param[1] = htole32(reg); 5428 iaq.iaq_param[3] = htole32(value); 5429 5430 if (ixl_atq_poll(sc, &iaq, 250) != 0) 5431 return ETIMEDOUT; 5432 5433 switch (htole16(iaq.iaq_retval)) { 5434 case IXL_AQ_RC_OK: 5435 /* success */ 5436 break; 5437 case IXL_AQ_RC_EACCES: 5438 return EPERM; 5439 case IXL_AQ_RC_EAGAIN: 5440 return EAGAIN; 5441 default: 5442 return EIO; 5443 } 5444 5445 return 0; 5446 } 5447 5448 static void 5449 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 5450 { 5451 int rv, retry, retry_limit; 5452 5453 retry_limit = sc->sc_rxctl_atq ? 5 : 0; 5454 5455 for (retry = 0; retry < retry_limit; retry++) { 5456 rv = ixl_rx_ctl_write(sc, reg, value); 5457 if (rv == 0) 5458 return; 5459 else if (rv == EAGAIN) 5460 delaymsec(1); 5461 else 5462 break; 5463 } 5464 5465 ixl_wr(sc, reg, value); 5466 } 5467 5468 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 5469 5470 #ifdef _MODULE 5471 #include "ioconf.c" 5472 #endif 5473 5474 static int 5475 if_ixl_modcmd(modcmd_t cmd, void *opaque) 5476 { 5477 int error = 0; 5478 5479 #ifdef _MODULE 5480 switch (cmd) { 5481 case MODULE_CMD_INIT: 5482 error = config_init_component(cfdriver_ioconf_if_ixl, 5483 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 5484 break; 5485 case MODULE_CMD_FINI: 5486 error = config_fini_component(cfdriver_ioconf_if_ixl, 5487 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 5488 break; 5489 default: 5490 error = ENOTTY; 5491 break; 5492 } 5493 #endif 5494 5495 return error; 5496 } 5497
Indexes created Sun Sep 21 20:09:37 GMT 2025