if_ixl.c revision 1.14
1 /* $NetBSD: if_ixl.c,v 1.14 2019/12/26 03:17:01 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2016,2017 David Gwynne <dlg (at) openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50 /* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76 #include <sys/cdefs.h> 77 78 #ifdef _KERNEL_OPT 79 #include "opt_net_mpsafe.h" 80 #endif 81 82 #include <sys/param.h> 83 #include <sys/types.h> 84 85 #include <sys/cpu.h> 86 #include <sys/device.h> 87 #include <sys/evcnt.h> 88 #include <sys/interrupt.h> 89 #include <sys/kmem.h> 90 #include <sys/malloc.h> 91 #include <sys/module.h> 92 #include <sys/mutex.h> 93 #include <sys/pcq.h> 94 #include <sys/syslog.h> 95 #include <sys/workqueue.h> 96 97 #include <sys/bus.h> 98 99 #include <net/bpf.h> 100 #include <net/if.h> 101 #include <net/if_dl.h> 102 #include <net/if_media.h> 103 #include <net/if_ether.h> 104 #include <net/rss_config.h> 105 106 #include <dev/pci/pcivar.h> 107 #include <dev/pci/pcidevs.h> 108 109 #include <dev/pci/if_ixlreg.h> 110 #include <dev/pci/if_ixlvar.h> 111 112 struct ixl_softc; /* defined */ 113 114 #define I40E_PF_RESET_WAIT_COUNT 200 115 #define I40E_AQ_LARGE_BUF 512 116 117 /* bitfields for Tx queue mapping in QTX_CTL */ 118 #define I40E_QTX_CTL_VF_QUEUE 0x0 119 #define I40E_QTX_CTL_VM_QUEUE 0x1 120 #define I40E_QTX_CTL_PF_QUEUE 0x2 121 122 #define I40E_QUEUE_TYPE_EOL 0x7ff 123 #define I40E_INTR_NOTX_QUEUE 0 124 125 #define I40E_QUEUE_TYPE_RX 0x0 126 #define I40E_QUEUE_TYPE_TX 0x1 127 #define I40E_QUEUE_TYPE_PE_CEQ 0x2 128 #define I40E_QUEUE_TYPE_UNKNOWN 0x3 129 130 #define I40E_ITR_INDEX_RX 0x0 131 #define I40E_ITR_INDEX_TX 0x1 132 #define I40E_ITR_INDEX_OTHER 0x2 133 #define I40E_ITR_INDEX_NONE 0x3 134 135 #define I40E_INTR_NOTX_QUEUE 0 136 #define I40E_INTR_NOTX_INTR 0 137 #define I40E_INTR_NOTX_RX_QUEUE 0 138 #define I40E_INTR_NOTX_TX_QUEUE 1 139 #define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 140 #define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 141 142 #define BIT_ULL(a) (1ULL << (a)) 143 #define IXL_RSS_HENA_DEFAULT_BASE \ 144 (BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 145 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 146 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 147 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 148 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 149 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 150 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 151 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 152 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 153 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 154 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 155 #define IXL_RSS_HENA_DEFAULT_XL710 IXL_RSS_HENA_DEFAULT_BASE 156 #define IXL_RSS_HENA_DEFAULT_X722 (IXL_RSS_HENA_DEFAULT_XL710 | \ 157 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 158 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 159 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 160 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 161 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 162 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 163 #define I40E_HASH_LUT_SIZE_128 0 164 #define IXL_RSS_KEY_SIZE_REG 13 165 166 #define IXL_ICR0_CRIT_ERR_MASK \ 167 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 168 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 169 I40E_PFINT_ICR0_PE_CRITERR_MASK) 170 171 #define IXL_TX_PKT_DESCS 8 172 #define IXL_TX_QUEUE_ALIGN 128 173 #define IXL_RX_QUEUE_ALIGN 128 174 175 #define IXL_HARDMTU 9712 /* 9726 - ETHER_HDR_LEN */ 176 177 #define IXL_PCIREG PCI_MAPREG_START 178 179 #define IXL_ITR0 0x0 180 #define IXL_ITR1 0x1 181 #define IXL_ITR2 0x2 182 #define IXL_NOITR 0x3 183 184 #define IXL_AQ_NUM 256 185 #define IXL_AQ_MASK (IXL_AQ_NUM - 1) 186 #define IXL_AQ_ALIGN 64 /* lol */ 187 #define IXL_AQ_BUFLEN 4096 188 189 #define IXL_HMC_ROUNDUP 512 190 #define IXL_HMC_PGSIZE 4096 191 #define IXL_HMC_DVASZ sizeof(uint64_t) 192 #define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 193 #define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 194 #define IXL_HMC_PDVALID 1ULL 195 196 #define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 197 198 struct ixl_aq_regs { 199 bus_size_t atq_tail; 200 bus_size_t atq_head; 201 bus_size_t atq_len; 202 bus_size_t atq_bal; 203 bus_size_t atq_bah; 204 205 bus_size_t arq_tail; 206 bus_size_t arq_head; 207 bus_size_t arq_len; 208 bus_size_t arq_bal; 209 bus_size_t arq_bah; 210 211 uint32_t atq_len_enable; 212 uint32_t atq_tail_mask; 213 uint32_t atq_head_mask; 214 215 uint32_t arq_len_enable; 216 uint32_t arq_tail_mask; 217 uint32_t arq_head_mask; 218 }; 219 220 struct ixl_phy_type { 221 uint64_t phy_type; 222 uint64_t ifm_type; 223 }; 224 225 struct ixl_speed_type { 226 uint8_t dev_speed; 227 uint64_t net_speed; 228 }; 229 230 struct ixl_aq_buf { 231 SIMPLEQ_ENTRY(ixl_aq_buf) 232 aqb_entry; 233 void *aqb_data; 234 bus_dmamap_t aqb_map; 235 bus_dma_segment_t aqb_seg; 236 size_t aqb_size; 237 int aqb_nsegs; 238 }; 239 SIMPLEQ_HEAD(ixl_aq_bufs, ixl_aq_buf); 240 241 struct ixl_dmamem { 242 bus_dmamap_t ixm_map; 243 bus_dma_segment_t ixm_seg; 244 int ixm_nsegs; 245 size_t ixm_size; 246 void *ixm_kva; 247 }; 248 249 #define IXL_DMA_MAP(_ixm) ((_ixm)->ixm_map) 250 #define IXL_DMA_DVA(_ixm) ((_ixm)->ixm_map->dm_segs[0].ds_addr) 251 #define IXL_DMA_KVA(_ixm) ((void *)(_ixm)->ixm_kva) 252 #define IXL_DMA_LEN(_ixm) ((_ixm)->ixm_size) 253 254 struct ixl_hmc_entry { 255 uint64_t hmc_base; 256 uint32_t hmc_count; 257 uint64_t hmc_size; 258 }; 259 260 enum ixl_hmc_types { 261 IXL_HMC_LAN_TX = 0, 262 IXL_HMC_LAN_RX, 263 IXL_HMC_FCOE_CTX, 264 IXL_HMC_FCOE_FILTER, 265 IXL_HMC_COUNT 266 }; 267 268 struct ixl_hmc_pack { 269 uint16_t offset; 270 uint16_t width; 271 uint16_t lsb; 272 }; 273 274 /* 275 * these hmc objects have weird sizes and alignments, so these are abstract 276 * representations of them that are nice for c to populate. 277 * 278 * the packing code relies on little-endian values being stored in the fields, 279 * no high bits in the fields being set, and the fields must be packed in the 280 * same order as they are in the ctx structure. 281 */ 282 283 struct ixl_hmc_rxq { 284 uint16_t head; 285 uint8_t cpuid; 286 uint64_t base; 287 #define IXL_HMC_RXQ_BASE_UNIT 128 288 uint16_t qlen; 289 uint16_t dbuff; 290 #define IXL_HMC_RXQ_DBUFF_UNIT 128 291 uint8_t hbuff; 292 #define IXL_HMC_RXQ_HBUFF_UNIT 64 293 uint8_t dtype; 294 #define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 295 #define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 296 #define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 297 uint8_t dsize; 298 #define IXL_HMC_RXQ_DSIZE_16 0 299 #define IXL_HMC_RXQ_DSIZE_32 1 300 uint8_t crcstrip; 301 uint8_t fc_ena; 302 uint8_t l2sel; 303 uint8_t hsplit_0; 304 uint8_t hsplit_1; 305 uint8_t showiv; 306 uint16_t rxmax; 307 uint8_t tphrdesc_ena; 308 uint8_t tphwdesc_ena; 309 uint8_t tphdata_ena; 310 uint8_t tphhead_ena; 311 uint8_t lrxqthresh; 312 uint8_t prefena; 313 }; 314 315 static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 316 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 317 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 318 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 319 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 320 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 321 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 322 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 323 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 324 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 325 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 326 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 327 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 328 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 329 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 330 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 331 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 332 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 333 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 334 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 335 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 336 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 337 }; 338 339 #define IXL_HMC_RXQ_MINSIZE (201 + 1) 340 341 struct ixl_hmc_txq { 342 uint16_t head; 343 uint8_t new_context; 344 uint64_t base; 345 #define IXL_HMC_TXQ_BASE_UNIT 128 346 uint8_t fc_ena; 347 uint8_t timesync_ena; 348 uint8_t fd_ena; 349 uint8_t alt_vlan_ena; 350 uint16_t thead_wb; 351 uint8_t cpuid; 352 uint8_t head_wb_ena; 353 #define IXL_HMC_TXQ_DESC_WB 0 354 #define IXL_HMC_TXQ_HEAD_WB 1 355 uint16_t qlen; 356 uint8_t tphrdesc_ena; 357 uint8_t tphrpacket_ena; 358 uint8_t tphwdesc_ena; 359 uint64_t head_wb_addr; 360 uint32_t crc; 361 uint16_t rdylist; 362 uint8_t rdylist_act; 363 }; 364 365 static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 366 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 367 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 368 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 369 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 370 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 371 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 372 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 373 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 374 /* line 1 */ 375 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 376 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 377 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 378 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 379 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 380 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 381 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 382 /* line 7 */ 383 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 384 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 385 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 386 }; 387 388 #define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 389 390 struct ixl_work { 391 struct work ixw_cookie; 392 void (*ixw_func)(void *); 393 void *ixw_arg; 394 unsigned int ixw_added; 395 }; 396 #define IXL_WORKQUEUE_PRI PRI_SOFTNET 397 398 struct ixl_tx_map { 399 struct mbuf *txm_m; 400 bus_dmamap_t txm_map; 401 unsigned int txm_eop; 402 }; 403 404 struct ixl_tx_ring { 405 kmutex_t txr_lock; 406 struct ixl_softc *txr_sc; 407 408 unsigned int txr_prod; 409 unsigned int txr_cons; 410 411 struct ixl_tx_map *txr_maps; 412 struct ixl_dmamem txr_mem; 413 414 bus_size_t txr_tail; 415 unsigned int txr_qid; 416 pcq_t *txr_intrq; 417 void *txr_si; 418 419 uint64_t txr_oerrors; /* if_oerrors */ 420 uint64_t txr_opackets; /* if_opackets */ 421 uint64_t txr_obytes; /* if_obytes */ 422 uint64_t txr_omcasts; /* if_omcasts */ 423 424 struct evcnt txr_defragged; 425 struct evcnt txr_defrag_failed; 426 struct evcnt txr_pcqdrop; 427 struct evcnt txr_transmitdef; 428 struct evcnt txr_intr; 429 struct evcnt txr_defer; 430 }; 431 432 struct ixl_rx_map { 433 struct mbuf *rxm_m; 434 bus_dmamap_t rxm_map; 435 }; 436 437 struct ixl_rx_ring { 438 kmutex_t rxr_lock; 439 440 unsigned int rxr_prod; 441 unsigned int rxr_cons; 442 443 struct ixl_rx_map *rxr_maps; 444 struct ixl_dmamem rxr_mem; 445 446 struct mbuf *rxr_m_head; 447 struct mbuf **rxr_m_tail; 448 449 bus_size_t rxr_tail; 450 unsigned int rxr_qid; 451 452 uint64_t rxr_ipackets; /* if_ipackets */ 453 uint64_t rxr_ibytes; /* if_ibytes */ 454 uint64_t rxr_iqdrops; /* iqdrops */ 455 uint64_t rxr_ierrors; /* if_ierrors */ 456 457 struct evcnt rxr_mgethdr_failed; 458 struct evcnt rxr_mgetcl_failed; 459 struct evcnt rxr_mbuf_load_failed; 460 struct evcnt rxr_intr; 461 struct evcnt rxr_defer; 462 }; 463 464 struct ixl_queue_pair { 465 struct ixl_softc *qp_sc; 466 struct ixl_tx_ring *qp_txr; 467 struct ixl_rx_ring *qp_rxr; 468 469 char qp_name[16]; 470 471 void *qp_si; 472 struct ixl_work qp_task; 473 bool qp_workqueue; 474 }; 475 476 struct ixl_atq { 477 struct ixl_aq_desc iatq_desc; 478 void (*iatq_fn)(struct ixl_softc *, 479 const struct ixl_aq_desc *); 480 }; 481 SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 482 483 struct ixl_product { 484 unsigned int vendor_id; 485 unsigned int product_id; 486 }; 487 488 /* 489 * Locking notes: 490 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 491 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 492 * - more than one lock of them cannot be held at once. 493 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 494 * (a spin mutex). 495 * - the lock cannot held with txr_lock or rxr_lock. 496 * + a field named sc_arq_* is not protected by any lock. 497 * - operations for sc_arq_* is done in one context related to 498 * sc_arq_task. 499 * + other fields in ixl_softc is protected by sc_cfg_lock 500 * (an adaptive mutex) 501 * - It must be held before another lock is held, and It can be 502 * released after the other lock is released. 503 * */ 504 505 struct ixl_softc { 506 device_t sc_dev; 507 struct ethercom sc_ec; 508 bool sc_attached; 509 bool sc_dead; 510 bool sc_rxctl_atq; 511 struct sysctllog *sc_sysctllog; 512 struct workqueue *sc_workq; 513 struct workqueue *sc_workq_txrx; 514 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 515 struct ifmedia sc_media; 516 uint64_t sc_media_status; 517 uint64_t sc_media_active; 518 kmutex_t sc_cfg_lock; 519 enum i40e_mac_type sc_mac_type; 520 uint32_t sc_rss_table_size; 521 uint32_t sc_rss_table_entry_width; 522 bool sc_txrx_workqueue; 523 u_int sc_tx_process_limit; 524 u_int sc_rx_process_limit; 525 u_int sc_tx_intr_process_limit; 526 u_int sc_rx_intr_process_limit; 527 528 int sc_cur_ec_capenable; 529 530 struct pci_attach_args sc_pa; 531 pci_intr_handle_t *sc_ihp; 532 void **sc_ihs; 533 unsigned int sc_nintrs; 534 535 bus_dma_tag_t sc_dmat; 536 bus_space_tag_t sc_memt; 537 bus_space_handle_t sc_memh; 538 bus_size_t sc_mems; 539 540 uint8_t sc_pf_id; 541 uint16_t sc_uplink_seid; /* le */ 542 uint16_t sc_downlink_seid; /* le */ 543 uint16_t sc_vsi_number; /* le */ 544 uint16_t sc_seid; 545 unsigned int sc_base_queue; 546 547 pci_intr_type_t sc_intrtype; 548 unsigned int sc_msix_vector_queue; 549 550 struct ixl_dmamem sc_scratch; 551 552 const struct ixl_aq_regs * 553 sc_aq_regs; 554 555 kmutex_t sc_atq_lock; 556 kcondvar_t sc_atq_cv; 557 struct ixl_dmamem sc_atq; 558 unsigned int sc_atq_prod; 559 unsigned int sc_atq_cons; 560 561 struct ixl_dmamem sc_arq; 562 struct ixl_work sc_arq_task; 563 struct ixl_aq_bufs sc_arq_idle; 564 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 565 unsigned int sc_arq_prod; 566 unsigned int sc_arq_cons; 567 568 struct ixl_work sc_link_state_task; 569 struct ixl_atq sc_link_state_atq; 570 571 struct ixl_dmamem sc_hmc_sd; 572 struct ixl_dmamem sc_hmc_pd; 573 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 574 575 unsigned int sc_tx_ring_ndescs; 576 unsigned int sc_rx_ring_ndescs; 577 unsigned int sc_nqueue_pairs; 578 unsigned int sc_nqueue_pairs_max; 579 unsigned int sc_nqueue_pairs_device; 580 struct ixl_queue_pair *sc_qps; 581 582 struct evcnt sc_event_atq; 583 struct evcnt sc_event_link; 584 struct evcnt sc_event_ecc_err; 585 struct evcnt sc_event_pci_exception; 586 struct evcnt sc_event_crit_err; 587 }; 588 589 #define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 590 #define IXL_TX_PROCESS_LIMIT 256 591 #define IXL_RX_PROCESS_LIMIT 256 592 #define IXL_TX_INTR_PROCESS_LIMIT 256 593 #define IXL_RX_INTR_PROCESS_LIMIT 0U 594 595 #define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx| \ 596 IFCAP_CSUM_TCPv4_Rx| \ 597 IFCAP_CSUM_UDPv4_Rx| \ 598 IFCAP_CSUM_TCPv6_Rx| \ 599 IFCAP_CSUM_UDPv6_Rx) 600 601 #define delaymsec(_x) DELAY(1000 * (_x)) 602 #ifdef IXL_DEBUG 603 #define DDPRINTF(sc, fmt, args...) \ 604 do { \ 605 if ((sc) != NULL) { \ 606 device_printf( \ 607 ((struct ixl_softc *)(sc))->sc_dev, \ 608 ""); \ 609 } \ 610 printf("%s:\t" fmt, __func__, ##args); \ 611 } while (0) 612 #else 613 #define DDPRINTF(sc, fmt, args...) __nothing 614 #endif 615 #define IXL_NOMSIX false 616 617 static enum i40e_mac_type 618 ixl_mactype(pci_product_id_t); 619 static void ixl_clear_hw(struct ixl_softc *); 620 static int ixl_pf_reset(struct ixl_softc *); 621 622 static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 623 bus_size_t, bus_size_t); 624 static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 625 626 static int ixl_arq_fill(struct ixl_softc *); 627 static void ixl_arq_unfill(struct ixl_softc *); 628 629 static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 630 unsigned int); 631 static void ixl_atq_set(struct ixl_atq *, 632 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 633 static int ixl_atq_post(struct ixl_softc *, struct ixl_atq *); 634 static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 635 static void ixl_atq_done(struct ixl_softc *); 636 static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 637 static int ixl_get_version(struct ixl_softc *); 638 static int ixl_get_hw_capabilities(struct ixl_softc *); 639 static int ixl_pxe_clear(struct ixl_softc *); 640 static int ixl_lldp_shut(struct ixl_softc *); 641 static int ixl_get_mac(struct ixl_softc *); 642 static int ixl_get_switch_config(struct ixl_softc *); 643 static int ixl_phy_mask_ints(struct ixl_softc *); 644 static int ixl_get_phy_types(struct ixl_softc *, uint64_t *); 645 static int ixl_restart_an(struct ixl_softc *); 646 static int ixl_hmc(struct ixl_softc *); 647 static void ixl_hmc_free(struct ixl_softc *); 648 static int ixl_get_vsi(struct ixl_softc *); 649 static int ixl_set_vsi(struct ixl_softc *); 650 static void ixl_set_filter_control(struct ixl_softc *); 651 static void ixl_get_link_status(void *); 652 static int ixl_get_link_status_poll(struct ixl_softc *); 653 static int ixl_set_link_status(struct ixl_softc *, 654 const struct ixl_aq_desc *); 655 static void ixl_config_rss(struct ixl_softc *); 656 static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 657 uint16_t, uint16_t); 658 static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *, 659 uint16_t, uint16_t); 660 static void ixl_arq(void *); 661 static void ixl_hmc_pack(void *, const void *, 662 const struct ixl_hmc_pack *, unsigned int); 663 static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 664 static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 665 666 static int ixl_match(device_t, cfdata_t, void *); 667 static void ixl_attach(device_t, device_t, void *); 668 static int ixl_detach(device_t, int); 669 670 static void ixl_media_add(struct ixl_softc *, uint64_t); 671 static int ixl_media_change(struct ifnet *); 672 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 673 static void ixl_watchdog(struct ifnet *); 674 static int ixl_ioctl(struct ifnet *, u_long, void *); 675 static void ixl_start(struct ifnet *); 676 static int ixl_transmit(struct ifnet *, struct mbuf *); 677 static void ixl_deferred_transmit(void *); 678 static int ixl_intr(void *); 679 static int ixl_queue_intr(void *); 680 static int ixl_other_intr(void *); 681 static void ixl_handle_queue(void *); 682 static void ixl_sched_handle_queue(struct ixl_softc *, 683 struct ixl_queue_pair *); 684 static int ixl_init(struct ifnet *); 685 static int ixl_init_locked(struct ixl_softc *); 686 static void ixl_stop(struct ifnet *, int); 687 static void ixl_stop_locked(struct ixl_softc *); 688 static int ixl_iff(struct ixl_softc *); 689 static int ixl_ifflags_cb(struct ethercom *); 690 static int ixl_setup_interrupts(struct ixl_softc *); 691 static int ixl_establish_intx(struct ixl_softc *); 692 static int ixl_establish_msix(struct ixl_softc *); 693 static void ixl_set_affinity_msix(struct ixl_softc *); 694 static void ixl_enable_queue_intr(struct ixl_softc *, 695 struct ixl_queue_pair *); 696 static void ixl_disable_queue_intr(struct ixl_softc *, 697 struct ixl_queue_pair *); 698 static void ixl_enable_other_intr(struct ixl_softc *); 699 static void ixl_disable_other_intr(struct ixl_softc *); 700 static void ixl_config_queue_intr(struct ixl_softc *); 701 static void ixl_config_other_intr(struct ixl_softc *); 702 703 static struct ixl_tx_ring * 704 ixl_txr_alloc(struct ixl_softc *, unsigned int); 705 static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 706 static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 707 static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 708 static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 709 static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 710 static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 711 static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 712 static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 713 714 static struct ixl_rx_ring * 715 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 716 static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 717 static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 718 static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 719 static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 720 static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 721 static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 722 static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 723 static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 724 725 static struct workqueue * 726 ixl_workq_create(const char *, pri_t, int, int); 727 static void ixl_workq_destroy(struct workqueue *); 728 static int ixl_workqs_teardown(device_t); 729 static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 730 static void ixl_work_add(struct workqueue *, struct ixl_work *); 731 static void ixl_work_wait(struct workqueue *, struct ixl_work *); 732 static void ixl_workq_work(struct work *, void *); 733 static const struct ixl_product * 734 ixl_lookup(const struct pci_attach_args *pa); 735 static void ixl_link_state_update(struct ixl_softc *, 736 const struct ixl_aq_desc *); 737 static int ixl_vlan_cb(struct ethercom *, uint16_t, bool); 738 static int ixl_setup_vlan_hwfilter(struct ixl_softc *); 739 static void ixl_teardown_vlan_hwfilter(struct ixl_softc *); 740 static int ixl_update_macvlan(struct ixl_softc *); 741 static int ixl_setup_interrupts(struct ixl_softc *);; 742 static void ixl_teardown_interrupts(struct ixl_softc *); 743 static int ixl_setup_stats(struct ixl_softc *); 744 static void ixl_teardown_stats(struct ixl_softc *); 745 static int ixl_setup_sysctls(struct ixl_softc *); 746 static void ixl_teardown_sysctls(struct ixl_softc *); 747 static int ixl_queue_pairs_alloc(struct ixl_softc *); 748 static void ixl_queue_pairs_free(struct ixl_softc *); 749 750 static const struct ixl_phy_type ixl_phy_type_map[] = { 751 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 752 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 753 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 754 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 755 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 756 { 1ULL << IXL_PHY_TYPE_XAUI | 757 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 758 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 759 { 1ULL << IXL_PHY_TYPE_XLAUI | 760 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 761 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 762 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 763 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 764 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 765 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 766 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 767 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 768 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 769 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 770 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 771 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 772 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 773 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 774 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 775 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 776 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 777 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 778 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 779 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 780 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 781 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 782 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 783 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 784 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_CR }, 785 }; 786 787 static const struct ixl_speed_type ixl_speed_type_map[] = { 788 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 789 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 790 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 791 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 792 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 793 }; 794 795 static const struct ixl_aq_regs ixl_pf_aq_regs = { 796 .atq_tail = I40E_PF_ATQT, 797 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 798 .atq_head = I40E_PF_ATQH, 799 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 800 .atq_len = I40E_PF_ATQLEN, 801 .atq_bal = I40E_PF_ATQBAL, 802 .atq_bah = I40E_PF_ATQBAH, 803 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 804 805 .arq_tail = I40E_PF_ARQT, 806 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 807 .arq_head = I40E_PF_ARQH, 808 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 809 .arq_len = I40E_PF_ARQLEN, 810 .arq_bal = I40E_PF_ARQBAL, 811 .arq_bah = I40E_PF_ARQBAH, 812 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 813 }; 814 815 #define ixl_rd(_s, _r) \ 816 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 817 #define ixl_wr(_s, _r, _v) \ 818 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 819 #define ixl_barrier(_s, _r, _l, _o) \ 820 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 821 #define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 822 #define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 823 824 static inline uint32_t 825 ixl_dmamem_hi(struct ixl_dmamem *ixm) 826 { 827 uint32_t retval; 828 uint64_t val; 829 830 if (sizeof(IXL_DMA_DVA(ixm)) > 4) { 831 val = (intptr_t)IXL_DMA_DVA(ixm); 832 retval = (uint32_t)(val >> 32); 833 } else { 834 retval = 0; 835 } 836 837 return retval; 838 } 839 840 static inline uint32_t 841 ixl_dmamem_lo(struct ixl_dmamem *ixm) 842 { 843 844 return (uint32_t)IXL_DMA_DVA(ixm); 845 } 846 847 static inline void 848 ixl_aq_dva(struct ixl_aq_desc *iaq, bus_addr_t addr) 849 { 850 uint64_t val; 851 852 if (sizeof(addr) > 4) { 853 val = (intptr_t)addr; 854 iaq->iaq_param[2] = htole32(val >> 32); 855 } else { 856 iaq->iaq_param[2] = htole32(0); 857 } 858 859 iaq->iaq_param[3] = htole32(addr); 860 } 861 862 static inline unsigned int 863 ixl_rxr_unrefreshed(unsigned int prod, unsigned int cons, unsigned int ndescs) 864 { 865 unsigned int num; 866 867 if (prod < cons) 868 num = cons - prod; 869 else 870 num = (ndescs - prod) + cons; 871 872 if (__predict_true(num > 0)) { 873 /* device cannot receive packets if all descripter is filled */ 874 num -= 1; 875 } 876 877 return num; 878 } 879 880 CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 881 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 882 DVF_DETACH_SHUTDOWN); 883 884 static const struct ixl_product ixl_products[] = { 885 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 886 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 887 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 888 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 889 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 890 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 891 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T }, 892 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 893 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 894 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 895 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 896 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 897 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 898 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 899 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 900 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 901 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 902 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 903 /* required last entry */ 904 {0, 0} 905 }; 906 907 static const struct ixl_product * 908 ixl_lookup(const struct pci_attach_args *pa) 909 { 910 const struct ixl_product *ixlp; 911 912 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 913 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 914 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 915 return ixlp; 916 } 917 918 return NULL; 919 } 920 921 static int 922 ixl_match(device_t parent, cfdata_t match, void *aux) 923 { 924 const struct pci_attach_args *pa = aux; 925 926 return (ixl_lookup(pa) != NULL) ? 1 : 0; 927 } 928 929 static void 930 ixl_attach(device_t parent, device_t self, void *aux) 931 { 932 struct ixl_softc *sc; 933 struct pci_attach_args *pa = aux; 934 struct ifnet *ifp; 935 pcireg_t memtype, reg; 936 uint32_t firstq, port, ari, func; 937 uint64_t phy_types = 0; 938 char xnamebuf[32]; 939 int tries, rv; 940 941 sc = device_private(self); 942 sc->sc_dev = self; 943 ifp = &sc->sc_ec.ec_if; 944 945 sc->sc_pa = *pa; 946 sc->sc_dmat = (pci_dma64_available(pa)) ? 947 pa->pa_dmat64 : pa->pa_dmat; 948 sc->sc_aq_regs = &ixl_pf_aq_regs; 949 950 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 951 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(reg)); 952 953 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 954 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 955 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 956 aprint_error(": unable to map registers\n"); 957 return; 958 } 959 960 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 961 962 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 963 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 964 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 965 sc->sc_base_queue = firstq; 966 967 ixl_clear_hw(sc); 968 if (ixl_pf_reset(sc) == -1) { 969 /* error printed by ixl pf_reset */ 970 goto unmap; 971 } 972 973 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 974 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 975 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 976 aprint_normal(": port %u", port); 977 978 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 979 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 980 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 981 982 func = ixl_rd(sc, I40E_PF_FUNC_RID); 983 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 984 985 /* initialise the adminq */ 986 987 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 988 989 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 990 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 991 aprint_error("\n" "%s: unable to allocate atq\n", 992 device_xname(self)); 993 goto unmap; 994 } 995 996 SIMPLEQ_INIT(&sc->sc_arq_idle); 997 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 998 sc->sc_arq_cons = 0; 999 sc->sc_arq_prod = 0; 1000 1001 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 1002 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1003 aprint_error("\n" "%s: unable to allocate arq\n", 1004 device_xname(self)); 1005 goto free_atq; 1006 } 1007 1008 if (!ixl_arq_fill(sc)) { 1009 aprint_error("\n" "%s: unable to fill arq descriptors\n", 1010 device_xname(self)); 1011 goto free_arq; 1012 } 1013 1014 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1015 0, IXL_DMA_LEN(&sc->sc_atq), 1016 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1017 1018 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1019 0, IXL_DMA_LEN(&sc->sc_arq), 1020 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1021 1022 for (tries = 0; tries < 10; tries++) { 1023 sc->sc_atq_cons = 0; 1024 sc->sc_atq_prod = 0; 1025 1026 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1027 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1028 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1029 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1030 1031 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1032 1033 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1034 ixl_dmamem_lo(&sc->sc_atq)); 1035 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1036 ixl_dmamem_hi(&sc->sc_atq)); 1037 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1038 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1039 1040 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1041 ixl_dmamem_lo(&sc->sc_arq)); 1042 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1043 ixl_dmamem_hi(&sc->sc_arq)); 1044 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1045 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1046 1047 rv = ixl_get_version(sc); 1048 if (rv == 0) 1049 break; 1050 if (rv != ETIMEDOUT) { 1051 aprint_error(", unable to get firmware version\n"); 1052 goto shutdown; 1053 } 1054 1055 delaymsec(100); 1056 } 1057 1058 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1059 1060 if (sc->sc_mac_type == I40E_MAC_X722) 1061 sc->sc_nqueue_pairs_device = 128; 1062 else 1063 sc->sc_nqueue_pairs_device = 64; 1064 1065 rv = ixl_get_hw_capabilities(sc); 1066 if (rv != 0) { 1067 aprint_error(", GET HW CAPABILITIES %s\n", 1068 rv == ETIMEDOUT ? "timeout" : "error"); 1069 goto shutdown; 1070 } 1071 1072 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max = 1073 MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1074 sc->sc_tx_ring_ndescs = 1024; 1075 sc->sc_rx_ring_ndescs = 1024; 1076 1077 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1078 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1079 1080 if (ixl_get_mac(sc) != 0) { 1081 /* error printed by ixl_get_mac */ 1082 goto shutdown; 1083 } 1084 1085 aprint_normal("\n"); 1086 aprint_naive("\n"); 1087 1088 aprint_normal_dev(self, "Ethernet address %s\n", 1089 ether_sprintf(sc->sc_enaddr)); 1090 1091 rv = ixl_pxe_clear(sc); 1092 if (rv != 0) { 1093 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1094 rv == ETIMEDOUT ? "timeout" : "error"); 1095 } 1096 1097 ixl_set_filter_control(sc); 1098 1099 if (ixl_hmc(sc) != 0) { 1100 /* error printed by ixl_hmc */ 1101 goto shutdown; 1102 } 1103 1104 if (ixl_lldp_shut(sc) != 0) { 1105 /* error printed by ixl_lldp_shut */ 1106 goto free_hmc; 1107 } 1108 1109 if (ixl_phy_mask_ints(sc) != 0) { 1110 /* error printed by ixl_phy_mask_ints */ 1111 goto free_hmc; 1112 } 1113 1114 if (ixl_restart_an(sc) != 0) { 1115 /* error printed by ixl_restart_an */ 1116 goto free_hmc; 1117 } 1118 1119 if (ixl_get_switch_config(sc) != 0) { 1120 /* error printed by ixl_get_switch_config */ 1121 goto free_hmc; 1122 } 1123 1124 if (ixl_get_phy_types(sc, &phy_types) != 0) { 1125 /* error printed by ixl_get_phy_abilities */ 1126 goto free_hmc; 1127 } 1128 1129 rv = ixl_get_link_status_poll(sc); 1130 if (rv != 0) { 1131 aprint_error_dev(self, "GET LINK STATUS %s\n", 1132 rv == ETIMEDOUT ? "timeout" : "error"); 1133 goto free_hmc; 1134 } 1135 1136 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1137 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1138 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1139 goto free_hmc; 1140 } 1141 1142 rv = ixl_get_vsi(sc); 1143 if (rv != 0) { 1144 aprint_error_dev(self, "GET VSI %s %d\n", 1145 rv == ETIMEDOUT ? "timeout" : "error", rv); 1146 goto free_scratch; 1147 } 1148 1149 rv = ixl_set_vsi(sc); 1150 if (rv != 0) { 1151 aprint_error_dev(self, "UPDATE VSI error %s %d\n", 1152 rv == ETIMEDOUT ? "timeout" : "error", rv); 1153 goto free_scratch; 1154 } 1155 1156 if (ixl_queue_pairs_alloc(sc) != 0) { 1157 /* error printed by ixl_queue_pairs_alloc */ 1158 goto free_scratch; 1159 } 1160 1161 if (ixl_setup_interrupts(sc) != 0) { 1162 /* error printed by ixl_setup_interrupts */ 1163 goto free_queue_pairs; 1164 } 1165 1166 if (ixl_setup_stats(sc) != 0) { 1167 aprint_error_dev(self, "failed to setup event counters\n"); 1168 goto teardown_intrs; 1169 } 1170 1171 if (ixl_setup_sysctls(sc) != 0) { 1172 /* error printed by ixl_setup_sysctls */ 1173 goto teardown_stats; 1174 } 1175 1176 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1177 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1178 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1179 if (sc->sc_workq == NULL) 1180 goto teardown_sysctls; 1181 1182 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1183 sc->sc_workq_txrx = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1184 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1185 if (sc->sc_workq_txrx == NULL) 1186 goto teardown_wqs; 1187 1188 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1189 cv_init(&sc->sc_atq_cv, xnamebuf); 1190 1191 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1192 1193 ifp->if_softc = sc; 1194 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1195 ifp->if_extflags = IFEF_MPSAFE; 1196 ifp->if_ioctl = ixl_ioctl; 1197 ifp->if_start = ixl_start; 1198 ifp->if_transmit = ixl_transmit; 1199 ifp->if_watchdog = ixl_watchdog; 1200 ifp->if_init = ixl_init; 1201 ifp->if_stop = ixl_stop; 1202 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1203 IFQ_SET_READY(&ifp->if_snd); 1204 ifp->if_capabilities |= IXL_IFCAP_RXCSUM; 1205 #if 0 1206 ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_UDPv4_Tx; 1207 #endif 1208 ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb); 1209 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_MTU; 1210 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1211 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1212 1213 sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities; 1214 /* Disable VLAN_HWFILTER by default */ 1215 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1216 1217 sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable; 1218 1219 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1220 ifmedia_init(&sc->sc_media, IFM_IMASK, ixl_media_change, 1221 ixl_media_status); 1222 1223 ixl_media_add(sc, phy_types); 1224 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1225 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1226 1227 if_attach(ifp); 1228 if_deferred_start_init(ifp, NULL); 1229 ether_ifattach(ifp, sc->sc_enaddr); 1230 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1231 1232 (void)ixl_get_link_status_poll(sc); 1233 ixl_work_set(&sc->sc_link_state_task, ixl_get_link_status, sc); 1234 1235 ixl_config_other_intr(sc); 1236 ixl_enable_other_intr(sc); 1237 1238 /* remove default mac filter and replace it so we can see vlans */ 1239 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 1240 if (rv != ENOENT) { 1241 aprint_debug_dev(self, 1242 "unable to remove macvlan %u\n", rv); 1243 } 1244 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1245 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1246 if (rv != ENOENT) { 1247 aprint_debug_dev(self, 1248 "unable to remove macvlan, ignore vlan %u\n", rv); 1249 } 1250 1251 if (ixl_update_macvlan(sc) != 0) { 1252 aprint_debug_dev(self, 1253 "couldn't enable vlan hardware filter\n"); 1254 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1255 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1256 } 1257 1258 sc->sc_txrx_workqueue = true; 1259 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1260 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1261 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1262 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1263 1264 if (pmf_device_register(self, NULL, NULL) != true) 1265 aprint_debug_dev(self, "couldn't establish power handler\n"); 1266 sc->sc_attached = true; 1267 return; 1268 1269 teardown_wqs: 1270 config_finalize_register(self, ixl_workqs_teardown); 1271 teardown_sysctls: 1272 ixl_teardown_sysctls(sc); 1273 teardown_stats: 1274 ixl_teardown_stats(sc); 1275 teardown_intrs: 1276 ixl_teardown_interrupts(sc); 1277 free_queue_pairs: 1278 ixl_queue_pairs_free(sc); 1279 free_scratch: 1280 ixl_dmamem_free(sc, &sc->sc_scratch); 1281 free_hmc: 1282 ixl_hmc_free(sc); 1283 shutdown: 1284 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1285 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1286 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1287 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1288 1289 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1290 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1291 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1292 1293 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1294 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1295 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1296 1297 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1298 0, IXL_DMA_LEN(&sc->sc_arq), 1299 BUS_DMASYNC_POSTREAD); 1300 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1301 0, IXL_DMA_LEN(&sc->sc_atq), 1302 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1303 1304 ixl_arq_unfill(sc); 1305 free_arq: 1306 ixl_dmamem_free(sc, &sc->sc_arq); 1307 free_atq: 1308 ixl_dmamem_free(sc, &sc->sc_atq); 1309 unmap: 1310 mutex_destroy(&sc->sc_atq_lock); 1311 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1312 mutex_destroy(&sc->sc_cfg_lock); 1313 sc->sc_mems = 0; 1314 1315 sc->sc_attached = false; 1316 } 1317 1318 static int 1319 ixl_detach(device_t self, int flags) 1320 { 1321 struct ixl_softc *sc = device_private(self); 1322 struct ifnet *ifp = &sc->sc_ec.ec_if; 1323 1324 if (!sc->sc_attached) 1325 return 0; 1326 1327 ixl_stop(ifp, 1); 1328 1329 ixl_disable_other_intr(sc); 1330 1331 /* wait for ATQ handler */ 1332 mutex_enter(&sc->sc_atq_lock); 1333 mutex_exit(&sc->sc_atq_lock); 1334 1335 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1336 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1337 1338 if (sc->sc_workq != NULL) { 1339 ixl_workq_destroy(sc->sc_workq); 1340 sc->sc_workq = NULL; 1341 } 1342 1343 if (sc->sc_workq_txrx != NULL) { 1344 ixl_workq_destroy(sc->sc_workq_txrx); 1345 sc->sc_workq_txrx = NULL; 1346 } 1347 1348 ifmedia_delete_instance(&sc->sc_media, IFM_INST_ANY); 1349 ether_ifdetach(ifp); 1350 if_detach(ifp); 1351 1352 ixl_teardown_interrupts(sc); 1353 ixl_teardown_stats(sc); 1354 1355 ixl_queue_pairs_free(sc); 1356 1357 ixl_dmamem_free(sc, &sc->sc_scratch); 1358 ixl_hmc_free(sc); 1359 1360 /* shutdown */ 1361 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1362 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1363 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1364 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1365 1366 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1367 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1368 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1369 1370 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1371 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1372 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1373 1374 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1375 0, IXL_DMA_LEN(&sc->sc_arq), 1376 BUS_DMASYNC_POSTREAD); 1377 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1378 0, IXL_DMA_LEN(&sc->sc_atq), 1379 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1380 1381 ixl_arq_unfill(sc); 1382 1383 ixl_dmamem_free(sc, &sc->sc_arq); 1384 ixl_dmamem_free(sc, &sc->sc_atq); 1385 1386 cv_destroy(&sc->sc_atq_cv); 1387 mutex_destroy(&sc->sc_atq_lock); 1388 1389 if (sc->sc_mems != 0) { 1390 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1391 sc->sc_mems = 0; 1392 } 1393 1394 mutex_destroy(&sc->sc_cfg_lock); 1395 1396 return 0; 1397 } 1398 1399 static int 1400 ixl_workqs_teardown(device_t self) 1401 { 1402 struct ixl_softc *sc = device_private(self); 1403 1404 if (sc->sc_workq != NULL) { 1405 ixl_workq_destroy(sc->sc_workq); 1406 sc->sc_workq = NULL; 1407 } 1408 1409 if (sc->sc_workq_txrx != NULL) { 1410 ixl_workq_destroy(sc->sc_workq_txrx); 1411 sc->sc_workq_txrx = NULL; 1412 } 1413 1414 return 0; 1415 } 1416 1417 static int 1418 ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 1419 { 1420 struct ifnet *ifp = &ec->ec_if; 1421 struct ixl_softc *sc = ifp->if_softc; 1422 int rv; 1423 1424 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1425 return 0; 1426 } 1427 1428 if (set) { 1429 rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid, 1430 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1431 if (rv == 0) { 1432 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 1433 vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1434 } 1435 } else { 1436 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid, 1437 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1438 (void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid, 1439 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1440 } 1441 1442 return rv; 1443 } 1444 1445 static void 1446 ixl_media_add(struct ixl_softc *sc, uint64_t phy_types) 1447 { 1448 struct ifmedia *ifm = &sc->sc_media; 1449 const struct ixl_phy_type *itype; 1450 unsigned int i; 1451 1452 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1453 itype = &ixl_phy_type_map[i]; 1454 1455 if (ISSET(phy_types, itype->phy_type)) { 1456 ifmedia_add(ifm, 1457 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1458 1459 if (itype->ifm_type == IFM_100_TX) { 1460 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1461 0, NULL); 1462 } 1463 } 1464 } 1465 } 1466 1467 static void 1468 ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1469 { 1470 struct ixl_softc *sc = ifp->if_softc; 1471 1472 ifmr->ifm_status = sc->sc_media_status; 1473 ifmr->ifm_active = sc->sc_media_active; 1474 1475 mutex_enter(&sc->sc_cfg_lock); 1476 if (ifp->if_link_state == LINK_STATE_UP) 1477 SET(ifmr->ifm_status, IFM_ACTIVE); 1478 mutex_exit(&sc->sc_cfg_lock); 1479 } 1480 1481 static int 1482 ixl_media_change(struct ifnet *ifp) 1483 { 1484 1485 return 0; 1486 } 1487 1488 static void 1489 ixl_watchdog(struct ifnet *ifp) 1490 { 1491 1492 } 1493 1494 static void 1495 ixl_del_all_multiaddr(struct ixl_softc *sc) 1496 { 1497 struct ethercom *ec = &sc->sc_ec; 1498 struct ether_multi *enm; 1499 struct ether_multistep step; 1500 1501 ETHER_LOCK(ec); 1502 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1503 ETHER_NEXT_MULTI(step, enm)) { 1504 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1505 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1506 } 1507 ETHER_UNLOCK(ec); 1508 } 1509 1510 static int 1511 ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1512 { 1513 struct ifnet *ifp = &sc->sc_ec.ec_if; 1514 int rv; 1515 1516 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1517 return 0; 1518 1519 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1520 ixl_del_all_multiaddr(sc); 1521 SET(ifp->if_flags, IFF_ALLMULTI); 1522 return ENETRESET; 1523 } 1524 1525 /* multicast address can not use VLAN HWFILTER */ 1526 rv = ixl_add_macvlan(sc, addrlo, 0, 1527 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1528 1529 if (rv == ENOSPC) { 1530 ixl_del_all_multiaddr(sc); 1531 SET(ifp->if_flags, IFF_ALLMULTI); 1532 return ENETRESET; 1533 } 1534 1535 return rv; 1536 } 1537 1538 static int 1539 ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1540 { 1541 struct ifnet *ifp = &sc->sc_ec.ec_if; 1542 struct ethercom *ec = &sc->sc_ec; 1543 struct ether_multi *enm, *enm_last; 1544 struct ether_multistep step; 1545 int error, rv = 0; 1546 1547 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1548 ixl_remove_macvlan(sc, addrlo, 0, 1549 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1550 return 0; 1551 } 1552 1553 ETHER_LOCK(ec); 1554 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1555 ETHER_NEXT_MULTI(step, enm)) { 1556 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1557 ETHER_ADDR_LEN) != 0) { 1558 goto out; 1559 } 1560 } 1561 1562 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1563 ETHER_NEXT_MULTI(step, enm)) { 1564 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1565 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1566 if (error != 0) 1567 break; 1568 } 1569 1570 if (enm != NULL) { 1571 enm_last = enm; 1572 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1573 ETHER_NEXT_MULTI(step, enm)) { 1574 if (enm == enm_last) 1575 break; 1576 1577 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1578 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1579 } 1580 } else { 1581 CLR(ifp->if_flags, IFF_ALLMULTI); 1582 rv = ENETRESET; 1583 } 1584 1585 out: 1586 ETHER_UNLOCK(ec); 1587 return rv; 1588 } 1589 1590 static int 1591 ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1592 { 1593 struct ifreq *ifr = (struct ifreq *)data; 1594 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1595 struct ixl_tx_ring *txr; 1596 struct ixl_rx_ring *rxr; 1597 const struct sockaddr *sa; 1598 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1599 int s, error = 0; 1600 unsigned int i; 1601 1602 switch (cmd) { 1603 case SIOCADDMULTI: 1604 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1605 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1606 error = ether_multiaddr(sa, addrlo, addrhi); 1607 if (error != 0) 1608 return error; 1609 1610 error = ixl_add_multi(sc, addrlo, addrhi); 1611 if (error != 0 && error != ENETRESET) { 1612 ether_delmulti(sa, &sc->sc_ec); 1613 error = EIO; 1614 } 1615 } 1616 break; 1617 1618 case SIOCDELMULTI: 1619 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1620 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1621 error = ether_multiaddr(sa, addrlo, addrhi); 1622 if (error != 0) 1623 return error; 1624 1625 error = ixl_del_multi(sc, addrlo, addrhi); 1626 } 1627 break; 1628 1629 case SIOCGIFDATA: 1630 case SIOCZIFDATA: 1631 ifp->if_ipackets = 0; 1632 ifp->if_ibytes = 0; 1633 ifp->if_iqdrops = 0; 1634 ifp->if_ierrors = 0; 1635 ifp->if_opackets = 0; 1636 ifp->if_obytes = 0; 1637 ifp->if_omcasts = 0; 1638 1639 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 1640 txr = sc->sc_qps[i].qp_txr; 1641 rxr = sc->sc_qps[i].qp_rxr; 1642 1643 mutex_enter(&rxr->rxr_lock); 1644 ifp->if_ipackets += rxr->rxr_ipackets; 1645 ifp->if_ibytes += rxr->rxr_ibytes; 1646 ifp->if_iqdrops += rxr->rxr_iqdrops; 1647 ifp->if_ierrors += rxr->rxr_ierrors; 1648 if (cmd == SIOCZIFDATA) { 1649 rxr->rxr_ipackets = 0; 1650 rxr->rxr_ibytes = 0; 1651 rxr->rxr_iqdrops = 0; 1652 rxr->rxr_ierrors = 0; 1653 } 1654 mutex_exit(&rxr->rxr_lock); 1655 1656 mutex_enter(&txr->txr_lock); 1657 ifp->if_opackets += txr->txr_opackets; 1658 ifp->if_obytes += txr->txr_opackets; 1659 ifp->if_omcasts += txr->txr_omcasts; 1660 if (cmd == SIOCZIFDATA) { 1661 txr->txr_opackets = 0; 1662 txr->txr_opackets = 0; 1663 txr->txr_omcasts = 0; 1664 } 1665 mutex_exit(&txr->txr_lock); 1666 } 1667 /* FALLTHROUGH */ 1668 default: 1669 s = splnet(); 1670 error = ether_ioctl(ifp, cmd, data); 1671 splx(s); 1672 } 1673 1674 if (error == ENETRESET) 1675 error = ixl_iff(sc); 1676 1677 return error; 1678 } 1679 1680 static enum i40e_mac_type 1681 ixl_mactype(pci_product_id_t id) 1682 { 1683 1684 switch (id) { 1685 case PCI_PRODUCT_INTEL_XL710_SFP: 1686 case PCI_PRODUCT_INTEL_XL710_KX_B: 1687 case PCI_PRODUCT_INTEL_XL710_KX_C: 1688 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1689 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1690 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1691 case PCI_PRODUCT_INTEL_X710_10G_T: 1692 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1693 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1694 case PCI_PRODUCT_INTEL_X710_T4_10G: 1695 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1696 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1697 return I40E_MAC_XL710; 1698 1699 case PCI_PRODUCT_INTEL_X722_KX: 1700 case PCI_PRODUCT_INTEL_X722_QSFP: 1701 case PCI_PRODUCT_INTEL_X722_SFP: 1702 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1703 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1704 case PCI_PRODUCT_INTEL_X722_I_SFP: 1705 return I40E_MAC_X722; 1706 } 1707 1708 return I40E_MAC_GENERIC; 1709 } 1710 1711 static inline void * 1712 ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1713 { 1714 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1715 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1716 1717 if (i >= e->hmc_count) 1718 return NULL; 1719 1720 kva += e->hmc_base; 1721 kva += i * e->hmc_size; 1722 1723 return kva; 1724 } 1725 1726 static inline size_t 1727 ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1728 { 1729 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1730 1731 return e->hmc_size; 1732 } 1733 1734 static void 1735 ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1736 { 1737 struct ixl_rx_ring *rxr = qp->qp_rxr; 1738 1739 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1740 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1741 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1742 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1743 ixl_flush(sc); 1744 } 1745 1746 static void 1747 ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1748 { 1749 struct ixl_rx_ring *rxr = qp->qp_rxr; 1750 1751 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1752 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1753 ixl_flush(sc); 1754 } 1755 1756 static void 1757 ixl_enable_other_intr(struct ixl_softc *sc) 1758 { 1759 1760 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1761 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1762 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1763 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1764 ixl_flush(sc); 1765 } 1766 1767 static void 1768 ixl_disable_other_intr(struct ixl_softc *sc) 1769 { 1770 1771 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1772 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1773 ixl_flush(sc); 1774 } 1775 1776 static int 1777 ixl_reinit(struct ixl_softc *sc) 1778 { 1779 struct ixl_rx_ring *rxr; 1780 struct ixl_tx_ring *txr; 1781 unsigned int i; 1782 uint32_t reg; 1783 1784 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1785 1786 if (ixl_get_vsi(sc) != 0) 1787 return EIO; 1788 1789 if (ixl_set_vsi(sc) != 0) 1790 return EIO; 1791 1792 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1793 txr = sc->sc_qps[i].qp_txr; 1794 rxr = sc->sc_qps[i].qp_rxr; 1795 1796 txr->txr_cons = txr->txr_prod = 0; 1797 rxr->rxr_cons = rxr->rxr_prod = 0; 1798 1799 ixl_txr_config(sc, txr); 1800 ixl_rxr_config(sc, rxr); 1801 } 1802 1803 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1804 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 1805 1806 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1807 txr = sc->sc_qps[i].qp_txr; 1808 rxr = sc->sc_qps[i].qp_rxr; 1809 1810 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 1811 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 1812 ixl_flush(sc); 1813 1814 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 1815 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 1816 1817 /* ixl_rxfill() needs lock held */ 1818 mutex_enter(&rxr->rxr_lock); 1819 ixl_rxfill(sc, rxr); 1820 mutex_exit(&rxr->rxr_lock); 1821 1822 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 1823 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 1824 ixl_wr(sc, I40E_QRX_ENA(i), reg); 1825 if (ixl_rxr_enabled(sc, rxr) != 0) 1826 goto stop; 1827 1828 ixl_txr_qdis(sc, txr, 1); 1829 1830 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 1831 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 1832 ixl_wr(sc, I40E_QTX_ENA(i), reg); 1833 1834 if (ixl_txr_enabled(sc, txr) != 0) 1835 goto stop; 1836 } 1837 1838 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1839 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 1840 1841 return 0; 1842 1843 stop: 1844 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1845 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 1846 1847 return ETIMEDOUT; 1848 } 1849 1850 static int 1851 ixl_init_locked(struct ixl_softc *sc) 1852 { 1853 struct ifnet *ifp = &sc->sc_ec.ec_if; 1854 unsigned int i; 1855 int error, eccap_change; 1856 1857 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1858 1859 if (ISSET(ifp->if_flags, IFF_RUNNING)) 1860 ixl_stop_locked(sc); 1861 1862 if (sc->sc_dead) { 1863 return ENXIO; 1864 } 1865 1866 eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable; 1867 if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING)) 1868 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 1869 1870 if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) { 1871 if (ixl_update_macvlan(sc) == 0) { 1872 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 1873 } else { 1874 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1875 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1876 } 1877 } 1878 1879 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 1880 sc->sc_nqueue_pairs = 1; 1881 else 1882 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1883 1884 error = ixl_reinit(sc); 1885 if (error) { 1886 ixl_stop_locked(sc); 1887 return error; 1888 } 1889 1890 SET(ifp->if_flags, IFF_RUNNING); 1891 CLR(ifp->if_flags, IFF_OACTIVE); 1892 1893 (void)ixl_get_link_status(sc); 1894 1895 ixl_config_rss(sc); 1896 ixl_config_queue_intr(sc); 1897 1898 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1899 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 1900 } 1901 1902 error = ixl_iff(sc); 1903 if (error) { 1904 ixl_stop_locked(sc); 1905 return error; 1906 } 1907 1908 return 0; 1909 } 1910 1911 static int 1912 ixl_init(struct ifnet *ifp) 1913 { 1914 struct ixl_softc *sc = ifp->if_softc; 1915 int error; 1916 1917 mutex_enter(&sc->sc_cfg_lock); 1918 error = ixl_init_locked(sc); 1919 mutex_exit(&sc->sc_cfg_lock); 1920 1921 return error; 1922 } 1923 1924 static int 1925 ixl_iff(struct ixl_softc *sc) 1926 { 1927 struct ifnet *ifp = &sc->sc_ec.ec_if; 1928 struct ixl_atq iatq; 1929 struct ixl_aq_desc *iaq; 1930 struct ixl_aq_vsi_promisc_param *param; 1931 uint16_t flag_add, flag_del; 1932 int error; 1933 1934 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 1935 return 0; 1936 1937 memset(&iatq, 0, sizeof(iatq)); 1938 1939 iaq = &iatq.iatq_desc; 1940 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 1941 1942 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 1943 param->flags = htole16(0); 1944 1945 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER) 1946 || ISSET(ifp->if_flags, IFF_PROMISC)) { 1947 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 1948 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 1949 } 1950 1951 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 1952 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 1953 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 1954 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1955 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 1956 } 1957 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 1958 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 1959 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 1960 param->seid = sc->sc_seid; 1961 1962 error = ixl_atq_exec(sc, &iatq); 1963 if (error) 1964 return error; 1965 1966 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 1967 return EIO; 1968 1969 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 1970 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1971 flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH; 1972 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH; 1973 } else { 1974 flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN; 1975 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN; 1976 } 1977 1978 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del); 1979 1980 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 1981 ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add); 1982 } 1983 return 0; 1984 } 1985 1986 static void 1987 ixl_stop_rendezvous(struct ixl_softc *sc) 1988 { 1989 struct ixl_tx_ring *txr; 1990 struct ixl_rx_ring *rxr; 1991 unsigned int i; 1992 1993 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1994 txr = sc->sc_qps[i].qp_txr; 1995 rxr = sc->sc_qps[i].qp_rxr; 1996 1997 mutex_enter(&txr->txr_lock); 1998 mutex_exit(&txr->txr_lock); 1999 2000 mutex_enter(&rxr->rxr_lock); 2001 mutex_exit(&rxr->rxr_lock); 2002 2003 ixl_work_wait(sc->sc_workq_txrx, 2004 &sc->sc_qps[i].qp_task); 2005 } 2006 } 2007 2008 static void 2009 ixl_stop_locked(struct ixl_softc *sc) 2010 { 2011 struct ifnet *ifp = &sc->sc_ec.ec_if; 2012 struct ixl_rx_ring *rxr; 2013 struct ixl_tx_ring *txr; 2014 unsigned int i; 2015 uint32_t reg; 2016 2017 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2018 2019 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 2020 2021 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2022 txr = sc->sc_qps[i].qp_txr; 2023 rxr = sc->sc_qps[i].qp_rxr; 2024 2025 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 2026 2027 mutex_enter(&txr->txr_lock); 2028 ixl_txr_qdis(sc, txr, 0); 2029 /* XXX wait at least 400 usec for all tx queues in one go */ 2030 ixl_flush(sc); 2031 DELAY(500); 2032 2033 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2034 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2035 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2036 /* XXX wait 50ms from completaion of the TX queue disable*/ 2037 ixl_flush(sc); 2038 DELAY(50); 2039 2040 if (ixl_txr_disabled(sc, txr) != 0) { 2041 mutex_exit(&txr->txr_lock); 2042 goto die; 2043 } 2044 mutex_exit(&txr->txr_lock); 2045 2046 mutex_enter(&rxr->rxr_lock); 2047 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2048 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2049 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2050 /* XXX wait 50ms from completion of the RX queue disable */ 2051 ixl_flush(sc); 2052 DELAY(50); 2053 2054 if (ixl_rxr_disabled(sc, rxr) != 0) { 2055 mutex_exit(&rxr->rxr_lock); 2056 goto die; 2057 } 2058 mutex_exit(&rxr->rxr_lock); 2059 } 2060 2061 ixl_stop_rendezvous(sc); 2062 2063 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2064 txr = sc->sc_qps[i].qp_txr; 2065 rxr = sc->sc_qps[i].qp_rxr; 2066 2067 ixl_txr_unconfig(sc, txr); 2068 ixl_rxr_unconfig(sc, rxr); 2069 2070 ixl_txr_clean(sc, txr); 2071 ixl_rxr_clean(sc, rxr); 2072 } 2073 2074 return; 2075 die: 2076 sc->sc_dead = true; 2077 log(LOG_CRIT, "%s: failed to shut down rings", 2078 device_xname(sc->sc_dev)); 2079 return; 2080 } 2081 2082 static void 2083 ixl_stop(struct ifnet *ifp, int disable) 2084 { 2085 struct ixl_softc *sc = ifp->if_softc; 2086 2087 mutex_enter(&sc->sc_cfg_lock); 2088 ixl_stop_locked(sc); 2089 mutex_exit(&sc->sc_cfg_lock); 2090 } 2091 2092 static int 2093 ixl_queue_pairs_alloc(struct ixl_softc *sc) 2094 { 2095 struct ixl_queue_pair *qp; 2096 unsigned int i; 2097 size_t sz; 2098 2099 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2100 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 2101 2102 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2103 qp = &sc->sc_qps[i]; 2104 2105 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2106 ixl_handle_queue, qp); 2107 if (qp->qp_si == NULL) 2108 goto free; 2109 2110 qp->qp_txr = ixl_txr_alloc(sc, i); 2111 if (qp->qp_txr == NULL) 2112 goto free; 2113 2114 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2115 if (qp->qp_rxr == NULL) 2116 goto free; 2117 2118 qp->qp_sc = sc; 2119 ixl_work_set(&qp->qp_task, ixl_handle_queue, qp); 2120 snprintf(qp->qp_name, sizeof(qp->qp_name), 2121 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2122 } 2123 2124 return 0; 2125 free: 2126 if (sc->sc_qps != NULL) { 2127 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2128 qp = &sc->sc_qps[i]; 2129 2130 if (qp->qp_txr != NULL) 2131 ixl_txr_free(sc, qp->qp_txr); 2132 if (qp->qp_rxr != NULL) 2133 ixl_rxr_free(sc, qp->qp_rxr); 2134 if (qp->qp_si != NULL) 2135 softint_disestablish(qp->qp_si); 2136 } 2137 2138 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2139 kmem_free(sc->sc_qps, sz); 2140 sc->sc_qps = NULL; 2141 } 2142 2143 return -1; 2144 } 2145 2146 static void 2147 ixl_queue_pairs_free(struct ixl_softc *sc) 2148 { 2149 struct ixl_queue_pair *qp; 2150 unsigned int i; 2151 size_t sz; 2152 2153 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2154 qp = &sc->sc_qps[i]; 2155 ixl_txr_free(sc, qp->qp_txr); 2156 ixl_rxr_free(sc, qp->qp_rxr); 2157 softint_disestablish(qp->qp_si); 2158 } 2159 2160 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2161 kmem_free(sc->sc_qps, sz); 2162 sc->sc_qps = NULL; 2163 } 2164 2165 static struct ixl_tx_ring * 2166 ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2167 { 2168 struct ixl_tx_ring *txr = NULL; 2169 struct ixl_tx_map *maps = NULL, *txm; 2170 unsigned int i; 2171 2172 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2173 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2174 KM_SLEEP); 2175 2176 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2177 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2178 IXL_TX_QUEUE_ALIGN) != 0) 2179 goto free; 2180 2181 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2182 txm = &maps[i]; 2183 2184 if (bus_dmamap_create(sc->sc_dmat, 2185 IXL_HARDMTU, IXL_TX_PKT_DESCS, IXL_HARDMTU, 0, 2186 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2187 goto uncreate; 2188 2189 txm->txm_eop = -1; 2190 txm->txm_m = NULL; 2191 } 2192 2193 txr->txr_cons = txr->txr_prod = 0; 2194 txr->txr_maps = maps; 2195 2196 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2197 if (txr->txr_intrq == NULL) 2198 goto uncreate; 2199 2200 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2201 ixl_deferred_transmit, txr); 2202 if (txr->txr_si == NULL) 2203 goto destroy_pcq; 2204 2205 txr->txr_tail = I40E_QTX_TAIL(qid); 2206 txr->txr_qid = qid; 2207 txr->txr_sc = sc; 2208 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2209 2210 return txr; 2211 2212 destroy_pcq: 2213 pcq_destroy(txr->txr_intrq); 2214 uncreate: 2215 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2216 txm = &maps[i]; 2217 2218 if (txm->txm_map == NULL) 2219 continue; 2220 2221 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2222 } 2223 2224 ixl_dmamem_free(sc, &txr->txr_mem); 2225 free: 2226 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2227 kmem_free(txr, sizeof(*txr)); 2228 2229 return NULL; 2230 } 2231 2232 static void 2233 ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2234 { 2235 unsigned int qid; 2236 bus_size_t reg; 2237 uint32_t r; 2238 2239 qid = txr->txr_qid + sc->sc_base_queue; 2240 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2241 qid %= 128; 2242 2243 r = ixl_rd(sc, reg); 2244 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2245 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2246 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2247 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2248 ixl_wr(sc, reg, r); 2249 } 2250 2251 static void 2252 ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2253 { 2254 struct ixl_hmc_txq txq; 2255 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2256 void *hmc; 2257 2258 memset(&txq, 0, sizeof(txq)); 2259 txq.head = htole16(txr->txr_cons); 2260 txq.new_context = 1; 2261 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2262 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2263 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2264 txq.tphrdesc_ena = 0; 2265 txq.tphrpacket_ena = 0; 2266 txq.tphwdesc_ena = 0; 2267 txq.rdylist = data->qs_handle[0]; 2268 2269 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2270 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2271 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2272 __arraycount(ixl_hmc_pack_txq)); 2273 } 2274 2275 static void 2276 ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2277 { 2278 void *hmc; 2279 2280 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2281 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2282 } 2283 2284 static void 2285 ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2286 { 2287 struct ixl_tx_map *maps, *txm; 2288 bus_dmamap_t map; 2289 unsigned int i; 2290 2291 maps = txr->txr_maps; 2292 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2293 txm = &maps[i]; 2294 2295 if (txm->txm_m == NULL) 2296 continue; 2297 2298 map = txm->txm_map; 2299 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2300 BUS_DMASYNC_POSTWRITE); 2301 bus_dmamap_unload(sc->sc_dmat, map); 2302 2303 m_freem(txm->txm_m); 2304 txm->txm_m = NULL; 2305 } 2306 } 2307 2308 static int 2309 ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2310 { 2311 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2312 uint32_t reg; 2313 int i; 2314 2315 for (i = 0; i < 10; i++) { 2316 reg = ixl_rd(sc, ena); 2317 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2318 return 0; 2319 2320 delaymsec(10); 2321 } 2322 2323 return ETIMEDOUT; 2324 } 2325 2326 static int 2327 ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2328 { 2329 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2330 uint32_t reg; 2331 int i; 2332 2333 KASSERT(mutex_owned(&txr->txr_lock)); 2334 2335 for (i = 0; i < 20; i++) { 2336 reg = ixl_rd(sc, ena); 2337 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2338 return 0; 2339 2340 delaymsec(10); 2341 } 2342 2343 return ETIMEDOUT; 2344 } 2345 2346 static void 2347 ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2348 { 2349 struct ixl_tx_map *maps, *txm; 2350 struct mbuf *m; 2351 unsigned int i; 2352 2353 softint_disestablish(txr->txr_si); 2354 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2355 m_freem(m); 2356 pcq_destroy(txr->txr_intrq); 2357 2358 maps = txr->txr_maps; 2359 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2360 txm = &maps[i]; 2361 2362 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2363 } 2364 2365 ixl_dmamem_free(sc, &txr->txr_mem); 2366 mutex_destroy(&txr->txr_lock); 2367 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2368 kmem_free(txr, sizeof(*txr)); 2369 } 2370 2371 static inline int 2372 ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2373 struct ixl_tx_ring *txr) 2374 { 2375 struct mbuf *m; 2376 int error; 2377 2378 KASSERT(mutex_owned(&txr->txr_lock)); 2379 2380 m = *m0; 2381 2382 error = bus_dmamap_load_mbuf(dmat, map, m, 2383 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2384 if (error != EFBIG) 2385 return error; 2386 2387 m = m_defrag(m, M_DONTWAIT); 2388 if (m != NULL) { 2389 *m0 = m; 2390 txr->txr_defragged.ev_count++; 2391 2392 error = bus_dmamap_load_mbuf(dmat, map, m, 2393 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2394 } else { 2395 txr->txr_defrag_failed.ev_count++; 2396 error = ENOBUFS; 2397 } 2398 2399 return error; 2400 } 2401 2402 static void 2403 ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2404 bool is_transmit) 2405 { 2406 struct ixl_softc *sc = ifp->if_softc; 2407 struct ixl_tx_desc *ring, *txd; 2408 struct ixl_tx_map *txm; 2409 bus_dmamap_t map; 2410 struct mbuf *m; 2411 uint64_t cmd, cmd_vlan; 2412 unsigned int prod, free, last, i; 2413 unsigned int mask; 2414 int post = 0; 2415 2416 KASSERT(mutex_owned(&txr->txr_lock)); 2417 2418 if (ifp->if_link_state != LINK_STATE_UP 2419 || !ISSET(ifp->if_flags, IFF_RUNNING) 2420 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2421 if (!is_transmit) 2422 IFQ_PURGE(&ifp->if_snd); 2423 return; 2424 } 2425 2426 prod = txr->txr_prod; 2427 free = txr->txr_cons; 2428 if (free <= prod) 2429 free += sc->sc_tx_ring_ndescs; 2430 free -= prod; 2431 2432 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2433 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2434 2435 ring = IXL_DMA_KVA(&txr->txr_mem); 2436 mask = sc->sc_tx_ring_ndescs - 1; 2437 last = prod; 2438 cmd = 0; 2439 txd = NULL; 2440 2441 for (;;) { 2442 if (free <= IXL_TX_PKT_DESCS) { 2443 if (!is_transmit) 2444 SET(ifp->if_flags, IFF_OACTIVE); 2445 break; 2446 } 2447 2448 if (is_transmit) 2449 m = pcq_get(txr->txr_intrq); 2450 else 2451 IFQ_DEQUEUE(&ifp->if_snd, m); 2452 2453 if (m == NULL) 2454 break; 2455 2456 txm = &txr->txr_maps[prod]; 2457 map = txm->txm_map; 2458 2459 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2460 txr->txr_oerrors++; 2461 m_freem(m); 2462 continue; 2463 } 2464 2465 if (vlan_has_tag(m)) { 2466 cmd_vlan = (uint64_t)vlan_get_tag(m) << 2467 IXL_TX_DESC_L2TAG1_SHIFT; 2468 cmd_vlan |= IXL_TX_DESC_CMD_IL2TAG1; 2469 } else { 2470 cmd_vlan = 0; 2471 } 2472 2473 bus_dmamap_sync(sc->sc_dmat, map, 0, 2474 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2475 2476 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2477 txd = &ring[prod]; 2478 2479 cmd = (uint64_t)map->dm_segs[i].ds_len << 2480 IXL_TX_DESC_BSIZE_SHIFT; 2481 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2482 cmd |= cmd_vlan; 2483 2484 txd->addr = htole64(map->dm_segs[i].ds_addr); 2485 txd->cmd = htole64(cmd); 2486 2487 last = prod; 2488 2489 prod++; 2490 prod &= mask; 2491 } 2492 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2493 txd->cmd = htole64(cmd); 2494 2495 txm->txm_m = m; 2496 txm->txm_eop = last; 2497 2498 bpf_mtap(ifp, m, BPF_D_OUT); 2499 2500 free -= i; 2501 post = 1; 2502 } 2503 2504 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2505 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2506 2507 if (post) { 2508 txr->txr_prod = prod; 2509 ixl_wr(sc, txr->txr_tail, prod); 2510 } 2511 } 2512 2513 static int 2514 ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2515 { 2516 struct ifnet *ifp = &sc->sc_ec.ec_if; 2517 struct ixl_tx_desc *ring, *txd; 2518 struct ixl_tx_map *txm; 2519 struct mbuf *m; 2520 bus_dmamap_t map; 2521 unsigned int cons, prod, last; 2522 unsigned int mask; 2523 uint64_t dtype; 2524 int done = 0, more = 0; 2525 2526 KASSERT(mutex_owned(&txr->txr_lock)); 2527 2528 prod = txr->txr_prod; 2529 cons = txr->txr_cons; 2530 2531 if (cons == prod) 2532 return 0; 2533 2534 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2535 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2536 2537 ring = IXL_DMA_KVA(&txr->txr_mem); 2538 mask = sc->sc_tx_ring_ndescs - 1; 2539 2540 do { 2541 if (txlimit-- <= 0) { 2542 more = 1; 2543 break; 2544 } 2545 2546 txm = &txr->txr_maps[cons]; 2547 last = txm->txm_eop; 2548 txd = &ring[last]; 2549 2550 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2551 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2552 break; 2553 2554 map = txm->txm_map; 2555 2556 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2557 BUS_DMASYNC_POSTWRITE); 2558 bus_dmamap_unload(sc->sc_dmat, map); 2559 2560 m = txm->txm_m; 2561 if (m != NULL) { 2562 txr->txr_opackets++; 2563 txr->txr_obytes += m->m_pkthdr.len; 2564 if (ISSET(m->m_flags, M_MCAST)) 2565 txr->txr_omcasts++; 2566 m_freem(m); 2567 } 2568 2569 txm->txm_m = NULL; 2570 txm->txm_eop = -1; 2571 2572 cons = last + 1; 2573 cons &= mask; 2574 done = 1; 2575 } while (cons != prod); 2576 2577 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2578 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2579 2580 txr->txr_cons = cons; 2581 2582 if (done) { 2583 softint_schedule(txr->txr_si); 2584 if (txr->txr_qid == 0) { 2585 CLR(ifp->if_flags, IFF_OACTIVE); 2586 if_schedule_deferred_start(ifp); 2587 } 2588 } 2589 2590 return more; 2591 } 2592 2593 static void 2594 ixl_start(struct ifnet *ifp) 2595 { 2596 struct ixl_softc *sc; 2597 struct ixl_tx_ring *txr; 2598 2599 sc = ifp->if_softc; 2600 txr = sc->sc_qps[0].qp_txr; 2601 2602 mutex_enter(&txr->txr_lock); 2603 ixl_tx_common_locked(ifp, txr, false); 2604 mutex_exit(&txr->txr_lock); 2605 } 2606 2607 static inline unsigned int 2608 ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2609 { 2610 u_int cpuid; 2611 2612 cpuid = cpu_index(curcpu()); 2613 2614 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2615 } 2616 2617 static int 2618 ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2619 { 2620 struct ixl_softc *sc; 2621 struct ixl_tx_ring *txr; 2622 unsigned int qid; 2623 2624 sc = ifp->if_softc; 2625 qid = ixl_select_txqueue(sc, m); 2626 2627 txr = sc->sc_qps[qid].qp_txr; 2628 2629 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2630 mutex_enter(&txr->txr_lock); 2631 txr->txr_pcqdrop.ev_count++; 2632 mutex_exit(&txr->txr_lock); 2633 2634 m_freem(m); 2635 return ENOBUFS; 2636 } 2637 2638 if (mutex_tryenter(&txr->txr_lock)) { 2639 ixl_tx_common_locked(ifp, txr, true); 2640 mutex_exit(&txr->txr_lock); 2641 } else { 2642 softint_schedule(txr->txr_si); 2643 } 2644 2645 return 0; 2646 } 2647 2648 static void 2649 ixl_deferred_transmit(void *xtxr) 2650 { 2651 struct ixl_tx_ring *txr = xtxr; 2652 struct ixl_softc *sc = txr->txr_sc; 2653 struct ifnet *ifp = &sc->sc_ec.ec_if; 2654 2655 mutex_enter(&txr->txr_lock); 2656 txr->txr_transmitdef.ev_count++; 2657 if (pcq_peek(txr->txr_intrq) != NULL) 2658 ixl_tx_common_locked(ifp, txr, true); 2659 mutex_exit(&txr->txr_lock); 2660 } 2661 2662 static struct ixl_rx_ring * 2663 ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2664 { 2665 struct ixl_rx_ring *rxr = NULL; 2666 struct ixl_rx_map *maps = NULL, *rxm; 2667 unsigned int i; 2668 2669 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2670 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2671 KM_SLEEP); 2672 2673 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2674 sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs, 2675 IXL_RX_QUEUE_ALIGN) != 0) 2676 goto free; 2677 2678 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2679 rxm = &maps[i]; 2680 2681 if (bus_dmamap_create(sc->sc_dmat, 2682 IXL_HARDMTU, 1, IXL_HARDMTU, 0, 2683 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2684 goto uncreate; 2685 2686 rxm->rxm_m = NULL; 2687 } 2688 2689 rxr->rxr_cons = rxr->rxr_prod = 0; 2690 rxr->rxr_m_head = NULL; 2691 rxr->rxr_m_tail = &rxr->rxr_m_head; 2692 rxr->rxr_maps = maps; 2693 2694 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2695 rxr->rxr_qid = qid; 2696 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2697 2698 return rxr; 2699 2700 uncreate: 2701 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2702 rxm = &maps[i]; 2703 2704 if (rxm->rxm_map == NULL) 2705 continue; 2706 2707 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2708 } 2709 2710 ixl_dmamem_free(sc, &rxr->rxr_mem); 2711 free: 2712 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2713 kmem_free(rxr, sizeof(*rxr)); 2714 2715 return NULL; 2716 } 2717 2718 static void 2719 ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2720 { 2721 struct ixl_rx_map *maps, *rxm; 2722 bus_dmamap_t map; 2723 unsigned int i; 2724 2725 maps = rxr->rxr_maps; 2726 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2727 rxm = &maps[i]; 2728 2729 if (rxm->rxm_m == NULL) 2730 continue; 2731 2732 map = rxm->rxm_map; 2733 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2734 BUS_DMASYNC_POSTWRITE); 2735 bus_dmamap_unload(sc->sc_dmat, map); 2736 2737 m_freem(rxm->rxm_m); 2738 rxm->rxm_m = NULL; 2739 } 2740 2741 m_freem(rxr->rxr_m_head); 2742 rxr->rxr_m_head = NULL; 2743 rxr->rxr_m_tail = &rxr->rxr_m_head; 2744 2745 rxr->rxr_prod = rxr->rxr_cons = 0; 2746 } 2747 2748 static int 2749 ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2750 { 2751 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 2752 uint32_t reg; 2753 int i; 2754 2755 for (i = 0; i < 10; i++) { 2756 reg = ixl_rd(sc, ena); 2757 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 2758 return 0; 2759 2760 delaymsec(10); 2761 } 2762 2763 return ETIMEDOUT; 2764 } 2765 2766 static int 2767 ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2768 { 2769 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 2770 uint32_t reg; 2771 int i; 2772 2773 KASSERT(mutex_owned(&rxr->rxr_lock)); 2774 2775 for (i = 0; i < 20; i++) { 2776 reg = ixl_rd(sc, ena); 2777 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 2778 return 0; 2779 2780 delaymsec(10); 2781 } 2782 2783 return ETIMEDOUT; 2784 } 2785 2786 static void 2787 ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2788 { 2789 struct ixl_hmc_rxq rxq; 2790 void *hmc; 2791 2792 memset(&rxq, 0, sizeof(rxq)); 2793 2794 rxq.head = htole16(rxr->rxr_cons); 2795 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 2796 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 2797 rxq.dbuff = htole16(MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 2798 rxq.hbuff = 0; 2799 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 2800 rxq.dsize = IXL_HMC_RXQ_DSIZE_32; 2801 rxq.crcstrip = 1; 2802 rxq.l2sel = 1; 2803 rxq.showiv = 1; 2804 rxq.rxmax = htole16(IXL_HARDMTU); 2805 rxq.tphrdesc_ena = 0; 2806 rxq.tphwdesc_ena = 0; 2807 rxq.tphdata_ena = 0; 2808 rxq.tphhead_ena = 0; 2809 rxq.lrxqthresh = 0; 2810 rxq.prefena = 1; 2811 2812 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 2813 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 2814 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 2815 __arraycount(ixl_hmc_pack_rxq)); 2816 } 2817 2818 static void 2819 ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2820 { 2821 void *hmc; 2822 2823 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 2824 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 2825 } 2826 2827 static void 2828 ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2829 { 2830 struct ixl_rx_map *maps, *rxm; 2831 unsigned int i; 2832 2833 maps = rxr->rxr_maps; 2834 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2835 rxm = &maps[i]; 2836 2837 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2838 } 2839 2840 ixl_dmamem_free(sc, &rxr->rxr_mem); 2841 mutex_destroy(&rxr->rxr_lock); 2842 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2843 kmem_free(rxr, sizeof(*rxr)); 2844 } 2845 2846 static inline void 2847 ixl_rx_csum(struct mbuf *m, uint64_t qword) 2848 { 2849 int flags_mask; 2850 2851 if (!ISSET(qword, IXL_RX_DESC_L3L4P)) { 2852 /* No L3 or L4 checksum was calculated */ 2853 return; 2854 } 2855 2856 switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) { 2857 case IXL_RX_DESC_PTYPE_IPV4FRAG: 2858 case IXL_RX_DESC_PTYPE_IPV4: 2859 case IXL_RX_DESC_PTYPE_SCTPV4: 2860 case IXL_RX_DESC_PTYPE_ICMPV4: 2861 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 2862 break; 2863 case IXL_RX_DESC_PTYPE_TCPV4: 2864 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 2865 flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD; 2866 break; 2867 case IXL_RX_DESC_PTYPE_UDPV4: 2868 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 2869 flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD; 2870 break; 2871 case IXL_RX_DESC_PTYPE_TCPV6: 2872 flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD; 2873 break; 2874 case IXL_RX_DESC_PTYPE_UDPV6: 2875 flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD; 2876 break; 2877 default: 2878 flags_mask = 0; 2879 } 2880 2881 m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 | 2882 M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6)); 2883 2884 if (ISSET(qword, IXL_RX_DESC_IPE)) { 2885 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD); 2886 } 2887 2888 if (ISSET(qword, IXL_RX_DESC_L4E)) { 2889 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD); 2890 } 2891 } 2892 2893 static int 2894 ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 2895 { 2896 struct ifnet *ifp = &sc->sc_ec.ec_if; 2897 struct ixl_rx_wb_desc_32 *ring, *rxd; 2898 struct ixl_rx_map *rxm; 2899 bus_dmamap_t map; 2900 unsigned int cons, prod; 2901 struct mbuf *m; 2902 uint64_t word, word0; 2903 unsigned int len; 2904 unsigned int mask; 2905 int done = 0, more = 0; 2906 2907 KASSERT(mutex_owned(&rxr->rxr_lock)); 2908 2909 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2910 return 0; 2911 2912 prod = rxr->rxr_prod; 2913 cons = rxr->rxr_cons; 2914 2915 if (cons == prod) 2916 return 0; 2917 2918 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 2919 0, IXL_DMA_LEN(&rxr->rxr_mem), 2920 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 2921 2922 ring = IXL_DMA_KVA(&rxr->rxr_mem); 2923 mask = sc->sc_rx_ring_ndescs - 1; 2924 2925 do { 2926 if (rxlimit-- <= 0) { 2927 more = 1; 2928 break; 2929 } 2930 2931 rxd = &ring[cons]; 2932 2933 word = le64toh(rxd->qword1); 2934 2935 if (!ISSET(word, IXL_RX_DESC_DD)) 2936 break; 2937 2938 rxm = &rxr->rxr_maps[cons]; 2939 2940 map = rxm->rxm_map; 2941 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2942 BUS_DMASYNC_POSTREAD); 2943 bus_dmamap_unload(sc->sc_dmat, map); 2944 2945 m = rxm->rxm_m; 2946 rxm->rxm_m = NULL; 2947 2948 KASSERT(m != NULL); 2949 2950 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 2951 m->m_len = len; 2952 m->m_pkthdr.len = 0; 2953 2954 m->m_next = NULL; 2955 *rxr->rxr_m_tail = m; 2956 rxr->rxr_m_tail = &m->m_next; 2957 2958 m = rxr->rxr_m_head; 2959 m->m_pkthdr.len += len; 2960 2961 if (ISSET(word, IXL_RX_DESC_EOP)) { 2962 word0 = le64toh(rxd->qword0); 2963 2964 if (ISSET(word, IXL_RX_DESC_L2TAG1P)) { 2965 vlan_set_tag(m, 2966 __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK)); 2967 } 2968 2969 if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0) 2970 ixl_rx_csum(m, word); 2971 2972 if (!ISSET(word, 2973 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 2974 m_set_rcvif(m, ifp); 2975 rxr->rxr_ipackets++; 2976 rxr->rxr_ibytes += m->m_pkthdr.len; 2977 if_percpuq_enqueue(ifp->if_percpuq, m); 2978 } else { 2979 rxr->rxr_ierrors++; 2980 m_freem(m); 2981 } 2982 2983 rxr->rxr_m_head = NULL; 2984 rxr->rxr_m_tail = &rxr->rxr_m_head; 2985 } 2986 2987 cons++; 2988 cons &= mask; 2989 2990 done = 1; 2991 } while (cons != prod); 2992 2993 if (done) { 2994 rxr->rxr_cons = cons; 2995 if (ixl_rxfill(sc, rxr) == -1) 2996 rxr->rxr_iqdrops++; 2997 } 2998 2999 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3000 0, IXL_DMA_LEN(&rxr->rxr_mem), 3001 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3002 3003 return more; 3004 } 3005 3006 static int 3007 ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3008 { 3009 struct ixl_rx_rd_desc_32 *ring, *rxd; 3010 struct ixl_rx_map *rxm; 3011 bus_dmamap_t map; 3012 struct mbuf *m; 3013 unsigned int prod; 3014 unsigned int slots; 3015 unsigned int mask; 3016 int post = 0, error = 0; 3017 3018 KASSERT(mutex_owned(&rxr->rxr_lock)); 3019 3020 prod = rxr->rxr_prod; 3021 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 3022 sc->sc_rx_ring_ndescs); 3023 3024 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3025 mask = sc->sc_rx_ring_ndescs - 1; 3026 3027 if (__predict_false(slots <= 0)) 3028 return -1; 3029 3030 do { 3031 rxm = &rxr->rxr_maps[prod]; 3032 3033 MGETHDR(m, M_DONTWAIT, MT_DATA); 3034 if (m == NULL) { 3035 rxr->rxr_mgethdr_failed.ev_count++; 3036 error = -1; 3037 break; 3038 } 3039 3040 MCLGET(m, M_DONTWAIT); 3041 if (!ISSET(m->m_flags, M_EXT)) { 3042 rxr->rxr_mgetcl_failed.ev_count++; 3043 error = -1; 3044 m_freem(m); 3045 break; 3046 } 3047 3048 m->m_len = m->m_pkthdr.len = MCLBYTES + ETHER_ALIGN; 3049 m_adj(m, ETHER_ALIGN); 3050 3051 map = rxm->rxm_map; 3052 3053 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 3054 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 3055 rxr->rxr_mbuf_load_failed.ev_count++; 3056 error = -1; 3057 m_freem(m); 3058 break; 3059 } 3060 3061 rxm->rxm_m = m; 3062 3063 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3064 BUS_DMASYNC_PREREAD); 3065 3066 rxd = &ring[prod]; 3067 3068 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 3069 rxd->haddr = htole64(0); 3070 3071 prod++; 3072 prod &= mask; 3073 3074 post = 1; 3075 3076 } while (--slots); 3077 3078 if (post) { 3079 rxr->rxr_prod = prod; 3080 ixl_wr(sc, rxr->rxr_tail, prod); 3081 } 3082 3083 return error; 3084 } 3085 3086 static inline int 3087 ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 3088 u_int txlimit, struct evcnt *txevcnt, 3089 u_int rxlimit, struct evcnt *rxevcnt) 3090 { 3091 struct ixl_tx_ring *txr = qp->qp_txr; 3092 struct ixl_rx_ring *rxr = qp->qp_rxr; 3093 int txmore, rxmore; 3094 int rv; 3095 3096 KASSERT(!mutex_owned(&txr->txr_lock)); 3097 KASSERT(!mutex_owned(&rxr->rxr_lock)); 3098 3099 mutex_enter(&txr->txr_lock); 3100 txevcnt->ev_count++; 3101 txmore = ixl_txeof(sc, txr, txlimit); 3102 mutex_exit(&txr->txr_lock); 3103 3104 mutex_enter(&rxr->rxr_lock); 3105 rxevcnt->ev_count++; 3106 rxmore = ixl_rxeof(sc, rxr, rxlimit); 3107 mutex_exit(&rxr->rxr_lock); 3108 3109 rv = txmore | (rxmore << 1); 3110 3111 return rv; 3112 } 3113 3114 static void 3115 ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 3116 { 3117 3118 if (qp->qp_workqueue) 3119 ixl_work_add(sc->sc_workq_txrx, &qp->qp_task); 3120 else 3121 softint_schedule(qp->qp_si); 3122 } 3123 3124 static int 3125 ixl_intr(void *xsc) 3126 { 3127 struct ixl_softc *sc = xsc; 3128 struct ixl_tx_ring *txr; 3129 struct ixl_rx_ring *rxr; 3130 uint32_t icr, rxintr, txintr; 3131 int rv = 0; 3132 unsigned int i; 3133 3134 KASSERT(sc != NULL); 3135 3136 ixl_enable_other_intr(sc); 3137 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3138 3139 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3140 atomic_inc_64(&sc->sc_event_atq.ev_count); 3141 ixl_atq_done(sc); 3142 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3143 rv = 1; 3144 } 3145 3146 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3147 atomic_inc_64(&sc->sc_event_link.ev_count); 3148 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3149 rv = 1; 3150 } 3151 3152 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 3153 txintr = icr & I40E_INTR_NOTX_TX_MASK; 3154 3155 if (txintr || rxintr) { 3156 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 3157 txr = sc->sc_qps[i].qp_txr; 3158 rxr = sc->sc_qps[i].qp_rxr; 3159 3160 ixl_handle_queue_common(sc, &sc->sc_qps[i], 3161 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 3162 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 3163 } 3164 rv = 1; 3165 } 3166 3167 return rv; 3168 } 3169 3170 static int 3171 ixl_queue_intr(void *xqp) 3172 { 3173 struct ixl_queue_pair *qp = xqp; 3174 struct ixl_tx_ring *txr = qp->qp_txr; 3175 struct ixl_rx_ring *rxr = qp->qp_rxr; 3176 struct ixl_softc *sc = qp->qp_sc; 3177 u_int txlimit, rxlimit; 3178 int more; 3179 3180 txlimit = sc->sc_tx_intr_process_limit; 3181 rxlimit = sc->sc_rx_intr_process_limit; 3182 qp->qp_workqueue = sc->sc_txrx_workqueue; 3183 3184 more = ixl_handle_queue_common(sc, qp, 3185 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3186 3187 if (more != 0) { 3188 ixl_sched_handle_queue(sc, qp); 3189 } else { 3190 /* for ALTQ */ 3191 if (txr->txr_qid == 0) 3192 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3193 softint_schedule(txr->txr_si); 3194 3195 ixl_enable_queue_intr(sc, qp); 3196 } 3197 3198 return 1; 3199 } 3200 3201 static void 3202 ixl_handle_queue(void *xqp) 3203 { 3204 struct ixl_queue_pair *qp = xqp; 3205 struct ixl_softc *sc = qp->qp_sc; 3206 struct ixl_tx_ring *txr = qp->qp_txr; 3207 struct ixl_rx_ring *rxr = qp->qp_rxr; 3208 u_int txlimit, rxlimit; 3209 int more; 3210 3211 txlimit = sc->sc_tx_process_limit; 3212 rxlimit = sc->sc_rx_process_limit; 3213 3214 more = ixl_handle_queue_common(sc, qp, 3215 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3216 3217 if (more != 0) 3218 ixl_sched_handle_queue(sc, qp); 3219 else 3220 ixl_enable_queue_intr(sc, qp); 3221 } 3222 3223 static inline void 3224 ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3225 { 3226 uint32_t hmc_idx, hmc_isvf; 3227 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3228 3229 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3230 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3231 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3232 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3233 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3234 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3235 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3236 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3237 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3238 3239 device_printf(sc->sc_dev, 3240 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3241 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3242 } 3243 3244 static int 3245 ixl_other_intr(void *xsc) 3246 { 3247 struct ixl_softc *sc = xsc; 3248 uint32_t icr, mask, reg; 3249 int rv; 3250 3251 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3252 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3253 3254 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3255 atomic_inc_64(&sc->sc_event_atq.ev_count); 3256 ixl_atq_done(sc); 3257 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3258 rv = 1; 3259 } 3260 3261 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3262 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3263 device_printf(sc->sc_dev, "link stat changed\n"); 3264 3265 atomic_inc_64(&sc->sc_event_link.ev_count); 3266 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3267 rv = 1; 3268 } 3269 3270 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3271 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3272 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3273 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3274 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3275 3276 device_printf(sc->sc_dev, "GRST: %s\n", 3277 reg == I40E_RESET_CORER ? "CORER" : 3278 reg == I40E_RESET_GLOBR ? "GLOBR" : 3279 reg == I40E_RESET_EMPR ? "EMPR" : 3280 "POR"); 3281 } 3282 3283 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3284 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3285 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3286 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3287 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3288 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3289 3290 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3291 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3292 device_printf(sc->sc_dev, "critical error\n"); 3293 } 3294 3295 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3296 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3297 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3298 ixl_print_hmc_error(sc, reg); 3299 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3300 } 3301 3302 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3303 ixl_flush(sc); 3304 ixl_enable_other_intr(sc); 3305 return rv; 3306 } 3307 3308 static void 3309 ixl_get_link_status_done(struct ixl_softc *sc, 3310 const struct ixl_aq_desc *iaq) 3311 { 3312 3313 ixl_link_state_update(sc, iaq); 3314 } 3315 3316 static void 3317 ixl_get_link_status(void *xsc) 3318 { 3319 struct ixl_softc *sc = xsc; 3320 struct ixl_aq_desc *iaq; 3321 struct ixl_aq_link_param *param; 3322 3323 memset(&sc->sc_link_state_atq, 0, sizeof(sc->sc_link_state_atq)); 3324 iaq = &sc->sc_link_state_atq.iatq_desc; 3325 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3326 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3327 param->notify = IXL_AQ_LINK_NOTIFY; 3328 3329 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 3330 (void)ixl_atq_post(sc, &sc->sc_link_state_atq); 3331 } 3332 3333 static void 3334 ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3335 { 3336 struct ifnet *ifp = &sc->sc_ec.ec_if; 3337 int link_state; 3338 3339 link_state = ixl_set_link_status(sc, iaq); 3340 3341 if (ifp->if_link_state != link_state) 3342 if_link_state_change(ifp, link_state); 3343 3344 if (link_state != LINK_STATE_DOWN) { 3345 if_schedule_deferred_start(ifp); 3346 } 3347 } 3348 3349 static void 3350 ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3351 const char *msg) 3352 { 3353 char buf[512]; 3354 size_t len; 3355 3356 len = sizeof(buf); 3357 buf[--len] = '\0'; 3358 3359 device_printf(sc->sc_dev, "%s\n", msg); 3360 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3361 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3362 buf, le16toh(iaq->iaq_opcode)); 3363 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3364 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3365 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3366 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3367 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3368 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3369 } 3370 3371 static void 3372 ixl_arq(void *xsc) 3373 { 3374 struct ixl_softc *sc = xsc; 3375 struct ixl_aq_desc *arq, *iaq; 3376 struct ixl_aq_buf *aqb; 3377 unsigned int cons = sc->sc_arq_cons; 3378 unsigned int prod; 3379 int done = 0; 3380 3381 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3382 sc->sc_aq_regs->arq_head_mask; 3383 3384 if (cons == prod) 3385 goto done; 3386 3387 arq = IXL_DMA_KVA(&sc->sc_arq); 3388 3389 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3390 0, IXL_DMA_LEN(&sc->sc_arq), 3391 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3392 3393 do { 3394 iaq = &arq[cons]; 3395 aqb = sc->sc_arq_live[cons]; 3396 3397 KASSERT(aqb != NULL); 3398 3399 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3400 BUS_DMASYNC_POSTREAD); 3401 3402 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3403 ixl_aq_dump(sc, iaq, "arq event"); 3404 3405 switch (iaq->iaq_opcode) { 3406 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3407 ixl_link_state_update(sc, iaq); 3408 break; 3409 } 3410 3411 memset(iaq, 0, sizeof(*iaq)); 3412 sc->sc_arq_live[cons] = NULL; 3413 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3414 3415 cons++; 3416 cons &= IXL_AQ_MASK; 3417 3418 done = 1; 3419 } while (cons != prod); 3420 3421 if (done) { 3422 sc->sc_arq_cons = cons; 3423 ixl_arq_fill(sc); 3424 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3425 0, IXL_DMA_LEN(&sc->sc_arq), 3426 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3427 } 3428 3429 done: 3430 ixl_enable_other_intr(sc); 3431 } 3432 3433 static void 3434 ixl_atq_set(struct ixl_atq *iatq, 3435 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3436 { 3437 3438 iatq->iatq_fn = fn; 3439 } 3440 3441 static int 3442 ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3443 { 3444 struct ixl_aq_desc *atq, *slot; 3445 unsigned int prod, cons, prod_next; 3446 3447 /* assert locked */ 3448 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3449 3450 atq = IXL_DMA_KVA(&sc->sc_atq); 3451 prod = sc->sc_atq_prod; 3452 cons = sc->sc_atq_cons; 3453 prod_next = (prod +1) & IXL_AQ_MASK; 3454 3455 if (cons == prod_next) 3456 return ENOMEM; 3457 3458 slot = &atq[prod]; 3459 3460 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3461 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3462 3463 *slot = iatq->iatq_desc; 3464 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3465 3466 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3467 ixl_aq_dump(sc, slot, "atq command"); 3468 3469 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3470 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3471 3472 sc->sc_atq_prod = prod_next; 3473 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3474 3475 return 0; 3476 } 3477 3478 static int 3479 ixl_atq_post(struct ixl_softc *sc, struct ixl_atq *iatq) 3480 { 3481 int rv; 3482 3483 mutex_enter(&sc->sc_atq_lock); 3484 rv = ixl_atq_post_locked(sc, iatq); 3485 mutex_exit(&sc->sc_atq_lock); 3486 3487 return rv; 3488 } 3489 3490 static void 3491 ixl_atq_done_locked(struct ixl_softc *sc) 3492 { 3493 struct ixl_aq_desc *atq, *slot; 3494 struct ixl_atq *iatq; 3495 unsigned int cons; 3496 unsigned int prod; 3497 3498 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3499 3500 prod = sc->sc_atq_prod; 3501 cons = sc->sc_atq_cons; 3502 3503 if (prod == cons) 3504 return; 3505 3506 atq = IXL_DMA_KVA(&sc->sc_atq); 3507 3508 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3509 0, IXL_DMA_LEN(&sc->sc_atq), 3510 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3511 3512 do { 3513 slot = &atq[cons]; 3514 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3515 break; 3516 3517 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3518 iatq->iatq_desc = *slot; 3519 3520 memset(slot, 0, sizeof(*slot)); 3521 3522 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3523 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3524 3525 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3526 3527 cons++; 3528 cons &= IXL_AQ_MASK; 3529 } while (cons != prod); 3530 3531 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3532 0, IXL_DMA_LEN(&sc->sc_atq), 3533 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3534 3535 sc->sc_atq_cons = cons; 3536 } 3537 3538 static void 3539 ixl_atq_done(struct ixl_softc *sc) 3540 { 3541 3542 mutex_enter(&sc->sc_atq_lock); 3543 ixl_atq_done_locked(sc); 3544 mutex_exit(&sc->sc_atq_lock); 3545 } 3546 3547 static void 3548 ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3549 { 3550 3551 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3552 3553 cv_signal(&sc->sc_atq_cv); 3554 } 3555 3556 static int 3557 ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3558 { 3559 int error; 3560 3561 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3562 3563 ixl_atq_set(iatq, ixl_wakeup); 3564 3565 mutex_enter(&sc->sc_atq_lock); 3566 error = ixl_atq_post_locked(sc, iatq); 3567 if (error) { 3568 mutex_exit(&sc->sc_atq_lock); 3569 return error; 3570 } 3571 3572 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3573 IXL_ATQ_EXEC_TIMEOUT); 3574 mutex_exit(&sc->sc_atq_lock); 3575 3576 return error; 3577 } 3578 3579 static int 3580 ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3581 { 3582 struct ixl_aq_desc *atq, *slot; 3583 unsigned int prod; 3584 unsigned int t = 0; 3585 3586 mutex_enter(&sc->sc_atq_lock); 3587 3588 atq = IXL_DMA_KVA(&sc->sc_atq); 3589 prod = sc->sc_atq_prod; 3590 slot = atq + prod; 3591 3592 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3593 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3594 3595 *slot = *iaq; 3596 slot->iaq_flags |= htole16(IXL_AQ_SI); 3597 3598 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3599 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3600 3601 prod++; 3602 prod &= IXL_AQ_MASK; 3603 sc->sc_atq_prod = prod; 3604 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3605 3606 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3607 delaymsec(1); 3608 3609 if (t++ > tm) { 3610 mutex_exit(&sc->sc_atq_lock); 3611 return ETIMEDOUT; 3612 } 3613 } 3614 3615 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3616 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3617 *iaq = *slot; 3618 memset(slot, 0, sizeof(*slot)); 3619 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3620 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3621 3622 sc->sc_atq_cons = prod; 3623 3624 mutex_exit(&sc->sc_atq_lock); 3625 3626 return 0; 3627 } 3628 3629 static int 3630 ixl_get_version(struct ixl_softc *sc) 3631 { 3632 struct ixl_aq_desc iaq; 3633 uint32_t fwbuild, fwver, apiver; 3634 uint16_t api_maj_ver, api_min_ver; 3635 3636 memset(&iaq, 0, sizeof(iaq)); 3637 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3638 3639 iaq.iaq_retval = le16toh(23); 3640 3641 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 3642 return ETIMEDOUT; 3643 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 3644 return EIO; 3645 3646 fwbuild = le32toh(iaq.iaq_param[1]); 3647 fwver = le32toh(iaq.iaq_param[2]); 3648 apiver = le32toh(iaq.iaq_param[3]); 3649 3650 api_maj_ver = (uint16_t)apiver; 3651 api_min_ver = (uint16_t)(apiver >> 16); 3652 3653 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 3654 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 3655 3656 sc->sc_rxctl_atq = true; 3657 if (sc->sc_mac_type == I40E_MAC_X722) { 3658 if (api_maj_ver == 1 && api_min_ver < 5) { 3659 sc->sc_rxctl_atq = false; 3660 } 3661 } 3662 3663 return 0; 3664 } 3665 3666 static int 3667 ixl_pxe_clear(struct ixl_softc *sc) 3668 { 3669 struct ixl_aq_desc iaq; 3670 int rv; 3671 3672 memset(&iaq, 0, sizeof(iaq)); 3673 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 3674 iaq.iaq_param[0] = htole32(0x2); 3675 3676 rv = ixl_atq_poll(sc, &iaq, 250); 3677 3678 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 3679 3680 if (rv != 0) 3681 return ETIMEDOUT; 3682 3683 switch (iaq.iaq_retval) { 3684 case htole16(IXL_AQ_RC_OK): 3685 case htole16(IXL_AQ_RC_EEXIST): 3686 break; 3687 default: 3688 return EIO; 3689 } 3690 3691 return 0; 3692 } 3693 3694 static int 3695 ixl_lldp_shut(struct ixl_softc *sc) 3696 { 3697 struct ixl_aq_desc iaq; 3698 3699 memset(&iaq, 0, sizeof(iaq)); 3700 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 3701 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 3702 3703 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 3704 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 3705 return -1; 3706 } 3707 3708 switch (iaq.iaq_retval) { 3709 case htole16(IXL_AQ_RC_EMODE): 3710 case htole16(IXL_AQ_RC_EPERM): 3711 /* ignore silently */ 3712 default: 3713 break; 3714 } 3715 3716 return 0; 3717 } 3718 3719 static void 3720 ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 3721 { 3722 uint16_t id; 3723 uint32_t number, logical_id; 3724 3725 id = le16toh(cap->cap_id); 3726 number = le32toh(cap->number); 3727 logical_id = le32toh(cap->logical_id); 3728 3729 switch (id) { 3730 case IXL_AQ_CAP_RSS: 3731 sc->sc_rss_table_size = number; 3732 sc->sc_rss_table_entry_width = logical_id; 3733 break; 3734 case IXL_AQ_CAP_RXQ: 3735 case IXL_AQ_CAP_TXQ: 3736 sc->sc_nqueue_pairs_device = MIN(number, 3737 sc->sc_nqueue_pairs_device); 3738 break; 3739 } 3740 } 3741 3742 static int 3743 ixl_get_hw_capabilities(struct ixl_softc *sc) 3744 { 3745 struct ixl_dmamem idm; 3746 struct ixl_aq_desc iaq; 3747 struct ixl_aq_capability *caps; 3748 size_t i, ncaps; 3749 bus_size_t caps_size; 3750 uint16_t status; 3751 int rv; 3752 3753 caps_size = sizeof(caps[0]) * 40; 3754 memset(&iaq, 0, sizeof(iaq)); 3755 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 3756 3757 do { 3758 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 3759 return -1; 3760 } 3761 3762 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3763 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3764 iaq.iaq_datalen = htole16(caps_size); 3765 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3766 3767 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 3768 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 3769 3770 rv = ixl_atq_poll(sc, &iaq, 250); 3771 3772 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 3773 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 3774 3775 if (rv != 0) { 3776 aprint_error(", HW capabilities timeout\n"); 3777 goto done; 3778 } 3779 3780 status = le16toh(iaq.iaq_retval); 3781 3782 if (status == IXL_AQ_RC_ENOMEM) { 3783 caps_size = le16toh(iaq.iaq_datalen); 3784 ixl_dmamem_free(sc, &idm); 3785 } 3786 } while (status == IXL_AQ_RC_ENOMEM); 3787 3788 if (status != IXL_AQ_RC_OK) { 3789 aprint_error(", HW capabilities error\n"); 3790 goto done; 3791 } 3792 3793 caps = IXL_DMA_KVA(&idm); 3794 ncaps = le16toh(iaq.iaq_param[1]); 3795 3796 for (i = 0; i < ncaps; i++) { 3797 ixl_parse_hw_capability(sc, &caps[i]); 3798 } 3799 3800 done: 3801 ixl_dmamem_free(sc, &idm); 3802 return rv; 3803 } 3804 3805 static int 3806 ixl_get_mac(struct ixl_softc *sc) 3807 { 3808 struct ixl_dmamem idm; 3809 struct ixl_aq_desc iaq; 3810 struct ixl_aq_mac_addresses *addrs; 3811 int rv; 3812 3813 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 3814 aprint_error(", unable to allocate mac addresses\n"); 3815 return -1; 3816 } 3817 3818 memset(&iaq, 0, sizeof(iaq)); 3819 iaq.iaq_flags = htole16(IXL_AQ_BUF); 3820 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 3821 iaq.iaq_datalen = htole16(sizeof(*addrs)); 3822 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3823 3824 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3825 BUS_DMASYNC_PREREAD); 3826 3827 rv = ixl_atq_poll(sc, &iaq, 250); 3828 3829 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3830 BUS_DMASYNC_POSTREAD); 3831 3832 if (rv != 0) { 3833 aprint_error(", MAC ADDRESS READ timeout\n"); 3834 rv = -1; 3835 goto done; 3836 } 3837 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3838 aprint_error(", MAC ADDRESS READ error\n"); 3839 rv = -1; 3840 goto done; 3841 } 3842 3843 addrs = IXL_DMA_KVA(&idm); 3844 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 3845 printf(", port address is not valid\n"); 3846 goto done; 3847 } 3848 3849 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 3850 rv = 0; 3851 3852 done: 3853 ixl_dmamem_free(sc, &idm); 3854 return rv; 3855 } 3856 3857 static int 3858 ixl_get_switch_config(struct ixl_softc *sc) 3859 { 3860 struct ixl_dmamem idm; 3861 struct ixl_aq_desc iaq; 3862 struct ixl_aq_switch_config *hdr; 3863 struct ixl_aq_switch_config_element *elms, *elm; 3864 unsigned int nelm, i; 3865 int rv; 3866 3867 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 3868 aprint_error_dev(sc->sc_dev, 3869 "unable to allocate switch config buffer\n"); 3870 return -1; 3871 } 3872 3873 memset(&iaq, 0, sizeof(iaq)); 3874 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3875 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3876 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 3877 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 3878 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 3879 3880 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3881 BUS_DMASYNC_PREREAD); 3882 3883 rv = ixl_atq_poll(sc, &iaq, 250); 3884 3885 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 3886 BUS_DMASYNC_POSTREAD); 3887 3888 if (rv != 0) { 3889 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 3890 rv = -1; 3891 goto done; 3892 } 3893 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3894 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 3895 rv = -1; 3896 goto done; 3897 } 3898 3899 hdr = IXL_DMA_KVA(&idm); 3900 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 3901 3902 nelm = le16toh(hdr->num_reported); 3903 if (nelm < 1) { 3904 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 3905 rv = -1; 3906 goto done; 3907 } 3908 3909 for (i = 0; i < nelm; i++) { 3910 elm = &elms[i]; 3911 3912 aprint_debug_dev(sc->sc_dev, 3913 "type %x revision %u seid %04x\n", 3914 elm->type, elm->revision, le16toh(elm->seid)); 3915 aprint_debug_dev(sc->sc_dev, 3916 "uplink %04x downlink %04x\n", 3917 le16toh(elm->uplink_seid), 3918 le16toh(elm->downlink_seid)); 3919 aprint_debug_dev(sc->sc_dev, 3920 "conntype %x scheduler %04x extra %04x\n", 3921 elm->connection_type, 3922 le16toh(elm->scheduler_id), 3923 le16toh(elm->element_info)); 3924 } 3925 3926 elm = &elms[0]; 3927 3928 sc->sc_uplink_seid = elm->uplink_seid; 3929 sc->sc_downlink_seid = elm->downlink_seid; 3930 sc->sc_seid = elm->seid; 3931 3932 if ((sc->sc_uplink_seid == htole16(0)) != 3933 (sc->sc_downlink_seid == htole16(0))) { 3934 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 3935 rv = -1; 3936 goto done; 3937 } 3938 3939 done: 3940 ixl_dmamem_free(sc, &idm); 3941 return rv; 3942 } 3943 3944 static int 3945 ixl_phy_mask_ints(struct ixl_softc *sc) 3946 { 3947 struct ixl_aq_desc iaq; 3948 3949 memset(&iaq, 0, sizeof(iaq)); 3950 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 3951 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 3952 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 3953 IXL_AQ_PHY_EV_MEDIA_NA)); 3954 3955 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 3956 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 3957 return -1; 3958 } 3959 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 3960 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 3961 return -1; 3962 } 3963 3964 return 0; 3965 } 3966 3967 static int 3968 ixl_get_phy_abilities(struct ixl_softc *sc,struct ixl_dmamem *idm) 3969 { 3970 struct ixl_aq_desc iaq; 3971 int rv; 3972 3973 memset(&iaq, 0, sizeof(iaq)); 3974 iaq.iaq_flags = htole16(IXL_AQ_BUF | 3975 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 3976 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 3977 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 3978 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 3979 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 3980 3981 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 3982 BUS_DMASYNC_PREREAD); 3983 3984 rv = ixl_atq_poll(sc, &iaq, 250); 3985 3986 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 3987 BUS_DMASYNC_POSTREAD); 3988 3989 if (rv != 0) 3990 return -1; 3991 3992 return le16toh(iaq.iaq_retval); 3993 } 3994 3995 static int 3996 ixl_get_phy_types(struct ixl_softc *sc, uint64_t *phy_types_ptr) 3997 { 3998 struct ixl_dmamem idm; 3999 struct ixl_aq_phy_abilities *phy; 4000 uint64_t phy_types; 4001 int rv; 4002 4003 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4004 aprint_error_dev(sc->sc_dev, 4005 "unable to allocate switch config buffer\n"); 4006 return -1; 4007 } 4008 4009 rv = ixl_get_phy_abilities(sc, &idm); 4010 switch (rv) { 4011 case -1: 4012 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 4013 goto done; 4014 case IXL_AQ_RC_OK: 4015 break; 4016 case IXL_AQ_RC_EIO: 4017 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 4018 break; 4019 default: 4020 aprint_error_dev(sc->sc_dev, 4021 "GET PHY ABILITIIES error %u\n", rv); 4022 goto done; 4023 } 4024 4025 phy = IXL_DMA_KVA(&idm); 4026 4027 phy_types = le32toh(phy->phy_type); 4028 phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 4029 4030 *phy_types_ptr = phy_types; 4031 4032 rv = 0; 4033 4034 done: 4035 ixl_dmamem_free(sc, &idm); 4036 return rv; 4037 } 4038 4039 static int 4040 ixl_get_link_status_poll(struct ixl_softc *sc) 4041 { 4042 struct ixl_aq_desc iaq; 4043 struct ixl_aq_link_param *param; 4044 int link; 4045 4046 memset(&iaq, 0, sizeof(iaq)); 4047 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 4048 param = (struct ixl_aq_link_param *)iaq.iaq_param; 4049 param->notify = IXL_AQ_LINK_NOTIFY; 4050 4051 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4052 return ETIMEDOUT; 4053 } 4054 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4055 return EIO; 4056 } 4057 4058 link = ixl_set_link_status(sc, &iaq); 4059 sc->sc_ec.ec_if.if_link_state = link; 4060 4061 return 0; 4062 } 4063 4064 static int 4065 ixl_get_vsi(struct ixl_softc *sc) 4066 { 4067 struct ixl_dmamem *vsi = &sc->sc_scratch; 4068 struct ixl_aq_desc iaq; 4069 struct ixl_aq_vsi_param *param; 4070 struct ixl_aq_vsi_reply *reply; 4071 int rv; 4072 4073 /* grumble, vsi info isn't "known" at compile time */ 4074 4075 memset(&iaq, 0, sizeof(iaq)); 4076 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4077 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4078 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 4079 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4080 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4081 4082 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4083 param->uplink_seid = sc->sc_seid; 4084 4085 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4086 BUS_DMASYNC_PREREAD); 4087 4088 rv = ixl_atq_poll(sc, &iaq, 250); 4089 4090 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4091 BUS_DMASYNC_POSTREAD); 4092 4093 if (rv != 0) { 4094 return ETIMEDOUT; 4095 } 4096 4097 switch (le16toh(iaq.iaq_retval)) { 4098 case IXL_AQ_RC_OK: 4099 break; 4100 case IXL_AQ_RC_ENOENT: 4101 return ENOENT; 4102 case IXL_AQ_RC_EACCES: 4103 return EACCES; 4104 default: 4105 return EIO; 4106 } 4107 4108 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 4109 sc->sc_vsi_number = reply->vsi_number; 4110 4111 return 0; 4112 } 4113 4114 static int 4115 ixl_set_vsi(struct ixl_softc *sc) 4116 { 4117 struct ixl_dmamem *vsi = &sc->sc_scratch; 4118 struct ixl_aq_desc iaq; 4119 struct ixl_aq_vsi_param *param; 4120 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 4121 unsigned int qnum; 4122 uint16_t val; 4123 int rv; 4124 4125 qnum = sc->sc_nqueue_pairs - 1; 4126 4127 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 4128 IXL_AQ_VSI_VALID_VLAN); 4129 4130 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 4131 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 4132 data->queue_mapping[0] = htole16(0); 4133 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 4134 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 4135 4136 val = le16toh(data->port_vlan_flags); 4137 CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK); 4138 SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL); 4139 4140 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { 4141 SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH); 4142 } else { 4143 SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING); 4144 } 4145 4146 data->port_vlan_flags = htole16(val); 4147 4148 /* grumble, vsi info isn't "known" at compile time */ 4149 4150 memset(&iaq, 0, sizeof(iaq)); 4151 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4152 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4153 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 4154 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4155 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4156 4157 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4158 param->uplink_seid = sc->sc_seid; 4159 4160 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4161 BUS_DMASYNC_PREWRITE); 4162 4163 rv = ixl_atq_poll(sc, &iaq, 250); 4164 4165 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4166 BUS_DMASYNC_POSTWRITE); 4167 4168 if (rv != 0) { 4169 return ETIMEDOUT; 4170 } 4171 4172 switch (le16toh(iaq.iaq_retval)) { 4173 case IXL_AQ_RC_OK: 4174 break; 4175 case IXL_AQ_RC_ENOENT: 4176 return ENOENT; 4177 case IXL_AQ_RC_EACCES: 4178 return EACCES; 4179 default: 4180 return EIO; 4181 } 4182 4183 return 0; 4184 } 4185 4186 static void 4187 ixl_set_filter_control(struct ixl_softc *sc) 4188 { 4189 uint32_t reg; 4190 4191 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4192 4193 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4194 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4195 4196 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4197 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4198 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4199 4200 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4201 } 4202 4203 static inline void 4204 ixl_get_default_rss_key(uint32_t *buf, size_t len) 4205 { 4206 size_t cplen; 4207 uint8_t rss_seed[RSS_KEYSIZE]; 4208 4209 rss_getkey(rss_seed); 4210 memset(buf, 0, len); 4211 4212 cplen = MIN(len, sizeof(rss_seed)); 4213 memcpy(buf, rss_seed, cplen); 4214 } 4215 4216 static void 4217 ixl_set_rss_key(struct ixl_softc *sc) 4218 { 4219 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4220 size_t i; 4221 4222 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4223 4224 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4225 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4226 } 4227 } 4228 4229 static void 4230 ixl_set_rss_pctype(struct ixl_softc *sc) 4231 { 4232 uint64_t set_hena = 0; 4233 uint32_t hena0, hena1; 4234 4235 if (sc->sc_mac_type == I40E_MAC_X722) 4236 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4237 else 4238 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4239 4240 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4241 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4242 4243 SET(hena0, set_hena); 4244 SET(hena1, set_hena >> 32); 4245 4246 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4247 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4248 } 4249 4250 static void 4251 ixl_set_rss_hlut(struct ixl_softc *sc) 4252 { 4253 unsigned int qid; 4254 uint8_t hlut_buf[512], lut_mask; 4255 uint32_t *hluts; 4256 size_t i, hluts_num; 4257 4258 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4259 4260 for (i = 0; i < sc->sc_rss_table_size; i++) { 4261 qid = i % sc->sc_nqueue_pairs; 4262 hlut_buf[i] = qid & lut_mask; 4263 } 4264 4265 hluts = (uint32_t *)hlut_buf; 4266 hluts_num = sc->sc_rss_table_size >> 2; 4267 for (i = 0; i < hluts_num; i++) { 4268 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4269 } 4270 ixl_flush(sc); 4271 } 4272 4273 static void 4274 ixl_config_rss(struct ixl_softc *sc) 4275 { 4276 4277 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4278 4279 ixl_set_rss_key(sc); 4280 ixl_set_rss_pctype(sc); 4281 ixl_set_rss_hlut(sc); 4282 } 4283 4284 static const struct ixl_phy_type * 4285 ixl_search_phy_type(uint8_t phy_type) 4286 { 4287 const struct ixl_phy_type *itype; 4288 uint64_t mask; 4289 unsigned int i; 4290 4291 if (phy_type >= 64) 4292 return NULL; 4293 4294 mask = 1ULL << phy_type; 4295 4296 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4297 itype = &ixl_phy_type_map[i]; 4298 4299 if (ISSET(itype->phy_type, mask)) 4300 return itype; 4301 } 4302 4303 return NULL; 4304 } 4305 4306 static uint64_t 4307 ixl_search_link_speed(uint8_t link_speed) 4308 { 4309 const struct ixl_speed_type *type; 4310 unsigned int i; 4311 4312 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4313 type = &ixl_speed_type_map[i]; 4314 4315 if (ISSET(type->dev_speed, link_speed)) 4316 return type->net_speed; 4317 } 4318 4319 return 0; 4320 } 4321 4322 static int 4323 ixl_restart_an(struct ixl_softc *sc) 4324 { 4325 struct ixl_aq_desc iaq; 4326 4327 memset(&iaq, 0, sizeof(iaq)); 4328 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4329 iaq.iaq_param[0] = 4330 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4331 4332 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4333 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4334 return -1; 4335 } 4336 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4337 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4338 return -1; 4339 } 4340 4341 return 0; 4342 } 4343 4344 static int 4345 ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4346 uint16_t vlan, uint16_t flags) 4347 { 4348 struct ixl_aq_desc iaq; 4349 struct ixl_aq_add_macvlan *param; 4350 struct ixl_aq_add_macvlan_elem *elem; 4351 4352 memset(&iaq, 0, sizeof(iaq)); 4353 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4354 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4355 iaq.iaq_datalen = htole16(sizeof(*elem)); 4356 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4357 4358 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4359 param->num_addrs = htole16(1); 4360 param->seid0 = htole16(0x8000) | sc->sc_seid; 4361 param->seid1 = 0; 4362 param->seid2 = 0; 4363 4364 elem = IXL_DMA_KVA(&sc->sc_scratch); 4365 memset(elem, 0, sizeof(*elem)); 4366 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4367 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4368 elem->vlan = htole16(vlan); 4369 4370 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4371 return IXL_AQ_RC_EINVAL; 4372 } 4373 4374 switch (le16toh(iaq.iaq_retval)) { 4375 case IXL_AQ_RC_OK: 4376 break; 4377 case IXL_AQ_RC_ENOSPC: 4378 return ENOSPC; 4379 case IXL_AQ_RC_ENOENT: 4380 return ENOENT; 4381 case IXL_AQ_RC_EACCES: 4382 return EACCES; 4383 case IXL_AQ_RC_EEXIST: 4384 return EEXIST; 4385 case IXL_AQ_RC_EINVAL: 4386 return EINVAL; 4387 default: 4388 return EIO; 4389 } 4390 4391 return 0; 4392 } 4393 4394 static int 4395 ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4396 uint16_t vlan, uint16_t flags) 4397 { 4398 struct ixl_aq_desc iaq; 4399 struct ixl_aq_remove_macvlan *param; 4400 struct ixl_aq_remove_macvlan_elem *elem; 4401 4402 memset(&iaq, 0, sizeof(iaq)); 4403 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4404 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 4405 iaq.iaq_datalen = htole16(sizeof(*elem)); 4406 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4407 4408 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 4409 param->num_addrs = htole16(1); 4410 param->seid0 = htole16(0x8000) | sc->sc_seid; 4411 param->seid1 = 0; 4412 param->seid2 = 0; 4413 4414 elem = IXL_DMA_KVA(&sc->sc_scratch); 4415 memset(elem, 0, sizeof(*elem)); 4416 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4417 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 4418 elem->vlan = htole16(vlan); 4419 4420 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4421 return EINVAL; 4422 } 4423 4424 switch (le16toh(iaq.iaq_retval)) { 4425 case IXL_AQ_RC_OK: 4426 break; 4427 case IXL_AQ_RC_ENOENT: 4428 return ENOENT; 4429 case IXL_AQ_RC_EACCES: 4430 return EACCES; 4431 case IXL_AQ_RC_EINVAL: 4432 return EINVAL; 4433 default: 4434 return EIO; 4435 } 4436 4437 return 0; 4438 } 4439 4440 static int 4441 ixl_hmc(struct ixl_softc *sc) 4442 { 4443 struct { 4444 uint32_t count; 4445 uint32_t minsize; 4446 bus_size_t objsiz; 4447 bus_size_t setoff; 4448 bus_size_t setcnt; 4449 } regs[] = { 4450 { 4451 0, 4452 IXL_HMC_TXQ_MINSIZE, 4453 I40E_GLHMC_LANTXOBJSZ, 4454 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 4455 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 4456 }, 4457 { 4458 0, 4459 IXL_HMC_RXQ_MINSIZE, 4460 I40E_GLHMC_LANRXOBJSZ, 4461 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 4462 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 4463 }, 4464 { 4465 0, 4466 0, 4467 I40E_GLHMC_FCOEDDPOBJSZ, 4468 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 4469 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 4470 }, 4471 { 4472 0, 4473 0, 4474 I40E_GLHMC_FCOEFOBJSZ, 4475 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 4476 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 4477 }, 4478 }; 4479 struct ixl_hmc_entry *e; 4480 uint64_t size, dva; 4481 uint8_t *kva; 4482 uint64_t *sdpage; 4483 unsigned int i; 4484 int npages, tables; 4485 uint32_t reg; 4486 4487 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 4488 4489 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 4490 ixl_rd(sc, I40E_GLHMC_LANQMAX); 4491 4492 size = 0; 4493 for (i = 0; i < __arraycount(regs); i++) { 4494 e = &sc->sc_hmc_entries[i]; 4495 4496 e->hmc_count = regs[i].count; 4497 reg = ixl_rd(sc, regs[i].objsiz); 4498 e->hmc_size = BIT_ULL(0x3F & reg); 4499 e->hmc_base = size; 4500 4501 if ((e->hmc_size * 8) < regs[i].minsize) { 4502 aprint_error_dev(sc->sc_dev, 4503 "kernel hmc entry is too big\n"); 4504 return -1; 4505 } 4506 4507 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 4508 } 4509 size = roundup(size, IXL_HMC_PGSIZE); 4510 npages = size / IXL_HMC_PGSIZE; 4511 4512 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 4513 4514 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 4515 aprint_error_dev(sc->sc_dev, 4516 "unable to allocate hmc pd memory\n"); 4517 return -1; 4518 } 4519 4520 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 4521 IXL_HMC_PGSIZE) != 0) { 4522 aprint_error_dev(sc->sc_dev, 4523 "unable to allocate hmc sd memory\n"); 4524 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 4525 return -1; 4526 } 4527 4528 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 4529 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 4530 4531 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 4532 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 4533 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4534 4535 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 4536 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 4537 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 4538 4539 for (i = 0; (int)i < npages; i++) { 4540 *sdpage = htole64(dva | IXL_HMC_PDVALID); 4541 sdpage++; 4542 4543 dva += IXL_HMC_PGSIZE; 4544 } 4545 4546 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 4547 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 4548 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 4549 4550 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 4551 for (i = 0; (int)i < tables; i++) { 4552 uint32_t count; 4553 4554 KASSERT(npages >= 0); 4555 4556 count = ((unsigned int)npages > IXL_HMC_PGS) ? 4557 IXL_HMC_PGS : (unsigned int)npages; 4558 4559 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 4560 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 4561 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 4562 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 4563 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 4564 ixl_wr(sc, I40E_PFHMC_SDCMD, 4565 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 4566 4567 npages -= IXL_HMC_PGS; 4568 dva += IXL_HMC_PGSIZE; 4569 } 4570 4571 for (i = 0; i < __arraycount(regs); i++) { 4572 e = &sc->sc_hmc_entries[i]; 4573 4574 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 4575 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 4576 } 4577 4578 return 0; 4579 } 4580 4581 static void 4582 ixl_hmc_free(struct ixl_softc *sc) 4583 { 4584 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 4585 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 4586 } 4587 4588 static void 4589 ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 4590 unsigned int npacking) 4591 { 4592 uint8_t *dst = d; 4593 const uint8_t *src = s; 4594 unsigned int i; 4595 4596 for (i = 0; i < npacking; i++) { 4597 const struct ixl_hmc_pack *pack = &packing[i]; 4598 unsigned int offset = pack->lsb / 8; 4599 unsigned int align = pack->lsb % 8; 4600 const uint8_t *in = src + pack->offset; 4601 uint8_t *out = dst + offset; 4602 int width = pack->width; 4603 unsigned int inbits = 0; 4604 4605 if (align) { 4606 inbits = (*in++) << align; 4607 *out++ |= (inbits & 0xff); 4608 inbits >>= 8; 4609 4610 width -= 8 - align; 4611 } 4612 4613 while (width >= 8) { 4614 inbits |= (*in++) << align; 4615 *out++ = (inbits & 0xff); 4616 inbits >>= 8; 4617 4618 width -= 8; 4619 } 4620 4621 if (width > 0) { 4622 inbits |= (*in) << align; 4623 *out |= (inbits & ((1 << width) - 1)); 4624 } 4625 } 4626 } 4627 4628 static struct ixl_aq_buf * 4629 ixl_aqb_alloc(struct ixl_softc *sc) 4630 { 4631 struct ixl_aq_buf *aqb; 4632 4633 aqb = malloc(sizeof(*aqb), M_DEVBUF, M_WAITOK); 4634 if (aqb == NULL) 4635 return NULL; 4636 4637 aqb->aqb_size = IXL_AQ_BUFLEN; 4638 4639 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 4640 aqb->aqb_size, 0, 4641 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 4642 goto free; 4643 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 4644 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 4645 BUS_DMA_WAITOK) != 0) 4646 goto destroy; 4647 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 4648 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 4649 goto dma_free; 4650 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 4651 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 4652 goto unmap; 4653 4654 return aqb; 4655 unmap: 4656 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 4657 dma_free: 4658 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 4659 destroy: 4660 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 4661 free: 4662 free(aqb, M_DEVBUF); 4663 4664 return NULL; 4665 } 4666 4667 static void 4668 ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 4669 { 4670 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 4671 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 4672 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 4673 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 4674 free(aqb, M_DEVBUF); 4675 } 4676 4677 static int 4678 ixl_arq_fill(struct ixl_softc *sc) 4679 { 4680 struct ixl_aq_buf *aqb; 4681 struct ixl_aq_desc *arq, *iaq; 4682 unsigned int prod = sc->sc_arq_prod; 4683 unsigned int n; 4684 int post = 0; 4685 4686 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 4687 IXL_AQ_NUM); 4688 arq = IXL_DMA_KVA(&sc->sc_arq); 4689 4690 if (__predict_false(n <= 0)) 4691 return 0; 4692 4693 do { 4694 aqb = sc->sc_arq_live[prod]; 4695 iaq = &arq[prod]; 4696 4697 if (aqb == NULL) { 4698 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 4699 if (aqb != NULL) { 4700 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 4701 ixl_aq_buf, aqb_entry); 4702 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 4703 break; 4704 } 4705 4706 sc->sc_arq_live[prod] = aqb; 4707 memset(aqb->aqb_data, 0, aqb->aqb_size); 4708 4709 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 4710 aqb->aqb_size, BUS_DMASYNC_PREREAD); 4711 4712 iaq->iaq_flags = htole16(IXL_AQ_BUF | 4713 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 4714 IXL_AQ_LB : 0)); 4715 iaq->iaq_opcode = 0; 4716 iaq->iaq_datalen = htole16(aqb->aqb_size); 4717 iaq->iaq_retval = 0; 4718 iaq->iaq_cookie = 0; 4719 iaq->iaq_param[0] = 0; 4720 iaq->iaq_param[1] = 0; 4721 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 4722 } 4723 4724 prod++; 4725 prod &= IXL_AQ_MASK; 4726 4727 post = 1; 4728 4729 } while (--n); 4730 4731 if (post) { 4732 sc->sc_arq_prod = prod; 4733 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 4734 } 4735 4736 return post; 4737 } 4738 4739 static void 4740 ixl_arq_unfill(struct ixl_softc *sc) 4741 { 4742 struct ixl_aq_buf *aqb; 4743 unsigned int i; 4744 4745 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 4746 aqb = sc->sc_arq_live[i]; 4747 if (aqb == NULL) 4748 continue; 4749 4750 sc->sc_arq_live[i] = NULL; 4751 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 4752 BUS_DMASYNC_POSTREAD); 4753 ixl_aqb_free(sc, aqb); 4754 } 4755 4756 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 4757 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 4758 ixl_aq_buf, aqb_entry); 4759 ixl_aqb_free(sc, aqb); 4760 } 4761 } 4762 4763 static void 4764 ixl_clear_hw(struct ixl_softc *sc) 4765 { 4766 uint32_t num_queues, base_queue; 4767 uint32_t num_pf_int; 4768 uint32_t num_vf_int; 4769 uint32_t num_vfs; 4770 uint32_t i, j; 4771 uint32_t val; 4772 uint32_t eol = 0x7ff; 4773 4774 /* get number of interrupts, queues, and vfs */ 4775 val = ixl_rd(sc, I40E_GLPCI_CNF2); 4776 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 4777 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 4778 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 4779 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 4780 4781 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 4782 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 4783 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 4784 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 4785 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 4786 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 4787 num_queues = (j - base_queue) + 1; 4788 else 4789 num_queues = 0; 4790 4791 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 4792 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 4793 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 4794 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 4795 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 4796 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 4797 num_vfs = (j - i) + 1; 4798 else 4799 num_vfs = 0; 4800 4801 /* stop all the interrupts */ 4802 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 4803 ixl_flush(sc); 4804 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 4805 for (i = 0; i < num_pf_int - 2; i++) 4806 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 4807 ixl_flush(sc); 4808 4809 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 4810 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4811 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 4812 for (i = 0; i < num_pf_int - 2; i++) 4813 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 4814 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4815 for (i = 0; i < num_vfs; i++) 4816 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 4817 for (i = 0; i < num_vf_int - 2; i++) 4818 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 4819 4820 /* warn the HW of the coming Tx disables */ 4821 for (i = 0; i < num_queues; i++) { 4822 uint32_t abs_queue_idx = base_queue + i; 4823 uint32_t reg_block = 0; 4824 4825 if (abs_queue_idx >= 128) { 4826 reg_block = abs_queue_idx / 128; 4827 abs_queue_idx %= 128; 4828 } 4829 4830 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 4831 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 4832 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 4833 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 4834 4835 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 4836 } 4837 delaymsec(400); 4838 4839 /* stop all the queues */ 4840 for (i = 0; i < num_queues; i++) { 4841 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 4842 ixl_wr(sc, I40E_QTX_ENA(i), 0); 4843 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 4844 ixl_wr(sc, I40E_QRX_ENA(i), 0); 4845 } 4846 4847 /* short wait for all queue disables to settle */ 4848 delaymsec(50); 4849 } 4850 4851 static int 4852 ixl_pf_reset(struct ixl_softc *sc) 4853 { 4854 uint32_t cnt = 0; 4855 uint32_t cnt1 = 0; 4856 uint32_t reg = 0, reg0 = 0; 4857 uint32_t grst_del; 4858 4859 /* 4860 * Poll for Global Reset steady state in case of recent GRST. 4861 * The grst delay value is in 100ms units, and we'll wait a 4862 * couple counts longer to be sure we don't just miss the end. 4863 */ 4864 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 4865 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 4866 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 4867 4868 grst_del = grst_del * 20; 4869 4870 for (cnt = 0; cnt < grst_del; cnt++) { 4871 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 4872 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 4873 break; 4874 delaymsec(100); 4875 } 4876 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 4877 aprint_error(", Global reset polling failed to complete\n"); 4878 return -1; 4879 } 4880 4881 /* Now Wait for the FW to be ready */ 4882 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 4883 reg = ixl_rd(sc, I40E_GLNVM_ULD); 4884 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4885 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 4886 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4887 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 4888 break; 4889 4890 delaymsec(10); 4891 } 4892 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 4893 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 4894 aprint_error(", wait for FW Reset complete timed out " 4895 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 4896 return -1; 4897 } 4898 4899 /* 4900 * If there was a Global Reset in progress when we got here, 4901 * we don't need to do the PF Reset 4902 */ 4903 if (cnt == 0) { 4904 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 4905 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 4906 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 4907 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 4908 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 4909 break; 4910 delaymsec(1); 4911 4912 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 4913 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 4914 aprint_error(", Core reset upcoming." 4915 " Skipping PF reset reset request\n"); 4916 return -1; 4917 } 4918 } 4919 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 4920 aprint_error(", PF reset polling failed to complete" 4921 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 4922 return -1; 4923 } 4924 } 4925 4926 return 0; 4927 } 4928 4929 static int 4930 ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 4931 bus_size_t size, bus_size_t align) 4932 { 4933 ixm->ixm_size = size; 4934 4935 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 4936 ixm->ixm_size, 0, 4937 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 4938 &ixm->ixm_map) != 0) 4939 return 1; 4940 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 4941 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 4942 BUS_DMA_WAITOK) != 0) 4943 goto destroy; 4944 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 4945 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 4946 goto free; 4947 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 4948 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 4949 goto unmap; 4950 4951 memset(ixm->ixm_kva, 0, ixm->ixm_size); 4952 4953 return 0; 4954 unmap: 4955 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 4956 free: 4957 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 4958 destroy: 4959 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 4960 return 1; 4961 } 4962 4963 static void 4964 ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 4965 { 4966 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 4967 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 4968 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 4969 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 4970 } 4971 4972 static int 4973 ixl_setup_vlan_hwfilter(struct ixl_softc *sc) 4974 { 4975 struct ethercom *ec = &sc->sc_ec; 4976 struct vlanid_list *vlanidp; 4977 int rv; 4978 4979 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 4980 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 4981 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 4982 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 4983 4984 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 4985 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 4986 if (rv != 0) 4987 return rv; 4988 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 4989 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 4990 if (rv != 0) 4991 return rv; 4992 4993 ETHER_LOCK(ec); 4994 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 4995 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 4996 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 4997 if (rv != 0) 4998 break; 4999 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 5000 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5001 if (rv != 0) 5002 break; 5003 } 5004 ETHER_UNLOCK(ec); 5005 5006 return rv; 5007 } 5008 5009 static void 5010 ixl_teardown_vlan_hwfilter(struct ixl_softc *sc) 5011 { 5012 struct vlanid_list *vlanidp; 5013 struct ethercom *ec = &sc->sc_ec; 5014 5015 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5016 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5017 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5018 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5019 5020 ETHER_LOCK(ec); 5021 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5022 ixl_remove_macvlan(sc, sc->sc_enaddr, 5023 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5024 ixl_remove_macvlan(sc, etherbroadcastaddr, 5025 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5026 } 5027 ETHER_UNLOCK(ec); 5028 5029 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5030 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5031 ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5032 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5033 } 5034 5035 static int 5036 ixl_update_macvlan(struct ixl_softc *sc) 5037 { 5038 int rv = 0; 5039 int next_ec_capenable = sc->sc_ec.ec_capenable; 5040 5041 if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 5042 rv = ixl_setup_vlan_hwfilter(sc); 5043 if (rv != 0) 5044 ixl_teardown_vlan_hwfilter(sc); 5045 } else { 5046 ixl_teardown_vlan_hwfilter(sc); 5047 } 5048 5049 return rv; 5050 } 5051 5052 static int 5053 ixl_ifflags_cb(struct ethercom *ec) 5054 { 5055 struct ifnet *ifp = &ec->ec_if; 5056 struct ixl_softc *sc = ifp->if_softc; 5057 int rv, change; 5058 5059 mutex_enter(&sc->sc_cfg_lock); 5060 5061 change = ec->ec_capenable ^ sc->sc_cur_ec_capenable; 5062 5063 if (ISSET(change, ETHERCAP_VLAN_HWTAGGING)) { 5064 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 5065 rv = ENETRESET; 5066 goto out; 5067 } 5068 5069 if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) { 5070 rv = ixl_update_macvlan(sc); 5071 if (rv == 0) { 5072 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 5073 } else { 5074 CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER); 5075 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 5076 } 5077 } 5078 5079 rv = ixl_iff(sc); 5080 out: 5081 mutex_exit(&sc->sc_cfg_lock); 5082 5083 return rv; 5084 } 5085 5086 static int 5087 ixl_set_link_status(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 5088 { 5089 const struct ixl_aq_link_status *status; 5090 const struct ixl_phy_type *itype; 5091 5092 uint64_t ifm_active = IFM_ETHER; 5093 uint64_t ifm_status = IFM_AVALID; 5094 int link_state = LINK_STATE_DOWN; 5095 uint64_t baudrate = 0; 5096 5097 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 5098 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) 5099 goto done; 5100 5101 ifm_active |= IFM_FDX; 5102 ifm_status |= IFM_ACTIVE; 5103 link_state = LINK_STATE_UP; 5104 5105 itype = ixl_search_phy_type(status->phy_type); 5106 if (itype != NULL) 5107 ifm_active |= itype->ifm_type; 5108 5109 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 5110 ifm_active |= IFM_ETH_TXPAUSE; 5111 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 5112 ifm_active |= IFM_ETH_RXPAUSE; 5113 5114 baudrate = ixl_search_link_speed(status->link_speed); 5115 5116 done: 5117 /* NET_ASSERT_LOCKED() except during attach */ 5118 sc->sc_media_active = ifm_active; 5119 sc->sc_media_status = ifm_status; 5120 5121 sc->sc_ec.ec_if.if_baudrate = baudrate; 5122 5123 return link_state; 5124 } 5125 5126 static int 5127 ixl_establish_intx(struct ixl_softc *sc) 5128 { 5129 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5130 pci_intr_handle_t *intr; 5131 char xnamebuf[32]; 5132 char intrbuf[PCI_INTRSTR_LEN]; 5133 char const *intrstr; 5134 5135 KASSERT(sc->sc_nintrs == 1); 5136 5137 intr = &sc->sc_ihp[0]; 5138 5139 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 5140 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 5141 device_xname(sc->sc_dev)); 5142 5143 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 5144 sc, xnamebuf); 5145 5146 if (sc->sc_ihs[0] == NULL) { 5147 aprint_error_dev(sc->sc_dev, 5148 "unable to establish interrupt at %s\n", intrstr); 5149 return -1; 5150 } 5151 5152 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 5153 return 0; 5154 } 5155 5156 static int 5157 ixl_establish_msix(struct ixl_softc *sc) 5158 { 5159 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5160 unsigned int vector = 0; 5161 unsigned int i; 5162 char xnamebuf[32]; 5163 char intrbuf[PCI_INTRSTR_LEN]; 5164 char const *intrstr; 5165 5166 /* the "other" intr is mapped to vector 0 */ 5167 vector = 0; 5168 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5169 intrbuf, sizeof(intrbuf)); 5170 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 5171 device_xname(sc->sc_dev)); 5172 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5173 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 5174 sc, xnamebuf); 5175 if (sc->sc_ihs[vector] == NULL) { 5176 aprint_error_dev(sc->sc_dev, 5177 "unable to establish interrupt at %s\n", intrstr); 5178 goto fail; 5179 } 5180 vector++; 5181 aprint_normal_dev(sc->sc_dev, "interrupt at %s\n", intrstr); 5182 5183 sc->sc_msix_vector_queue = vector; 5184 5185 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5186 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5187 intrbuf, sizeof(intrbuf)); 5188 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 5189 device_xname(sc->sc_dev), i); 5190 5191 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5192 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 5193 (void *)&sc->sc_qps[i], xnamebuf); 5194 5195 if (sc->sc_ihs[vector] == NULL) { 5196 aprint_error_dev(sc->sc_dev, 5197 "unable to establish interrupt at %s\n", intrstr); 5198 goto fail; 5199 } 5200 vector++; 5201 aprint_normal_dev(sc->sc_dev, 5202 "interrupt at %s\n", intrstr); 5203 } 5204 5205 return 0; 5206 fail: 5207 for (i = 0; i < vector; i++) { 5208 pci_intr_disestablish(pc, sc->sc_ihs[i]); 5209 } 5210 5211 sc->sc_msix_vector_queue = 0; 5212 sc->sc_msix_vector_queue = 0; 5213 5214 return -1; 5215 } 5216 5217 static void 5218 ixl_set_affinity_msix(struct ixl_softc *sc) 5219 { 5220 kcpuset_t *affinity; 5221 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5222 int affinity_to, r; 5223 unsigned int i, vector; 5224 char intrbuf[PCI_INTRSTR_LEN]; 5225 char const *intrstr; 5226 5227 affinity_to = 0; 5228 kcpuset_create(&affinity, false); 5229 5230 vector = sc->sc_msix_vector_queue; 5231 5232 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5233 affinity_to = i % ncpu; 5234 5235 kcpuset_zero(affinity); 5236 kcpuset_set(affinity, affinity_to); 5237 5238 intrstr = pci_intr_string(pc, sc->sc_ihp[vector + i], 5239 intrbuf, sizeof(intrbuf)); 5240 r = interrupt_distribute(sc->sc_ihs[vector + i], 5241 affinity, NULL); 5242 if (r == 0) { 5243 aprint_normal_dev(sc->sc_dev, 5244 "for TXRX%u interrupting at %s affinity to %u\n", 5245 i, intrstr, affinity_to); 5246 } else { 5247 aprint_normal_dev(sc->sc_dev, 5248 "for TXRX%u interrupting at %s\n", 5249 i, intrstr); 5250 } 5251 } 5252 5253 vector = 0; /* vector 0 means "other" interrupt */ 5254 affinity_to = (affinity_to + 1) % ncpu; 5255 kcpuset_zero(affinity); 5256 kcpuset_set(affinity, affinity_to); 5257 5258 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5259 intrbuf, sizeof(intrbuf)); 5260 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5261 if (r == 0) { 5262 aprint_normal_dev(sc->sc_dev, 5263 "for other interrupting at %s affinity to %u\n", 5264 intrstr, affinity_to); 5265 } else { 5266 aprint_normal_dev(sc->sc_dev, 5267 "for other interrupting at %s", intrstr); 5268 } 5269 5270 kcpuset_destroy(affinity); 5271 } 5272 5273 static void 5274 ixl_config_queue_intr(struct ixl_softc *sc) 5275 { 5276 unsigned int i, vector; 5277 5278 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5279 vector = sc->sc_msix_vector_queue; 5280 } else { 5281 vector = I40E_INTR_NOTX_INTR; 5282 5283 ixl_wr(sc, I40E_PFINT_LNKLST0, 5284 (I40E_INTR_NOTX_QUEUE << 5285 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5286 (I40E_QUEUE_TYPE_RX << 5287 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5288 } 5289 5290 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5291 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5292 ixl_flush(sc); 5293 5294 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5295 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5296 (I40E_QUEUE_TYPE_RX << 5297 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5298 5299 ixl_wr(sc, I40E_QINT_RQCTL(i), 5300 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5301 (I40E_ITR_INDEX_RX << 5302 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5303 (I40E_INTR_NOTX_RX_QUEUE << 5304 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5305 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5306 (I40E_QUEUE_TYPE_TX << 5307 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5308 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5309 5310 ixl_wr(sc, I40E_QINT_TQCTL(i), 5311 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5312 (I40E_ITR_INDEX_TX << 5313 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5314 (I40E_INTR_NOTX_TX_QUEUE << 5315 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5316 (I40E_QUEUE_TYPE_EOL << 5317 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5318 (I40E_QUEUE_TYPE_RX << 5319 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5320 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5321 5322 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) 5323 vector++; 5324 } 5325 ixl_flush(sc); 5326 5327 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), 0x7a); 5328 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), 0x7a); 5329 ixl_flush(sc); 5330 } 5331 5332 static void 5333 ixl_config_other_intr(struct ixl_softc *sc) 5334 { 5335 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5336 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5337 5338 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5339 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5340 I40E_PFINT_ICR0_ENA_GRST_MASK | 5341 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5342 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5343 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5344 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5345 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5346 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5347 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5348 5349 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5350 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5351 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5352 (I40E_ITR_INDEX_OTHER << 5353 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5354 ixl_flush(sc); 5355 } 5356 5357 static int 5358 ixl_setup_interrupts(struct ixl_softc *sc) 5359 { 5360 struct pci_attach_args *pa = &sc->sc_pa; 5361 pci_intr_type_t max_type, intr_type; 5362 int counts[PCI_INTR_TYPE_SIZE]; 5363 int error; 5364 unsigned int i; 5365 bool retry, nomsix = IXL_NOMSIX; 5366 5367 memset(counts, 0, sizeof(counts)); 5368 max_type = PCI_INTR_TYPE_MSIX; 5369 /* QPs + other interrupt */ 5370 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5371 counts[PCI_INTR_TYPE_INTX] = 1; 5372 5373 if (nomsix) 5374 counts[PCI_INTR_TYPE_MSIX] = 0; 5375 5376 do { 5377 retry = false; 5378 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5379 if (error != 0) { 5380 aprint_error_dev(sc->sc_dev, 5381 "couldn't map interrupt\n"); 5382 break; 5383 } 5384 for (i = 0; i < sc->sc_nintrs; i++) { 5385 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5386 PCI_INTR_MPSAFE, true); 5387 } 5388 5389 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5390 sc->sc_nintrs = counts[intr_type]; 5391 KASSERT(sc->sc_nintrs > 0); 5392 5393 sc->sc_ihs = kmem_alloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5394 KM_SLEEP); 5395 5396 if (intr_type == PCI_INTR_TYPE_MSIX) { 5397 error = ixl_establish_msix(sc); 5398 if (error) { 5399 counts[PCI_INTR_TYPE_MSIX] = 0; 5400 retry = true; 5401 } else { 5402 ixl_set_affinity_msix(sc); 5403 } 5404 } else if (intr_type == PCI_INTR_TYPE_INTX) { 5405 error = ixl_establish_intx(sc); 5406 } else { 5407 error = -1; 5408 } 5409 5410 if (error) { 5411 kmem_free(sc->sc_ihs, 5412 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5413 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5414 } else { 5415 sc->sc_intrtype = intr_type; 5416 } 5417 } while (retry); 5418 5419 return error; 5420 } 5421 5422 static void 5423 ixl_teardown_interrupts(struct ixl_softc *sc) 5424 { 5425 struct pci_attach_args *pa = &sc->sc_pa; 5426 unsigned int i; 5427 5428 for (i = 0; i < sc->sc_nintrs; i++) { 5429 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 5430 } 5431 5432 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 5433 5434 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 5435 sc->sc_ihs = NULL; 5436 sc->sc_nintrs = 0; 5437 } 5438 5439 static int 5440 ixl_setup_stats(struct ixl_softc *sc) 5441 { 5442 struct ixl_queue_pair *qp; 5443 struct ixl_tx_ring *txr; 5444 struct ixl_rx_ring *rxr; 5445 unsigned int i; 5446 5447 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5448 qp = &sc->sc_qps[i]; 5449 txr = qp->qp_txr; 5450 rxr = qp->qp_rxr; 5451 5452 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 5453 NULL, qp->qp_name, "m_defrag successed"); 5454 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 5455 NULL, qp->qp_name, "m_defrag_failed"); 5456 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 5457 NULL, qp->qp_name, "Dropped in pcq"); 5458 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 5459 NULL, qp->qp_name, "Deferred transmit"); 5460 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 5461 NULL, qp->qp_name, "Interrupt on queue"); 5462 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 5463 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5464 5465 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 5466 NULL, qp->qp_name, "MGETHDR failed"); 5467 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 5468 NULL, qp->qp_name, "MCLGET failed"); 5469 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 5470 EVCNT_TYPE_MISC, NULL, qp->qp_name, 5471 "bus_dmamap_load_mbuf failed"); 5472 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 5473 NULL, qp->qp_name, "Interrupt on queue"); 5474 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 5475 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 5476 } 5477 5478 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 5479 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 5480 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 5481 NULL, device_xname(sc->sc_dev), "Link status event"); 5482 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 5483 NULL, device_xname(sc->sc_dev), "ECC error"); 5484 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 5485 NULL, device_xname(sc->sc_dev), "PCI exception"); 5486 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 5487 NULL, device_xname(sc->sc_dev), "Critical error"); 5488 5489 return 0; 5490 } 5491 5492 static void 5493 ixl_teardown_stats(struct ixl_softc *sc) 5494 { 5495 struct ixl_tx_ring *txr; 5496 struct ixl_rx_ring *rxr; 5497 unsigned int i; 5498 5499 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5500 txr = sc->sc_qps[i].qp_txr; 5501 rxr = sc->sc_qps[i].qp_rxr; 5502 5503 evcnt_detach(&txr->txr_defragged); 5504 evcnt_detach(&txr->txr_defrag_failed); 5505 evcnt_detach(&txr->txr_pcqdrop); 5506 evcnt_detach(&txr->txr_transmitdef); 5507 evcnt_detach(&txr->txr_intr); 5508 evcnt_detach(&txr->txr_defer); 5509 5510 evcnt_detach(&rxr->rxr_mgethdr_failed); 5511 evcnt_detach(&rxr->rxr_mgetcl_failed); 5512 evcnt_detach(&rxr->rxr_mbuf_load_failed); 5513 evcnt_detach(&rxr->rxr_intr); 5514 evcnt_detach(&rxr->rxr_defer); 5515 } 5516 5517 evcnt_detach(&sc->sc_event_atq); 5518 evcnt_detach(&sc->sc_event_link); 5519 evcnt_detach(&sc->sc_event_ecc_err); 5520 evcnt_detach(&sc->sc_event_pci_exception); 5521 evcnt_detach(&sc->sc_event_crit_err); 5522 } 5523 5524 static int 5525 ixl_setup_sysctls(struct ixl_softc *sc) 5526 { 5527 const char *devname; 5528 struct sysctllog **log; 5529 const struct sysctlnode *rnode, *rxnode, *txnode; 5530 int error; 5531 5532 log = &sc->sc_sysctllog; 5533 devname = device_xname(sc->sc_dev); 5534 5535 error = sysctl_createv(log, 0, NULL, &rnode, 5536 0, CTLTYPE_NODE, devname, 5537 SYSCTL_DESCR("ixl information and settings"), 5538 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 5539 if (error) 5540 goto out; 5541 5542 error = sysctl_createv(log, 0, &rnode, NULL, 5543 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 5544 SYSCTL_DESCR("Use workqueue for packet processing"), 5545 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 5546 if (error) 5547 goto out; 5548 5549 error = sysctl_createv(log, 0, &rnode, &rxnode, 5550 0, CTLTYPE_NODE, "rx", 5551 SYSCTL_DESCR("ixl information and settings for Rx"), 5552 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 5553 if (error) 5554 goto out; 5555 5556 error = sysctl_createv(log, 0, &rxnode, NULL, 5557 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 5558 SYSCTL_DESCR("max number of Rx packets" 5559 " to process for interrupt processing"), 5560 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 5561 if (error) 5562 goto out; 5563 5564 error = sysctl_createv(log, 0, &rxnode, NULL, 5565 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 5566 SYSCTL_DESCR("max number of Rx packets" 5567 " to process for deferred processing"), 5568 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 5569 if (error) 5570 goto out; 5571 5572 error = sysctl_createv(log, 0, &rnode, &txnode, 5573 0, CTLTYPE_NODE, "tx", 5574 SYSCTL_DESCR("ixl information and settings for Tx"), 5575 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 5576 if (error) 5577 goto out; 5578 5579 error = sysctl_createv(log, 0, &txnode, NULL, 5580 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 5581 SYSCTL_DESCR("max number of Tx packets" 5582 " to process for interrupt processing"), 5583 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 5584 if (error) 5585 goto out; 5586 5587 error = sysctl_createv(log, 0, &txnode, NULL, 5588 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 5589 SYSCTL_DESCR("max number of Tx packets" 5590 " to process for deferred processing"), 5591 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 5592 if (error) 5593 goto out; 5594 5595 out: 5596 if (error) { 5597 aprint_error_dev(sc->sc_dev, 5598 "unable to create sysctl node\n"); 5599 sysctl_teardown(log); 5600 } 5601 5602 return error; 5603 } 5604 5605 static void 5606 ixl_teardown_sysctls(struct ixl_softc *sc) 5607 { 5608 5609 sysctl_teardown(&sc->sc_sysctllog); 5610 } 5611 5612 static struct workqueue * 5613 ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 5614 { 5615 struct workqueue *wq; 5616 int error; 5617 5618 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 5619 prio, ipl, flags); 5620 5621 if (error) 5622 return NULL; 5623 5624 return wq; 5625 } 5626 5627 static void 5628 ixl_workq_destroy(struct workqueue *wq) 5629 { 5630 5631 workqueue_destroy(wq); 5632 } 5633 5634 static void 5635 ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 5636 { 5637 5638 memset(work, 0, sizeof(*work)); 5639 work->ixw_func = func; 5640 work->ixw_arg = arg; 5641 } 5642 5643 static void 5644 ixl_work_add(struct workqueue *wq, struct ixl_work *work) 5645 { 5646 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 5647 return; 5648 5649 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 5650 } 5651 5652 static void 5653 ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 5654 { 5655 5656 workqueue_wait(wq, &work->ixw_cookie); 5657 } 5658 5659 static void 5660 ixl_workq_work(struct work *wk, void *context) 5661 { 5662 struct ixl_work *work; 5663 5664 work = container_of(wk, struct ixl_work, ixw_cookie); 5665 5666 atomic_swap_uint(&work->ixw_added, 0); 5667 kpreempt_disable(); 5668 work->ixw_func(work->ixw_arg); 5669 kpreempt_enable(); 5670 } 5671 5672 static int 5673 ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 5674 { 5675 struct ixl_aq_desc iaq; 5676 5677 memset(&iaq, 0, sizeof(iaq)); 5678 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 5679 iaq.iaq_param[1] = htole32(reg); 5680 5681 if (ixl_atq_poll(sc, &iaq, 250) != 0) 5682 return ETIMEDOUT; 5683 5684 switch (htole16(iaq.iaq_retval)) { 5685 case IXL_AQ_RC_OK: 5686 /* success */ 5687 break; 5688 case IXL_AQ_RC_EACCES: 5689 return EPERM; 5690 case IXL_AQ_RC_EAGAIN: 5691 return EAGAIN; 5692 default: 5693 return EIO; 5694 } 5695 5696 *rv = htole32(iaq.iaq_param[3]); 5697 return 0; 5698 } 5699 5700 static uint32_t 5701 ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 5702 { 5703 uint32_t val; 5704 int rv, retry, retry_limit; 5705 5706 retry_limit = sc->sc_rxctl_atq ? 5 : 0; 5707 5708 for (retry = 0; retry < retry_limit; retry++) { 5709 rv = ixl_rx_ctl_read(sc, reg, &val); 5710 if (rv == 0) 5711 return val; 5712 else if (rv == EAGAIN) 5713 delaymsec(1); 5714 else 5715 break; 5716 } 5717 5718 val = ixl_rd(sc, reg); 5719 5720 return val; 5721 } 5722 5723 static int 5724 ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 5725 { 5726 struct ixl_aq_desc iaq; 5727 5728 memset(&iaq, 0, sizeof(iaq)); 5729 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 5730 iaq.iaq_param[1] = htole32(reg); 5731 iaq.iaq_param[3] = htole32(value); 5732 5733 if (ixl_atq_poll(sc, &iaq, 250) != 0) 5734 return ETIMEDOUT; 5735 5736 switch (htole16(iaq.iaq_retval)) { 5737 case IXL_AQ_RC_OK: 5738 /* success */ 5739 break; 5740 case IXL_AQ_RC_EACCES: 5741 return EPERM; 5742 case IXL_AQ_RC_EAGAIN: 5743 return EAGAIN; 5744 default: 5745 return EIO; 5746 } 5747 5748 return 0; 5749 } 5750 5751 static void 5752 ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 5753 { 5754 int rv, retry, retry_limit; 5755 5756 retry_limit = sc->sc_rxctl_atq ? 5 : 0; 5757 5758 for (retry = 0; retry < retry_limit; retry++) { 5759 rv = ixl_rx_ctl_write(sc, reg, value); 5760 if (rv == 0) 5761 return; 5762 else if (rv == EAGAIN) 5763 delaymsec(1); 5764 else 5765 break; 5766 } 5767 5768 ixl_wr(sc, reg, value); 5769 } 5770 5771 MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 5772 5773 #ifdef _MODULE 5774 #include "ioconf.c" 5775 #endif 5776 5777 static int 5778 if_ixl_modcmd(modcmd_t cmd, void *opaque) 5779 { 5780 int error = 0; 5781 5782 #ifdef _MODULE 5783 switch (cmd) { 5784 case MODULE_CMD_INIT: 5785 error = config_init_component(cfdriver_ioconf_if_ixl, 5786 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 5787 break; 5788 case MODULE_CMD_FINI: 5789 error = config_fini_component(cfdriver_ioconf_if_ixl, 5790 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 5791 break; 5792 default: 5793 error = ENOTTY; 5794 break; 5795 } 5796 #endif 5797 5798 return error; 5799 } 5800
Indexes created Sun Sep 21 16:09:51 GMT 2025