if_vmx.c revision 1.9
1 /* $NetBSD: if_vmx.c,v 1.9 2022/07/06 06:32:50 msaitoh Exp $ */ 2 /* $OpenBSD: if_vmx.c,v 1.16 2014/01/22 06:04:17 brad Exp $ */ 3 4 /* 5 * Copyright (c) 2013 Tsubai Masanari 6 * Copyright (c) 2013 Bryan Venteicher <bryanv (at) FreeBSD.org> 7 * 8 * Permission to use, copy, modify, and distribute this software for any 9 * purpose with or without fee is hereby granted, provided that the above 10 * copyright notice and this permission notice appear in all copies. 11 * 12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 19 */ 20 21 #include <sys/cdefs.h> 22 __KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1.9 2022/07/06 06:32:50 msaitoh Exp $"); 23 24 #include <sys/param.h> 25 #include <sys/cpu.h> 26 #include <sys/kernel.h> 27 #include <sys/kmem.h> 28 #include <sys/bitops.h> 29 #include <sys/bus.h> 30 #include <sys/device.h> 31 #include <sys/mbuf.h> 32 #include <sys/module.h> 33 #include <sys/sockio.h> 34 #include <sys/pcq.h> 35 #include <sys/workqueue.h> 36 #include <sys/interrupt.h> 37 38 #include <net/bpf.h> 39 #include <net/if.h> 40 #include <net/if_ether.h> 41 #include <net/if_media.h> 42 43 #include <netinet/if_inarp.h> 44 #include <netinet/in_systm.h> /* for <netinet/ip.h> */ 45 #include <netinet/in.h> /* for <netinet/ip.h> */ 46 #include <netinet/ip.h> /* for struct ip */ 47 #include <netinet/ip6.h> /* for struct ip6_hdr */ 48 #include <netinet/tcp.h> /* for struct tcphdr */ 49 #include <netinet/udp.h> /* for struct udphdr */ 50 51 #include <dev/pci/pcivar.h> 52 #include <dev/pci/pcireg.h> 53 #include <dev/pci/pcidevs.h> 54 55 #include <dev/pci/if_vmxreg.h> 56 57 #define VMXNET3_DRIVER_VERSION 0x00010000 58 59 /* 60 * Max descriptors per Tx packet. We must limit the size of the 61 * any TSO packets based on the number of segments. 62 */ 63 #define VMXNET3_TX_MAXSEGS 32 64 #define VMXNET3_TX_MAXSIZE (VMXNET3_TX_MAXSEGS * MCLBYTES) 65 66 /* 67 * Maximum support Tx segments size. The length field in the 68 * Tx descriptor is 14 bits. 69 */ 70 #define VMXNET3_TX_MAXSEGSIZE (1 << 14) 71 72 /* 73 * The maximum number of Rx segments we accept. 74 */ 75 #define VMXNET3_MAX_RX_SEGS 0 /* no segments */ 76 77 /* 78 * Predetermined size of the multicast MACs filter table. If the 79 * number of multicast addresses exceeds this size, then the 80 * ALL_MULTI mode is use instead. 81 */ 82 #define VMXNET3_MULTICAST_MAX 32 83 84 /* 85 * Our Tx watchdog timeout. 86 */ 87 #define VMXNET3_WATCHDOG_TIMEOUT 5 88 89 /* 90 * Default value for vmx_intr_{rx,tx}_process_limit which is used for 91 * max number of packets to process for interrupt handler 92 */ 93 #define VMXNET3_RX_INTR_PROCESS_LIMIT 0U 94 #define VMXNET3_TX_INTR_PROCESS_LIMIT 256 95 96 /* 97 * Default value for vmx_{rx,tx}_process_limit which is used for 98 * max number of packets to process for deferred processing 99 */ 100 #define VMXNET3_RX_PROCESS_LIMIT 256 101 #define VMXNET3_TX_PROCESS_LIMIT 256 102 103 #define VMXNET3_WORKQUEUE_PRI PRI_SOFTNET 104 105 /* 106 * IP protocols that we can perform Tx checksum offloading of. 107 */ 108 #define VMXNET3_CSUM_OFFLOAD \ 109 (M_CSUM_TCPv4 | M_CSUM_UDPv4) 110 #define VMXNET3_CSUM_OFFLOAD_IPV6 \ 111 (M_CSUM_TCPv6 | M_CSUM_UDPv6) 112 113 #define VMXNET3_CSUM_ALL_OFFLOAD \ 114 (VMXNET3_CSUM_OFFLOAD | VMXNET3_CSUM_OFFLOAD_IPV6 | M_CSUM_TSOv4 | M_CSUM_TSOv6) 115 116 #define VMXNET3_RXRINGS_PERQ 2 117 118 #define VMXNET3_CORE_LOCK(_sc) mutex_enter((_sc)->vmx_mtx) 119 #define VMXNET3_CORE_UNLOCK(_sc) mutex_exit((_sc)->vmx_mtx) 120 #define VMXNET3_CORE_LOCK_ASSERT(_sc) mutex_owned((_sc)->vmx_mtx) 121 122 #define VMXNET3_RXQ_LOCK(_rxq) mutex_enter((_rxq)->vxrxq_mtx) 123 #define VMXNET3_RXQ_UNLOCK(_rxq) mutex_exit((_rxq)->vxrxq_mtx) 124 #define VMXNET3_RXQ_LOCK_ASSERT(_rxq) \ 125 mutex_owned((_rxq)->vxrxq_mtx) 126 127 #define VMXNET3_TXQ_LOCK(_txq) mutex_enter((_txq)->vxtxq_mtx) 128 #define VMXNET3_TXQ_TRYLOCK(_txq) mutex_tryenter((_txq)->vxtxq_mtx) 129 #define VMXNET3_TXQ_UNLOCK(_txq) mutex_exit((_txq)->vxtxq_mtx) 130 #define VMXNET3_TXQ_LOCK_ASSERT(_txq) \ 131 mutex_owned((_txq)->vxtxq_mtx) 132 133 struct vmxnet3_dma_alloc { 134 bus_addr_t dma_paddr; 135 void *dma_vaddr; 136 bus_dmamap_t dma_map; 137 bus_size_t dma_size; 138 bus_dma_segment_t dma_segs[1]; 139 }; 140 141 struct vmxnet3_txbuf { 142 bus_dmamap_t vtxb_dmamap; 143 struct mbuf *vtxb_m; 144 }; 145 146 struct vmxnet3_txring { 147 struct vmxnet3_txbuf *vxtxr_txbuf; 148 struct vmxnet3_txdesc *vxtxr_txd; 149 u_int vxtxr_head; 150 u_int vxtxr_next; 151 u_int vxtxr_ndesc; 152 int vxtxr_gen; 153 struct vmxnet3_dma_alloc vxtxr_dma; 154 }; 155 156 struct vmxnet3_rxbuf { 157 bus_dmamap_t vrxb_dmamap; 158 struct mbuf *vrxb_m; 159 }; 160 161 struct vmxnet3_rxring { 162 struct vmxnet3_rxbuf *vxrxr_rxbuf; 163 struct vmxnet3_rxdesc *vxrxr_rxd; 164 u_int vxrxr_fill; 165 u_int vxrxr_ndesc; 166 int vxrxr_gen; 167 int vxrxr_rid; 168 struct vmxnet3_dma_alloc vxrxr_dma; 169 bus_dmamap_t vxrxr_spare_dmap; 170 }; 171 172 struct vmxnet3_comp_ring { 173 union { 174 struct vmxnet3_txcompdesc *txcd; 175 struct vmxnet3_rxcompdesc *rxcd; 176 } vxcr_u; 177 u_int vxcr_next; 178 u_int vxcr_ndesc; 179 int vxcr_gen; 180 struct vmxnet3_dma_alloc vxcr_dma; 181 }; 182 183 struct vmxnet3_txq_stats { 184 #if 0 185 uint64_t vmtxs_opackets; /* if_opackets */ 186 uint64_t vmtxs_obytes; /* if_obytes */ 187 uint64_t vmtxs_omcasts; /* if_omcasts */ 188 #endif 189 uint64_t vmtxs_csum; 190 uint64_t vmtxs_tso; 191 uint64_t vmtxs_full; 192 uint64_t vmtxs_offload_failed; 193 }; 194 195 struct vmxnet3_txqueue { 196 kmutex_t *vxtxq_mtx; 197 struct vmxnet3_softc *vxtxq_sc; 198 int vxtxq_watchdog; 199 pcq_t *vxtxq_interq; 200 struct vmxnet3_txring vxtxq_cmd_ring; 201 struct vmxnet3_comp_ring vxtxq_comp_ring; 202 struct vmxnet3_txq_stats vxtxq_stats; 203 struct vmxnet3_txq_shared *vxtxq_ts; 204 char vxtxq_name[16]; 205 206 void *vxtxq_si; 207 208 struct evcnt vxtxq_intr; 209 struct evcnt vxtxq_defer; 210 struct evcnt vxtxq_deferreq; 211 struct evcnt vxtxq_pcqdrop; 212 struct evcnt vxtxq_transmitdef; 213 struct evcnt vxtxq_watchdogto; 214 struct evcnt vxtxq_defragged; 215 struct evcnt vxtxq_defrag_failed; 216 }; 217 218 #if 0 219 struct vmxnet3_rxq_stats { 220 uint64_t vmrxs_ipackets; /* if_ipackets */ 221 uint64_t vmrxs_ibytes; /* if_ibytes */ 222 uint64_t vmrxs_iqdrops; /* if_iqdrops */ 223 uint64_t vmrxs_ierrors; /* if_ierrors */ 224 }; 225 #endif 226 227 struct vmxnet3_rxqueue { 228 kmutex_t *vxrxq_mtx; 229 struct vmxnet3_softc *vxrxq_sc; 230 struct mbuf *vxrxq_mhead; 231 struct mbuf *vxrxq_mtail; 232 struct vmxnet3_rxring vxrxq_cmd_ring[VMXNET3_RXRINGS_PERQ]; 233 struct vmxnet3_comp_ring vxrxq_comp_ring; 234 #if 0 235 struct vmxnet3_rxq_stats vxrxq_stats; 236 #endif 237 struct vmxnet3_rxq_shared *vxrxq_rs; 238 char vxrxq_name[16]; 239 240 struct evcnt vxrxq_intr; 241 struct evcnt vxrxq_defer; 242 struct evcnt vxrxq_deferreq; 243 struct evcnt vxrxq_mgetcl_failed; 244 struct evcnt vxrxq_mbuf_load_failed; 245 }; 246 247 struct vmxnet3_queue { 248 int vxq_id; 249 int vxq_intr_idx; 250 251 struct vmxnet3_txqueue vxq_txqueue; 252 struct vmxnet3_rxqueue vxq_rxqueue; 253 254 void *vxq_si; 255 bool vxq_workqueue; 256 bool vxq_wq_enqueued; 257 struct work vxq_wq_cookie; 258 }; 259 260 struct vmxnet3_softc { 261 device_t vmx_dev; 262 struct ethercom vmx_ethercom; 263 struct ifmedia vmx_media; 264 struct vmxnet3_driver_shared *vmx_ds; 265 int vmx_flags; 266 #define VMXNET3_FLAG_NO_MSIX (1 << 0) 267 #define VMXNET3_FLAG_RSS (1 << 1) 268 #define VMXNET3_FLAG_ATTACHED (1 << 2) 269 270 struct vmxnet3_queue *vmx_queue; 271 272 struct pci_attach_args *vmx_pa; 273 pci_chipset_tag_t vmx_pc; 274 275 bus_space_tag_t vmx_iot0; 276 bus_space_tag_t vmx_iot1; 277 bus_space_handle_t vmx_ioh0; 278 bus_space_handle_t vmx_ioh1; 279 bus_size_t vmx_ios0; 280 bus_size_t vmx_ios1; 281 bus_dma_tag_t vmx_dmat; 282 283 int vmx_link_active; 284 int vmx_ntxqueues; 285 int vmx_nrxqueues; 286 int vmx_ntxdescs; 287 int vmx_nrxdescs; 288 int vmx_max_rxsegs; 289 290 struct evcnt vmx_event_intr; 291 struct evcnt vmx_event_link; 292 struct evcnt vmx_event_txqerror; 293 struct evcnt vmx_event_rxqerror; 294 struct evcnt vmx_event_dic; 295 struct evcnt vmx_event_debug; 296 297 int vmx_intr_type; 298 int vmx_intr_mask_mode; 299 int vmx_event_intr_idx; 300 int vmx_nintrs; 301 pci_intr_handle_t *vmx_intrs; /* legacy use vmx_intrs[0] */ 302 void *vmx_ihs[VMXNET3_MAX_INTRS]; 303 304 kmutex_t *vmx_mtx; 305 306 uint8_t *vmx_mcast; 307 void *vmx_qs; 308 struct vmxnet3_rss_shared *vmx_rss; 309 callout_t vmx_tick; 310 struct vmxnet3_dma_alloc vmx_ds_dma; 311 struct vmxnet3_dma_alloc vmx_qs_dma; 312 struct vmxnet3_dma_alloc vmx_mcast_dma; 313 struct vmxnet3_dma_alloc vmx_rss_dma; 314 int vmx_max_ntxqueues; 315 int vmx_max_nrxqueues; 316 uint8_t vmx_lladdr[ETHER_ADDR_LEN]; 317 318 u_int vmx_rx_intr_process_limit; 319 u_int vmx_tx_intr_process_limit; 320 u_int vmx_rx_process_limit; 321 u_int vmx_tx_process_limit; 322 struct sysctllog *vmx_sysctllog; 323 324 bool vmx_txrx_workqueue; 325 struct workqueue *vmx_queue_wq; 326 }; 327 328 #define VMXNET3_STAT 329 330 #ifdef VMXNET3_STAT 331 struct { 332 u_int txhead; 333 u_int txdone; 334 u_int maxtxlen; 335 u_int rxdone; 336 u_int rxfill; 337 u_int intr; 338 } vmxstat; 339 #endif 340 341 typedef enum { 342 VMXNET3_BARRIER_RD, 343 VMXNET3_BARRIER_WR, 344 } vmxnet3_barrier_t; 345 346 #define JUMBO_LEN (MCLBYTES - ETHER_ALIGN) /* XXX */ 347 #define DMAADDR(map) ((map)->dm_segs[0].ds_addr) 348 349 #define vtophys(va) 0 /* XXX ok? */ 350 351 static int vmxnet3_match(device_t, cfdata_t, void *); 352 static void vmxnet3_attach(device_t, device_t, void *); 353 static int vmxnet3_detach(device_t, int); 354 355 static int vmxnet3_alloc_pci_resources(struct vmxnet3_softc *); 356 static void vmxnet3_free_pci_resources(struct vmxnet3_softc *); 357 static int vmxnet3_check_version(struct vmxnet3_softc *); 358 static void vmxnet3_check_multiqueue(struct vmxnet3_softc *); 359 360 static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 361 static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 362 static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 363 static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 364 static void vmxnet3_free_interrupts(struct vmxnet3_softc *); 365 366 static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 367 static int vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *); 368 static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 369 static void vmxnet3_set_interrupt_idx(struct vmxnet3_softc *); 370 static int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 371 static int vmxnet3_setup_sysctl(struct vmxnet3_softc *); 372 373 static int vmxnet3_setup_stats(struct vmxnet3_softc *); 374 static void vmxnet3_teardown_stats(struct vmxnet3_softc *); 375 376 static int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 377 static int vmxnet3_init_txq(struct vmxnet3_softc *, int); 378 static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 379 static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 380 static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 381 static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 382 383 static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 384 static void vmxnet3_free_shared_data(struct vmxnet3_softc *); 385 static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 386 static void vmxnet3_free_txq_data(struct vmxnet3_softc *); 387 static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 388 static void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 389 static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 390 static void vmxnet3_free_queue_data(struct vmxnet3_softc *); 391 static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 392 static void vmxnet3_free_mcast_table(struct vmxnet3_softc *); 393 static void vmxnet3_init_shared_data(struct vmxnet3_softc *); 394 static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); 395 static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 396 static int vmxnet3_alloc_data(struct vmxnet3_softc *); 397 static void vmxnet3_free_data(struct vmxnet3_softc *); 398 static int vmxnet3_setup_interface(struct vmxnet3_softc *); 399 400 static void vmxnet3_evintr(struct vmxnet3_softc *); 401 static bool vmxnet3_txq_eof(struct vmxnet3_txqueue *, u_int); 402 static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxqueue *, 403 struct vmxnet3_rxring *); 404 static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 405 struct vmxnet3_rxring *, int); 406 static void vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *); 407 static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 408 static void vmxnet3_rxq_input(struct vmxnet3_rxqueue *, 409 struct vmxnet3_rxcompdesc *, struct mbuf *); 410 static bool vmxnet3_rxq_eof(struct vmxnet3_rxqueue *, u_int); 411 static int vmxnet3_legacy_intr(void *); 412 static int vmxnet3_txrxq_intr(void *); 413 static void vmxnet3_handle_queue(void *); 414 static void vmxnet3_handle_queue_work(struct work *, void *); 415 static int vmxnet3_event_intr(void *); 416 417 static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 418 static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 419 static void vmxnet3_stop_locked(struct vmxnet3_softc *); 420 static void vmxnet3_stop_rendezvous(struct vmxnet3_softc *); 421 static void vmxnet3_stop(struct ifnet *, int); 422 423 static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 424 static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 425 static int vmxnet3_reinit_queues(struct vmxnet3_softc *); 426 static int vmxnet3_enable_device(struct vmxnet3_softc *); 427 static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 428 static int vmxnet3_reinit(struct vmxnet3_softc *); 429 430 static int vmxnet3_init_locked(struct vmxnet3_softc *); 431 static int vmxnet3_init(struct ifnet *); 432 433 static int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *, struct mbuf *, int *, int *); 434 static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, bus_dmamap_t); 435 static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 436 static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 437 static void vmxnet3_start_locked(struct ifnet *); 438 static void vmxnet3_start(struct ifnet *); 439 static void vmxnet3_transmit_locked(struct ifnet *, struct vmxnet3_txqueue *); 440 static int vmxnet3_transmit(struct ifnet *, struct mbuf *); 441 static void vmxnet3_deferred_transmit(void *); 442 443 static void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 444 static int vmxnet3_ioctl(struct ifnet *, u_long, void *); 445 static int vmxnet3_ifflags_cb(struct ethercom *); 446 447 static int vmxnet3_watchdog(struct vmxnet3_txqueue *); 448 static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); 449 static void vmxnet3_tick(void *); 450 static void vmxnet3_if_link_status(struct vmxnet3_softc *); 451 static bool vmxnet3_cmd_link_status(struct ifnet *); 452 static void vmxnet3_ifmedia_status(struct ifnet *, struct ifmediareq *); 453 static int vmxnet3_ifmedia_change(struct ifnet *); 454 static void vmxnet3_set_lladdr(struct vmxnet3_softc *); 455 static void vmxnet3_get_lladdr(struct vmxnet3_softc *); 456 457 static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 458 static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 459 460 static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, bus_size_t, 461 struct vmxnet3_dma_alloc *); 462 static void vmxnet3_dma_free(struct vmxnet3_softc *, struct vmxnet3_dma_alloc *); 463 464 CFATTACH_DECL3_NEW(vmx, sizeof(struct vmxnet3_softc), 465 vmxnet3_match, vmxnet3_attach, vmxnet3_detach, NULL, NULL, NULL, 0); 466 467 /* round down to the nearest power of 2 */ 468 static int 469 vmxnet3_calc_queue_size(int n) 470 { 471 472 if (__predict_false(n <= 0)) 473 return 1; 474 475 return (1U << (fls32(n) - 1)); 476 } 477 478 static inline void 479 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 480 { 481 482 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 483 } 484 485 static inline uint32_t 486 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 487 { 488 489 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 490 } 491 492 static inline void 493 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 494 { 495 496 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 497 } 498 499 static inline void 500 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 501 { 502 503 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 504 } 505 506 static inline uint32_t 507 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 508 { 509 510 vmxnet3_write_cmd(sc, cmd); 511 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 512 } 513 514 static inline void 515 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 516 { 517 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 518 } 519 520 static inline void 521 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 522 { 523 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 524 } 525 526 static inline void 527 vmxnet3_rxr_increment_fill(struct vmxnet3_rxring *rxr) 528 { 529 530 if (++rxr->vxrxr_fill == rxr->vxrxr_ndesc) { 531 rxr->vxrxr_fill = 0; 532 rxr->vxrxr_gen ^= 1; 533 } 534 } 535 536 static inline int 537 vmxnet3_txring_avail(struct vmxnet3_txring *txr) 538 { 539 int avail = txr->vxtxr_next - txr->vxtxr_head - 1; 540 return (avail < 0 ? (int)txr->vxtxr_ndesc + avail : avail); 541 } 542 543 /* 544 * Since this is a purely paravirtualized device, we do not have 545 * to worry about DMA coherency. But at times, we must make sure 546 * both the compiler and CPU do not reorder memory operations. 547 */ 548 static inline void 549 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 550 { 551 552 switch (type) { 553 case VMXNET3_BARRIER_RD: 554 membar_consumer(); 555 break; 556 case VMXNET3_BARRIER_WR: 557 membar_producer(); 558 break; 559 default: 560 panic("%s: bad barrier type %d", __func__, type); 561 } 562 } 563 564 static int 565 vmxnet3_match(device_t parent, cfdata_t match, void *aux) 566 { 567 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 568 569 if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_VMWARE && 570 PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_VMWARE_VMXNET3) 571 return 1; 572 573 return 0; 574 } 575 576 static void 577 vmxnet3_attach(device_t parent, device_t self, void *aux) 578 { 579 struct vmxnet3_softc *sc = device_private(self); 580 struct pci_attach_args *pa = aux; 581 pcireg_t preg; 582 int error; 583 int candidate; 584 585 sc->vmx_dev = self; 586 sc->vmx_pa = pa; 587 sc->vmx_pc = pa->pa_pc; 588 if (pci_dma64_available(pa)) 589 sc->vmx_dmat = pa->pa_dmat64; 590 else 591 sc->vmx_dmat = pa->pa_dmat; 592 593 pci_aprint_devinfo_fancy(pa, "Ethernet controller", "vmxnet3", 1); 594 595 preg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 596 preg |= PCI_COMMAND_MASTER_ENABLE; 597 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, preg); 598 599 sc->vmx_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 600 callout_init(&sc->vmx_tick, CALLOUT_MPSAFE); 601 602 candidate = MIN(MIN(VMXNET3_MAX_TX_QUEUES, VMXNET3_MAX_RX_QUEUES), 603 ncpu); 604 sc->vmx_max_ntxqueues = sc->vmx_max_nrxqueues = 605 vmxnet3_calc_queue_size(candidate); 606 sc->vmx_ntxdescs = 512; 607 sc->vmx_nrxdescs = 256; 608 sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 609 610 error = vmxnet3_alloc_pci_resources(sc); 611 if (error) 612 return; 613 614 error = vmxnet3_check_version(sc); 615 if (error) 616 return; 617 618 error = vmxnet3_alloc_rxtx_queues(sc); 619 if (error) 620 return; 621 622 error = vmxnet3_alloc_interrupts(sc); 623 if (error) 624 return; 625 626 vmxnet3_check_multiqueue(sc); 627 628 error = vmxnet3_alloc_data(sc); 629 if (error) 630 return; 631 632 error = vmxnet3_setup_interface(sc); 633 if (error) 634 return; 635 636 error = vmxnet3_setup_interrupts(sc); 637 if (error) 638 return; 639 640 error = vmxnet3_setup_sysctl(sc); 641 if (error) 642 return; 643 644 error = vmxnet3_setup_stats(sc); 645 if (error) 646 return; 647 648 sc->vmx_flags |= VMXNET3_FLAG_ATTACHED; 649 } 650 651 static int 652 vmxnet3_detach(device_t self, int flags) 653 { 654 struct vmxnet3_softc *sc; 655 struct ifnet *ifp; 656 657 sc = device_private(self); 658 ifp = &sc->vmx_ethercom.ec_if; 659 660 if (sc->vmx_flags & VMXNET3_FLAG_ATTACHED) { 661 VMXNET3_CORE_LOCK(sc); 662 vmxnet3_stop_locked(sc); 663 callout_halt(&sc->vmx_tick, sc->vmx_mtx); 664 callout_destroy(&sc->vmx_tick); 665 VMXNET3_CORE_UNLOCK(sc); 666 667 ether_ifdetach(ifp); 668 if_detach(ifp); 669 ifmedia_fini(&sc->vmx_media); 670 } 671 672 vmxnet3_teardown_stats(sc); 673 sysctl_teardown(&sc->vmx_sysctllog); 674 675 vmxnet3_free_interrupts(sc); 676 677 vmxnet3_free_data(sc); 678 vmxnet3_free_pci_resources(sc); 679 vmxnet3_free_rxtx_queues(sc); 680 681 if (sc->vmx_mtx) 682 mutex_obj_free(sc->vmx_mtx); 683 684 return (0); 685 } 686 687 static int 688 vmxnet3_alloc_pci_resources(struct vmxnet3_softc *sc) 689 { 690 struct pci_attach_args *pa = sc->vmx_pa; 691 pcireg_t memtype; 692 693 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 694 if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->vmx_iot0, &sc->vmx_ioh0, 695 NULL, &sc->vmx_ios0)) { 696 aprint_error_dev(sc->vmx_dev, "failed to map BAR0\n"); 697 return (ENXIO); 698 } 699 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(1)); 700 if (pci_mapreg_map(pa, PCI_BAR(1), memtype, 0, &sc->vmx_iot1, &sc->vmx_ioh1, 701 NULL, &sc->vmx_ios1)) { 702 aprint_error_dev(sc->vmx_dev, "failed to map BAR1\n"); 703 return (ENXIO); 704 } 705 706 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, NULL, NULL)) { 707 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 708 return (0); 709 } 710 711 return (0); 712 } 713 714 static void 715 vmxnet3_free_pci_resources(struct vmxnet3_softc *sc) 716 { 717 718 if (sc->vmx_ios0) { 719 bus_space_unmap(sc->vmx_iot0, sc->vmx_ioh0, sc->vmx_ios0); 720 sc->vmx_ios0 = 0; 721 } 722 723 if (sc->vmx_ios1) { 724 bus_space_unmap(sc->vmx_iot1, sc->vmx_ioh1, sc->vmx_ios1); 725 sc->vmx_ios1 = 0; 726 } 727 } 728 729 static int 730 vmxnet3_check_version(struct vmxnet3_softc *sc) 731 { 732 u_int ver; 733 734 ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 735 if ((ver & 0x1) == 0) { 736 aprint_error_dev(sc->vmx_dev, 737 "unsupported hardware version 0x%x\n", ver); 738 return (ENOTSUP); 739 } 740 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 741 742 ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 743 if ((ver & 0x1) == 0) { 744 aprint_error_dev(sc->vmx_dev, 745 "incompatiable UPT version 0x%x\n", ver); 746 return (ENOTSUP); 747 } 748 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 749 750 return (0); 751 } 752 753 static void 754 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc) 755 { 756 757 if (sc->vmx_intr_type != VMXNET3_IT_MSIX) 758 goto out; 759 760 /* Just use the maximum configured for now. */ 761 sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; 762 sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; 763 764 if (sc->vmx_nrxqueues > 1) 765 sc->vmx_flags |= VMXNET3_FLAG_RSS; 766 767 return; 768 769 out: 770 sc->vmx_ntxqueues = 1; 771 sc->vmx_nrxqueues = 1; 772 } 773 774 static int 775 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 776 { 777 int required; 778 struct pci_attach_args *pa = sc->vmx_pa; 779 780 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 781 return (1); 782 783 /* Allocate an additional vector for the events interrupt. */ 784 required = MIN(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues) + 1; 785 786 if (pci_msix_count(pa->pa_pc, pa->pa_tag) < required) 787 return (1); 788 789 if (pci_msix_alloc_exact(pa, &sc->vmx_intrs, required) == 0) { 790 sc->vmx_nintrs = required; 791 return (0); 792 } 793 794 return (1); 795 } 796 797 static int 798 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 799 { 800 int nmsi, required; 801 struct pci_attach_args *pa = sc->vmx_pa; 802 803 required = 1; 804 805 nmsi = pci_msi_count(pa->pa_pc, pa->pa_tag); 806 if (nmsi < required) 807 return (1); 808 809 if (pci_msi_alloc_exact(pa, &sc->vmx_intrs, required) == 0) { 810 sc->vmx_nintrs = required; 811 return (0); 812 } 813 814 return (1); 815 } 816 817 static int 818 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 819 { 820 821 if (pci_intx_alloc(sc->vmx_pa, &sc->vmx_intrs) == 0) { 822 sc->vmx_nintrs = 1; 823 return (0); 824 } 825 826 return (1); 827 } 828 829 static int 830 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 831 { 832 u_int config; 833 int error; 834 835 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 836 837 sc->vmx_intr_type = config & 0x03; 838 sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 839 840 switch (sc->vmx_intr_type) { 841 case VMXNET3_IT_AUTO: 842 sc->vmx_intr_type = VMXNET3_IT_MSIX; 843 /* FALLTHROUGH */ 844 case VMXNET3_IT_MSIX: 845 error = vmxnet3_alloc_msix_interrupts(sc); 846 if (error == 0) 847 break; 848 sc->vmx_intr_type = VMXNET3_IT_MSI; 849 /* FALLTHROUGH */ 850 case VMXNET3_IT_MSI: 851 error = vmxnet3_alloc_msi_interrupts(sc); 852 if (error == 0) 853 break; 854 sc->vmx_intr_type = VMXNET3_IT_LEGACY; 855 /* FALLTHROUGH */ 856 case VMXNET3_IT_LEGACY: 857 error = vmxnet3_alloc_legacy_interrupts(sc); 858 if (error == 0) 859 break; 860 /* FALLTHROUGH */ 861 default: 862 sc->vmx_intr_type = -1; 863 aprint_error_dev(sc->vmx_dev, "cannot allocate any interrupt resources\n"); 864 return (ENXIO); 865 } 866 867 return (error); 868 } 869 870 static void 871 vmxnet3_free_interrupts(struct vmxnet3_softc *sc) 872 { 873 pci_chipset_tag_t pc = sc->vmx_pc; 874 int i; 875 876 workqueue_destroy(sc->vmx_queue_wq); 877 for (i = 0; i < sc->vmx_ntxqueues; i++) { 878 struct vmxnet3_queue *vmxq = &sc->vmx_queue[i]; 879 880 softint_disestablish(vmxq->vxq_si); 881 vmxq->vxq_si = NULL; 882 } 883 for (i = 0; i < sc->vmx_nintrs; i++) { 884 pci_intr_disestablish(pc, sc->vmx_ihs[i]); 885 } 886 pci_intr_release(pc, sc->vmx_intrs, sc->vmx_nintrs); 887 } 888 889 static int 890 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 891 { 892 pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; 893 struct vmxnet3_queue *vmxq; 894 pci_intr_handle_t *intr; 895 void **ihs; 896 int intr_idx, i, use_queues, error; 897 kcpuset_t *affinity; 898 const char *intrstr; 899 char intrbuf[PCI_INTRSTR_LEN]; 900 char xnamebuf[32]; 901 902 intr = sc->vmx_intrs; 903 intr_idx = 0; 904 ihs = sc->vmx_ihs; 905 906 /* See vmxnet3_alloc_msix_interrupts() */ 907 use_queues = MIN(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues); 908 for (i = 0; i < use_queues; i++, intr++, ihs++, intr_idx++) { 909 snprintf(xnamebuf, 32, "%s: txrx %d", device_xname(sc->vmx_dev), i); 910 911 vmxq = &sc->vmx_queue[i]; 912 913 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 914 915 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 916 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 917 vmxnet3_txrxq_intr, vmxq, xnamebuf); 918 if (*ihs == NULL) { 919 aprint_error_dev(sc->vmx_dev, 920 "unable to establish txrx interrupt at %s\n", intrstr); 921 return (-1); 922 } 923 aprint_normal_dev(sc->vmx_dev, "txrx interrupting at %s\n", intrstr); 924 925 kcpuset_create(&affinity, true); 926 kcpuset_set(affinity, intr_idx % ncpu); 927 error = interrupt_distribute(*ihs, affinity, NULL); 928 if (error) { 929 aprint_normal_dev(sc->vmx_dev, 930 "%s cannot be changed affinity, use default CPU\n", 931 intrstr); 932 } 933 kcpuset_destroy(affinity); 934 935 vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 936 vmxnet3_handle_queue, vmxq); 937 if (vmxq->vxq_si == NULL) { 938 aprint_error_dev(sc->vmx_dev, 939 "softint_establish for vxq_si failed\n"); 940 return (-1); 941 } 942 943 vmxq->vxq_intr_idx = intr_idx; 944 } 945 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(sc->vmx_dev)); 946 error = workqueue_create(&sc->vmx_queue_wq, xnamebuf, 947 vmxnet3_handle_queue_work, sc, VMXNET3_WORKQUEUE_PRI, IPL_NET, 948 WQ_PERCPU | WQ_MPSAFE); 949 if (error) { 950 aprint_error_dev(sc->vmx_dev, "workqueue_create failed\n"); 951 return (-1); 952 } 953 sc->vmx_txrx_workqueue = false; 954 955 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 956 957 snprintf(xnamebuf, 32, "%s: link", device_xname(sc->vmx_dev)); 958 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 959 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 960 vmxnet3_event_intr, sc, xnamebuf); 961 if (*ihs == NULL) { 962 aprint_error_dev(sc->vmx_dev, 963 "unable to establish event interrupt at %s\n", intrstr); 964 return (-1); 965 } 966 aprint_normal_dev(sc->vmx_dev, "event interrupting at %s\n", intrstr); 967 968 sc->vmx_event_intr_idx = intr_idx; 969 970 return (0); 971 } 972 973 static int 974 vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *sc) 975 { 976 pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; 977 pci_intr_handle_t *intr; 978 void **ihs; 979 struct vmxnet3_queue *vmxq; 980 int i; 981 const char *intrstr; 982 char intrbuf[PCI_INTRSTR_LEN]; 983 char xnamebuf[32]; 984 985 intr = &sc->vmx_intrs[0]; 986 ihs = sc->vmx_ihs; 987 vmxq = &sc->vmx_queue[0]; 988 989 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 990 991 snprintf(xnamebuf, 32, "%s: msi", device_xname(sc->vmx_dev)); 992 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 993 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 994 vmxnet3_legacy_intr, sc, xnamebuf); 995 if (*ihs == NULL) { 996 aprint_error_dev(sc->vmx_dev, 997 "unable to establish interrupt at %s\n", intrstr); 998 return (-1); 999 } 1000 aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr); 1001 1002 vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1003 vmxnet3_handle_queue, vmxq); 1004 if (vmxq->vxq_si == NULL) { 1005 aprint_error_dev(sc->vmx_dev, 1006 "softint_establish for vxq_si failed\n"); 1007 return (-1); 1008 } 1009 1010 for (i = 0; i < MIN(sc->vmx_nrxqueues, sc->vmx_nrxqueues); i++) 1011 sc->vmx_queue[i].vxq_intr_idx = 0; 1012 sc->vmx_event_intr_idx = 0; 1013 1014 return (0); 1015 } 1016 1017 static int 1018 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 1019 { 1020 pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; 1021 pci_intr_handle_t *intr; 1022 void **ihs; 1023 struct vmxnet3_queue *vmxq; 1024 int i; 1025 const char *intrstr; 1026 char intrbuf[PCI_INTRSTR_LEN]; 1027 char xnamebuf[32]; 1028 1029 intr = &sc->vmx_intrs[0]; 1030 ihs = sc->vmx_ihs; 1031 vmxq = &sc->vmx_queue[0]; 1032 1033 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1034 1035 snprintf(xnamebuf, 32, "%s:legacy", device_xname(sc->vmx_dev)); 1036 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1037 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1038 vmxnet3_legacy_intr, sc, xnamebuf); 1039 if (*ihs == NULL) { 1040 aprint_error_dev(sc->vmx_dev, 1041 "unable to establish interrupt at %s\n", intrstr); 1042 return (-1); 1043 } 1044 aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr); 1045 1046 vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1047 vmxnet3_handle_queue, vmxq); 1048 if (vmxq->vxq_si == NULL) { 1049 aprint_error_dev(sc->vmx_dev, 1050 "softint_establish for vxq_si failed\n"); 1051 return (-1); 1052 } 1053 1054 for (i = 0; i < MIN(sc->vmx_nrxqueues, sc->vmx_nrxqueues); i++) 1055 sc->vmx_queue[i].vxq_intr_idx = 0; 1056 sc->vmx_event_intr_idx = 0; 1057 1058 return (0); 1059 } 1060 1061 static void 1062 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 1063 { 1064 struct vmxnet3_queue *vmxq; 1065 struct vmxnet3_txqueue *txq; 1066 struct vmxnet3_txq_shared *txs; 1067 struct vmxnet3_rxqueue *rxq; 1068 struct vmxnet3_rxq_shared *rxs; 1069 int i; 1070 1071 sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 1072 1073 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1074 vmxq = &sc->vmx_queue[i]; 1075 txq = &vmxq->vxq_txqueue; 1076 txs = txq->vxtxq_ts; 1077 txs->intr_idx = vmxq->vxq_intr_idx; 1078 } 1079 1080 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1081 vmxq = &sc->vmx_queue[i]; 1082 rxq = &vmxq->vxq_rxqueue; 1083 rxs = rxq->vxrxq_rs; 1084 rxs->intr_idx = vmxq->vxq_intr_idx; 1085 } 1086 } 1087 1088 static int 1089 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 1090 { 1091 int error; 1092 1093 switch (sc->vmx_intr_type) { 1094 case VMXNET3_IT_MSIX: 1095 error = vmxnet3_setup_msix_interrupts(sc); 1096 break; 1097 case VMXNET3_IT_MSI: 1098 error = vmxnet3_setup_msi_interrupt(sc); 1099 break; 1100 case VMXNET3_IT_LEGACY: 1101 error = vmxnet3_setup_legacy_interrupt(sc); 1102 break; 1103 default: 1104 panic("%s: invalid interrupt type %d", __func__, 1105 sc->vmx_intr_type); 1106 } 1107 1108 if (error == 0) 1109 vmxnet3_set_interrupt_idx(sc); 1110 1111 return (error); 1112 } 1113 1114 static int 1115 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 1116 { 1117 struct vmxnet3_rxqueue *rxq; 1118 struct vmxnet3_rxring *rxr; 1119 int i; 1120 1121 rxq = &sc->vmx_queue[q].vxq_rxqueue; 1122 1123 snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 1124 device_xname(sc->vmx_dev), q); 1125 rxq->vxrxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */); 1126 1127 rxq->vxrxq_sc = sc; 1128 1129 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1130 rxr = &rxq->vxrxq_cmd_ring[i]; 1131 rxr->vxrxr_rid = i; 1132 rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 1133 rxr->vxrxr_rxbuf = kmem_zalloc(rxr->vxrxr_ndesc * 1134 sizeof(struct vmxnet3_rxbuf), KM_SLEEP); 1135 1136 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 1137 } 1138 1139 return (0); 1140 } 1141 1142 static int 1143 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 1144 { 1145 struct vmxnet3_txqueue *txq; 1146 struct vmxnet3_txring *txr; 1147 1148 txq = &sc->vmx_queue[q].vxq_txqueue; 1149 txr = &txq->vxtxq_cmd_ring; 1150 1151 snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 1152 device_xname(sc->vmx_dev), q); 1153 txq->vxtxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */); 1154 1155 txq->vxtxq_sc = sc; 1156 1157 txq->vxtxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1158 vmxnet3_deferred_transmit, txq); 1159 if (txq->vxtxq_si == NULL) { 1160 mutex_obj_free(txq->vxtxq_mtx); 1161 aprint_error_dev(sc->vmx_dev, 1162 "softint_establish for vxtxq_si failed\n"); 1163 return ENOMEM; 1164 } 1165 1166 txr->vxtxr_ndesc = sc->vmx_ntxdescs; 1167 txr->vxtxr_txbuf = kmem_zalloc(txr->vxtxr_ndesc * 1168 sizeof(struct vmxnet3_txbuf), KM_SLEEP); 1169 1170 txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 1171 1172 txq->vxtxq_interq = pcq_create(sc->vmx_ntxdescs, KM_SLEEP); 1173 1174 return (0); 1175 } 1176 1177 static int 1178 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 1179 { 1180 int i, error, max_nqueues; 1181 1182 KASSERT(!cpu_intr_p()); 1183 KASSERT(!cpu_softintr_p()); 1184 1185 /* 1186 * Only attempt to create multiple queues if MSIX is available. 1187 * This check prevents us from allocating queue structures that 1188 * we will not use. 1189 * 1190 * FreeBSD: 1191 * MSIX is disabled by default because its apparently broken for 1192 * devices passed through by at least ESXi 5.1. 1193 * The hw.pci.honor_msi_blacklist tunable must be set to zero for MSIX. 1194 */ 1195 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { 1196 sc->vmx_max_nrxqueues = 1; 1197 sc->vmx_max_ntxqueues = 1; 1198 } 1199 1200 max_nqueues = MAX(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues); 1201 sc->vmx_queue = kmem_zalloc(sizeof(struct vmxnet3_queue) * max_nqueues, 1202 KM_SLEEP); 1203 1204 for (i = 0; i < max_nqueues; i++) { 1205 struct vmxnet3_queue *vmxq = &sc->vmx_queue[i]; 1206 vmxq->vxq_id = i; 1207 } 1208 1209 for (i = 0; i < sc->vmx_max_nrxqueues; i++) { 1210 error = vmxnet3_init_rxq(sc, i); 1211 if (error) 1212 return (error); 1213 } 1214 1215 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1216 error = vmxnet3_init_txq(sc, i); 1217 if (error) 1218 return (error); 1219 } 1220 1221 return (0); 1222 } 1223 1224 static void 1225 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 1226 { 1227 struct vmxnet3_rxring *rxr; 1228 int i; 1229 1230 rxq->vxrxq_sc = NULL; 1231 1232 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1233 rxr = &rxq->vxrxq_cmd_ring[i]; 1234 1235 if (rxr->vxrxr_rxbuf != NULL) { 1236 kmem_free(rxr->vxrxr_rxbuf, 1237 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxbuf)); 1238 rxr->vxrxr_rxbuf = NULL; 1239 } 1240 } 1241 1242 if (rxq->vxrxq_mtx != NULL) 1243 mutex_obj_free(rxq->vxrxq_mtx); 1244 } 1245 1246 static void 1247 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 1248 { 1249 struct vmxnet3_txring *txr; 1250 struct mbuf *m; 1251 1252 txr = &txq->vxtxq_cmd_ring; 1253 1254 txq->vxtxq_sc = NULL; 1255 1256 softint_disestablish(txq->vxtxq_si); 1257 1258 while ((m = pcq_get(txq->vxtxq_interq)) != NULL) 1259 m_freem(m); 1260 pcq_destroy(txq->vxtxq_interq); 1261 1262 if (txr->vxtxr_txbuf != NULL) { 1263 kmem_free(txr->vxtxr_txbuf, 1264 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txbuf)); 1265 txr->vxtxr_txbuf = NULL; 1266 } 1267 1268 if (txq->vxtxq_mtx != NULL) 1269 mutex_obj_free(txq->vxtxq_mtx); 1270 } 1271 1272 static void 1273 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 1274 { 1275 int i; 1276 1277 if (sc->vmx_queue != NULL) { 1278 int max_nqueues; 1279 1280 for (i = 0; i < sc->vmx_max_nrxqueues; i++) 1281 vmxnet3_destroy_rxq(&sc->vmx_queue[i].vxq_rxqueue); 1282 1283 for (i = 0; i < sc->vmx_max_ntxqueues; i++) 1284 vmxnet3_destroy_txq(&sc->vmx_queue[i].vxq_txqueue); 1285 1286 max_nqueues = MAX(sc->vmx_max_nrxqueues, sc->vmx_max_ntxqueues); 1287 kmem_free(sc->vmx_queue, 1288 sizeof(struct vmxnet3_queue) * max_nqueues); 1289 } 1290 } 1291 1292 static int 1293 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 1294 { 1295 device_t dev; 1296 uint8_t *kva; 1297 size_t size; 1298 int i, error; 1299 1300 dev = sc->vmx_dev; 1301 1302 size = sizeof(struct vmxnet3_driver_shared); 1303 error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 1304 if (error) { 1305 device_printf(dev, "cannot alloc shared memory\n"); 1306 return (error); 1307 } 1308 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 1309 1310 size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 1311 sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 1312 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 1313 if (error) { 1314 device_printf(dev, "cannot alloc queue shared memory\n"); 1315 return (error); 1316 } 1317 sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 1318 kva = sc->vmx_qs; 1319 1320 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1321 sc->vmx_queue[i].vxq_txqueue.vxtxq_ts = 1322 (struct vmxnet3_txq_shared *) kva; 1323 kva += sizeof(struct vmxnet3_txq_shared); 1324 } 1325 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1326 sc->vmx_queue[i].vxq_rxqueue.vxrxq_rs = 1327 (struct vmxnet3_rxq_shared *) kva; 1328 kva += sizeof(struct vmxnet3_rxq_shared); 1329 } 1330 1331 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1332 size = sizeof(struct vmxnet3_rss_shared); 1333 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); 1334 if (error) { 1335 device_printf(dev, "cannot alloc rss shared memory\n"); 1336 return (error); 1337 } 1338 sc->vmx_rss = 1339 (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; 1340 } 1341 1342 return (0); 1343 } 1344 1345 static void 1346 vmxnet3_free_shared_data(struct vmxnet3_softc *sc) 1347 { 1348 1349 if (sc->vmx_rss != NULL) { 1350 vmxnet3_dma_free(sc, &sc->vmx_rss_dma); 1351 sc->vmx_rss = NULL; 1352 } 1353 1354 if (sc->vmx_qs != NULL) { 1355 vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 1356 sc->vmx_qs = NULL; 1357 } 1358 1359 if (sc->vmx_ds != NULL) { 1360 vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 1361 sc->vmx_ds = NULL; 1362 } 1363 } 1364 1365 static int 1366 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 1367 { 1368 device_t dev; 1369 struct vmxnet3_txqueue *txq; 1370 struct vmxnet3_txring *txr; 1371 struct vmxnet3_comp_ring *txc; 1372 size_t descsz, compsz; 1373 u_int i; 1374 int q, error; 1375 1376 dev = sc->vmx_dev; 1377 1378 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1379 txq = &sc->vmx_queue[q].vxq_txqueue; 1380 txr = &txq->vxtxq_cmd_ring; 1381 txc = &txq->vxtxq_comp_ring; 1382 1383 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 1384 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 1385 1386 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 1387 if (error) { 1388 device_printf(dev, "cannot alloc Tx descriptors for " 1389 "queue %d error %d\n", q, error); 1390 return (error); 1391 } 1392 txr->vxtxr_txd = 1393 (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1394 1395 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1396 if (error) { 1397 device_printf(dev, "cannot alloc Tx comp descriptors " 1398 "for queue %d error %d\n", q, error); 1399 return (error); 1400 } 1401 txc->vxcr_u.txcd = 1402 (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1403 1404 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1405 error = bus_dmamap_create(sc->vmx_dmat, VMXNET3_TX_MAXSIZE, 1406 VMXNET3_TX_MAXSEGS, VMXNET3_TX_MAXSEGSIZE, 0, BUS_DMA_NOWAIT, 1407 &txr->vxtxr_txbuf[i].vtxb_dmamap); 1408 if (error) { 1409 device_printf(dev, "unable to create Tx buf " 1410 "dmamap for queue %d idx %d\n", q, i); 1411 return (error); 1412 } 1413 } 1414 } 1415 1416 return (0); 1417 } 1418 1419 static void 1420 vmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1421 { 1422 struct vmxnet3_txqueue *txq; 1423 struct vmxnet3_txring *txr; 1424 struct vmxnet3_comp_ring *txc; 1425 struct vmxnet3_txbuf *txb; 1426 u_int i; 1427 int q; 1428 1429 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1430 txq = &sc->vmx_queue[q].vxq_txqueue; 1431 txr = &txq->vxtxq_cmd_ring; 1432 txc = &txq->vxtxq_comp_ring; 1433 1434 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1435 txb = &txr->vxtxr_txbuf[i]; 1436 if (txb->vtxb_dmamap != NULL) { 1437 bus_dmamap_destroy(sc->vmx_dmat, 1438 txb->vtxb_dmamap); 1439 txb->vtxb_dmamap = NULL; 1440 } 1441 } 1442 1443 if (txc->vxcr_u.txcd != NULL) { 1444 vmxnet3_dma_free(sc, &txc->vxcr_dma); 1445 txc->vxcr_u.txcd = NULL; 1446 } 1447 1448 if (txr->vxtxr_txd != NULL) { 1449 vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1450 txr->vxtxr_txd = NULL; 1451 } 1452 } 1453 } 1454 1455 static int 1456 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1457 { 1458 device_t dev; 1459 struct vmxnet3_rxqueue *rxq; 1460 struct vmxnet3_rxring *rxr; 1461 struct vmxnet3_comp_ring *rxc; 1462 int descsz, compsz; 1463 u_int i, j; 1464 int q, error; 1465 1466 dev = sc->vmx_dev; 1467 1468 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1469 rxq = &sc->vmx_queue[q].vxq_rxqueue; 1470 rxc = &rxq->vxrxq_comp_ring; 1471 compsz = 0; 1472 1473 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1474 rxr = &rxq->vxrxq_cmd_ring[i]; 1475 1476 descsz = rxr->vxrxr_ndesc * 1477 sizeof(struct vmxnet3_rxdesc); 1478 compsz += rxr->vxrxr_ndesc * 1479 sizeof(struct vmxnet3_rxcompdesc); 1480 1481 error = vmxnet3_dma_malloc(sc, descsz, 512, 1482 &rxr->vxrxr_dma); 1483 if (error) { 1484 device_printf(dev, "cannot allocate Rx " 1485 "descriptors for queue %d/%d error %d\n", 1486 i, q, error); 1487 return (error); 1488 } 1489 rxr->vxrxr_rxd = 1490 (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1491 } 1492 1493 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1494 if (error) { 1495 device_printf(dev, "cannot alloc Rx comp descriptors " 1496 "for queue %d error %d\n", q, error); 1497 return (error); 1498 } 1499 rxc->vxcr_u.rxcd = 1500 (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1501 1502 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1503 rxr = &rxq->vxrxq_cmd_ring[i]; 1504 1505 error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1, 1506 JUMBO_LEN, 0, BUS_DMA_NOWAIT, 1507 &rxr->vxrxr_spare_dmap); 1508 if (error) { 1509 device_printf(dev, "unable to create spare " 1510 "dmamap for queue %d/%d error %d\n", 1511 q, i, error); 1512 return (error); 1513 } 1514 1515 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1516 error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1, 1517 JUMBO_LEN, 0, BUS_DMA_NOWAIT, 1518 &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1519 if (error) { 1520 device_printf(dev, "unable to create " 1521 "dmamap for queue %d/%d slot %d " 1522 "error %d\n", 1523 q, i, j, error); 1524 return (error); 1525 } 1526 } 1527 } 1528 } 1529 1530 return (0); 1531 } 1532 1533 static void 1534 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1535 { 1536 struct vmxnet3_rxqueue *rxq; 1537 struct vmxnet3_rxring *rxr; 1538 struct vmxnet3_comp_ring *rxc; 1539 struct vmxnet3_rxbuf *rxb; 1540 u_int i, j; 1541 int q; 1542 1543 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1544 rxq = &sc->vmx_queue[q].vxq_rxqueue; 1545 rxc = &rxq->vxrxq_comp_ring; 1546 1547 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1548 rxr = &rxq->vxrxq_cmd_ring[i]; 1549 1550 if (rxr->vxrxr_spare_dmap != NULL) { 1551 bus_dmamap_destroy(sc->vmx_dmat, 1552 rxr->vxrxr_spare_dmap); 1553 rxr->vxrxr_spare_dmap = NULL; 1554 } 1555 1556 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1557 rxb = &rxr->vxrxr_rxbuf[j]; 1558 if (rxb->vrxb_dmamap != NULL) { 1559 bus_dmamap_destroy(sc->vmx_dmat, 1560 rxb->vrxb_dmamap); 1561 rxb->vrxb_dmamap = NULL; 1562 } 1563 } 1564 } 1565 1566 if (rxc->vxcr_u.rxcd != NULL) { 1567 vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1568 rxc->vxcr_u.rxcd = NULL; 1569 } 1570 1571 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1572 rxr = &rxq->vxrxq_cmd_ring[i]; 1573 1574 if (rxr->vxrxr_rxd != NULL) { 1575 vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1576 rxr->vxrxr_rxd = NULL; 1577 } 1578 } 1579 } 1580 } 1581 1582 static int 1583 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1584 { 1585 int error; 1586 1587 error = vmxnet3_alloc_txq_data(sc); 1588 if (error) 1589 return (error); 1590 1591 error = vmxnet3_alloc_rxq_data(sc); 1592 if (error) 1593 return (error); 1594 1595 return (0); 1596 } 1597 1598 static void 1599 vmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1600 { 1601 1602 if (sc->vmx_queue != NULL) { 1603 vmxnet3_free_rxq_data(sc); 1604 vmxnet3_free_txq_data(sc); 1605 } 1606 } 1607 1608 static int 1609 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1610 { 1611 int error; 1612 1613 error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1614 32, &sc->vmx_mcast_dma); 1615 if (error) 1616 device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1617 else 1618 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1619 1620 return (error); 1621 } 1622 1623 static void 1624 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1625 { 1626 1627 if (sc->vmx_mcast != NULL) { 1628 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1629 sc->vmx_mcast = NULL; 1630 } 1631 } 1632 1633 static void 1634 vmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1635 { 1636 struct vmxnet3_driver_shared *ds; 1637 struct vmxnet3_txqueue *txq; 1638 struct vmxnet3_txq_shared *txs; 1639 struct vmxnet3_rxqueue *rxq; 1640 struct vmxnet3_rxq_shared *rxs; 1641 int i; 1642 1643 ds = sc->vmx_ds; 1644 1645 /* 1646 * Initialize fields of the shared data that remains the same across 1647 * reinits. Note the shared data is zero'd when allocated. 1648 */ 1649 1650 ds->magic = VMXNET3_REV1_MAGIC; 1651 1652 /* DriverInfo */ 1653 ds->version = VMXNET3_DRIVER_VERSION; 1654 ds->guest = VMXNET3_GOS_FREEBSD | 1655 #ifdef __LP64__ 1656 VMXNET3_GOS_64BIT; 1657 #else 1658 VMXNET3_GOS_32BIT; 1659 #endif 1660 ds->vmxnet3_revision = 1; 1661 ds->upt_version = 1; 1662 1663 /* Misc. conf */ 1664 ds->driver_data = vtophys(sc); 1665 ds->driver_data_len = sizeof(struct vmxnet3_softc); 1666 ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1667 ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1668 ds->nrxsg_max = sc->vmx_max_rxsegs; 1669 1670 /* RSS conf */ 1671 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1672 ds->rss.version = 1; 1673 ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; 1674 ds->rss.len = sc->vmx_rss_dma.dma_size; 1675 } 1676 1677 /* Interrupt control. */ 1678 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1679 ds->nintr = sc->vmx_nintrs; 1680 ds->evintr = sc->vmx_event_intr_idx; 1681 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1682 1683 for (i = 0; i < sc->vmx_nintrs; i++) 1684 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1685 1686 /* Receive filter. */ 1687 ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1688 ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1689 1690 /* Tx queues */ 1691 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1692 txq = &sc->vmx_queue[i].vxq_txqueue; 1693 txs = txq->vxtxq_ts; 1694 1695 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1696 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1697 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1698 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1699 txs->driver_data = vtophys(txq); 1700 txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1701 } 1702 1703 /* Rx queues */ 1704 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1705 rxq = &sc->vmx_queue[i].vxq_rxqueue; 1706 rxs = rxq->vxrxq_rs; 1707 1708 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1709 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1710 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1711 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1712 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1713 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1714 rxs->driver_data = vtophys(rxq); 1715 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1716 } 1717 } 1718 1719 static void 1720 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) 1721 { 1722 /* 1723 * Use the same key as the Linux driver until FreeBSD can do 1724 * RSS (presumably Toeplitz) in software. 1725 */ 1726 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 1727 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 1728 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 1729 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 1730 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 1731 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, 1732 }; 1733 1734 struct vmxnet3_rss_shared *rss; 1735 int i; 1736 1737 rss = sc->vmx_rss; 1738 1739 rss->hash_type = 1740 UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | 1741 UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; 1742 rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 1743 rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; 1744 rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 1745 memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); 1746 1747 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) 1748 rss->ind_table[i] = i % sc->vmx_nrxqueues; 1749 } 1750 1751 static void 1752 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1753 { 1754 struct ifnet *ifp; 1755 struct vmxnet3_driver_shared *ds; 1756 1757 ifp = &sc->vmx_ethercom.ec_if; 1758 ds = sc->vmx_ds; 1759 1760 ds->mtu = ifp->if_mtu; 1761 ds->ntxqueue = sc->vmx_ntxqueues; 1762 ds->nrxqueue = sc->vmx_nrxqueues; 1763 1764 ds->upt_features = 0; 1765 if (ifp->if_capenable & 1766 (IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 1767 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) 1768 ds->upt_features |= UPT1_F_CSUM; 1769 if (sc->vmx_ethercom.ec_capenable & ETHERCAP_VLAN_HWTAGGING) 1770 ds->upt_features |= UPT1_F_VLAN; 1771 1772 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1773 ds->upt_features |= UPT1_F_RSS; 1774 vmxnet3_reinit_rss_shared_data(sc); 1775 } 1776 1777 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1778 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1779 (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1780 } 1781 1782 static int 1783 vmxnet3_alloc_data(struct vmxnet3_softc *sc) 1784 { 1785 int error; 1786 1787 error = vmxnet3_alloc_shared_data(sc); 1788 if (error) 1789 return (error); 1790 1791 error = vmxnet3_alloc_queue_data(sc); 1792 if (error) 1793 return (error); 1794 1795 error = vmxnet3_alloc_mcast_table(sc); 1796 if (error) 1797 return (error); 1798 1799 vmxnet3_init_shared_data(sc); 1800 1801 return (0); 1802 } 1803 1804 static void 1805 vmxnet3_free_data(struct vmxnet3_softc *sc) 1806 { 1807 1808 vmxnet3_free_mcast_table(sc); 1809 vmxnet3_free_queue_data(sc); 1810 vmxnet3_free_shared_data(sc); 1811 } 1812 1813 static int 1814 vmxnet3_setup_interface(struct vmxnet3_softc *sc) 1815 { 1816 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 1817 1818 vmxnet3_get_lladdr(sc); 1819 aprint_normal_dev(sc->vmx_dev, "Ethernet address %s\n", 1820 ether_sprintf(sc->vmx_lladdr)); 1821 vmxnet3_set_lladdr(sc); 1822 1823 strlcpy(ifp->if_xname, device_xname(sc->vmx_dev), IFNAMSIZ); 1824 ifp->if_softc = sc; 1825 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; 1826 ifp->if_extflags = IFEF_MPSAFE; 1827 ifp->if_ioctl = vmxnet3_ioctl; 1828 ifp->if_start = vmxnet3_start; 1829 ifp->if_transmit = vmxnet3_transmit; 1830 ifp->if_watchdog = NULL; 1831 ifp->if_init = vmxnet3_init; 1832 ifp->if_stop = vmxnet3_stop; 1833 sc->vmx_ethercom.ec_if.if_capabilities |=IFCAP_CSUM_IPv4_Rx | 1834 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1835 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 1836 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 1837 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 1838 1839 ifp->if_capenable = ifp->if_capabilities; 1840 1841 sc->vmx_ethercom.ec_if.if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1842 1843 sc->vmx_ethercom.ec_capabilities |= 1844 ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING | ETHERCAP_JUMBO_MTU; 1845 sc->vmx_ethercom.ec_capenable |= ETHERCAP_VLAN_HWTAGGING; 1846 1847 IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs); 1848 IFQ_SET_READY(&ifp->if_snd); 1849 1850 /* Initialize ifmedia structures. */ 1851 sc->vmx_ethercom.ec_ifmedia = &sc->vmx_media; 1852 ifmedia_init_with_lock(&sc->vmx_media, IFM_IMASK, vmxnet3_ifmedia_change, 1853 vmxnet3_ifmedia_status, sc->vmx_mtx); 1854 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1855 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL); 1856 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_10G_T, 0, NULL); 1857 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1858 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_1000_T, 0, NULL); 1859 ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1860 1861 if_attach(ifp); 1862 if_deferred_start_init(ifp, NULL); 1863 ether_ifattach(ifp, sc->vmx_lladdr); 1864 ether_set_ifflags_cb(&sc->vmx_ethercom, vmxnet3_ifflags_cb); 1865 vmxnet3_cmd_link_status(ifp); 1866 1867 /* should set before setting interrupts */ 1868 sc->vmx_rx_intr_process_limit = VMXNET3_RX_INTR_PROCESS_LIMIT; 1869 sc->vmx_rx_process_limit = VMXNET3_RX_PROCESS_LIMIT; 1870 sc->vmx_tx_intr_process_limit = VMXNET3_TX_INTR_PROCESS_LIMIT; 1871 sc->vmx_tx_process_limit = VMXNET3_TX_PROCESS_LIMIT; 1872 1873 return (0); 1874 } 1875 1876 static int 1877 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 1878 { 1879 const char *devname; 1880 struct sysctllog **log; 1881 const struct sysctlnode *rnode, *rxnode, *txnode; 1882 int error; 1883 1884 log = &sc->vmx_sysctllog; 1885 devname = device_xname(sc->vmx_dev); 1886 1887 error = sysctl_createv(log, 0, NULL, &rnode, 1888 0, CTLTYPE_NODE, devname, 1889 SYSCTL_DESCR("vmxnet3 information and settings"), 1890 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 1891 if (error) 1892 goto out; 1893 error = sysctl_createv(log, 0, &rnode, NULL, 1894 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 1895 SYSCTL_DESCR("Use workqueue for packet processing"), 1896 NULL, 0, &sc->vmx_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 1897 if (error) 1898 goto out; 1899 1900 error = sysctl_createv(log, 0, &rnode, &rxnode, 1901 0, CTLTYPE_NODE, "rx", 1902 SYSCTL_DESCR("vmxnet3 information and settings for Rx"), 1903 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 1904 if (error) 1905 goto out; 1906 error = sysctl_createv(log, 0, &rxnode, NULL, 1907 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 1908 SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), 1909 NULL, 0, &sc->vmx_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 1910 if (error) 1911 goto out; 1912 error = sysctl_createv(log, 0, &rxnode, NULL, 1913 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 1914 SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), 1915 NULL, 0, &sc->vmx_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 1916 if (error) 1917 goto out; 1918 1919 error = sysctl_createv(log, 0, &rnode, &txnode, 1920 0, CTLTYPE_NODE, "tx", 1921 SYSCTL_DESCR("vmxnet3 information and settings for Tx"), 1922 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 1923 if (error) 1924 goto out; 1925 error = sysctl_createv(log, 0, &txnode, NULL, 1926 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 1927 SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), 1928 NULL, 0, &sc->vmx_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 1929 if (error) 1930 goto out; 1931 error = sysctl_createv(log, 0, &txnode, NULL, 1932 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 1933 SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), 1934 NULL, 0, &sc->vmx_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 1935 1936 out: 1937 if (error) { 1938 aprint_error_dev(sc->vmx_dev, 1939 "unable to create sysctl node\n"); 1940 sysctl_teardown(log); 1941 } 1942 return error; 1943 } 1944 1945 static int 1946 vmxnet3_setup_stats(struct vmxnet3_softc *sc) 1947 { 1948 struct vmxnet3_queue *vmxq; 1949 struct vmxnet3_txqueue *txq; 1950 struct vmxnet3_rxqueue *rxq; 1951 int i; 1952 1953 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1954 vmxq = &sc->vmx_queue[i]; 1955 txq = &vmxq->vxq_txqueue; 1956 evcnt_attach_dynamic(&txq->vxtxq_intr, EVCNT_TYPE_INTR, 1957 NULL, txq->vxtxq_name, "Interrupt on queue"); 1958 evcnt_attach_dynamic(&txq->vxtxq_defer, EVCNT_TYPE_MISC, 1959 NULL, txq->vxtxq_name, "Handled queue in softint/workqueue"); 1960 evcnt_attach_dynamic(&txq->vxtxq_deferreq, EVCNT_TYPE_MISC, 1961 NULL, txq->vxtxq_name, "Requested in softint/workqueue"); 1962 evcnt_attach_dynamic(&txq->vxtxq_pcqdrop, EVCNT_TYPE_MISC, 1963 NULL, txq->vxtxq_name, "Dropped in pcq"); 1964 evcnt_attach_dynamic(&txq->vxtxq_transmitdef, EVCNT_TYPE_MISC, 1965 NULL, txq->vxtxq_name, "Deferred transmit"); 1966 evcnt_attach_dynamic(&txq->vxtxq_watchdogto, EVCNT_TYPE_MISC, 1967 NULL, txq->vxtxq_name, "Watchdog timeout"); 1968 evcnt_attach_dynamic(&txq->vxtxq_defragged, EVCNT_TYPE_MISC, 1969 NULL, txq->vxtxq_name, "m_defrag successed"); 1970 evcnt_attach_dynamic(&txq->vxtxq_defrag_failed, EVCNT_TYPE_MISC, 1971 NULL, txq->vxtxq_name, "m_defrag failed"); 1972 } 1973 1974 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1975 vmxq = &sc->vmx_queue[i]; 1976 rxq = &vmxq->vxq_rxqueue; 1977 evcnt_attach_dynamic(&rxq->vxrxq_intr, EVCNT_TYPE_INTR, 1978 NULL, rxq->vxrxq_name, "Interrupt on queue"); 1979 evcnt_attach_dynamic(&rxq->vxrxq_defer, EVCNT_TYPE_MISC, 1980 NULL, rxq->vxrxq_name, "Handled queue in softint/workqueue"); 1981 evcnt_attach_dynamic(&rxq->vxrxq_deferreq, EVCNT_TYPE_MISC, 1982 NULL, rxq->vxrxq_name, "Requested in softint/workqueue"); 1983 evcnt_attach_dynamic(&rxq->vxrxq_mgetcl_failed, EVCNT_TYPE_MISC, 1984 NULL, rxq->vxrxq_name, "MCLGET failed"); 1985 evcnt_attach_dynamic(&rxq->vxrxq_mbuf_load_failed, EVCNT_TYPE_MISC, 1986 NULL, rxq->vxrxq_name, "bus_dmamap_load_mbuf failed"); 1987 } 1988 1989 evcnt_attach_dynamic(&sc->vmx_event_intr, EVCNT_TYPE_INTR, 1990 NULL, device_xname(sc->vmx_dev), "Interrupt for other events"); 1991 evcnt_attach_dynamic(&sc->vmx_event_link, EVCNT_TYPE_MISC, 1992 NULL, device_xname(sc->vmx_dev), "Link status event"); 1993 evcnt_attach_dynamic(&sc->vmx_event_txqerror, EVCNT_TYPE_MISC, 1994 NULL, device_xname(sc->vmx_dev), "Tx queue error event"); 1995 evcnt_attach_dynamic(&sc->vmx_event_rxqerror, EVCNT_TYPE_MISC, 1996 NULL, device_xname(sc->vmx_dev), "Rx queue error event"); 1997 evcnt_attach_dynamic(&sc->vmx_event_dic, EVCNT_TYPE_MISC, 1998 NULL, device_xname(sc->vmx_dev), "Device impl change event"); 1999 evcnt_attach_dynamic(&sc->vmx_event_debug, EVCNT_TYPE_MISC, 2000 NULL, device_xname(sc->vmx_dev), "Debug event"); 2001 2002 return 0; 2003 } 2004 2005 static void 2006 vmxnet3_teardown_stats(struct vmxnet3_softc *sc) 2007 { 2008 struct vmxnet3_queue *vmxq; 2009 struct vmxnet3_txqueue *txq; 2010 struct vmxnet3_rxqueue *rxq; 2011 int i; 2012 2013 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2014 vmxq = &sc->vmx_queue[i]; 2015 txq = &vmxq->vxq_txqueue; 2016 evcnt_detach(&txq->vxtxq_intr); 2017 evcnt_detach(&txq->vxtxq_defer); 2018 evcnt_detach(&txq->vxtxq_deferreq); 2019 evcnt_detach(&txq->vxtxq_pcqdrop); 2020 evcnt_detach(&txq->vxtxq_transmitdef); 2021 evcnt_detach(&txq->vxtxq_watchdogto); 2022 evcnt_detach(&txq->vxtxq_defragged); 2023 evcnt_detach(&txq->vxtxq_defrag_failed); 2024 } 2025 2026 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2027 vmxq = &sc->vmx_queue[i]; 2028 rxq = &vmxq->vxq_rxqueue; 2029 evcnt_detach(&rxq->vxrxq_intr); 2030 evcnt_detach(&rxq->vxrxq_defer); 2031 evcnt_detach(&rxq->vxrxq_deferreq); 2032 evcnt_detach(&rxq->vxrxq_mgetcl_failed); 2033 evcnt_detach(&rxq->vxrxq_mbuf_load_failed); 2034 } 2035 2036 evcnt_detach(&sc->vmx_event_intr); 2037 evcnt_detach(&sc->vmx_event_link); 2038 evcnt_detach(&sc->vmx_event_txqerror); 2039 evcnt_detach(&sc->vmx_event_rxqerror); 2040 evcnt_detach(&sc->vmx_event_dic); 2041 evcnt_detach(&sc->vmx_event_debug); 2042 } 2043 2044 static void 2045 vmxnet3_evintr(struct vmxnet3_softc *sc) 2046 { 2047 device_t dev; 2048 struct vmxnet3_txq_shared *ts; 2049 struct vmxnet3_rxq_shared *rs; 2050 uint32_t event; 2051 int reset; 2052 2053 dev = sc->vmx_dev; 2054 reset = 0; 2055 2056 VMXNET3_CORE_LOCK(sc); 2057 2058 /* Clear events. */ 2059 event = sc->vmx_ds->event; 2060 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 2061 2062 if (event & VMXNET3_EVENT_LINK) { 2063 sc->vmx_event_link.ev_count++; 2064 vmxnet3_if_link_status(sc); 2065 if (sc->vmx_link_active != 0) 2066 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); 2067 } 2068 2069 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 2070 if (event & VMXNET3_EVENT_TQERROR) 2071 sc->vmx_event_txqerror.ev_count++; 2072 if (event & VMXNET3_EVENT_RQERROR) 2073 sc->vmx_event_rxqerror.ev_count++; 2074 2075 reset = 1; 2076 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 2077 ts = sc->vmx_queue[0].vxq_txqueue.vxtxq_ts; 2078 if (ts->stopped != 0) 2079 device_printf(dev, "Tx queue error %#x\n", ts->error); 2080 rs = sc->vmx_queue[0].vxq_rxqueue.vxrxq_rs; 2081 if (rs->stopped != 0) 2082 device_printf(dev, "Rx queue error %#x\n", rs->error); 2083 device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 2084 } 2085 2086 if (event & VMXNET3_EVENT_DIC) { 2087 sc->vmx_event_dic.ev_count++; 2088 device_printf(dev, "device implementation change event\n"); 2089 } 2090 if (event & VMXNET3_EVENT_DEBUG) { 2091 sc->vmx_event_debug.ev_count++; 2092 device_printf(dev, "debug event\n"); 2093 } 2094 2095 if (reset != 0) 2096 vmxnet3_init_locked(sc); 2097 2098 VMXNET3_CORE_UNLOCK(sc); 2099 } 2100 2101 static bool 2102 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq, u_int limit) 2103 { 2104 struct vmxnet3_softc *sc; 2105 struct vmxnet3_txring *txr; 2106 struct vmxnet3_comp_ring *txc; 2107 struct vmxnet3_txcompdesc *txcd; 2108 struct vmxnet3_txbuf *txb; 2109 struct ifnet *ifp; 2110 struct mbuf *m; 2111 u_int sop; 2112 bool more = false; 2113 2114 sc = txq->vxtxq_sc; 2115 txr = &txq->vxtxq_cmd_ring; 2116 txc = &txq->vxtxq_comp_ring; 2117 ifp = &sc->vmx_ethercom.ec_if; 2118 2119 VMXNET3_TXQ_LOCK_ASSERT(txq); 2120 2121 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2122 for (;;) { 2123 if (limit-- == 0) { 2124 more = true; 2125 break; 2126 } 2127 2128 txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 2129 if (txcd->gen != txc->vxcr_gen) 2130 break; 2131 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2132 2133 if (++txc->vxcr_next == txc->vxcr_ndesc) { 2134 txc->vxcr_next = 0; 2135 txc->vxcr_gen ^= 1; 2136 } 2137 2138 sop = txr->vxtxr_next; 2139 txb = &txr->vxtxr_txbuf[sop]; 2140 2141 if ((m = txb->vtxb_m) != NULL) { 2142 bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap, 2143 0, txb->vtxb_dmamap->dm_mapsize, 2144 BUS_DMASYNC_POSTWRITE); 2145 bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap); 2146 2147 if_statinc_ref(nsr, if_opackets); 2148 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 2149 if (m->m_flags & M_MCAST) 2150 if_statinc_ref(nsr, if_omcasts); 2151 2152 m_freem(m); 2153 txb->vtxb_m = NULL; 2154 } 2155 2156 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 2157 } 2158 IF_STAT_PUTREF(ifp); 2159 2160 if (txr->vxtxr_head == txr->vxtxr_next) 2161 txq->vxtxq_watchdog = 0; 2162 2163 return more; 2164 } 2165 2166 static int 2167 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq, 2168 struct vmxnet3_rxring *rxr) 2169 { 2170 struct mbuf *m; 2171 struct vmxnet3_rxdesc *rxd; 2172 struct vmxnet3_rxbuf *rxb; 2173 bus_dma_tag_t tag; 2174 bus_dmamap_t dmap; 2175 int idx, btype, error; 2176 2177 tag = sc->vmx_dmat; 2178 dmap = rxr->vxrxr_spare_dmap; 2179 idx = rxr->vxrxr_fill; 2180 rxd = &rxr->vxrxr_rxd[idx]; 2181 rxb = &rxr->vxrxr_rxbuf[idx]; 2182 2183 /* Don't allocate buffers for ring 2 for now. */ 2184 if (rxr->vxrxr_rid != 0) 2185 return -1; 2186 btype = VMXNET3_BTYPE_HEAD; 2187 2188 MGETHDR(m, M_DONTWAIT, MT_DATA); 2189 if (m == NULL) 2190 return (ENOBUFS); 2191 2192 MCLGET(m, M_DONTWAIT); 2193 if ((m->m_flags & M_EXT) == 0) { 2194 rxq->vxrxq_mgetcl_failed.ev_count++; 2195 m_freem(m); 2196 return (ENOBUFS); 2197 } 2198 2199 m->m_pkthdr.len = m->m_len = JUMBO_LEN; 2200 m_adj(m, ETHER_ALIGN); 2201 2202 error = bus_dmamap_load_mbuf(sc->vmx_dmat, dmap, m, BUS_DMA_NOWAIT); 2203 if (error) { 2204 m_freem(m); 2205 rxq->vxrxq_mbuf_load_failed.ev_count++; 2206 return (error); 2207 } 2208 2209 if (rxb->vrxb_m != NULL) { 2210 bus_dmamap_sync(tag, rxb->vrxb_dmamap, 2211 0, rxb->vrxb_dmamap->dm_mapsize, 2212 BUS_DMASYNC_POSTREAD); 2213 bus_dmamap_unload(tag, rxb->vrxb_dmamap); 2214 } 2215 2216 rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 2217 rxb->vrxb_dmamap = dmap; 2218 rxb->vrxb_m = m; 2219 2220 rxd->addr = DMAADDR(dmap); 2221 rxd->len = m->m_pkthdr.len; 2222 rxd->btype = btype; 2223 rxd->gen = rxr->vxrxr_gen; 2224 2225 vmxnet3_rxr_increment_fill(rxr); 2226 return (0); 2227 } 2228 2229 static void 2230 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 2231 struct vmxnet3_rxring *rxr, int idx) 2232 { 2233 struct vmxnet3_rxdesc *rxd; 2234 2235 rxd = &rxr->vxrxr_rxd[idx]; 2236 rxd->gen = rxr->vxrxr_gen; 2237 vmxnet3_rxr_increment_fill(rxr); 2238 } 2239 2240 static void 2241 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 2242 { 2243 struct vmxnet3_softc *sc; 2244 struct vmxnet3_rxring *rxr; 2245 struct vmxnet3_comp_ring *rxc; 2246 struct vmxnet3_rxcompdesc *rxcd; 2247 int idx, eof; 2248 2249 sc = rxq->vxrxq_sc; 2250 rxc = &rxq->vxrxq_comp_ring; 2251 2252 do { 2253 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2254 if (rxcd->gen != rxc->vxcr_gen) 2255 break; /* Not expected. */ 2256 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2257 2258 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2259 rxc->vxcr_next = 0; 2260 rxc->vxcr_gen ^= 1; 2261 } 2262 2263 idx = rxcd->rxd_idx; 2264 eof = rxcd->eop; 2265 if (rxcd->qid < sc->vmx_nrxqueues) 2266 rxr = &rxq->vxrxq_cmd_ring[0]; 2267 else 2268 rxr = &rxq->vxrxq_cmd_ring[1]; 2269 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2270 } while (!eof); 2271 } 2272 2273 static void 2274 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2275 { 2276 if (rxcd->no_csum) 2277 return; 2278 2279 if (rxcd->ipv4) { 2280 m->m_pkthdr.csum_flags |= M_CSUM_IPv4; 2281 if (rxcd->ipcsum_ok == 0) 2282 m->m_pkthdr.csum_flags |= M_CSUM_IPv4_BAD; 2283 } 2284 2285 if (rxcd->fragment) 2286 return; 2287 2288 if (rxcd->tcp) { 2289 m->m_pkthdr.csum_flags |= 2290 rxcd->ipv4 ? M_CSUM_TCPv4 : M_CSUM_TCPv6; 2291 if ((rxcd->csum_ok) == 0) 2292 m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; 2293 } 2294 2295 if (rxcd->udp) { 2296 m->m_pkthdr.csum_flags |= 2297 rxcd->ipv4 ? M_CSUM_UDPv4 : M_CSUM_UDPv6 ; 2298 if ((rxcd->csum_ok) == 0) 2299 m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; 2300 } 2301 } 2302 2303 static void 2304 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 2305 struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2306 { 2307 struct vmxnet3_softc *sc; 2308 struct ifnet *ifp; 2309 2310 sc = rxq->vxrxq_sc; 2311 ifp = &sc->vmx_ethercom.ec_if; 2312 2313 if (rxcd->error) { 2314 if_statinc(ifp, if_ierrors); 2315 m_freem(m); 2316 return; 2317 } 2318 2319 if (!rxcd->no_csum) 2320 vmxnet3_rx_csum(rxcd, m); 2321 if (rxcd->vlan) 2322 vlan_set_tag(m, rxcd->vtag); 2323 2324 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2325 if_statinc_ref(nsr, if_ipackets); 2326 if_statadd_ref(nsr, if_ibytes, m->m_pkthdr.len); 2327 IF_STAT_PUTREF(ifp); 2328 2329 if_percpuq_enqueue(ifp->if_percpuq, m); 2330 } 2331 2332 static bool 2333 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq, u_int limit) 2334 { 2335 struct vmxnet3_softc *sc; 2336 struct ifnet *ifp; 2337 struct vmxnet3_rxring *rxr; 2338 struct vmxnet3_comp_ring *rxc; 2339 struct vmxnet3_rxdesc *rxd __diagused; 2340 struct vmxnet3_rxcompdesc *rxcd; 2341 struct mbuf *m, *m_head, *m_tail; 2342 u_int idx, length; 2343 bool more = false; 2344 2345 sc = rxq->vxrxq_sc; 2346 ifp = &sc->vmx_ethercom.ec_if; 2347 rxc = &rxq->vxrxq_comp_ring; 2348 2349 VMXNET3_RXQ_LOCK_ASSERT(rxq); 2350 2351 if ((ifp->if_flags & IFF_RUNNING) == 0) 2352 return more; 2353 2354 m_head = rxq->vxrxq_mhead; 2355 rxq->vxrxq_mhead = NULL; 2356 m_tail = rxq->vxrxq_mtail; 2357 rxq->vxrxq_mtail = NULL; 2358 KASSERT(m_head == NULL || m_tail != NULL); 2359 2360 for (;;) { 2361 if (limit-- == 0) { 2362 more = true; 2363 break; 2364 } 2365 2366 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2367 if (rxcd->gen != rxc->vxcr_gen) { 2368 rxq->vxrxq_mhead = m_head; 2369 rxq->vxrxq_mtail = m_tail; 2370 break; 2371 } 2372 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2373 2374 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2375 rxc->vxcr_next = 0; 2376 rxc->vxcr_gen ^= 1; 2377 } 2378 2379 idx = rxcd->rxd_idx; 2380 length = rxcd->len; 2381 if (rxcd->qid < sc->vmx_nrxqueues) 2382 rxr = &rxq->vxrxq_cmd_ring[0]; 2383 else 2384 rxr = &rxq->vxrxq_cmd_ring[1]; 2385 rxd = &rxr->vxrxr_rxd[idx]; 2386 2387 m = rxr->vxrxr_rxbuf[idx].vrxb_m; 2388 KASSERT(m != NULL); 2389 2390 /* 2391 * The host may skip descriptors. We detect this when this 2392 * descriptor does not match the previous fill index. Catch 2393 * up with the host now. 2394 */ 2395 if (__predict_false(rxr->vxrxr_fill != idx)) { 2396 while (rxr->vxrxr_fill != idx) { 2397 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 2398 rxr->vxrxr_gen; 2399 vmxnet3_rxr_increment_fill(rxr); 2400 } 2401 } 2402 2403 if (rxcd->sop) { 2404 /* start of frame w/o head buffer */ 2405 KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD); 2406 /* start of frame not in ring 0 */ 2407 KASSERT(rxr == &rxq->vxrxq_cmd_ring[0]); 2408 /* duplicate start of frame? */ 2409 KASSERT(m_head == NULL); 2410 2411 if (length == 0) { 2412 /* Just ignore this descriptor. */ 2413 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2414 goto nextp; 2415 } 2416 2417 if (vmxnet3_newbuf(sc, rxq, rxr) != 0) { 2418 if_statinc(ifp, if_iqdrops); 2419 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2420 if (!rxcd->eop) 2421 vmxnet3_rxq_discard_chain(rxq); 2422 goto nextp; 2423 } 2424 2425 m_set_rcvif(m, ifp); 2426 m->m_pkthdr.len = m->m_len = length; 2427 m->m_pkthdr.csum_flags = 0; 2428 m_head = m_tail = m; 2429 2430 } else { 2431 /* non start of frame w/o body buffer */ 2432 KASSERT(rxd->btype == VMXNET3_BTYPE_BODY); 2433 /* frame not started? */ 2434 KASSERT(m_head != NULL); 2435 2436 if (vmxnet3_newbuf(sc, rxq, rxr) != 0) { 2437 if_statinc(ifp, if_iqdrops); 2438 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2439 if (!rxcd->eop) 2440 vmxnet3_rxq_discard_chain(rxq); 2441 m_freem(m_head); 2442 m_head = m_tail = NULL; 2443 goto nextp; 2444 } 2445 2446 m->m_len = length; 2447 m_head->m_pkthdr.len += length; 2448 m_tail->m_next = m; 2449 m_tail = m; 2450 } 2451 2452 if (rxcd->eop) { 2453 vmxnet3_rxq_input(rxq, rxcd, m_head); 2454 m_head = m_tail = NULL; 2455 2456 /* Must recheck after dropping the Rx lock. */ 2457 if ((ifp->if_flags & IFF_RUNNING) == 0) 2458 break; 2459 } 2460 2461 nextp: 2462 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 2463 int qid = rxcd->qid; 2464 bus_size_t r; 2465 2466 idx = (idx + 1) % rxr->vxrxr_ndesc; 2467 if (qid >= sc->vmx_nrxqueues) { 2468 qid -= sc->vmx_nrxqueues; 2469 r = VMXNET3_BAR0_RXH2(qid); 2470 } else 2471 r = VMXNET3_BAR0_RXH1(qid); 2472 vmxnet3_write_bar0(sc, r, idx); 2473 } 2474 } 2475 2476 return more; 2477 } 2478 2479 static inline void 2480 vmxnet3_sched_handle_queue(struct vmxnet3_softc *sc, struct vmxnet3_queue *vmxq) 2481 { 2482 2483 if (vmxq->vxq_workqueue) { 2484 /* 2485 * When this function is called, "vmxq" is owned by one CPU. 2486 * so, atomic operation is not required here. 2487 */ 2488 if (!vmxq->vxq_wq_enqueued) { 2489 vmxq->vxq_wq_enqueued = true; 2490 workqueue_enqueue(sc->vmx_queue_wq, 2491 &vmxq->vxq_wq_cookie, curcpu()); 2492 } 2493 } else { 2494 softint_schedule(vmxq->vxq_si); 2495 } 2496 } 2497 2498 static int 2499 vmxnet3_legacy_intr(void *xsc) 2500 { 2501 struct vmxnet3_softc *sc; 2502 struct vmxnet3_queue *vmxq; 2503 struct vmxnet3_txqueue *txq; 2504 struct vmxnet3_rxqueue *rxq; 2505 u_int txlimit, rxlimit; 2506 bool txmore, rxmore; 2507 2508 sc = xsc; 2509 vmxq = &sc->vmx_queue[0]; 2510 txq = &vmxq->vxq_txqueue; 2511 rxq = &vmxq->vxq_rxqueue; 2512 txlimit = sc->vmx_tx_intr_process_limit; 2513 rxlimit = sc->vmx_rx_intr_process_limit; 2514 2515 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 2516 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 2517 return (0); 2518 } 2519 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2520 vmxnet3_disable_all_intrs(sc); 2521 2522 if (sc->vmx_ds->event != 0) 2523 vmxnet3_evintr(sc); 2524 2525 VMXNET3_TXQ_LOCK(txq); 2526 txmore = vmxnet3_txq_eof(txq, txlimit); 2527 VMXNET3_TXQ_UNLOCK(txq); 2528 2529 VMXNET3_RXQ_LOCK(rxq); 2530 rxmore = vmxnet3_rxq_eof(rxq, rxlimit); 2531 VMXNET3_RXQ_UNLOCK(rxq); 2532 2533 if (txmore || rxmore) 2534 vmxnet3_sched_handle_queue(sc, vmxq); 2535 else { 2536 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); 2537 vmxnet3_enable_all_intrs(sc); 2538 } 2539 2540 return (1); 2541 } 2542 2543 static int 2544 vmxnet3_txrxq_intr(void *xvmxq) 2545 { 2546 struct vmxnet3_softc *sc; 2547 struct vmxnet3_queue *vmxq; 2548 struct vmxnet3_txqueue *txq; 2549 struct vmxnet3_rxqueue *rxq; 2550 u_int txlimit, rxlimit; 2551 bool txmore, rxmore; 2552 2553 vmxq = xvmxq; 2554 txq = &vmxq->vxq_txqueue; 2555 rxq = &vmxq->vxq_rxqueue; 2556 sc = txq->vxtxq_sc; 2557 txlimit = sc->vmx_tx_intr_process_limit; 2558 rxlimit = sc->vmx_rx_intr_process_limit; 2559 vmxq->vxq_workqueue = sc->vmx_txrx_workqueue; 2560 2561 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2562 vmxnet3_disable_intr(sc, vmxq->vxq_intr_idx); 2563 2564 VMXNET3_TXQ_LOCK(txq); 2565 txq->vxtxq_intr.ev_count++; 2566 txmore = vmxnet3_txq_eof(txq, txlimit); 2567 VMXNET3_TXQ_UNLOCK(txq); 2568 2569 VMXNET3_RXQ_LOCK(rxq); 2570 rxq->vxrxq_intr.ev_count++; 2571 rxmore = vmxnet3_rxq_eof(rxq, rxlimit); 2572 VMXNET3_RXQ_UNLOCK(rxq); 2573 2574 if (txmore || rxmore) 2575 vmxnet3_sched_handle_queue(sc, vmxq); 2576 else { 2577 /* for ALTQ */ 2578 if (vmxq->vxq_id == 0) 2579 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); 2580 softint_schedule(txq->vxtxq_si); 2581 2582 vmxnet3_enable_intr(sc, vmxq->vxq_intr_idx); 2583 } 2584 2585 return (1); 2586 } 2587 2588 static void 2589 vmxnet3_handle_queue(void *xvmxq) 2590 { 2591 struct vmxnet3_softc *sc; 2592 struct vmxnet3_queue *vmxq; 2593 struct vmxnet3_txqueue *txq; 2594 struct vmxnet3_rxqueue *rxq; 2595 u_int txlimit, rxlimit; 2596 bool txmore, rxmore; 2597 2598 vmxq = xvmxq; 2599 txq = &vmxq->vxq_txqueue; 2600 rxq = &vmxq->vxq_rxqueue; 2601 sc = txq->vxtxq_sc; 2602 txlimit = sc->vmx_tx_process_limit; 2603 rxlimit = sc->vmx_rx_process_limit; 2604 2605 VMXNET3_TXQ_LOCK(txq); 2606 txq->vxtxq_defer.ev_count++; 2607 txmore = vmxnet3_txq_eof(txq, txlimit); 2608 if (txmore) 2609 txq->vxtxq_deferreq.ev_count++; 2610 /* for ALTQ */ 2611 if (vmxq->vxq_id == 0) 2612 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); 2613 softint_schedule(txq->vxtxq_si); 2614 VMXNET3_TXQ_UNLOCK(txq); 2615 2616 VMXNET3_RXQ_LOCK(rxq); 2617 rxq->vxrxq_defer.ev_count++; 2618 rxmore = vmxnet3_rxq_eof(rxq, rxlimit); 2619 if (rxmore) 2620 rxq->vxrxq_deferreq.ev_count++; 2621 VMXNET3_RXQ_UNLOCK(rxq); 2622 2623 if (txmore || rxmore) 2624 vmxnet3_sched_handle_queue(sc, vmxq); 2625 else 2626 vmxnet3_enable_intr(sc, vmxq->vxq_intr_idx); 2627 } 2628 2629 static void 2630 vmxnet3_handle_queue_work(struct work *wk, void *context) 2631 { 2632 struct vmxnet3_queue *vmxq; 2633 2634 vmxq = container_of(wk, struct vmxnet3_queue, vxq_wq_cookie); 2635 vmxq->vxq_wq_enqueued = false; 2636 vmxnet3_handle_queue(vmxq); 2637 } 2638 2639 static int 2640 vmxnet3_event_intr(void *xsc) 2641 { 2642 struct vmxnet3_softc *sc; 2643 2644 sc = xsc; 2645 2646 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2647 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 2648 2649 sc->vmx_event_intr.ev_count++; 2650 2651 if (sc->vmx_ds->event != 0) 2652 vmxnet3_evintr(sc); 2653 2654 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2655 2656 return (1); 2657 } 2658 2659 static void 2660 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2661 { 2662 struct vmxnet3_txring *txr; 2663 struct vmxnet3_txbuf *txb; 2664 u_int i; 2665 2666 txr = &txq->vxtxq_cmd_ring; 2667 2668 for (i = 0; i < txr->vxtxr_ndesc; i++) { 2669 txb = &txr->vxtxr_txbuf[i]; 2670 2671 if (txb->vtxb_m == NULL) 2672 continue; 2673 2674 bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap, 2675 0, txb->vtxb_dmamap->dm_mapsize, 2676 BUS_DMASYNC_POSTWRITE); 2677 bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap); 2678 m_freem(txb->vtxb_m); 2679 txb->vtxb_m = NULL; 2680 } 2681 } 2682 2683 static void 2684 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2685 { 2686 struct vmxnet3_rxring *rxr; 2687 struct vmxnet3_rxbuf *rxb; 2688 u_int i, j; 2689 2690 if (rxq->vxrxq_mhead != NULL) { 2691 m_freem(rxq->vxrxq_mhead); 2692 rxq->vxrxq_mhead = NULL; 2693 rxq->vxrxq_mtail = NULL; 2694 } 2695 2696 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2697 rxr = &rxq->vxrxq_cmd_ring[i]; 2698 2699 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2700 rxb = &rxr->vxrxr_rxbuf[j]; 2701 2702 if (rxb->vrxb_m == NULL) 2703 continue; 2704 2705 bus_dmamap_sync(sc->vmx_dmat, rxb->vrxb_dmamap, 2706 0, rxb->vrxb_dmamap->dm_mapsize, 2707 BUS_DMASYNC_POSTREAD); 2708 bus_dmamap_unload(sc->vmx_dmat, rxb->vrxb_dmamap); 2709 m_freem(rxb->vrxb_m); 2710 rxb->vrxb_m = NULL; 2711 } 2712 } 2713 } 2714 2715 static void 2716 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2717 { 2718 struct vmxnet3_rxqueue *rxq; 2719 struct vmxnet3_txqueue *txq; 2720 struct vmxnet3_queue *vmxq; 2721 int i; 2722 2723 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2724 rxq = &sc->vmx_queue[i].vxq_rxqueue; 2725 VMXNET3_RXQ_LOCK(rxq); 2726 VMXNET3_RXQ_UNLOCK(rxq); 2727 } 2728 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2729 txq = &sc->vmx_queue[i].vxq_txqueue; 2730 VMXNET3_TXQ_LOCK(txq); 2731 VMXNET3_TXQ_UNLOCK(txq); 2732 } 2733 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2734 vmxq = &sc->vmx_queue[i]; 2735 workqueue_wait(sc->vmx_queue_wq, &vmxq->vxq_wq_cookie); 2736 } 2737 } 2738 2739 static void 2740 vmxnet3_stop_locked(struct vmxnet3_softc *sc) 2741 { 2742 struct ifnet *ifp; 2743 int q; 2744 2745 ifp = &sc->vmx_ethercom.ec_if; 2746 VMXNET3_CORE_LOCK_ASSERT(sc); 2747 2748 ifp->if_flags &= ~IFF_RUNNING; 2749 sc->vmx_link_active = 0; 2750 callout_stop(&sc->vmx_tick); 2751 2752 /* Disable interrupts. */ 2753 vmxnet3_disable_all_intrs(sc); 2754 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2755 2756 vmxnet3_stop_rendezvous(sc); 2757 2758 for (q = 0; q < sc->vmx_ntxqueues; q++) 2759 vmxnet3_txstop(sc, &sc->vmx_queue[q].vxq_txqueue); 2760 for (q = 0; q < sc->vmx_nrxqueues; q++) 2761 vmxnet3_rxstop(sc, &sc->vmx_queue[q].vxq_rxqueue); 2762 2763 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2764 } 2765 2766 static void 2767 vmxnet3_stop(struct ifnet *ifp, int disable) 2768 { 2769 struct vmxnet3_softc *sc = ifp->if_softc; 2770 2771 VMXNET3_CORE_LOCK(sc); 2772 vmxnet3_stop_locked(sc); 2773 VMXNET3_CORE_UNLOCK(sc); 2774 } 2775 2776 static void 2777 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2778 { 2779 struct vmxnet3_txring *txr; 2780 struct vmxnet3_comp_ring *txc; 2781 2782 txr = &txq->vxtxq_cmd_ring; 2783 txr->vxtxr_head = 0; 2784 txr->vxtxr_next = 0; 2785 txr->vxtxr_gen = VMXNET3_INIT_GEN; 2786 memset(txr->vxtxr_txd, 0, 2787 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2788 2789 txc = &txq->vxtxq_comp_ring; 2790 txc->vxcr_next = 0; 2791 txc->vxcr_gen = VMXNET3_INIT_GEN; 2792 memset(txc->vxcr_u.txcd, 0, 2793 txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2794 } 2795 2796 static int 2797 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2798 { 2799 struct vmxnet3_rxring *rxr; 2800 struct vmxnet3_comp_ring *rxc; 2801 u_int i, populate, idx; 2802 int error; 2803 2804 /* LRO and jumbo frame is not supported yet */ 2805 populate = 1; 2806 2807 for (i = 0; i < populate; i++) { 2808 rxr = &rxq->vxrxq_cmd_ring[i]; 2809 rxr->vxrxr_fill = 0; 2810 rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2811 memset(rxr->vxrxr_rxd, 0, 2812 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2813 2814 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2815 error = vmxnet3_newbuf(sc, rxq, rxr); 2816 if (error) 2817 return (error); 2818 } 2819 } 2820 2821 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2822 rxr = &rxq->vxrxq_cmd_ring[i]; 2823 rxr->vxrxr_fill = 0; 2824 rxr->vxrxr_gen = 0; 2825 memset(rxr->vxrxr_rxd, 0, 2826 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2827 } 2828 2829 rxc = &rxq->vxrxq_comp_ring; 2830 rxc->vxcr_next = 0; 2831 rxc->vxcr_gen = VMXNET3_INIT_GEN; 2832 memset(rxc->vxcr_u.rxcd, 0, 2833 rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2834 2835 return (0); 2836 } 2837 2838 static int 2839 vmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2840 { 2841 device_t dev; 2842 int q, error; 2843 dev = sc->vmx_dev; 2844 2845 for (q = 0; q < sc->vmx_ntxqueues; q++) 2846 vmxnet3_txinit(sc, &sc->vmx_queue[q].vxq_txqueue); 2847 2848 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2849 error = vmxnet3_rxinit(sc, &sc->vmx_queue[q].vxq_rxqueue); 2850 if (error) { 2851 device_printf(dev, "cannot populate Rx queue %d\n", q); 2852 return (error); 2853 } 2854 } 2855 2856 return (0); 2857 } 2858 2859 static int 2860 vmxnet3_enable_device(struct vmxnet3_softc *sc) 2861 { 2862 int q; 2863 2864 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2865 device_printf(sc->vmx_dev, "device enable command failed!\n"); 2866 return (1); 2867 } 2868 2869 /* Reset the Rx queue heads. */ 2870 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2871 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2872 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2873 } 2874 2875 return (0); 2876 } 2877 2878 static void 2879 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2880 { 2881 2882 vmxnet3_set_rxfilter(sc); 2883 2884 memset(sc->vmx_ds->vlan_filter, 0, sizeof(sc->vmx_ds->vlan_filter)); 2885 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2886 } 2887 2888 static int 2889 vmxnet3_reinit(struct vmxnet3_softc *sc) 2890 { 2891 2892 vmxnet3_set_lladdr(sc); 2893 vmxnet3_reinit_shared_data(sc); 2894 2895 if (vmxnet3_reinit_queues(sc) != 0) 2896 return (ENXIO); 2897 2898 if (vmxnet3_enable_device(sc) != 0) 2899 return (ENXIO); 2900 2901 vmxnet3_reinit_rxfilters(sc); 2902 2903 return (0); 2904 } 2905 2906 static int 2907 vmxnet3_init_locked(struct vmxnet3_softc *sc) 2908 { 2909 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 2910 int error; 2911 2912 vmxnet3_stop_locked(sc); 2913 2914 error = vmxnet3_reinit(sc); 2915 if (error) { 2916 vmxnet3_stop_locked(sc); 2917 return (error); 2918 } 2919 2920 ifp->if_flags |= IFF_RUNNING; 2921 vmxnet3_if_link_status(sc); 2922 2923 vmxnet3_enable_all_intrs(sc); 2924 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2925 2926 return (0); 2927 } 2928 2929 static int 2930 vmxnet3_init(struct ifnet *ifp) 2931 { 2932 struct vmxnet3_softc *sc = ifp->if_softc; 2933 int error; 2934 2935 VMXNET3_CORE_LOCK(sc); 2936 error = vmxnet3_init_locked(sc); 2937 VMXNET3_CORE_UNLOCK(sc); 2938 2939 return (error); 2940 } 2941 2942 static int 2943 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, 2944 int *start, int *csum_start) 2945 { 2946 struct ether_header *eh; 2947 struct mbuf *mp; 2948 int offset, csum_off, iphl, offp; 2949 bool v4; 2950 2951 eh = mtod(m, struct ether_header *); 2952 switch (htons(eh->ether_type)) { 2953 case ETHERTYPE_IP: 2954 case ETHERTYPE_IPV6: 2955 offset = ETHER_HDR_LEN; 2956 break; 2957 case ETHERTYPE_VLAN: 2958 offset = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2959 break; 2960 default: 2961 m_freem(m); 2962 return (EINVAL); 2963 } 2964 2965 if ((m->m_pkthdr.csum_flags & 2966 (M_CSUM_TSOv4 | M_CSUM_UDPv4 | M_CSUM_TCPv4)) != 0) { 2967 iphl = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); 2968 v4 = true; 2969 } else { 2970 iphl = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); 2971 v4 = false; 2972 } 2973 *start = offset + iphl; 2974 2975 if (m->m_pkthdr.csum_flags & 2976 (M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_TSOv4 | M_CSUM_TSOv6)) { 2977 csum_off = offsetof(struct tcphdr, th_sum); 2978 } else { 2979 csum_off = offsetof(struct udphdr, uh_sum); 2980 } 2981 2982 *csum_start = *start + csum_off; 2983 mp = m_pulldown(m, 0, *csum_start + 2, &offp); 2984 if (!mp) { 2985 /* m is already freed */ 2986 return ENOBUFS; 2987 } 2988 2989 if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) { 2990 struct tcphdr *tcp; 2991 2992 txq->vxtxq_stats.vmtxs_tso++; 2993 tcp = (void *)(mtod(mp, char *) + offp + *start); 2994 2995 if (v4) { 2996 struct ip *ip; 2997 2998 ip = (void *)(mtod(mp, char *) + offp + offset); 2999 tcp->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 3000 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3001 } else { 3002 struct ip6_hdr *ip6; 3003 3004 ip6 = (void *)(mtod(mp, char *) + offp + offset); 3005 tcp->th_sum = in6_cksum_phdr(&ip6->ip6_src, 3006 &ip6->ip6_dst, 0, htonl(IPPROTO_TCP)); 3007 } 3008 3009 /* 3010 * For TSO, the size of the protocol header is also 3011 * included in the descriptor header size. 3012 */ 3013 *start += (tcp->th_off << 2); 3014 } else 3015 txq->vxtxq_stats.vmtxs_csum++; 3016 3017 return (0); 3018 } 3019 3020 static int 3021 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 3022 bus_dmamap_t dmap) 3023 { 3024 struct mbuf *m; 3025 bus_dma_tag_t tag; 3026 int error; 3027 3028 m = *m0; 3029 tag = txq->vxtxq_sc->vmx_dmat; 3030 3031 error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT); 3032 if (error == 0 || error != EFBIG) 3033 return (error); 3034 3035 m = m_defrag(m, M_NOWAIT); 3036 if (m != NULL) { 3037 *m0 = m; 3038 error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT); 3039 } else 3040 error = ENOBUFS; 3041 3042 if (error) { 3043 m_freem(*m0); 3044 *m0 = NULL; 3045 txq->vxtxq_defrag_failed.ev_count++; 3046 } else 3047 txq->vxtxq_defragged.ev_count++; 3048 3049 return (error); 3050 } 3051 3052 static void 3053 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 3054 { 3055 3056 bus_dmamap_unload(txq->vxtxq_sc->vmx_dmat, dmap); 3057 } 3058 3059 static int 3060 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 3061 { 3062 struct vmxnet3_softc *sc; 3063 struct vmxnet3_txring *txr; 3064 struct vmxnet3_txdesc *txd, *sop; 3065 struct mbuf *m; 3066 bus_dmamap_t dmap; 3067 bus_dma_segment_t *segs; 3068 int i, gen, start, csum_start, nsegs, error; 3069 3070 sc = txq->vxtxq_sc; 3071 start = 0; 3072 txd = NULL; 3073 txr = &txq->vxtxq_cmd_ring; 3074 dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 3075 csum_start = 0; /* GCC */ 3076 3077 error = vmxnet3_txq_load_mbuf(txq, m0, dmap); 3078 if (error) 3079 return (error); 3080 3081 nsegs = dmap->dm_nsegs; 3082 segs = dmap->dm_segs; 3083 3084 m = *m0; 3085 KASSERT(m->m_flags & M_PKTHDR); 3086 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS); 3087 3088 if (vmxnet3_txring_avail(txr) < nsegs) { 3089 txq->vxtxq_stats.vmtxs_full++; 3090 vmxnet3_txq_unload_mbuf(txq, dmap); 3091 return (ENOSPC); 3092 } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 3093 error = vmxnet3_txq_offload_ctx(txq, m, &start, &csum_start); 3094 if (error) { 3095 /* m is already freed */ 3096 txq->vxtxq_stats.vmtxs_offload_failed++; 3097 vmxnet3_txq_unload_mbuf(txq, dmap); 3098 *m0 = NULL; 3099 return (error); 3100 } 3101 } 3102 3103 txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m; 3104 sop = &txr->vxtxr_txd[txr->vxtxr_head]; 3105 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 3106 3107 for (i = 0; i < nsegs; i++) { 3108 txd = &txr->vxtxr_txd[txr->vxtxr_head]; 3109 3110 txd->addr = segs[i].ds_addr; 3111 txd->len = segs[i].ds_len; 3112 txd->gen = gen; 3113 txd->dtype = 0; 3114 txd->offload_mode = VMXNET3_OM_NONE; 3115 txd->offload_pos = 0; 3116 txd->hlen = 0; 3117 txd->eop = 0; 3118 txd->compreq = 0; 3119 txd->vtag_mode = 0; 3120 txd->vtag = 0; 3121 3122 if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 3123 txr->vxtxr_head = 0; 3124 txr->vxtxr_gen ^= 1; 3125 } 3126 gen = txr->vxtxr_gen; 3127 } 3128 txd->eop = 1; 3129 txd->compreq = 1; 3130 3131 if (vlan_has_tag(m)) { 3132 sop->vtag_mode = 1; 3133 sop->vtag = vlan_get_tag(m); 3134 } 3135 3136 if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) { 3137 sop->offload_mode = VMXNET3_OM_TSO; 3138 sop->hlen = start; 3139 sop->offload_pos = m->m_pkthdr.segsz; 3140 } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 3141 VMXNET3_CSUM_OFFLOAD_IPV6)) { 3142 sop->offload_mode = VMXNET3_OM_CSUM; 3143 sop->hlen = start; 3144 sop->offload_pos = csum_start; 3145 } 3146 3147 /* Finally, change the ownership. */ 3148 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 3149 sop->gen ^= 1; 3150 3151 txq->vxtxq_ts->npending += nsegs; 3152 if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 3153 struct vmxnet3_queue *vmxq; 3154 vmxq = container_of(txq, struct vmxnet3_queue, vxq_txqueue); 3155 txq->vxtxq_ts->npending = 0; 3156 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(vmxq->vxq_id), 3157 txr->vxtxr_head); 3158 } 3159 3160 return (0); 3161 } 3162 3163 #define VMXNET3_TX_START 1 3164 #define VMXNET3_TX_TRANSMIT 2 3165 static inline void 3166 vmxnet3_tx_common_locked(struct ifnet *ifp, struct vmxnet3_txqueue *txq, int txtype) 3167 { 3168 struct vmxnet3_softc *sc; 3169 struct vmxnet3_txring *txr; 3170 struct mbuf *m_head; 3171 int tx; 3172 3173 sc = ifp->if_softc; 3174 txr = &txq->vxtxq_cmd_ring; 3175 tx = 0; 3176 3177 VMXNET3_TXQ_LOCK_ASSERT(txq); 3178 3179 if ((ifp->if_flags & IFF_RUNNING) == 0 || 3180 sc->vmx_link_active == 0) 3181 return; 3182 3183 for (;;) { 3184 if (txtype == VMXNET3_TX_START) 3185 IFQ_POLL(&ifp->if_snd, m_head); 3186 else 3187 m_head = pcq_peek(txq->vxtxq_interq); 3188 if (m_head == NULL) 3189 break; 3190 3191 if (vmxnet3_txring_avail(txr) < VMXNET3_TX_MAXSEGS) 3192 break; 3193 3194 if (txtype == VMXNET3_TX_START) 3195 IFQ_DEQUEUE(&ifp->if_snd, m_head); 3196 else 3197 m_head = pcq_get(txq->vxtxq_interq); 3198 if (m_head == NULL) 3199 break; 3200 3201 if (vmxnet3_txq_encap(txq, &m_head) != 0) { 3202 if (m_head != NULL) 3203 m_freem(m_head); 3204 break; 3205 } 3206 3207 tx++; 3208 bpf_mtap(ifp, m_head, BPF_D_OUT); 3209 } 3210 3211 if (tx > 0) 3212 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 3213 } 3214 3215 static void 3216 vmxnet3_start_locked(struct ifnet *ifp) 3217 { 3218 struct vmxnet3_softc *sc; 3219 struct vmxnet3_txqueue *txq; 3220 3221 sc = ifp->if_softc; 3222 txq = &sc->vmx_queue[0].vxq_txqueue; 3223 3224 vmxnet3_tx_common_locked(ifp, txq, VMXNET3_TX_START); 3225 } 3226 3227 void 3228 vmxnet3_start(struct ifnet *ifp) 3229 { 3230 struct vmxnet3_softc *sc; 3231 struct vmxnet3_txqueue *txq; 3232 3233 sc = ifp->if_softc; 3234 txq = &sc->vmx_queue[0].vxq_txqueue; 3235 3236 VMXNET3_TXQ_LOCK(txq); 3237 vmxnet3_start_locked(ifp); 3238 VMXNET3_TXQ_UNLOCK(txq); 3239 } 3240 3241 static int 3242 vmxnet3_select_txqueue(struct ifnet *ifp, struct mbuf *m __unused) 3243 { 3244 struct vmxnet3_softc *sc; 3245 u_int cpuid; 3246 3247 sc = ifp->if_softc; 3248 cpuid = cpu_index(curcpu()); 3249 /* 3250 * Furure work 3251 * We should select txqueue to even up the load even if ncpu is 3252 * different from sc->vmx_ntxqueues. Currently, the load is not 3253 * even, that is, when ncpu is six and ntxqueues is four, the load 3254 * of vmx_queue[0] and vmx_queue[1] is higher than vmx_queue[2] and 3255 * vmx_queue[3] because CPU#4 always uses vmx_queue[0] and CPU#5 always 3256 * uses vmx_queue[1]. 3257 * Furthermore, we should not use random value to select txqueue to 3258 * avoid reordering. We should use flow information of mbuf. 3259 */ 3260 return cpuid % sc->vmx_ntxqueues; 3261 } 3262 3263 static void 3264 vmxnet3_transmit_locked(struct ifnet *ifp, struct vmxnet3_txqueue *txq) 3265 { 3266 3267 vmxnet3_tx_common_locked(ifp, txq, VMXNET3_TX_TRANSMIT); 3268 } 3269 3270 static int 3271 vmxnet3_transmit(struct ifnet *ifp, struct mbuf *m) 3272 { 3273 struct vmxnet3_softc *sc; 3274 struct vmxnet3_txqueue *txq; 3275 int qid; 3276 3277 qid = vmxnet3_select_txqueue(ifp, m); 3278 sc = ifp->if_softc; 3279 txq = &sc->vmx_queue[qid].vxq_txqueue; 3280 3281 if (__predict_false(!pcq_put(txq->vxtxq_interq, m))) { 3282 VMXNET3_TXQ_LOCK(txq); 3283 txq->vxtxq_pcqdrop.ev_count++; 3284 VMXNET3_TXQ_UNLOCK(txq); 3285 m_freem(m); 3286 return ENOBUFS; 3287 } 3288 3289 if (VMXNET3_TXQ_TRYLOCK(txq)) { 3290 vmxnet3_transmit_locked(ifp, txq); 3291 VMXNET3_TXQ_UNLOCK(txq); 3292 } else { 3293 kpreempt_disable(); 3294 softint_schedule(txq->vxtxq_si); 3295 kpreempt_enable(); 3296 } 3297 3298 return 0; 3299 } 3300 3301 static void 3302 vmxnet3_deferred_transmit(void *arg) 3303 { 3304 struct vmxnet3_txqueue *txq = arg; 3305 struct vmxnet3_softc *sc = txq->vxtxq_sc; 3306 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 3307 3308 VMXNET3_TXQ_LOCK(txq); 3309 txq->vxtxq_transmitdef.ev_count++; 3310 if (pcq_peek(txq->vxtxq_interq) != NULL) 3311 vmxnet3_transmit_locked(ifp, txq); 3312 VMXNET3_TXQ_UNLOCK(txq); 3313 } 3314 3315 static void 3316 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 3317 { 3318 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 3319 struct ethercom *ec = &sc->vmx_ethercom; 3320 struct vmxnet3_driver_shared *ds = sc->vmx_ds; 3321 struct ether_multi *enm; 3322 struct ether_multistep step; 3323 u_int mode; 3324 uint8_t *p; 3325 3326 ds->mcast_tablelen = 0; 3327 ETHER_LOCK(ec); 3328 CLR(ec->ec_flags, ETHER_F_ALLMULTI); 3329 ETHER_UNLOCK(ec); 3330 3331 /* 3332 * Always accept broadcast frames. 3333 * Always accept frames destined to our station address. 3334 */ 3335 mode = VMXNET3_RXMODE_BCAST | VMXNET3_RXMODE_UCAST; 3336 3337 ETHER_LOCK(ec); 3338 if (ISSET(ifp->if_flags, IFF_PROMISC) || 3339 ec->ec_multicnt > VMXNET3_MULTICAST_MAX) 3340 goto allmulti; 3341 3342 p = sc->vmx_mcast; 3343 ETHER_FIRST_MULTI(step, ec, enm); 3344 while (enm != NULL) { 3345 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) { 3346 /* 3347 * We must listen to a range of multicast addresses. 3348 * For now, just accept all multicasts, rather than 3349 * trying to set only those filter bits needed to match 3350 * the range. (At this time, the only use of address 3351 * ranges is for IP multicast routing, for which the 3352 * range is big enough to require all bits set.) 3353 */ 3354 goto allmulti; 3355 } 3356 memcpy(p, enm->enm_addrlo, ETHER_ADDR_LEN); 3357 3358 p += ETHER_ADDR_LEN; 3359 3360 ETHER_NEXT_MULTI(step, enm); 3361 } 3362 3363 if (ec->ec_multicnt > 0) { 3364 SET(mode, VMXNET3_RXMODE_MCAST); 3365 ds->mcast_tablelen = p - sc->vmx_mcast; 3366 } 3367 ETHER_UNLOCK(ec); 3368 3369 goto setit; 3370 3371 allmulti: 3372 SET(ec->ec_flags, ETHER_F_ALLMULTI); 3373 ETHER_UNLOCK(ec); 3374 SET(mode, (VMXNET3_RXMODE_ALLMULTI | VMXNET3_RXMODE_MCAST)); 3375 if (ifp->if_flags & IFF_PROMISC) 3376 SET(mode, VMXNET3_RXMODE_PROMISC); 3377 3378 setit: 3379 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 3380 ds->rxmode = mode; 3381 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 3382 } 3383 3384 static int 3385 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, void *data) 3386 { 3387 struct vmxnet3_softc *sc = ifp->if_softc; 3388 struct ifreq *ifr = (struct ifreq *)data; 3389 int s, error = 0; 3390 3391 switch (cmd) { 3392 case SIOCSIFMTU: { 3393 int nmtu = ifr->ifr_mtu; 3394 3395 if (nmtu < VMXNET3_MIN_MTU || nmtu > VMXNET3_MAX_MTU) { 3396 error = EINVAL; 3397 break; 3398 } 3399 if (ifp->if_mtu != (uint64_t)nmtu) { 3400 s = splnet(); 3401 error = ether_ioctl(ifp, cmd, data); 3402 splx(s); 3403 if (error == ENETRESET) 3404 error = vmxnet3_init(ifp); 3405 } 3406 break; 3407 } 3408 3409 default: 3410 s = splnet(); 3411 error = ether_ioctl(ifp, cmd, data); 3412 splx(s); 3413 } 3414 3415 if (error == ENETRESET) { 3416 VMXNET3_CORE_LOCK(sc); 3417 if (ifp->if_flags & IFF_RUNNING) 3418 vmxnet3_set_rxfilter(sc); 3419 VMXNET3_CORE_UNLOCK(sc); 3420 error = 0; 3421 } 3422 3423 return error; 3424 } 3425 3426 static int 3427 vmxnet3_ifflags_cb(struct ethercom *ec) 3428 { 3429 struct vmxnet3_softc *sc; 3430 3431 sc = ec->ec_if.if_softc; 3432 3433 VMXNET3_CORE_LOCK(sc); 3434 vmxnet3_set_rxfilter(sc); 3435 VMXNET3_CORE_UNLOCK(sc); 3436 3437 vmxnet3_if_link_status(sc); 3438 3439 return 0; 3440 } 3441 3442 static int 3443 vmxnet3_watchdog(struct vmxnet3_txqueue *txq) 3444 { 3445 struct vmxnet3_softc *sc; 3446 struct vmxnet3_queue *vmxq; 3447 3448 sc = txq->vxtxq_sc; 3449 vmxq = container_of(txq, struct vmxnet3_queue, vxq_txqueue); 3450 3451 VMXNET3_TXQ_LOCK(txq); 3452 if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 3453 VMXNET3_TXQ_UNLOCK(txq); 3454 return (0); 3455 } 3456 txq->vxtxq_watchdogto.ev_count++; 3457 VMXNET3_TXQ_UNLOCK(txq); 3458 3459 device_printf(sc->vmx_dev, "watchdog timeout on queue %d\n", 3460 vmxq->vxq_id); 3461 return (1); 3462 } 3463 3464 static void 3465 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) 3466 { 3467 3468 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 3469 } 3470 3471 static void 3472 vmxnet3_tick(void *xsc) 3473 { 3474 struct vmxnet3_softc *sc; 3475 int i, timedout; 3476 3477 sc = xsc; 3478 timedout = 0; 3479 3480 VMXNET3_CORE_LOCK(sc); 3481 3482 vmxnet3_refresh_host_stats(sc); 3483 3484 for (i = 0; i < sc->vmx_ntxqueues; i++) 3485 timedout |= vmxnet3_watchdog(&sc->vmx_queue[i].vxq_txqueue); 3486 3487 if (timedout != 0) 3488 vmxnet3_init_locked(sc); 3489 else 3490 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 3491 3492 VMXNET3_CORE_UNLOCK(sc); 3493 } 3494 3495 /* 3496 * update link state of ifnet and softc 3497 */ 3498 static void 3499 vmxnet3_if_link_status(struct vmxnet3_softc *sc) 3500 { 3501 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 3502 u_int link; 3503 bool up; 3504 3505 up = vmxnet3_cmd_link_status(ifp); 3506 if (up) { 3507 sc->vmx_link_active = 1; 3508 link = LINK_STATE_UP; 3509 } else { 3510 sc->vmx_link_active = 0; 3511 link = LINK_STATE_DOWN; 3512 } 3513 3514 if_link_state_change(ifp, link); 3515 } 3516 3517 /* 3518 * check vmx(4) state by VMXNET3_CMD and update ifp->if_baudrate 3519 * returns 3520 * - true: link up 3521 * - flase: link down 3522 */ 3523 static bool 3524 vmxnet3_cmd_link_status(struct ifnet *ifp) 3525 { 3526 struct vmxnet3_softc *sc = ifp->if_softc; 3527 u_int x, speed; 3528 3529 x = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 3530 if ((x & 1) == 0) 3531 return false; 3532 3533 speed = x >> 16; 3534 ifp->if_baudrate = IF_Mbps(speed); 3535 return true; 3536 } 3537 3538 static void 3539 vmxnet3_ifmedia_status(struct ifnet *ifp, struct ifmediareq *ifmr) 3540 { 3541 bool up; 3542 3543 ifmr->ifm_status = IFM_AVALID; 3544 ifmr->ifm_active = IFM_ETHER; 3545 3546 up = vmxnet3_cmd_link_status(ifp); 3547 if (!up) 3548 return; 3549 3550 ifmr->ifm_status |= IFM_ACTIVE; 3551 3552 if (ifp->if_baudrate >= IF_Gbps(10ULL)) 3553 ifmr->ifm_active |= IFM_10G_T; 3554 } 3555 3556 static int 3557 vmxnet3_ifmedia_change(struct ifnet *ifp) 3558 { 3559 return 0; 3560 } 3561 3562 static void 3563 vmxnet3_set_lladdr(struct vmxnet3_softc *sc) 3564 { 3565 uint32_t ml, mh; 3566 3567 ml = sc->vmx_lladdr[0]; 3568 ml |= sc->vmx_lladdr[1] << 8; 3569 ml |= sc->vmx_lladdr[2] << 16; 3570 ml |= sc->vmx_lladdr[3] << 24; 3571 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 3572 3573 mh = sc->vmx_lladdr[4]; 3574 mh |= sc->vmx_lladdr[5] << 8; 3575 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 3576 } 3577 3578 static void 3579 vmxnet3_get_lladdr(struct vmxnet3_softc *sc) 3580 { 3581 uint32_t ml, mh; 3582 3583 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 3584 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 3585 3586 sc->vmx_lladdr[0] = ml; 3587 sc->vmx_lladdr[1] = ml >> 8; 3588 sc->vmx_lladdr[2] = ml >> 16; 3589 sc->vmx_lladdr[3] = ml >> 24; 3590 sc->vmx_lladdr[4] = mh; 3591 sc->vmx_lladdr[5] = mh >> 8; 3592 } 3593 3594 static void 3595 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3596 { 3597 int i; 3598 3599 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3600 for (i = 0; i < sc->vmx_nintrs; i++) 3601 vmxnet3_enable_intr(sc, i); 3602 } 3603 3604 static void 3605 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3606 { 3607 int i; 3608 3609 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3610 for (i = 0; i < sc->vmx_nintrs; i++) 3611 vmxnet3_disable_intr(sc, i); 3612 } 3613 3614 static int 3615 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3616 struct vmxnet3_dma_alloc *dma) 3617 { 3618 bus_dma_tag_t t = sc->vmx_dmat; 3619 bus_dma_segment_t *segs = dma->dma_segs; 3620 int n, error; 3621 3622 memset(dma, 0, sizeof(*dma)); 3623 3624 error = bus_dmamem_alloc(t, size, align, 0, segs, 1, &n, BUS_DMA_NOWAIT); 3625 if (error) { 3626 aprint_error_dev(sc->vmx_dev, "bus_dmamem_alloc failed: %d\n", error); 3627 goto fail1; 3628 } 3629 KASSERT(n == 1); 3630 3631 error = bus_dmamem_map(t, segs, 1, size, &dma->dma_vaddr, BUS_DMA_NOWAIT); 3632 if (error) { 3633 aprint_error_dev(sc->vmx_dev, "bus_dmamem_map failed: %d\n", error); 3634 goto fail2; 3635 } 3636 3637 error = bus_dmamap_create(t, size, 1, size, 0, BUS_DMA_NOWAIT, &dma->dma_map); 3638 if (error) { 3639 aprint_error_dev(sc->vmx_dev, "bus_dmamap_create failed: %d\n", error); 3640 goto fail3; 3641 } 3642 3643 error = bus_dmamap_load(t, dma->dma_map, dma->dma_vaddr, size, NULL, 3644 BUS_DMA_NOWAIT); 3645 if (error) { 3646 aprint_error_dev(sc->vmx_dev, "bus_dmamap_load failed: %d\n", error); 3647 goto fail4; 3648 } 3649 3650 memset(dma->dma_vaddr, 0, size); 3651 dma->dma_paddr = DMAADDR(dma->dma_map); 3652 dma->dma_size = size; 3653 3654 return (0); 3655 fail4: 3656 bus_dmamap_destroy(t, dma->dma_map); 3657 fail3: 3658 bus_dmamem_unmap(t, dma->dma_vaddr, size); 3659 fail2: 3660 bus_dmamem_free(t, segs, 1); 3661 fail1: 3662 return (error); 3663 } 3664 3665 static void 3666 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 3667 { 3668 bus_dma_tag_t t = sc->vmx_dmat; 3669 3670 bus_dmamap_unload(t, dma->dma_map); 3671 bus_dmamap_destroy(t, dma->dma_map); 3672 bus_dmamem_unmap(t, dma->dma_vaddr, dma->dma_size); 3673 bus_dmamem_free(t, dma->dma_segs, 1); 3674 3675 memset(dma, 0, sizeof(*dma)); 3676 } 3677 3678 MODULE(MODULE_CLASS_DRIVER, if_vmx, "pci"); 3679 3680 #ifdef _MODULE 3681 #include "ioconf.c" 3682 #endif 3683 3684 static int 3685 if_vmx_modcmd(modcmd_t cmd, void *opaque) 3686 { 3687 int error = 0; 3688 3689 switch (cmd) { 3690 case MODULE_CMD_INIT: 3691 #ifdef _MODULE 3692 error = config_init_component(cfdriver_ioconf_if_vmx, 3693 cfattach_ioconf_if_vmx, cfdata_ioconf_if_vmx); 3694 #endif 3695 return error; 3696 case MODULE_CMD_FINI: 3697 #ifdef _MODULE 3698 error = config_fini_component(cfdriver_ioconf_if_vmx, 3699 cfattach_ioconf_if_vmx, cfdata_ioconf_if_vmx); 3700 #endif 3701 return error; 3702 default: 3703 return ENOTTY; 3704 } 3705 } 3706 3707
Indexes created Tue Sep 23 22:09:46 GMT 2025