if_vmx.c revision 1.7
1 /* $NetBSD: if_vmx.c,v 1.7 2022/03/30 02:45:14 knakahara Exp $ */ 2 /* $OpenBSD: if_vmx.c,v 1.16 2014/01/22 06:04:17 brad Exp $ */ 3 4 /* 5 * Copyright (c) 2013 Tsubai Masanari 6 * Copyright (c) 2013 Bryan Venteicher <bryanv (at) FreeBSD.org> 7 * 8 * Permission to use, copy, modify, and distribute this software for any 9 * purpose with or without fee is hereby granted, provided that the above 10 * copyright notice and this permission notice appear in all copies. 11 * 12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 19 */ 20 21 #include <sys/cdefs.h> 22 __KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1.7 2022/03/30 02:45:14 knakahara Exp $"); 23 24 #include <sys/param.h> 25 #include <sys/cpu.h> 26 #include <sys/kernel.h> 27 #include <sys/kmem.h> 28 #include <sys/bitops.h> 29 #include <sys/bus.h> 30 #include <sys/device.h> 31 #include <sys/mbuf.h> 32 #include <sys/module.h> 33 #include <sys/sockio.h> 34 #include <sys/pcq.h> 35 #include <sys/workqueue.h> 36 #include <sys/interrupt.h> 37 38 #include <net/bpf.h> 39 #include <net/if.h> 40 #include <net/if_ether.h> 41 #include <net/if_media.h> 42 43 #include <netinet/if_inarp.h> 44 #include <netinet/in_systm.h> /* for <netinet/ip.h> */ 45 #include <netinet/in.h> /* for <netinet/ip.h> */ 46 #include <netinet/ip.h> /* for struct ip */ 47 #include <netinet/ip6.h> /* for struct ip6_hdr */ 48 #include <netinet/tcp.h> /* for struct tcphdr */ 49 #include <netinet/udp.h> /* for struct udphdr */ 50 51 #include <dev/pci/pcivar.h> 52 #include <dev/pci/pcireg.h> 53 #include <dev/pci/pcidevs.h> 54 55 #include <dev/pci/if_vmxreg.h> 56 57 #define VMXNET3_DRIVER_VERSION 0x00010000 58 59 /* 60 * Max descriptors per Tx packet. We must limit the size of the 61 * any TSO packets based on the number of segments. 62 */ 63 #define VMXNET3_TX_MAXSEGS 32 64 #define VMXNET3_TX_MAXSIZE (VMXNET3_TX_MAXSEGS * MCLBYTES) 65 66 /* 67 * Maximum support Tx segments size. The length field in the 68 * Tx descriptor is 14 bits. 69 */ 70 #define VMXNET3_TX_MAXSEGSIZE (1 << 14) 71 72 /* 73 * The maximum number of Rx segments we accept. 74 */ 75 #define VMXNET3_MAX_RX_SEGS 0 /* no segments */ 76 77 /* 78 * Predetermined size of the multicast MACs filter table. If the 79 * number of multicast addresses exceeds this size, then the 80 * ALL_MULTI mode is use instead. 81 */ 82 #define VMXNET3_MULTICAST_MAX 32 83 84 /* 85 * Our Tx watchdog timeout. 86 */ 87 #define VMXNET3_WATCHDOG_TIMEOUT 5 88 89 /* 90 * Default value for vmx_intr_{rx,tx}_process_limit which is used for 91 * max number of packets to process for interrupt handler 92 */ 93 #define VMXNET3_RX_INTR_PROCESS_LIMIT 0U 94 #define VMXNET3_TX_INTR_PROCESS_LIMIT 256 95 96 /* 97 * Default value for vmx_{rx,tx}_process_limit which is used for 98 * max number of packets to process for deferred processing 99 */ 100 #define VMXNET3_RX_PROCESS_LIMIT 256 101 #define VMXNET3_TX_PROCESS_LIMIT 256 102 103 #define VMXNET3_WORKQUEUE_PRI PRI_SOFTNET 104 105 /* 106 * IP protocols that we can perform Tx checksum offloading of. 107 */ 108 #define VMXNET3_CSUM_OFFLOAD \ 109 (M_CSUM_TCPv4 | M_CSUM_UDPv4) 110 #define VMXNET3_CSUM_OFFLOAD_IPV6 \ 111 (M_CSUM_TCPv6 | M_CSUM_UDPv6) 112 113 #define VMXNET3_CSUM_ALL_OFFLOAD \ 114 (VMXNET3_CSUM_OFFLOAD | VMXNET3_CSUM_OFFLOAD_IPV6 | M_CSUM_TSOv4 | M_CSUM_TSOv6) 115 116 #define VMXNET3_RXRINGS_PERQ 2 117 118 #define VMXNET3_CORE_LOCK(_sc) mutex_enter((_sc)->vmx_mtx) 119 #define VMXNET3_CORE_UNLOCK(_sc) mutex_exit((_sc)->vmx_mtx) 120 #define VMXNET3_CORE_LOCK_ASSERT(_sc) mutex_owned((_sc)->vmx_mtx) 121 122 #define VMXNET3_RXQ_LOCK(_rxq) mutex_enter((_rxq)->vxrxq_mtx) 123 #define VMXNET3_RXQ_UNLOCK(_rxq) mutex_exit((_rxq)->vxrxq_mtx) 124 #define VMXNET3_RXQ_LOCK_ASSERT(_rxq) \ 125 mutex_owned((_rxq)->vxrxq_mtx) 126 127 #define VMXNET3_TXQ_LOCK(_txq) mutex_enter((_txq)->vxtxq_mtx) 128 #define VMXNET3_TXQ_TRYLOCK(_txq) mutex_tryenter((_txq)->vxtxq_mtx) 129 #define VMXNET3_TXQ_UNLOCK(_txq) mutex_exit((_txq)->vxtxq_mtx) 130 #define VMXNET3_TXQ_LOCK_ASSERT(_txq) \ 131 mutex_owned((_txq)->vxtxq_mtx) 132 133 struct vmxnet3_dma_alloc { 134 bus_addr_t dma_paddr; 135 void *dma_vaddr; 136 bus_dmamap_t dma_map; 137 bus_size_t dma_size; 138 bus_dma_segment_t dma_segs[1]; 139 }; 140 141 struct vmxnet3_txbuf { 142 bus_dmamap_t vtxb_dmamap; 143 struct mbuf *vtxb_m; 144 }; 145 146 struct vmxnet3_txring { 147 struct vmxnet3_txbuf *vxtxr_txbuf; 148 struct vmxnet3_txdesc *vxtxr_txd; 149 u_int vxtxr_head; 150 u_int vxtxr_next; 151 u_int vxtxr_ndesc; 152 int vxtxr_gen; 153 struct vmxnet3_dma_alloc vxtxr_dma; 154 }; 155 156 struct vmxnet3_rxbuf { 157 bus_dmamap_t vrxb_dmamap; 158 struct mbuf *vrxb_m; 159 }; 160 161 struct vmxnet3_rxring { 162 struct vmxnet3_rxbuf *vxrxr_rxbuf; 163 struct vmxnet3_rxdesc *vxrxr_rxd; 164 u_int vxrxr_fill; 165 u_int vxrxr_ndesc; 166 int vxrxr_gen; 167 int vxrxr_rid; 168 struct vmxnet3_dma_alloc vxrxr_dma; 169 bus_dmamap_t vxrxr_spare_dmap; 170 }; 171 172 struct vmxnet3_comp_ring { 173 union { 174 struct vmxnet3_txcompdesc *txcd; 175 struct vmxnet3_rxcompdesc *rxcd; 176 } vxcr_u; 177 u_int vxcr_next; 178 u_int vxcr_ndesc; 179 int vxcr_gen; 180 struct vmxnet3_dma_alloc vxcr_dma; 181 }; 182 183 struct vmxnet3_txq_stats { 184 #if 0 185 uint64_t vmtxs_opackets; /* if_opackets */ 186 uint64_t vmtxs_obytes; /* if_obytes */ 187 uint64_t vmtxs_omcasts; /* if_omcasts */ 188 #endif 189 uint64_t vmtxs_csum; 190 uint64_t vmtxs_tso; 191 uint64_t vmtxs_full; 192 uint64_t vmtxs_offload_failed; 193 }; 194 195 struct vmxnet3_txqueue { 196 kmutex_t *vxtxq_mtx; 197 struct vmxnet3_softc *vxtxq_sc; 198 int vxtxq_watchdog; 199 pcq_t *vxtxq_interq; 200 struct vmxnet3_txring vxtxq_cmd_ring; 201 struct vmxnet3_comp_ring vxtxq_comp_ring; 202 struct vmxnet3_txq_stats vxtxq_stats; 203 struct vmxnet3_txq_shared *vxtxq_ts; 204 char vxtxq_name[16]; 205 206 void *vxtxq_si; 207 208 struct evcnt vxtxq_intr; 209 struct evcnt vxtxq_defer; 210 struct evcnt vxtxq_deferreq; 211 struct evcnt vxtxq_pcqdrop; 212 struct evcnt vxtxq_transmitdef; 213 struct evcnt vxtxq_watchdogto; 214 struct evcnt vxtxq_defragged; 215 struct evcnt vxtxq_defrag_failed; 216 }; 217 218 #if 0 219 struct vmxnet3_rxq_stats { 220 uint64_t vmrxs_ipackets; /* if_ipackets */ 221 uint64_t vmrxs_ibytes; /* if_ibytes */ 222 uint64_t vmrxs_iqdrops; /* if_iqdrops */ 223 uint64_t vmrxs_ierrors; /* if_ierrors */ 224 }; 225 #endif 226 227 struct vmxnet3_rxqueue { 228 kmutex_t *vxrxq_mtx; 229 struct vmxnet3_softc *vxrxq_sc; 230 struct mbuf *vxrxq_mhead; 231 struct mbuf *vxrxq_mtail; 232 struct vmxnet3_rxring vxrxq_cmd_ring[VMXNET3_RXRINGS_PERQ]; 233 struct vmxnet3_comp_ring vxrxq_comp_ring; 234 #if 0 235 struct vmxnet3_rxq_stats vxrxq_stats; 236 #endif 237 struct vmxnet3_rxq_shared *vxrxq_rs; 238 char vxrxq_name[16]; 239 240 struct evcnt vxrxq_intr; 241 struct evcnt vxrxq_defer; 242 struct evcnt vxrxq_deferreq; 243 struct evcnt vxrxq_mgetcl_failed; 244 struct evcnt vxrxq_mbuf_load_failed; 245 }; 246 247 struct vmxnet3_queue { 248 int vxq_id; 249 int vxq_intr_idx; 250 251 struct vmxnet3_txqueue vxq_txqueue; 252 struct vmxnet3_rxqueue vxq_rxqueue; 253 254 void *vxq_si; 255 bool vxq_workqueue; 256 bool vxq_wq_enqueued; 257 struct work vxq_wq_cookie; 258 }; 259 260 struct vmxnet3_softc { 261 device_t vmx_dev; 262 struct ethercom vmx_ethercom; 263 struct ifmedia vmx_media; 264 struct vmxnet3_driver_shared *vmx_ds; 265 int vmx_flags; 266 #define VMXNET3_FLAG_NO_MSIX (1 << 0) 267 #define VMXNET3_FLAG_RSS (1 << 1) 268 #define VMXNET3_FLAG_ATTACHED (1 << 2) 269 270 struct vmxnet3_queue *vmx_queue; 271 272 struct pci_attach_args *vmx_pa; 273 pci_chipset_tag_t vmx_pc; 274 275 bus_space_tag_t vmx_iot0; 276 bus_space_tag_t vmx_iot1; 277 bus_space_handle_t vmx_ioh0; 278 bus_space_handle_t vmx_ioh1; 279 bus_size_t vmx_ios0; 280 bus_size_t vmx_ios1; 281 bus_dma_tag_t vmx_dmat; 282 283 int vmx_link_active; 284 int vmx_ntxqueues; 285 int vmx_nrxqueues; 286 int vmx_ntxdescs; 287 int vmx_nrxdescs; 288 int vmx_max_rxsegs; 289 290 struct evcnt vmx_event_intr; 291 struct evcnt vmx_event_link; 292 struct evcnt vmx_event_txqerror; 293 struct evcnt vmx_event_rxqerror; 294 struct evcnt vmx_event_dic; 295 struct evcnt vmx_event_debug; 296 297 int vmx_intr_type; 298 int vmx_intr_mask_mode; 299 int vmx_event_intr_idx; 300 int vmx_nintrs; 301 pci_intr_handle_t *vmx_intrs; /* legacy use vmx_intrs[0] */ 302 void *vmx_ihs[VMXNET3_MAX_INTRS]; 303 304 kmutex_t *vmx_mtx; 305 306 uint8_t *vmx_mcast; 307 void *vmx_qs; 308 struct vmxnet3_rss_shared *vmx_rss; 309 callout_t vmx_tick; 310 struct vmxnet3_dma_alloc vmx_ds_dma; 311 struct vmxnet3_dma_alloc vmx_qs_dma; 312 struct vmxnet3_dma_alloc vmx_mcast_dma; 313 struct vmxnet3_dma_alloc vmx_rss_dma; 314 int vmx_max_ntxqueues; 315 int vmx_max_nrxqueues; 316 uint8_t vmx_lladdr[ETHER_ADDR_LEN]; 317 318 u_int vmx_rx_intr_process_limit; 319 u_int vmx_tx_intr_process_limit; 320 u_int vmx_rx_process_limit; 321 u_int vmx_tx_process_limit; 322 struct sysctllog *vmx_sysctllog; 323 324 bool vmx_txrx_workqueue; 325 struct workqueue *vmx_queue_wq; 326 }; 327 328 #define VMXNET3_STAT 329 330 #ifdef VMXNET3_STAT 331 struct { 332 u_int txhead; 333 u_int txdone; 334 u_int maxtxlen; 335 u_int rxdone; 336 u_int rxfill; 337 u_int intr; 338 } vmxstat; 339 #endif 340 341 typedef enum { 342 VMXNET3_BARRIER_RD, 343 VMXNET3_BARRIER_WR, 344 } vmxnet3_barrier_t; 345 346 #define JUMBO_LEN (MCLBYTES - ETHER_ALIGN) /* XXX */ 347 #define DMAADDR(map) ((map)->dm_segs[0].ds_addr) 348 349 #define vtophys(va) 0 /* XXX ok? */ 350 351 static int vmxnet3_match(device_t, cfdata_t, void *); 352 static void vmxnet3_attach(device_t, device_t, void *); 353 static int vmxnet3_detach(device_t, int); 354 355 static int vmxnet3_alloc_pci_resources(struct vmxnet3_softc *); 356 static void vmxnet3_free_pci_resources(struct vmxnet3_softc *); 357 static int vmxnet3_check_version(struct vmxnet3_softc *); 358 static void vmxnet3_check_multiqueue(struct vmxnet3_softc *); 359 360 static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 361 static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 362 static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 363 static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 364 static void vmxnet3_free_interrupts(struct vmxnet3_softc *); 365 366 static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 367 static int vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *); 368 static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 369 static void vmxnet3_set_interrupt_idx(struct vmxnet3_softc *); 370 static int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 371 static int vmxnet3_setup_sysctl(struct vmxnet3_softc *); 372 373 static int vmxnet3_setup_stats(struct vmxnet3_softc *); 374 static void vmxnet3_teardown_stats(struct vmxnet3_softc *); 375 376 static int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 377 static int vmxnet3_init_txq(struct vmxnet3_softc *, int); 378 static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 379 static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 380 static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 381 static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 382 383 static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 384 static void vmxnet3_free_shared_data(struct vmxnet3_softc *); 385 static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 386 static void vmxnet3_free_txq_data(struct vmxnet3_softc *); 387 static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 388 static void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 389 static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 390 static void vmxnet3_free_queue_data(struct vmxnet3_softc *); 391 static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 392 static void vmxnet3_free_mcast_table(struct vmxnet3_softc *); 393 static void vmxnet3_init_shared_data(struct vmxnet3_softc *); 394 static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); 395 static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 396 static int vmxnet3_alloc_data(struct vmxnet3_softc *); 397 static void vmxnet3_free_data(struct vmxnet3_softc *); 398 static int vmxnet3_setup_interface(struct vmxnet3_softc *); 399 400 static void vmxnet3_evintr(struct vmxnet3_softc *); 401 static bool vmxnet3_txq_eof(struct vmxnet3_txqueue *, u_int); 402 static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxqueue *, 403 struct vmxnet3_rxring *); 404 static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 405 struct vmxnet3_rxring *, int); 406 static void vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *); 407 static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 408 static void vmxnet3_rxq_input(struct vmxnet3_rxqueue *, 409 struct vmxnet3_rxcompdesc *, struct mbuf *); 410 static bool vmxnet3_rxq_eof(struct vmxnet3_rxqueue *, u_int); 411 static int vmxnet3_legacy_intr(void *); 412 static int vmxnet3_txrxq_intr(void *); 413 static void vmxnet3_handle_queue(void *); 414 static void vmxnet3_handle_queue_work(struct work *, void *); 415 static int vmxnet3_event_intr(void *); 416 417 static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 418 static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 419 static void vmxnet3_stop_locked(struct vmxnet3_softc *); 420 static void vmxnet3_stop_rendezvous(struct vmxnet3_softc *); 421 static void vmxnet3_stop(struct ifnet *, int); 422 423 static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 424 static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 425 static int vmxnet3_reinit_queues(struct vmxnet3_softc *); 426 static int vmxnet3_enable_device(struct vmxnet3_softc *); 427 static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 428 static int vmxnet3_reinit(struct vmxnet3_softc *); 429 430 static int vmxnet3_init_locked(struct vmxnet3_softc *); 431 static int vmxnet3_init(struct ifnet *); 432 433 static int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *, struct mbuf *, int *, int *); 434 static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, bus_dmamap_t); 435 static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 436 static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 437 static void vmxnet3_start_locked(struct ifnet *); 438 static void vmxnet3_start(struct ifnet *); 439 static void vmxnet3_transmit_locked(struct ifnet *, struct vmxnet3_txqueue *); 440 static int vmxnet3_transmit(struct ifnet *, struct mbuf *); 441 static void vmxnet3_deferred_transmit(void *); 442 443 static void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 444 static int vmxnet3_ioctl(struct ifnet *, u_long, void *); 445 static int vmxnet3_ifflags_cb(struct ethercom *); 446 447 static int vmxnet3_watchdog(struct vmxnet3_txqueue *); 448 static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); 449 static void vmxnet3_tick(void *); 450 static void vmxnet3_if_link_status(struct vmxnet3_softc *); 451 static bool vmxnet3_cmd_link_status(struct ifnet *); 452 static void vmxnet3_ifmedia_status(struct ifnet *, struct ifmediareq *); 453 static int vmxnet3_ifmedia_change(struct ifnet *); 454 static void vmxnet3_set_lladdr(struct vmxnet3_softc *); 455 static void vmxnet3_get_lladdr(struct vmxnet3_softc *); 456 457 static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 458 static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 459 460 static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, bus_size_t, 461 struct vmxnet3_dma_alloc *); 462 static void vmxnet3_dma_free(struct vmxnet3_softc *, struct vmxnet3_dma_alloc *); 463 464 CFATTACH_DECL3_NEW(vmx, sizeof(struct vmxnet3_softc), 465 vmxnet3_match, vmxnet3_attach, vmxnet3_detach, NULL, NULL, NULL, 0); 466 467 /* round down to the nearest power of 2 */ 468 static int 469 vmxnet3_calc_queue_size(int n) 470 { 471 472 if (__predict_false(n <= 0)) 473 return 1; 474 475 return (1U << (fls32(n) - 1)); 476 } 477 478 static inline void 479 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 480 { 481 482 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 483 } 484 485 static inline uint32_t 486 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 487 { 488 489 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 490 } 491 492 static inline void 493 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 494 { 495 496 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 497 } 498 499 static inline void 500 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 501 { 502 503 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 504 } 505 506 static inline uint32_t 507 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 508 { 509 510 vmxnet3_write_cmd(sc, cmd); 511 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 512 } 513 514 static inline void 515 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 516 { 517 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 518 } 519 520 static inline void 521 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 522 { 523 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 524 } 525 526 static inline void 527 vmxnet3_rxr_increment_fill(struct vmxnet3_rxring *rxr) 528 { 529 530 if (++rxr->vxrxr_fill == rxr->vxrxr_ndesc) { 531 rxr->vxrxr_fill = 0; 532 rxr->vxrxr_gen ^= 1; 533 } 534 } 535 536 static inline int 537 vmxnet3_txring_avail(struct vmxnet3_txring *txr) 538 { 539 int avail = txr->vxtxr_next - txr->vxtxr_head - 1; 540 return (avail < 0 ? (int)txr->vxtxr_ndesc + avail : avail); 541 } 542 543 /* 544 * Since this is a purely paravirtualized device, we do not have 545 * to worry about DMA coherency. But at times, we must make sure 546 * both the compiler and CPU do not reorder memory operations. 547 */ 548 static inline void 549 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 550 { 551 552 switch (type) { 553 case VMXNET3_BARRIER_RD: 554 membar_consumer(); 555 break; 556 case VMXNET3_BARRIER_WR: 557 membar_producer(); 558 break; 559 default: 560 panic("%s: bad barrier type %d", __func__, type); 561 } 562 } 563 564 static int 565 vmxnet3_match(device_t parent, cfdata_t match, void *aux) 566 { 567 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 568 569 if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_VMWARE && 570 PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_VMWARE_VMXNET3) 571 return 1; 572 573 return 0; 574 } 575 576 static void 577 vmxnet3_attach(device_t parent, device_t self, void *aux) 578 { 579 struct vmxnet3_softc *sc = device_private(self); 580 struct pci_attach_args *pa = aux; 581 pcireg_t preg; 582 int error; 583 int candidate; 584 585 sc->vmx_dev = self; 586 sc->vmx_pa = pa; 587 sc->vmx_pc = pa->pa_pc; 588 if (pci_dma64_available(pa)) 589 sc->vmx_dmat = pa->pa_dmat64; 590 else 591 sc->vmx_dmat = pa->pa_dmat; 592 593 pci_aprint_devinfo_fancy(pa, "Ethernet controller", "vmxnet3", 1); 594 595 preg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 596 preg |= PCI_COMMAND_MASTER_ENABLE; 597 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, preg); 598 599 sc->vmx_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 600 callout_init(&sc->vmx_tick, CALLOUT_MPSAFE); 601 602 candidate = MIN(MIN(VMXNET3_MAX_TX_QUEUES, VMXNET3_MAX_RX_QUEUES), 603 ncpu); 604 sc->vmx_max_ntxqueues = sc->vmx_max_nrxqueues = 605 vmxnet3_calc_queue_size(candidate); 606 sc->vmx_ntxdescs = 512; 607 sc->vmx_nrxdescs = 256; 608 sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 609 610 error = vmxnet3_alloc_pci_resources(sc); 611 if (error) 612 return; 613 614 error = vmxnet3_check_version(sc); 615 if (error) 616 return; 617 618 error = vmxnet3_alloc_rxtx_queues(sc); 619 if (error) 620 return; 621 622 error = vmxnet3_alloc_interrupts(sc); 623 if (error) 624 return; 625 626 vmxnet3_check_multiqueue(sc); 627 628 error = vmxnet3_alloc_data(sc); 629 if (error) 630 return; 631 632 error = vmxnet3_setup_interface(sc); 633 if (error) 634 return; 635 636 error = vmxnet3_setup_interrupts(sc); 637 if (error) 638 return; 639 640 error = vmxnet3_setup_sysctl(sc); 641 if (error) 642 return; 643 644 error = vmxnet3_setup_stats(sc); 645 if (error) 646 return; 647 648 sc->vmx_flags |= VMXNET3_FLAG_ATTACHED; 649 } 650 651 static int 652 vmxnet3_detach(device_t self, int flags) 653 { 654 struct vmxnet3_softc *sc; 655 struct ifnet *ifp; 656 657 sc = device_private(self); 658 ifp = &sc->vmx_ethercom.ec_if; 659 660 if (sc->vmx_flags & VMXNET3_FLAG_ATTACHED) { 661 VMXNET3_CORE_LOCK(sc); 662 vmxnet3_stop_locked(sc); 663 callout_halt(&sc->vmx_tick, sc->vmx_mtx); 664 callout_destroy(&sc->vmx_tick); 665 VMXNET3_CORE_UNLOCK(sc); 666 667 ether_ifdetach(ifp); 668 if_detach(ifp); 669 ifmedia_fini(&sc->vmx_media); 670 } 671 672 vmxnet3_teardown_stats(sc); 673 sysctl_teardown(&sc->vmx_sysctllog); 674 675 vmxnet3_free_interrupts(sc); 676 677 vmxnet3_free_data(sc); 678 vmxnet3_free_pci_resources(sc); 679 vmxnet3_free_rxtx_queues(sc); 680 681 if (sc->vmx_mtx) 682 mutex_obj_free(sc->vmx_mtx); 683 684 return (0); 685 } 686 687 static int 688 vmxnet3_alloc_pci_resources(struct vmxnet3_softc *sc) 689 { 690 struct pci_attach_args *pa = sc->vmx_pa; 691 pcireg_t memtype; 692 693 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 694 if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->vmx_iot0, &sc->vmx_ioh0, 695 NULL, &sc->vmx_ios0)) { 696 aprint_error_dev(sc->vmx_dev, "failed to map BAR0\n"); 697 return (ENXIO); 698 } 699 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(1)); 700 if (pci_mapreg_map(pa, PCI_BAR(1), memtype, 0, &sc->vmx_iot1, &sc->vmx_ioh1, 701 NULL, &sc->vmx_ios1)) { 702 aprint_error_dev(sc->vmx_dev, "failed to map BAR1\n"); 703 return (ENXIO); 704 } 705 706 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, NULL, NULL)) { 707 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 708 return (0); 709 } 710 711 return (0); 712 } 713 714 static void 715 vmxnet3_free_pci_resources(struct vmxnet3_softc *sc) 716 { 717 718 if (sc->vmx_ios0) { 719 bus_space_unmap(sc->vmx_iot0, sc->vmx_ioh0, sc->vmx_ios0); 720 sc->vmx_ios0 = 0; 721 } 722 723 if (sc->vmx_ios1) { 724 bus_space_unmap(sc->vmx_iot1, sc->vmx_ioh1, sc->vmx_ios1); 725 sc->vmx_ios1 = 0; 726 } 727 } 728 729 static int 730 vmxnet3_check_version(struct vmxnet3_softc *sc) 731 { 732 u_int ver; 733 734 ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 735 if ((ver & 0x1) == 0) { 736 aprint_error_dev(sc->vmx_dev, 737 "unsupported hardware version 0x%x\n", ver); 738 return (ENOTSUP); 739 } 740 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 741 742 ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 743 if ((ver & 0x1) == 0) { 744 aprint_error_dev(sc->vmx_dev, 745 "incompatiable UPT version 0x%x\n", ver); 746 return (ENOTSUP); 747 } 748 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 749 750 return (0); 751 } 752 753 static void 754 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc) 755 { 756 757 if (sc->vmx_intr_type != VMXNET3_IT_MSIX) 758 goto out; 759 760 /* Just use the maximum configured for now. */ 761 sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; 762 sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; 763 764 if (sc->vmx_nrxqueues > 1) 765 sc->vmx_flags |= VMXNET3_FLAG_RSS; 766 767 return; 768 769 out: 770 sc->vmx_ntxqueues = 1; 771 sc->vmx_nrxqueues = 1; 772 } 773 774 static int 775 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 776 { 777 int required; 778 struct pci_attach_args *pa = sc->vmx_pa; 779 780 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 781 return (1); 782 783 /* Allocate an additional vector for the events interrupt. */ 784 required = MIN(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues) + 1; 785 786 if (pci_msix_count(pa->pa_pc, pa->pa_tag) < required) 787 return (1); 788 789 if (pci_msix_alloc_exact(pa, &sc->vmx_intrs, required) == 0) { 790 sc->vmx_nintrs = required; 791 return (0); 792 } 793 794 return (1); 795 } 796 797 static int 798 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 799 { 800 int nmsi, required; 801 struct pci_attach_args *pa = sc->vmx_pa; 802 803 required = 1; 804 805 nmsi = pci_msi_count(pa->pa_pc, pa->pa_tag); 806 if (nmsi < required) 807 return (1); 808 809 if (pci_msi_alloc_exact(pa, &sc->vmx_intrs, required) == 0) { 810 sc->vmx_nintrs = required; 811 return (0); 812 } 813 814 return (1); 815 } 816 817 static int 818 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 819 { 820 821 if (pci_intx_alloc(sc->vmx_pa, &sc->vmx_intrs) == 0) { 822 sc->vmx_nintrs = 1; 823 return (0); 824 } 825 826 return (1); 827 } 828 829 static int 830 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 831 { 832 u_int config; 833 int error; 834 835 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 836 837 sc->vmx_intr_type = config & 0x03; 838 sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 839 840 switch (sc->vmx_intr_type) { 841 case VMXNET3_IT_AUTO: 842 sc->vmx_intr_type = VMXNET3_IT_MSIX; 843 /* FALLTHROUGH */ 844 case VMXNET3_IT_MSIX: 845 error = vmxnet3_alloc_msix_interrupts(sc); 846 if (error == 0) 847 break; 848 sc->vmx_intr_type = VMXNET3_IT_MSI; 849 /* FALLTHROUGH */ 850 case VMXNET3_IT_MSI: 851 error = vmxnet3_alloc_msi_interrupts(sc); 852 if (error == 0) 853 break; 854 sc->vmx_intr_type = VMXNET3_IT_LEGACY; 855 /* FALLTHROUGH */ 856 case VMXNET3_IT_LEGACY: 857 error = vmxnet3_alloc_legacy_interrupts(sc); 858 if (error == 0) 859 break; 860 /* FALLTHROUGH */ 861 default: 862 sc->vmx_intr_type = -1; 863 aprint_error_dev(sc->vmx_dev, "cannot allocate any interrupt resources\n"); 864 return (ENXIO); 865 } 866 867 return (error); 868 } 869 870 static void 871 vmxnet3_free_interrupts(struct vmxnet3_softc *sc) 872 { 873 pci_chipset_tag_t pc = sc->vmx_pc; 874 int i; 875 876 workqueue_destroy(sc->vmx_queue_wq); 877 for (i = 0; i < sc->vmx_ntxqueues; i++) { 878 struct vmxnet3_queue *vmxq = &sc->vmx_queue[i]; 879 880 softint_disestablish(vmxq->vxq_si); 881 vmxq->vxq_si = NULL; 882 } 883 for (i = 0; i < sc->vmx_nintrs; i++) { 884 pci_intr_disestablish(pc, sc->vmx_ihs[i]); 885 } 886 pci_intr_release(pc, sc->vmx_intrs, sc->vmx_nintrs); 887 } 888 889 static int 890 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 891 { 892 pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; 893 struct vmxnet3_queue *vmxq; 894 pci_intr_handle_t *intr; 895 void **ihs; 896 int intr_idx, i, use_queues, error; 897 kcpuset_t *affinity; 898 const char *intrstr; 899 char intrbuf[PCI_INTRSTR_LEN]; 900 char xnamebuf[32]; 901 902 intr = sc->vmx_intrs; 903 intr_idx = 0; 904 ihs = sc->vmx_ihs; 905 906 /* See vmxnet3_alloc_msix_interrupts() */ 907 use_queues = MIN(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues); 908 for (i = 0; i < use_queues; i++, intr++, ihs++, intr_idx++) { 909 snprintf(xnamebuf, 32, "%s: txrx %d", device_xname(sc->vmx_dev), i); 910 911 vmxq = &sc->vmx_queue[i]; 912 913 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 914 915 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 916 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 917 vmxnet3_txrxq_intr, vmxq, xnamebuf); 918 if (*ihs == NULL) { 919 aprint_error_dev(sc->vmx_dev, 920 "unable to establish txrx interrupt at %s\n", intrstr); 921 return (-1); 922 } 923 aprint_normal_dev(sc->vmx_dev, "txrx interrupting at %s\n", intrstr); 924 925 kcpuset_create(&affinity, true); 926 kcpuset_set(affinity, intr_idx % ncpu); 927 error = interrupt_distribute(*ihs, affinity, NULL); 928 if (error) { 929 aprint_normal_dev(sc->vmx_dev, 930 "%s cannot be changed affinity, use default CPU\n", 931 intrstr); 932 } 933 kcpuset_destroy(affinity); 934 935 vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 936 vmxnet3_handle_queue, vmxq); 937 if (vmxq->vxq_si == NULL) { 938 aprint_error_dev(sc->vmx_dev, 939 "softint_establish for vxq_si failed\n"); 940 return (-1); 941 } 942 943 vmxq->vxq_intr_idx = intr_idx; 944 } 945 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(sc->vmx_dev)); 946 error = workqueue_create(&sc->vmx_queue_wq, xnamebuf, 947 vmxnet3_handle_queue_work, sc, VMXNET3_WORKQUEUE_PRI, IPL_NET, 948 WQ_PERCPU | WQ_MPSAFE); 949 if (error) { 950 aprint_error_dev(sc->vmx_dev, "workqueue_create failed\n"); 951 return (-1); 952 } 953 sc->vmx_txrx_workqueue = false; 954 955 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 956 957 snprintf(xnamebuf, 32, "%s: link", device_xname(sc->vmx_dev)); 958 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 959 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 960 vmxnet3_event_intr, sc, xnamebuf); 961 if (*ihs == NULL) { 962 aprint_error_dev(sc->vmx_dev, 963 "unable to establish event interrupt at %s\n", intrstr); 964 return (-1); 965 } 966 aprint_normal_dev(sc->vmx_dev, "event interrupting at %s\n", intrstr); 967 968 sc->vmx_event_intr_idx = intr_idx; 969 970 return (0); 971 } 972 973 static int 974 vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *sc) 975 { 976 pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; 977 pci_intr_handle_t *intr; 978 void **ihs; 979 struct vmxnet3_queue *vmxq; 980 int i; 981 const char *intrstr; 982 char intrbuf[PCI_INTRSTR_LEN]; 983 char xnamebuf[32]; 984 985 intr = &sc->vmx_intrs[0]; 986 ihs = sc->vmx_ihs; 987 vmxq = &sc->vmx_queue[0]; 988 989 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 990 991 snprintf(xnamebuf, 32, "%s: msi", device_xname(sc->vmx_dev)); 992 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 993 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 994 vmxnet3_legacy_intr, sc, xnamebuf); 995 if (*ihs == NULL) { 996 aprint_error_dev(sc->vmx_dev, 997 "unable to establish interrupt at %s\n", intrstr); 998 return (-1); 999 } 1000 aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr); 1001 1002 vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1003 vmxnet3_handle_queue, vmxq); 1004 if (vmxq->vxq_si == NULL) { 1005 aprint_error_dev(sc->vmx_dev, 1006 "softint_establish for vxq_si failed\n"); 1007 return (-1); 1008 } 1009 1010 for (i = 0; i < MIN(sc->vmx_nrxqueues, sc->vmx_nrxqueues); i++) 1011 sc->vmx_queue[i].vxq_intr_idx = 0; 1012 sc->vmx_event_intr_idx = 0; 1013 1014 return (0); 1015 } 1016 1017 static int 1018 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 1019 { 1020 pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; 1021 pci_intr_handle_t *intr; 1022 void **ihs; 1023 struct vmxnet3_queue *vmxq; 1024 int i; 1025 const char *intrstr; 1026 char intrbuf[PCI_INTRSTR_LEN]; 1027 char xnamebuf[32]; 1028 1029 intr = &sc->vmx_intrs[0]; 1030 ihs = sc->vmx_ihs; 1031 vmxq = &sc->vmx_queue[0]; 1032 1033 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1034 1035 snprintf(xnamebuf, 32, "%s:legacy", device_xname(sc->vmx_dev)); 1036 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1037 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1038 vmxnet3_legacy_intr, sc, xnamebuf); 1039 if (*ihs == NULL) { 1040 aprint_error_dev(sc->vmx_dev, 1041 "unable to establish interrupt at %s\n", intrstr); 1042 return (-1); 1043 } 1044 aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr); 1045 1046 vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1047 vmxnet3_handle_queue, vmxq); 1048 if (vmxq->vxq_si == NULL) { 1049 aprint_error_dev(sc->vmx_dev, 1050 "softint_establish for vxq_si failed\n"); 1051 return (-1); 1052 } 1053 1054 for (i = 0; i < MIN(sc->vmx_nrxqueues, sc->vmx_nrxqueues); i++) 1055 sc->vmx_queue[i].vxq_intr_idx = 0; 1056 sc->vmx_event_intr_idx = 0; 1057 1058 return (0); 1059 } 1060 1061 static void 1062 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 1063 { 1064 struct vmxnet3_queue *vmxq; 1065 struct vmxnet3_txqueue *txq; 1066 struct vmxnet3_txq_shared *txs; 1067 struct vmxnet3_rxqueue *rxq; 1068 struct vmxnet3_rxq_shared *rxs; 1069 int i; 1070 1071 sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 1072 1073 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1074 vmxq = &sc->vmx_queue[i]; 1075 txq = &vmxq->vxq_txqueue; 1076 txs = txq->vxtxq_ts; 1077 txs->intr_idx = vmxq->vxq_intr_idx; 1078 } 1079 1080 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1081 vmxq = &sc->vmx_queue[i]; 1082 rxq = &vmxq->vxq_rxqueue; 1083 rxs = rxq->vxrxq_rs; 1084 rxs->intr_idx = vmxq->vxq_intr_idx; 1085 } 1086 } 1087 1088 static int 1089 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 1090 { 1091 int error; 1092 1093 switch (sc->vmx_intr_type) { 1094 case VMXNET3_IT_MSIX: 1095 error = vmxnet3_setup_msix_interrupts(sc); 1096 break; 1097 case VMXNET3_IT_MSI: 1098 error = vmxnet3_setup_msi_interrupt(sc); 1099 break; 1100 case VMXNET3_IT_LEGACY: 1101 error = vmxnet3_setup_legacy_interrupt(sc); 1102 break; 1103 default: 1104 panic("%s: invalid interrupt type %d", __func__, 1105 sc->vmx_intr_type); 1106 } 1107 1108 if (error == 0) 1109 vmxnet3_set_interrupt_idx(sc); 1110 1111 return (error); 1112 } 1113 1114 static int 1115 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 1116 { 1117 struct vmxnet3_rxqueue *rxq; 1118 struct vmxnet3_rxring *rxr; 1119 int i; 1120 1121 rxq = &sc->vmx_queue[q].vxq_rxqueue; 1122 1123 snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 1124 device_xname(sc->vmx_dev), q); 1125 rxq->vxrxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */); 1126 1127 rxq->vxrxq_sc = sc; 1128 1129 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1130 rxr = &rxq->vxrxq_cmd_ring[i]; 1131 rxr->vxrxr_rid = i; 1132 rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 1133 rxr->vxrxr_rxbuf = kmem_zalloc(rxr->vxrxr_ndesc * 1134 sizeof(struct vmxnet3_rxbuf), KM_SLEEP); 1135 1136 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 1137 } 1138 1139 return (0); 1140 } 1141 1142 static int 1143 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 1144 { 1145 struct vmxnet3_txqueue *txq; 1146 struct vmxnet3_txring *txr; 1147 1148 txq = &sc->vmx_queue[q].vxq_txqueue; 1149 txr = &txq->vxtxq_cmd_ring; 1150 1151 snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 1152 device_xname(sc->vmx_dev), q); 1153 txq->vxtxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */); 1154 1155 txq->vxtxq_sc = sc; 1156 1157 txq->vxtxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1158 vmxnet3_deferred_transmit, txq); 1159 if (txq->vxtxq_si == NULL) { 1160 mutex_obj_free(txq->vxtxq_mtx); 1161 aprint_error_dev(sc->vmx_dev, 1162 "softint_establish for vxtxq_si failed\n"); 1163 return ENOMEM; 1164 } 1165 1166 txr->vxtxr_ndesc = sc->vmx_ntxdescs; 1167 txr->vxtxr_txbuf = kmem_zalloc(txr->vxtxr_ndesc * 1168 sizeof(struct vmxnet3_txbuf), KM_SLEEP); 1169 1170 txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 1171 1172 txq->vxtxq_interq = pcq_create(sc->vmx_ntxdescs, KM_SLEEP); 1173 1174 return (0); 1175 } 1176 1177 static int 1178 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 1179 { 1180 int i, error, max_nqueues; 1181 1182 KASSERT(!cpu_intr_p()); 1183 KASSERT(!cpu_softintr_p()); 1184 1185 /* 1186 * Only attempt to create multiple queues if MSIX is available. 1187 * This check prevents us from allocating queue structures that 1188 * we will not use. 1189 * 1190 * FreeBSD: 1191 * MSIX is disabled by default because its apparently broken for 1192 * devices passed through by at least ESXi 5.1. 1193 * The hw.pci.honor_msi_blacklist tunable must be set to zero for MSIX. 1194 */ 1195 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { 1196 sc->vmx_max_nrxqueues = 1; 1197 sc->vmx_max_ntxqueues = 1; 1198 } 1199 1200 max_nqueues = MAX(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues); 1201 sc->vmx_queue = kmem_zalloc(sizeof(struct vmxnet3_queue) * max_nqueues, 1202 KM_SLEEP); 1203 1204 for (i = 0; i < max_nqueues; i++) { 1205 struct vmxnet3_queue *vmxq = &sc->vmx_queue[i]; 1206 vmxq->vxq_id = i; 1207 } 1208 1209 for (i = 0; i < sc->vmx_max_nrxqueues; i++) { 1210 error = vmxnet3_init_rxq(sc, i); 1211 if (error) 1212 return (error); 1213 } 1214 1215 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1216 error = vmxnet3_init_txq(sc, i); 1217 if (error) 1218 return (error); 1219 } 1220 1221 return (0); 1222 } 1223 1224 static void 1225 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 1226 { 1227 struct vmxnet3_rxring *rxr; 1228 int i; 1229 1230 rxq->vxrxq_sc = NULL; 1231 1232 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1233 rxr = &rxq->vxrxq_cmd_ring[i]; 1234 1235 if (rxr->vxrxr_rxbuf != NULL) { 1236 kmem_free(rxr->vxrxr_rxbuf, 1237 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxbuf)); 1238 rxr->vxrxr_rxbuf = NULL; 1239 } 1240 } 1241 1242 if (rxq->vxrxq_mtx != NULL) 1243 mutex_obj_free(rxq->vxrxq_mtx); 1244 } 1245 1246 static void 1247 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 1248 { 1249 struct vmxnet3_txring *txr; 1250 struct mbuf *m; 1251 1252 txr = &txq->vxtxq_cmd_ring; 1253 1254 txq->vxtxq_sc = NULL; 1255 1256 softint_disestablish(txq->vxtxq_si); 1257 1258 while ((m = pcq_get(txq->vxtxq_interq)) != NULL) 1259 m_freem(m); 1260 pcq_destroy(txq->vxtxq_interq); 1261 1262 if (txr->vxtxr_txbuf != NULL) { 1263 kmem_free(txr->vxtxr_txbuf, 1264 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txbuf)); 1265 txr->vxtxr_txbuf = NULL; 1266 } 1267 1268 if (txq->vxtxq_mtx != NULL) 1269 mutex_obj_free(txq->vxtxq_mtx); 1270 } 1271 1272 static void 1273 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 1274 { 1275 int i; 1276 1277 if (sc->vmx_queue != NULL) { 1278 int max_nqueues; 1279 1280 for (i = 0; i < sc->vmx_max_nrxqueues; i++) 1281 vmxnet3_destroy_rxq(&sc->vmx_queue[i].vxq_rxqueue); 1282 1283 for (i = 0; i < sc->vmx_max_ntxqueues; i++) 1284 vmxnet3_destroy_txq(&sc->vmx_queue[i].vxq_txqueue); 1285 1286 max_nqueues = MAX(sc->vmx_max_nrxqueues, sc->vmx_max_ntxqueues); 1287 kmem_free(sc->vmx_queue, 1288 sizeof(struct vmxnet3_queue) * max_nqueues); 1289 } 1290 } 1291 1292 static int 1293 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 1294 { 1295 device_t dev; 1296 uint8_t *kva; 1297 size_t size; 1298 int i, error; 1299 1300 dev = sc->vmx_dev; 1301 1302 size = sizeof(struct vmxnet3_driver_shared); 1303 error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 1304 if (error) { 1305 device_printf(dev, "cannot alloc shared memory\n"); 1306 return (error); 1307 } 1308 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 1309 1310 size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 1311 sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 1312 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 1313 if (error) { 1314 device_printf(dev, "cannot alloc queue shared memory\n"); 1315 return (error); 1316 } 1317 sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 1318 kva = sc->vmx_qs; 1319 1320 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1321 sc->vmx_queue[i].vxq_txqueue.vxtxq_ts = 1322 (struct vmxnet3_txq_shared *) kva; 1323 kva += sizeof(struct vmxnet3_txq_shared); 1324 } 1325 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1326 sc->vmx_queue[i].vxq_rxqueue.vxrxq_rs = 1327 (struct vmxnet3_rxq_shared *) kva; 1328 kva += sizeof(struct vmxnet3_rxq_shared); 1329 } 1330 1331 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1332 size = sizeof(struct vmxnet3_rss_shared); 1333 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); 1334 if (error) { 1335 device_printf(dev, "cannot alloc rss shared memory\n"); 1336 return (error); 1337 } 1338 sc->vmx_rss = 1339 (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; 1340 } 1341 1342 return (0); 1343 } 1344 1345 static void 1346 vmxnet3_free_shared_data(struct vmxnet3_softc *sc) 1347 { 1348 1349 if (sc->vmx_rss != NULL) { 1350 vmxnet3_dma_free(sc, &sc->vmx_rss_dma); 1351 sc->vmx_rss = NULL; 1352 } 1353 1354 if (sc->vmx_qs != NULL) { 1355 vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 1356 sc->vmx_qs = NULL; 1357 } 1358 1359 if (sc->vmx_ds != NULL) { 1360 vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 1361 sc->vmx_ds = NULL; 1362 } 1363 } 1364 1365 static int 1366 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 1367 { 1368 device_t dev; 1369 struct vmxnet3_txqueue *txq; 1370 struct vmxnet3_txring *txr; 1371 struct vmxnet3_comp_ring *txc; 1372 size_t descsz, compsz; 1373 u_int i; 1374 int q, error; 1375 1376 dev = sc->vmx_dev; 1377 1378 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1379 txq = &sc->vmx_queue[q].vxq_txqueue; 1380 txr = &txq->vxtxq_cmd_ring; 1381 txc = &txq->vxtxq_comp_ring; 1382 1383 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 1384 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 1385 1386 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 1387 if (error) { 1388 device_printf(dev, "cannot alloc Tx descriptors for " 1389 "queue %d error %d\n", q, error); 1390 return (error); 1391 } 1392 txr->vxtxr_txd = 1393 (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1394 1395 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1396 if (error) { 1397 device_printf(dev, "cannot alloc Tx comp descriptors " 1398 "for queue %d error %d\n", q, error); 1399 return (error); 1400 } 1401 txc->vxcr_u.txcd = 1402 (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1403 1404 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1405 error = bus_dmamap_create(sc->vmx_dmat, VMXNET3_TX_MAXSIZE, 1406 VMXNET3_TX_MAXSEGS, VMXNET3_TX_MAXSEGSIZE, 0, BUS_DMA_NOWAIT, 1407 &txr->vxtxr_txbuf[i].vtxb_dmamap); 1408 if (error) { 1409 device_printf(dev, "unable to create Tx buf " 1410 "dmamap for queue %d idx %d\n", q, i); 1411 return (error); 1412 } 1413 } 1414 } 1415 1416 return (0); 1417 } 1418 1419 static void 1420 vmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1421 { 1422 struct vmxnet3_txqueue *txq; 1423 struct vmxnet3_txring *txr; 1424 struct vmxnet3_comp_ring *txc; 1425 struct vmxnet3_txbuf *txb; 1426 u_int i; 1427 int q; 1428 1429 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1430 txq = &sc->vmx_queue[q].vxq_txqueue; 1431 txr = &txq->vxtxq_cmd_ring; 1432 txc = &txq->vxtxq_comp_ring; 1433 1434 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1435 txb = &txr->vxtxr_txbuf[i]; 1436 if (txb->vtxb_dmamap != NULL) { 1437 bus_dmamap_destroy(sc->vmx_dmat, 1438 txb->vtxb_dmamap); 1439 txb->vtxb_dmamap = NULL; 1440 } 1441 } 1442 1443 if (txc->vxcr_u.txcd != NULL) { 1444 vmxnet3_dma_free(sc, &txc->vxcr_dma); 1445 txc->vxcr_u.txcd = NULL; 1446 } 1447 1448 if (txr->vxtxr_txd != NULL) { 1449 vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1450 txr->vxtxr_txd = NULL; 1451 } 1452 } 1453 } 1454 1455 static int 1456 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1457 { 1458 device_t dev; 1459 struct vmxnet3_rxqueue *rxq; 1460 struct vmxnet3_rxring *rxr; 1461 struct vmxnet3_comp_ring *rxc; 1462 int descsz, compsz; 1463 u_int i, j; 1464 int q, error; 1465 1466 dev = sc->vmx_dev; 1467 1468 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1469 rxq = &sc->vmx_queue[q].vxq_rxqueue; 1470 rxc = &rxq->vxrxq_comp_ring; 1471 compsz = 0; 1472 1473 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1474 rxr = &rxq->vxrxq_cmd_ring[i]; 1475 1476 descsz = rxr->vxrxr_ndesc * 1477 sizeof(struct vmxnet3_rxdesc); 1478 compsz += rxr->vxrxr_ndesc * 1479 sizeof(struct vmxnet3_rxcompdesc); 1480 1481 error = vmxnet3_dma_malloc(sc, descsz, 512, 1482 &rxr->vxrxr_dma); 1483 if (error) { 1484 device_printf(dev, "cannot allocate Rx " 1485 "descriptors for queue %d/%d error %d\n", 1486 i, q, error); 1487 return (error); 1488 } 1489 rxr->vxrxr_rxd = 1490 (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1491 } 1492 1493 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1494 if (error) { 1495 device_printf(dev, "cannot alloc Rx comp descriptors " 1496 "for queue %d error %d\n", q, error); 1497 return (error); 1498 } 1499 rxc->vxcr_u.rxcd = 1500 (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1501 1502 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1503 rxr = &rxq->vxrxq_cmd_ring[i]; 1504 1505 error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1, 1506 JUMBO_LEN, 0, BUS_DMA_NOWAIT, 1507 &rxr->vxrxr_spare_dmap); 1508 if (error) { 1509 device_printf(dev, "unable to create spare " 1510 "dmamap for queue %d/%d error %d\n", 1511 q, i, error); 1512 return (error); 1513 } 1514 1515 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1516 error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1, 1517 JUMBO_LEN, 0, BUS_DMA_NOWAIT, 1518 &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1519 if (error) { 1520 device_printf(dev, "unable to create " 1521 "dmamap for queue %d/%d slot %d " 1522 "error %d\n", 1523 q, i, j, error); 1524 return (error); 1525 } 1526 } 1527 } 1528 } 1529 1530 return (0); 1531 } 1532 1533 static void 1534 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1535 { 1536 struct vmxnet3_rxqueue *rxq; 1537 struct vmxnet3_rxring *rxr; 1538 struct vmxnet3_comp_ring *rxc; 1539 struct vmxnet3_rxbuf *rxb; 1540 u_int i, j; 1541 int q; 1542 1543 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1544 rxq = &sc->vmx_queue[q].vxq_rxqueue; 1545 rxc = &rxq->vxrxq_comp_ring; 1546 1547 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1548 rxr = &rxq->vxrxq_cmd_ring[i]; 1549 1550 if (rxr->vxrxr_spare_dmap != NULL) { 1551 bus_dmamap_destroy(sc->vmx_dmat, 1552 rxr->vxrxr_spare_dmap); 1553 rxr->vxrxr_spare_dmap = NULL; 1554 } 1555 1556 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1557 rxb = &rxr->vxrxr_rxbuf[j]; 1558 if (rxb->vrxb_dmamap != NULL) { 1559 bus_dmamap_destroy(sc->vmx_dmat, 1560 rxb->vrxb_dmamap); 1561 rxb->vrxb_dmamap = NULL; 1562 } 1563 } 1564 } 1565 1566 if (rxc->vxcr_u.rxcd != NULL) { 1567 vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1568 rxc->vxcr_u.rxcd = NULL; 1569 } 1570 1571 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1572 rxr = &rxq->vxrxq_cmd_ring[i]; 1573 1574 if (rxr->vxrxr_rxd != NULL) { 1575 vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1576 rxr->vxrxr_rxd = NULL; 1577 } 1578 } 1579 } 1580 } 1581 1582 static int 1583 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1584 { 1585 int error; 1586 1587 error = vmxnet3_alloc_txq_data(sc); 1588 if (error) 1589 return (error); 1590 1591 error = vmxnet3_alloc_rxq_data(sc); 1592 if (error) 1593 return (error); 1594 1595 return (0); 1596 } 1597 1598 static void 1599 vmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1600 { 1601 1602 if (sc->vmx_queue != NULL) { 1603 vmxnet3_free_rxq_data(sc); 1604 vmxnet3_free_txq_data(sc); 1605 } 1606 } 1607 1608 static int 1609 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1610 { 1611 int error; 1612 1613 error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1614 32, &sc->vmx_mcast_dma); 1615 if (error) 1616 device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1617 else 1618 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1619 1620 return (error); 1621 } 1622 1623 static void 1624 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1625 { 1626 1627 if (sc->vmx_mcast != NULL) { 1628 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1629 sc->vmx_mcast = NULL; 1630 } 1631 } 1632 1633 static void 1634 vmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1635 { 1636 struct vmxnet3_driver_shared *ds; 1637 struct vmxnet3_txqueue *txq; 1638 struct vmxnet3_txq_shared *txs; 1639 struct vmxnet3_rxqueue *rxq; 1640 struct vmxnet3_rxq_shared *rxs; 1641 int i; 1642 1643 ds = sc->vmx_ds; 1644 1645 /* 1646 * Initialize fields of the shared data that remains the same across 1647 * reinits. Note the shared data is zero'd when allocated. 1648 */ 1649 1650 ds->magic = VMXNET3_REV1_MAGIC; 1651 1652 /* DriverInfo */ 1653 ds->version = VMXNET3_DRIVER_VERSION; 1654 ds->guest = VMXNET3_GOS_FREEBSD | 1655 #ifdef __LP64__ 1656 VMXNET3_GOS_64BIT; 1657 #else 1658 VMXNET3_GOS_32BIT; 1659 #endif 1660 ds->vmxnet3_revision = 1; 1661 ds->upt_version = 1; 1662 1663 /* Misc. conf */ 1664 ds->driver_data = vtophys(sc); 1665 ds->driver_data_len = sizeof(struct vmxnet3_softc); 1666 ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1667 ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1668 ds->nrxsg_max = sc->vmx_max_rxsegs; 1669 1670 /* RSS conf */ 1671 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1672 ds->rss.version = 1; 1673 ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; 1674 ds->rss.len = sc->vmx_rss_dma.dma_size; 1675 } 1676 1677 /* Interrupt control. */ 1678 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1679 ds->nintr = sc->vmx_nintrs; 1680 ds->evintr = sc->vmx_event_intr_idx; 1681 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1682 1683 for (i = 0; i < sc->vmx_nintrs; i++) 1684 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1685 1686 /* Receive filter. */ 1687 ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1688 ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1689 1690 /* Tx queues */ 1691 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1692 txq = &sc->vmx_queue[i].vxq_txqueue; 1693 txs = txq->vxtxq_ts; 1694 1695 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1696 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1697 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1698 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1699 txs->driver_data = vtophys(txq); 1700 txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1701 } 1702 1703 /* Rx queues */ 1704 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1705 rxq = &sc->vmx_queue[i].vxq_rxqueue; 1706 rxs = rxq->vxrxq_rs; 1707 1708 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1709 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1710 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1711 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1712 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1713 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1714 rxs->driver_data = vtophys(rxq); 1715 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1716 } 1717 } 1718 1719 static void 1720 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) 1721 { 1722 /* 1723 * Use the same key as the Linux driver until FreeBSD can do 1724 * RSS (presumably Toeplitz) in software. 1725 */ 1726 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 1727 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 1728 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 1729 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 1730 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 1731 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, 1732 }; 1733 1734 struct vmxnet3_rss_shared *rss; 1735 int i; 1736 1737 rss = sc->vmx_rss; 1738 1739 rss->hash_type = 1740 UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | 1741 UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; 1742 rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 1743 rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; 1744 rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 1745 memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); 1746 1747 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) 1748 rss->ind_table[i] = i % sc->vmx_nrxqueues; 1749 } 1750 1751 static void 1752 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1753 { 1754 struct ifnet *ifp; 1755 struct vmxnet3_driver_shared *ds; 1756 1757 ifp = &sc->vmx_ethercom.ec_if; 1758 ds = sc->vmx_ds; 1759 1760 ds->mtu = ifp->if_mtu; 1761 ds->ntxqueue = sc->vmx_ntxqueues; 1762 ds->nrxqueue = sc->vmx_nrxqueues; 1763 1764 ds->upt_features = 0; 1765 if (ifp->if_capenable & 1766 (IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 1767 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) 1768 ds->upt_features |= UPT1_F_CSUM; 1769 if (sc->vmx_ethercom.ec_capenable & ETHERCAP_VLAN_HWTAGGING) 1770 ds->upt_features |= UPT1_F_VLAN; 1771 1772 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1773 ds->upt_features |= UPT1_F_RSS; 1774 vmxnet3_reinit_rss_shared_data(sc); 1775 } 1776 1777 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1778 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1779 (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1780 } 1781 1782 static int 1783 vmxnet3_alloc_data(struct vmxnet3_softc *sc) 1784 { 1785 int error; 1786 1787 error = vmxnet3_alloc_shared_data(sc); 1788 if (error) 1789 return (error); 1790 1791 error = vmxnet3_alloc_queue_data(sc); 1792 if (error) 1793 return (error); 1794 1795 error = vmxnet3_alloc_mcast_table(sc); 1796 if (error) 1797 return (error); 1798 1799 vmxnet3_init_shared_data(sc); 1800 1801 return (0); 1802 } 1803 1804 static void 1805 vmxnet3_free_data(struct vmxnet3_softc *sc) 1806 { 1807 1808 vmxnet3_free_mcast_table(sc); 1809 vmxnet3_free_queue_data(sc); 1810 vmxnet3_free_shared_data(sc); 1811 } 1812 1813 static int 1814 vmxnet3_setup_interface(struct vmxnet3_softc *sc) 1815 { 1816 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 1817 1818 vmxnet3_get_lladdr(sc); 1819 aprint_normal_dev(sc->vmx_dev, "Ethernet address %s\n", 1820 ether_sprintf(sc->vmx_lladdr)); 1821 vmxnet3_set_lladdr(sc); 1822 1823 strlcpy(ifp->if_xname, device_xname(sc->vmx_dev), IFNAMSIZ); 1824 ifp->if_softc = sc; 1825 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; 1826 ifp->if_extflags = IFEF_MPSAFE; 1827 ifp->if_ioctl = vmxnet3_ioctl; 1828 ifp->if_start = vmxnet3_start; 1829 ifp->if_transmit = vmxnet3_transmit; 1830 ifp->if_watchdog = NULL; 1831 ifp->if_init = vmxnet3_init; 1832 ifp->if_stop = vmxnet3_stop; 1833 sc->vmx_ethercom.ec_if.if_capabilities |=IFCAP_CSUM_IPv4_Rx | 1834 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1835 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 1836 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 1837 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 1838 1839 ifp->if_capenable = ifp->if_capabilities; 1840 1841 sc->vmx_ethercom.ec_if.if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1842 1843 sc->vmx_ethercom.ec_capabilities |= 1844 ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING | ETHERCAP_JUMBO_MTU; 1845 sc->vmx_ethercom.ec_capenable |= ETHERCAP_VLAN_HWTAGGING; 1846 1847 IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs); 1848 IFQ_SET_READY(&ifp->if_snd); 1849 1850 /* Initialize ifmedia structures. */ 1851 sc->vmx_ethercom.ec_ifmedia = &sc->vmx_media; 1852 ifmedia_init_with_lock(&sc->vmx_media, IFM_IMASK, vmxnet3_ifmedia_change, 1853 vmxnet3_ifmedia_status, sc->vmx_mtx); 1854 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1855 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL); 1856 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_10G_T, 0, NULL); 1857 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1858 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_1000_T, 0, NULL); 1859 ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1860 1861 if_attach(ifp); 1862 if_deferred_start_init(ifp, NULL); 1863 ether_ifattach(ifp, sc->vmx_lladdr); 1864 ether_set_ifflags_cb(&sc->vmx_ethercom, vmxnet3_ifflags_cb); 1865 vmxnet3_cmd_link_status(ifp); 1866 1867 /* should set before setting interrupts */ 1868 sc->vmx_rx_intr_process_limit = VMXNET3_RX_INTR_PROCESS_LIMIT; 1869 sc->vmx_rx_process_limit = VMXNET3_RX_PROCESS_LIMIT; 1870 sc->vmx_tx_intr_process_limit = VMXNET3_TX_INTR_PROCESS_LIMIT; 1871 sc->vmx_tx_process_limit = VMXNET3_TX_PROCESS_LIMIT; 1872 1873 return (0); 1874 } 1875 1876 static int 1877 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 1878 { 1879 const char *devname; 1880 struct sysctllog **log; 1881 const struct sysctlnode *rnode, *rxnode, *txnode; 1882 int error; 1883 1884 log = &sc->vmx_sysctllog; 1885 devname = device_xname(sc->vmx_dev); 1886 1887 error = sysctl_createv(log, 0, NULL, &rnode, 1888 0, CTLTYPE_NODE, devname, 1889 SYSCTL_DESCR("vmxnet3 information and settings"), 1890 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 1891 if (error) 1892 goto out; 1893 error = sysctl_createv(log, 0, &rnode, NULL, 1894 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 1895 SYSCTL_DESCR("Use workqueue for packet processing"), 1896 NULL, 0, &sc->vmx_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 1897 if (error) 1898 goto out; 1899 1900 error = sysctl_createv(log, 0, &rnode, &rxnode, 1901 0, CTLTYPE_NODE, "rx", 1902 SYSCTL_DESCR("vmxnet3 information and settings for Rx"), 1903 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 1904 if (error) 1905 goto out; 1906 error = sysctl_createv(log, 0, &rxnode, NULL, 1907 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 1908 SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), 1909 NULL, 0, &sc->vmx_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 1910 if (error) 1911 goto out; 1912 error = sysctl_createv(log, 0, &rxnode, NULL, 1913 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 1914 SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), 1915 NULL, 0, &sc->vmx_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 1916 if (error) 1917 goto out; 1918 1919 error = sysctl_createv(log, 0, &rnode, &txnode, 1920 0, CTLTYPE_NODE, "tx", 1921 SYSCTL_DESCR("vmxnet3 information and settings for Tx"), 1922 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 1923 if (error) 1924 goto out; 1925 error = sysctl_createv(log, 0, &txnode, NULL, 1926 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 1927 SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), 1928 NULL, 0, &sc->vmx_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 1929 if (error) 1930 goto out; 1931 error = sysctl_createv(log, 0, &txnode, NULL, 1932 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 1933 SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), 1934 NULL, 0, &sc->vmx_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 1935 1936 out: 1937 if (error) { 1938 aprint_error_dev(sc->vmx_dev, 1939 "unable to create sysctl node\n"); 1940 sysctl_teardown(log); 1941 } 1942 return error; 1943 } 1944 1945 static int 1946 vmxnet3_setup_stats(struct vmxnet3_softc *sc) 1947 { 1948 struct vmxnet3_queue *vmxq; 1949 struct vmxnet3_txqueue *txq; 1950 struct vmxnet3_rxqueue *rxq; 1951 int i; 1952 1953 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1954 vmxq = &sc->vmx_queue[i]; 1955 txq = &vmxq->vxq_txqueue; 1956 evcnt_attach_dynamic(&txq->vxtxq_intr, EVCNT_TYPE_INTR, 1957 NULL, txq->vxtxq_name, "Interrupt on queue"); 1958 evcnt_attach_dynamic(&txq->vxtxq_defer, EVCNT_TYPE_MISC, 1959 NULL, txq->vxtxq_name, "Handled queue in softint/workqueue"); 1960 evcnt_attach_dynamic(&txq->vxtxq_deferreq, EVCNT_TYPE_MISC, 1961 NULL, txq->vxtxq_name, "Requested in softint/workqueue"); 1962 evcnt_attach_dynamic(&txq->vxtxq_pcqdrop, EVCNT_TYPE_MISC, 1963 NULL, txq->vxtxq_name, "Dropped in pcq"); 1964 evcnt_attach_dynamic(&txq->vxtxq_transmitdef, EVCNT_TYPE_MISC, 1965 NULL, txq->vxtxq_name, "Deferred transmit"); 1966 evcnt_attach_dynamic(&txq->vxtxq_watchdogto, EVCNT_TYPE_MISC, 1967 NULL, txq->vxtxq_name, "Watchdog timeout"); 1968 evcnt_attach_dynamic(&txq->vxtxq_defragged, EVCNT_TYPE_MISC, 1969 NULL, txq->vxtxq_name, "m_defrag successed"); 1970 evcnt_attach_dynamic(&txq->vxtxq_defrag_failed, EVCNT_TYPE_MISC, 1971 NULL, txq->vxtxq_name, "m_defrag failed"); 1972 } 1973 1974 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1975 vmxq = &sc->vmx_queue[i]; 1976 rxq = &vmxq->vxq_rxqueue; 1977 evcnt_attach_dynamic(&rxq->vxrxq_intr, EVCNT_TYPE_INTR, 1978 NULL, rxq->vxrxq_name, "Interrupt on queue"); 1979 evcnt_attach_dynamic(&rxq->vxrxq_defer, EVCNT_TYPE_MISC, 1980 NULL, rxq->vxrxq_name, "Handled queue in softint/workqueue"); 1981 evcnt_attach_dynamic(&rxq->vxrxq_deferreq, EVCNT_TYPE_MISC, 1982 NULL, rxq->vxrxq_name, "Requested in softint/workqueue"); 1983 evcnt_attach_dynamic(&rxq->vxrxq_mgetcl_failed, EVCNT_TYPE_MISC, 1984 NULL, rxq->vxrxq_name, "MCLGET failed"); 1985 evcnt_attach_dynamic(&rxq->vxrxq_mbuf_load_failed, EVCNT_TYPE_MISC, 1986 NULL, rxq->vxrxq_name, "bus_dmamap_load_mbuf failed"); 1987 } 1988 1989 evcnt_attach_dynamic(&sc->vmx_event_intr, EVCNT_TYPE_INTR, 1990 NULL, device_xname(sc->vmx_dev), "Interrupt for other events"); 1991 evcnt_attach_dynamic(&sc->vmx_event_link, EVCNT_TYPE_MISC, 1992 NULL, device_xname(sc->vmx_dev), "Link status event"); 1993 evcnt_attach_dynamic(&sc->vmx_event_txqerror, EVCNT_TYPE_MISC, 1994 NULL, device_xname(sc->vmx_dev), "Tx queue error event"); 1995 evcnt_attach_dynamic(&sc->vmx_event_rxqerror, EVCNT_TYPE_MISC, 1996 NULL, device_xname(sc->vmx_dev), "Rx queue error event"); 1997 evcnt_attach_dynamic(&sc->vmx_event_dic, EVCNT_TYPE_MISC, 1998 NULL, device_xname(sc->vmx_dev), "Device impl change event"); 1999 evcnt_attach_dynamic(&sc->vmx_event_debug, EVCNT_TYPE_MISC, 2000 NULL, device_xname(sc->vmx_dev), "Debug event"); 2001 2002 return 0; 2003 } 2004 2005 static void 2006 vmxnet3_teardown_stats(struct vmxnet3_softc *sc) 2007 { 2008 struct vmxnet3_queue *vmxq; 2009 struct vmxnet3_txqueue *txq; 2010 struct vmxnet3_rxqueue *rxq; 2011 int i; 2012 2013 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2014 vmxq = &sc->vmx_queue[i]; 2015 txq = &vmxq->vxq_txqueue; 2016 evcnt_detach(&txq->vxtxq_intr); 2017 evcnt_detach(&txq->vxtxq_defer); 2018 evcnt_detach(&txq->vxtxq_deferreq); 2019 evcnt_detach(&txq->vxtxq_pcqdrop); 2020 evcnt_detach(&txq->vxtxq_transmitdef); 2021 evcnt_detach(&txq->vxtxq_watchdogto); 2022 evcnt_detach(&txq->vxtxq_defragged); 2023 evcnt_detach(&txq->vxtxq_defrag_failed); 2024 } 2025 2026 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2027 vmxq = &sc->vmx_queue[i]; 2028 rxq = &vmxq->vxq_rxqueue; 2029 evcnt_detach(&rxq->vxrxq_intr); 2030 evcnt_detach(&rxq->vxrxq_defer); 2031 evcnt_detach(&rxq->vxrxq_deferreq); 2032 evcnt_detach(&rxq->vxrxq_mgetcl_failed); 2033 evcnt_detach(&rxq->vxrxq_mbuf_load_failed); 2034 } 2035 2036 evcnt_detach(&sc->vmx_event_intr); 2037 evcnt_detach(&sc->vmx_event_link); 2038 evcnt_detach(&sc->vmx_event_txqerror); 2039 evcnt_detach(&sc->vmx_event_rxqerror); 2040 evcnt_detach(&sc->vmx_event_dic); 2041 evcnt_detach(&sc->vmx_event_debug); 2042 } 2043 2044 static void 2045 vmxnet3_evintr(struct vmxnet3_softc *sc) 2046 { 2047 device_t dev; 2048 struct vmxnet3_txq_shared *ts; 2049 struct vmxnet3_rxq_shared *rs; 2050 uint32_t event; 2051 int reset; 2052 2053 dev = sc->vmx_dev; 2054 reset = 0; 2055 2056 VMXNET3_CORE_LOCK(sc); 2057 2058 /* Clear events. */ 2059 event = sc->vmx_ds->event; 2060 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 2061 2062 if (event & VMXNET3_EVENT_LINK) { 2063 sc->vmx_event_link.ev_count++; 2064 vmxnet3_if_link_status(sc); 2065 if (sc->vmx_link_active != 0) 2066 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); 2067 } 2068 2069 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 2070 if (event & VMXNET3_EVENT_TQERROR) 2071 sc->vmx_event_txqerror.ev_count++; 2072 if (event & VMXNET3_EVENT_RQERROR) 2073 sc->vmx_event_rxqerror.ev_count++; 2074 2075 reset = 1; 2076 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 2077 ts = sc->vmx_queue[0].vxq_txqueue.vxtxq_ts; 2078 if (ts->stopped != 0) 2079 device_printf(dev, "Tx queue error %#x\n", ts->error); 2080 rs = sc->vmx_queue[0].vxq_rxqueue.vxrxq_rs; 2081 if (rs->stopped != 0) 2082 device_printf(dev, "Rx queue error %#x\n", rs->error); 2083 device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 2084 } 2085 2086 if (event & VMXNET3_EVENT_DIC) { 2087 sc->vmx_event_dic.ev_count++; 2088 device_printf(dev, "device implementation change event\n"); 2089 } 2090 if (event & VMXNET3_EVENT_DEBUG) { 2091 sc->vmx_event_debug.ev_count++; 2092 device_printf(dev, "debug event\n"); 2093 } 2094 2095 if (reset != 0) 2096 vmxnet3_init_locked(sc); 2097 2098 VMXNET3_CORE_UNLOCK(sc); 2099 } 2100 2101 static bool 2102 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq, u_int limit) 2103 { 2104 struct vmxnet3_softc *sc; 2105 struct vmxnet3_txring *txr; 2106 struct vmxnet3_comp_ring *txc; 2107 struct vmxnet3_txcompdesc *txcd; 2108 struct vmxnet3_txbuf *txb; 2109 struct ifnet *ifp; 2110 struct mbuf *m; 2111 u_int sop; 2112 bool more = false; 2113 2114 sc = txq->vxtxq_sc; 2115 txr = &txq->vxtxq_cmd_ring; 2116 txc = &txq->vxtxq_comp_ring; 2117 ifp = &sc->vmx_ethercom.ec_if; 2118 2119 VMXNET3_TXQ_LOCK_ASSERT(txq); 2120 2121 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2122 for (;;) { 2123 if (limit-- == 0) { 2124 more = true; 2125 break; 2126 } 2127 2128 txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 2129 if (txcd->gen != txc->vxcr_gen) 2130 break; 2131 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2132 2133 if (++txc->vxcr_next == txc->vxcr_ndesc) { 2134 txc->vxcr_next = 0; 2135 txc->vxcr_gen ^= 1; 2136 } 2137 2138 sop = txr->vxtxr_next; 2139 txb = &txr->vxtxr_txbuf[sop]; 2140 2141 if ((m = txb->vtxb_m) != NULL) { 2142 bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap, 2143 0, txb->vtxb_dmamap->dm_mapsize, 2144 BUS_DMASYNC_POSTWRITE); 2145 bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap); 2146 2147 if_statinc_ref(nsr, if_opackets); 2148 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 2149 if (m->m_flags & M_MCAST) 2150 if_statinc_ref(nsr, if_omcasts); 2151 2152 m_freem(m); 2153 txb->vtxb_m = NULL; 2154 } 2155 2156 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 2157 } 2158 IF_STAT_PUTREF(ifp); 2159 2160 if (txr->vxtxr_head == txr->vxtxr_next) 2161 txq->vxtxq_watchdog = 0; 2162 2163 return more; 2164 } 2165 2166 static int 2167 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq, 2168 struct vmxnet3_rxring *rxr) 2169 { 2170 struct mbuf *m; 2171 struct vmxnet3_rxdesc *rxd; 2172 struct vmxnet3_rxbuf *rxb; 2173 bus_dma_tag_t tag; 2174 bus_dmamap_t dmap; 2175 int idx, btype, error; 2176 2177 tag = sc->vmx_dmat; 2178 dmap = rxr->vxrxr_spare_dmap; 2179 idx = rxr->vxrxr_fill; 2180 rxd = &rxr->vxrxr_rxd[idx]; 2181 rxb = &rxr->vxrxr_rxbuf[idx]; 2182 2183 /* Don't allocate buffers for ring 2 for now. */ 2184 if (rxr->vxrxr_rid != 0) 2185 return -1; 2186 btype = VMXNET3_BTYPE_HEAD; 2187 2188 MGETHDR(m, M_DONTWAIT, MT_DATA); 2189 if (m == NULL) 2190 return (ENOBUFS); 2191 2192 MCLGET(m, M_DONTWAIT); 2193 if ((m->m_flags & M_EXT) == 0) { 2194 rxq->vxrxq_mgetcl_failed.ev_count++; 2195 m_freem(m); 2196 return (ENOBUFS); 2197 } 2198 2199 m->m_pkthdr.len = m->m_len = JUMBO_LEN; 2200 m_adj(m, ETHER_ALIGN); 2201 2202 error = bus_dmamap_load_mbuf(sc->vmx_dmat, dmap, m, BUS_DMA_NOWAIT); 2203 if (error) { 2204 m_freem(m); 2205 rxq->vxrxq_mbuf_load_failed.ev_count++; 2206 return (error); 2207 } 2208 2209 if (rxb->vrxb_m != NULL) { 2210 bus_dmamap_sync(tag, rxb->vrxb_dmamap, 2211 0, rxb->vrxb_dmamap->dm_mapsize, 2212 BUS_DMASYNC_POSTREAD); 2213 bus_dmamap_unload(tag, rxb->vrxb_dmamap); 2214 } 2215 2216 rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 2217 rxb->vrxb_dmamap = dmap; 2218 rxb->vrxb_m = m; 2219 2220 rxd->addr = DMAADDR(dmap); 2221 rxd->len = m->m_pkthdr.len; 2222 rxd->btype = btype; 2223 rxd->gen = rxr->vxrxr_gen; 2224 2225 vmxnet3_rxr_increment_fill(rxr); 2226 return (0); 2227 } 2228 2229 static void 2230 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 2231 struct vmxnet3_rxring *rxr, int idx) 2232 { 2233 struct vmxnet3_rxdesc *rxd; 2234 2235 rxd = &rxr->vxrxr_rxd[idx]; 2236 rxd->gen = rxr->vxrxr_gen; 2237 vmxnet3_rxr_increment_fill(rxr); 2238 } 2239 2240 static void 2241 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 2242 { 2243 struct vmxnet3_softc *sc; 2244 struct vmxnet3_rxring *rxr; 2245 struct vmxnet3_comp_ring *rxc; 2246 struct vmxnet3_rxcompdesc *rxcd; 2247 int idx, eof; 2248 2249 sc = rxq->vxrxq_sc; 2250 rxc = &rxq->vxrxq_comp_ring; 2251 2252 do { 2253 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2254 if (rxcd->gen != rxc->vxcr_gen) 2255 break; /* Not expected. */ 2256 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2257 2258 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2259 rxc->vxcr_next = 0; 2260 rxc->vxcr_gen ^= 1; 2261 } 2262 2263 idx = rxcd->rxd_idx; 2264 eof = rxcd->eop; 2265 if (rxcd->qid < sc->vmx_nrxqueues) 2266 rxr = &rxq->vxrxq_cmd_ring[0]; 2267 else 2268 rxr = &rxq->vxrxq_cmd_ring[1]; 2269 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2270 } while (!eof); 2271 } 2272 2273 static void 2274 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2275 { 2276 if (rxcd->no_csum) 2277 return; 2278 2279 if (rxcd->ipv4) { 2280 m->m_pkthdr.csum_flags |= M_CSUM_IPv4; 2281 if (rxcd->ipcsum_ok == 0) 2282 m->m_pkthdr.csum_flags |= M_CSUM_IPv4_BAD; 2283 } 2284 2285 if (rxcd->fragment) 2286 return; 2287 2288 if (rxcd->tcp) { 2289 m->m_pkthdr.csum_flags |= 2290 rxcd->ipv4 ? M_CSUM_TCPv4 : M_CSUM_TCPv6; 2291 if ((rxcd->csum_ok) == 0) 2292 m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; 2293 } 2294 2295 if (rxcd->udp) { 2296 m->m_pkthdr.csum_flags |= 2297 rxcd->ipv4 ? M_CSUM_UDPv4 : M_CSUM_UDPv6 ; 2298 if ((rxcd->csum_ok) == 0) 2299 m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; 2300 } 2301 } 2302 2303 static void 2304 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 2305 struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2306 { 2307 struct vmxnet3_softc *sc; 2308 struct ifnet *ifp; 2309 2310 sc = rxq->vxrxq_sc; 2311 ifp = &sc->vmx_ethercom.ec_if; 2312 2313 if (rxcd->error) { 2314 if_statinc(ifp, if_ierrors); 2315 m_freem(m); 2316 return; 2317 } 2318 2319 if (!rxcd->no_csum) 2320 vmxnet3_rx_csum(rxcd, m); 2321 if (rxcd->vlan) 2322 vlan_set_tag(m, rxcd->vtag); 2323 2324 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2325 if_statinc_ref(nsr, if_ipackets); 2326 if_statadd_ref(nsr, if_ibytes, m->m_pkthdr.len); 2327 IF_STAT_PUTREF(ifp); 2328 2329 if_percpuq_enqueue(ifp->if_percpuq, m); 2330 } 2331 2332 static bool 2333 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq, u_int limit) 2334 { 2335 struct vmxnet3_softc *sc; 2336 struct ifnet *ifp; 2337 struct vmxnet3_rxring *rxr; 2338 struct vmxnet3_comp_ring *rxc; 2339 struct vmxnet3_rxdesc *rxd __diagused; 2340 struct vmxnet3_rxcompdesc *rxcd; 2341 struct mbuf *m, *m_head, *m_tail; 2342 u_int idx, length; 2343 bool more = false; 2344 2345 sc = rxq->vxrxq_sc; 2346 ifp = &sc->vmx_ethercom.ec_if; 2347 rxc = &rxq->vxrxq_comp_ring; 2348 2349 VMXNET3_RXQ_LOCK_ASSERT(rxq); 2350 2351 if ((ifp->if_flags & IFF_RUNNING) == 0) 2352 return more; 2353 2354 m_head = rxq->vxrxq_mhead; 2355 rxq->vxrxq_mhead = NULL; 2356 m_tail = rxq->vxrxq_mtail; 2357 rxq->vxrxq_mtail = NULL; 2358 KASSERT(m_head == NULL || m_tail != NULL); 2359 2360 for (;;) { 2361 if (limit-- == 0) { 2362 more = true; 2363 break; 2364 } 2365 2366 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2367 if (rxcd->gen != rxc->vxcr_gen) { 2368 rxq->vxrxq_mhead = m_head; 2369 rxq->vxrxq_mtail = m_tail; 2370 break; 2371 } 2372 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2373 2374 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2375 rxc->vxcr_next = 0; 2376 rxc->vxcr_gen ^= 1; 2377 } 2378 2379 idx = rxcd->rxd_idx; 2380 length = rxcd->len; 2381 if (rxcd->qid < sc->vmx_nrxqueues) 2382 rxr = &rxq->vxrxq_cmd_ring[0]; 2383 else 2384 rxr = &rxq->vxrxq_cmd_ring[1]; 2385 rxd = &rxr->vxrxr_rxd[idx]; 2386 2387 m = rxr->vxrxr_rxbuf[idx].vrxb_m; 2388 KASSERT(m != NULL); 2389 2390 /* 2391 * The host may skip descriptors. We detect this when this 2392 * descriptor does not match the previous fill index. Catch 2393 * up with the host now. 2394 */ 2395 if (__predict_false(rxr->vxrxr_fill != idx)) { 2396 while (rxr->vxrxr_fill != idx) { 2397 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 2398 rxr->vxrxr_gen; 2399 vmxnet3_rxr_increment_fill(rxr); 2400 } 2401 } 2402 2403 if (rxcd->sop) { 2404 /* start of frame w/o head buffer */ 2405 KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD); 2406 /* start of frame not in ring 0 */ 2407 KASSERT(rxr == &rxq->vxrxq_cmd_ring[0]); 2408 /* duplicate start of frame? */ 2409 KASSERT(m_head == NULL); 2410 2411 if (length == 0) { 2412 /* Just ignore this descriptor. */ 2413 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2414 goto nextp; 2415 } 2416 2417 if (vmxnet3_newbuf(sc, rxq, rxr) != 0) { 2418 if_statinc(ifp, if_iqdrops); 2419 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2420 if (!rxcd->eop) 2421 vmxnet3_rxq_discard_chain(rxq); 2422 goto nextp; 2423 } 2424 2425 m_set_rcvif(m, ifp); 2426 m->m_pkthdr.len = m->m_len = length; 2427 m->m_pkthdr.csum_flags = 0; 2428 m_head = m_tail = m; 2429 2430 } else { 2431 /* non start of frame w/o body buffer */ 2432 KASSERT(rxd->btype == VMXNET3_BTYPE_BODY); 2433 /* frame not started? */ 2434 KASSERT(m_head != NULL); 2435 2436 if (vmxnet3_newbuf(sc, rxq, rxr) != 0) { 2437 if_statinc(ifp, if_iqdrops); 2438 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2439 if (!rxcd->eop) 2440 vmxnet3_rxq_discard_chain(rxq); 2441 m_freem(m_head); 2442 m_head = m_tail = NULL; 2443 goto nextp; 2444 } 2445 2446 m->m_len = length; 2447 m_head->m_pkthdr.len += length; 2448 m_tail->m_next = m; 2449 m_tail = m; 2450 } 2451 2452 if (rxcd->eop) { 2453 vmxnet3_rxq_input(rxq, rxcd, m_head); 2454 m_head = m_tail = NULL; 2455 2456 /* Must recheck after dropping the Rx lock. */ 2457 if ((ifp->if_flags & IFF_RUNNING) == 0) 2458 break; 2459 } 2460 2461 nextp: 2462 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 2463 int qid = rxcd->qid; 2464 bus_size_t r; 2465 2466 idx = (idx + 1) % rxr->vxrxr_ndesc; 2467 if (qid >= sc->vmx_nrxqueues) { 2468 qid -= sc->vmx_nrxqueues; 2469 r = VMXNET3_BAR0_RXH2(qid); 2470 } else 2471 r = VMXNET3_BAR0_RXH1(qid); 2472 vmxnet3_write_bar0(sc, r, idx); 2473 } 2474 } 2475 2476 return more; 2477 } 2478 2479 static inline void 2480 vmxnet3_sched_handle_queue(struct vmxnet3_softc *sc, struct vmxnet3_queue *vmxq) 2481 { 2482 2483 if (vmxq->vxq_workqueue) { 2484 /* 2485 * When this function is called, "vmxq" is owned by one CPU. 2486 * so, atomic operation is not required here. 2487 */ 2488 if (!vmxq->vxq_wq_enqueued) { 2489 vmxq->vxq_wq_enqueued = true; 2490 workqueue_enqueue(sc->vmx_queue_wq, 2491 &vmxq->vxq_wq_cookie, curcpu()); 2492 } 2493 } else { 2494 softint_schedule(vmxq->vxq_si); 2495 } 2496 } 2497 2498 static int 2499 vmxnet3_legacy_intr(void *xsc) 2500 { 2501 struct vmxnet3_softc *sc; 2502 struct vmxnet3_rxqueue *rxq; 2503 struct vmxnet3_txqueue *txq; 2504 u_int txlimit, rxlimit; 2505 bool txmore, rxmore; 2506 2507 sc = xsc; 2508 rxq = &sc->vmx_queue[0].vxq_rxqueue; 2509 txq = &sc->vmx_queue[0].vxq_txqueue; 2510 txlimit = sc->vmx_tx_intr_process_limit; 2511 rxlimit = sc->vmx_rx_intr_process_limit; 2512 2513 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 2514 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 2515 return (0); 2516 } 2517 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2518 vmxnet3_disable_all_intrs(sc); 2519 2520 if (sc->vmx_ds->event != 0) 2521 vmxnet3_evintr(sc); 2522 2523 VMXNET3_RXQ_LOCK(rxq); 2524 rxmore = vmxnet3_rxq_eof(rxq, rxlimit); 2525 VMXNET3_RXQ_UNLOCK(rxq); 2526 2527 VMXNET3_TXQ_LOCK(txq); 2528 txmore = vmxnet3_txq_eof(txq, txlimit); 2529 VMXNET3_TXQ_UNLOCK(txq); 2530 2531 if (txmore || rxmore) { 2532 vmxnet3_sched_handle_queue(sc, &sc->vmx_queue[0]); 2533 } else { 2534 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); 2535 vmxnet3_enable_all_intrs(sc); 2536 } 2537 return (1); 2538 } 2539 2540 static int 2541 vmxnet3_txrxq_intr(void *xvmxq) 2542 { 2543 struct vmxnet3_softc *sc; 2544 struct vmxnet3_queue *vmxq; 2545 struct vmxnet3_txqueue *txq; 2546 struct vmxnet3_rxqueue *rxq; 2547 u_int txlimit, rxlimit; 2548 bool txmore, rxmore; 2549 2550 vmxq = xvmxq; 2551 txq = &vmxq->vxq_txqueue; 2552 rxq = &vmxq->vxq_rxqueue; 2553 sc = txq->vxtxq_sc; 2554 txlimit = sc->vmx_tx_intr_process_limit; 2555 rxlimit = sc->vmx_rx_intr_process_limit; 2556 vmxq->vxq_workqueue = sc->vmx_txrx_workqueue; 2557 2558 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2559 vmxnet3_disable_intr(sc, vmxq->vxq_intr_idx); 2560 2561 VMXNET3_TXQ_LOCK(txq); 2562 txq->vxtxq_intr.ev_count++; 2563 txmore = vmxnet3_txq_eof(txq, txlimit); 2564 VMXNET3_TXQ_UNLOCK(txq); 2565 2566 VMXNET3_RXQ_LOCK(rxq); 2567 rxq->vxrxq_intr.ev_count++; 2568 rxmore = vmxnet3_rxq_eof(rxq, rxlimit); 2569 VMXNET3_RXQ_UNLOCK(rxq); 2570 2571 if (txmore || rxmore) { 2572 vmxnet3_sched_handle_queue(sc, vmxq); 2573 } else { 2574 /* for ALTQ */ 2575 if (vmxq->vxq_id == 0) 2576 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); 2577 softint_schedule(txq->vxtxq_si); 2578 2579 vmxnet3_enable_intr(sc, vmxq->vxq_intr_idx); 2580 } 2581 2582 return (1); 2583 } 2584 2585 static void 2586 vmxnet3_handle_queue(void *xvmxq) 2587 { 2588 struct vmxnet3_softc *sc; 2589 struct vmxnet3_queue *vmxq; 2590 struct vmxnet3_txqueue *txq; 2591 struct vmxnet3_rxqueue *rxq; 2592 u_int txlimit, rxlimit; 2593 bool txmore, rxmore; 2594 2595 vmxq = xvmxq; 2596 txq = &vmxq->vxq_txqueue; 2597 rxq = &vmxq->vxq_rxqueue; 2598 sc = txq->vxtxq_sc; 2599 txlimit = sc->vmx_tx_process_limit; 2600 rxlimit = sc->vmx_rx_process_limit; 2601 2602 VMXNET3_TXQ_LOCK(txq); 2603 txq->vxtxq_defer.ev_count++; 2604 txmore = vmxnet3_txq_eof(txq, txlimit); 2605 if (txmore) 2606 txq->vxtxq_deferreq.ev_count++; 2607 /* for ALTQ */ 2608 if (vmxq->vxq_id == 0) 2609 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); 2610 softint_schedule(txq->vxtxq_si); 2611 VMXNET3_TXQ_UNLOCK(txq); 2612 2613 VMXNET3_RXQ_LOCK(rxq); 2614 rxq->vxrxq_defer.ev_count++; 2615 rxmore = vmxnet3_rxq_eof(rxq, rxlimit); 2616 if (rxmore) 2617 rxq->vxrxq_deferreq.ev_count++; 2618 VMXNET3_RXQ_UNLOCK(rxq); 2619 2620 if (txmore || rxmore) 2621 vmxnet3_sched_handle_queue(sc, vmxq); 2622 else 2623 vmxnet3_enable_intr(sc, vmxq->vxq_intr_idx); 2624 } 2625 2626 static void 2627 vmxnet3_handle_queue_work(struct work *wk, void *context) 2628 { 2629 struct vmxnet3_queue *vmxq; 2630 2631 vmxq = container_of(wk, struct vmxnet3_queue, vxq_wq_cookie); 2632 vmxq->vxq_wq_enqueued = false; 2633 vmxnet3_handle_queue(vmxq); 2634 } 2635 2636 static int 2637 vmxnet3_event_intr(void *xsc) 2638 { 2639 struct vmxnet3_softc *sc; 2640 2641 sc = xsc; 2642 2643 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2644 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 2645 2646 sc->vmx_event_intr.ev_count++; 2647 2648 if (sc->vmx_ds->event != 0) 2649 vmxnet3_evintr(sc); 2650 2651 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2652 2653 return (1); 2654 } 2655 2656 static void 2657 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2658 { 2659 struct vmxnet3_txring *txr; 2660 struct vmxnet3_txbuf *txb; 2661 u_int i; 2662 2663 txr = &txq->vxtxq_cmd_ring; 2664 2665 for (i = 0; i < txr->vxtxr_ndesc; i++) { 2666 txb = &txr->vxtxr_txbuf[i]; 2667 2668 if (txb->vtxb_m == NULL) 2669 continue; 2670 2671 bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap, 2672 0, txb->vtxb_dmamap->dm_mapsize, 2673 BUS_DMASYNC_POSTWRITE); 2674 bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap); 2675 m_freem(txb->vtxb_m); 2676 txb->vtxb_m = NULL; 2677 } 2678 } 2679 2680 static void 2681 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2682 { 2683 struct vmxnet3_rxring *rxr; 2684 struct vmxnet3_rxbuf *rxb; 2685 u_int i, j; 2686 2687 if (rxq->vxrxq_mhead != NULL) { 2688 m_freem(rxq->vxrxq_mhead); 2689 rxq->vxrxq_mhead = NULL; 2690 rxq->vxrxq_mtail = NULL; 2691 } 2692 2693 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2694 rxr = &rxq->vxrxq_cmd_ring[i]; 2695 2696 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2697 rxb = &rxr->vxrxr_rxbuf[j]; 2698 2699 if (rxb->vrxb_m == NULL) 2700 continue; 2701 2702 bus_dmamap_sync(sc->vmx_dmat, rxb->vrxb_dmamap, 2703 0, rxb->vrxb_dmamap->dm_mapsize, 2704 BUS_DMASYNC_POSTREAD); 2705 bus_dmamap_unload(sc->vmx_dmat, rxb->vrxb_dmamap); 2706 m_freem(rxb->vrxb_m); 2707 rxb->vrxb_m = NULL; 2708 } 2709 } 2710 } 2711 2712 static void 2713 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2714 { 2715 struct vmxnet3_rxqueue *rxq; 2716 struct vmxnet3_txqueue *txq; 2717 struct vmxnet3_queue *vmxq; 2718 int i; 2719 2720 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2721 rxq = &sc->vmx_queue[i].vxq_rxqueue; 2722 VMXNET3_RXQ_LOCK(rxq); 2723 VMXNET3_RXQ_UNLOCK(rxq); 2724 } 2725 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2726 txq = &sc->vmx_queue[i].vxq_txqueue; 2727 VMXNET3_TXQ_LOCK(txq); 2728 VMXNET3_TXQ_UNLOCK(txq); 2729 } 2730 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2731 vmxq = &sc->vmx_queue[i]; 2732 workqueue_wait(sc->vmx_queue_wq, &vmxq->vxq_wq_cookie); 2733 } 2734 } 2735 2736 static void 2737 vmxnet3_stop_locked(struct vmxnet3_softc *sc) 2738 { 2739 struct ifnet *ifp; 2740 int q; 2741 2742 ifp = &sc->vmx_ethercom.ec_if; 2743 VMXNET3_CORE_LOCK_ASSERT(sc); 2744 2745 ifp->if_flags &= ~IFF_RUNNING; 2746 sc->vmx_link_active = 0; 2747 callout_stop(&sc->vmx_tick); 2748 2749 /* Disable interrupts. */ 2750 vmxnet3_disable_all_intrs(sc); 2751 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2752 2753 vmxnet3_stop_rendezvous(sc); 2754 2755 for (q = 0; q < sc->vmx_ntxqueues; q++) 2756 vmxnet3_txstop(sc, &sc->vmx_queue[q].vxq_txqueue); 2757 for (q = 0; q < sc->vmx_nrxqueues; q++) 2758 vmxnet3_rxstop(sc, &sc->vmx_queue[q].vxq_rxqueue); 2759 2760 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2761 } 2762 2763 static void 2764 vmxnet3_stop(struct ifnet *ifp, int disable) 2765 { 2766 struct vmxnet3_softc *sc = ifp->if_softc; 2767 2768 VMXNET3_CORE_LOCK(sc); 2769 vmxnet3_stop_locked(sc); 2770 VMXNET3_CORE_UNLOCK(sc); 2771 } 2772 2773 static void 2774 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2775 { 2776 struct vmxnet3_txring *txr; 2777 struct vmxnet3_comp_ring *txc; 2778 2779 txr = &txq->vxtxq_cmd_ring; 2780 txr->vxtxr_head = 0; 2781 txr->vxtxr_next = 0; 2782 txr->vxtxr_gen = VMXNET3_INIT_GEN; 2783 memset(txr->vxtxr_txd, 0, 2784 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2785 2786 txc = &txq->vxtxq_comp_ring; 2787 txc->vxcr_next = 0; 2788 txc->vxcr_gen = VMXNET3_INIT_GEN; 2789 memset(txc->vxcr_u.txcd, 0, 2790 txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2791 } 2792 2793 static int 2794 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2795 { 2796 struct vmxnet3_rxring *rxr; 2797 struct vmxnet3_comp_ring *rxc; 2798 u_int i, populate, idx; 2799 int error; 2800 2801 /* LRO and jumbo frame is not supported yet */ 2802 populate = 1; 2803 2804 for (i = 0; i < populate; i++) { 2805 rxr = &rxq->vxrxq_cmd_ring[i]; 2806 rxr->vxrxr_fill = 0; 2807 rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2808 memset(rxr->vxrxr_rxd, 0, 2809 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2810 2811 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2812 error = vmxnet3_newbuf(sc, rxq, rxr); 2813 if (error) 2814 return (error); 2815 } 2816 } 2817 2818 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2819 rxr = &rxq->vxrxq_cmd_ring[i]; 2820 rxr->vxrxr_fill = 0; 2821 rxr->vxrxr_gen = 0; 2822 memset(rxr->vxrxr_rxd, 0, 2823 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2824 } 2825 2826 rxc = &rxq->vxrxq_comp_ring; 2827 rxc->vxcr_next = 0; 2828 rxc->vxcr_gen = VMXNET3_INIT_GEN; 2829 memset(rxc->vxcr_u.rxcd, 0, 2830 rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2831 2832 return (0); 2833 } 2834 2835 static int 2836 vmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2837 { 2838 device_t dev; 2839 int q, error; 2840 dev = sc->vmx_dev; 2841 2842 for (q = 0; q < sc->vmx_ntxqueues; q++) 2843 vmxnet3_txinit(sc, &sc->vmx_queue[q].vxq_txqueue); 2844 2845 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2846 error = vmxnet3_rxinit(sc, &sc->vmx_queue[q].vxq_rxqueue); 2847 if (error) { 2848 device_printf(dev, "cannot populate Rx queue %d\n", q); 2849 return (error); 2850 } 2851 } 2852 2853 return (0); 2854 } 2855 2856 static int 2857 vmxnet3_enable_device(struct vmxnet3_softc *sc) 2858 { 2859 int q; 2860 2861 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2862 device_printf(sc->vmx_dev, "device enable command failed!\n"); 2863 return (1); 2864 } 2865 2866 /* Reset the Rx queue heads. */ 2867 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2868 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2869 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2870 } 2871 2872 return (0); 2873 } 2874 2875 static void 2876 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2877 { 2878 2879 vmxnet3_set_rxfilter(sc); 2880 2881 memset(sc->vmx_ds->vlan_filter, 0, sizeof(sc->vmx_ds->vlan_filter)); 2882 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2883 } 2884 2885 static int 2886 vmxnet3_reinit(struct vmxnet3_softc *sc) 2887 { 2888 2889 vmxnet3_set_lladdr(sc); 2890 vmxnet3_reinit_shared_data(sc); 2891 2892 if (vmxnet3_reinit_queues(sc) != 0) 2893 return (ENXIO); 2894 2895 if (vmxnet3_enable_device(sc) != 0) 2896 return (ENXIO); 2897 2898 vmxnet3_reinit_rxfilters(sc); 2899 2900 return (0); 2901 } 2902 2903 static int 2904 vmxnet3_init_locked(struct vmxnet3_softc *sc) 2905 { 2906 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 2907 int error; 2908 2909 vmxnet3_stop_locked(sc); 2910 2911 error = vmxnet3_reinit(sc); 2912 if (error) { 2913 vmxnet3_stop_locked(sc); 2914 return (error); 2915 } 2916 2917 ifp->if_flags |= IFF_RUNNING; 2918 vmxnet3_if_link_status(sc); 2919 2920 vmxnet3_enable_all_intrs(sc); 2921 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2922 2923 return (0); 2924 } 2925 2926 static int 2927 vmxnet3_init(struct ifnet *ifp) 2928 { 2929 struct vmxnet3_softc *sc = ifp->if_softc; 2930 int error; 2931 2932 VMXNET3_CORE_LOCK(sc); 2933 error = vmxnet3_init_locked(sc); 2934 VMXNET3_CORE_UNLOCK(sc); 2935 2936 return (error); 2937 } 2938 2939 static int 2940 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, 2941 int *start, int *csum_start) 2942 { 2943 struct ether_header *eh; 2944 struct mbuf *mp; 2945 int offset, csum_off, iphl, offp; 2946 bool v4; 2947 2948 eh = mtod(m, struct ether_header *); 2949 switch (htons(eh->ether_type)) { 2950 case ETHERTYPE_IP: 2951 case ETHERTYPE_IPV6: 2952 offset = ETHER_HDR_LEN; 2953 break; 2954 case ETHERTYPE_VLAN: 2955 offset = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2956 break; 2957 default: 2958 m_freem(m); 2959 return (EINVAL); 2960 } 2961 2962 if ((m->m_pkthdr.csum_flags & 2963 (M_CSUM_TSOv4 | M_CSUM_UDPv4 | M_CSUM_TCPv4)) != 0) { 2964 iphl = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); 2965 v4 = true; 2966 } else { 2967 iphl = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); 2968 v4 = false; 2969 } 2970 *start = offset + iphl; 2971 2972 if (m->m_pkthdr.csum_flags & 2973 (M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_TSOv4 | M_CSUM_TSOv6)) { 2974 csum_off = offsetof(struct tcphdr, th_sum); 2975 } else { 2976 csum_off = offsetof(struct udphdr, uh_sum); 2977 } 2978 2979 *csum_start = *start + csum_off; 2980 mp = m_pulldown(m, 0, *csum_start + 2, &offp); 2981 if (!mp) { 2982 /* m is already freed */ 2983 return ENOBUFS; 2984 } 2985 2986 if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) { 2987 struct tcphdr *tcp; 2988 2989 txq->vxtxq_stats.vmtxs_tso++; 2990 tcp = (void *)(mtod(mp, char *) + offp + *start); 2991 2992 if (v4) { 2993 struct ip *ip; 2994 2995 ip = (void *)(mtod(mp, char *) + offp + offset); 2996 tcp->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 2997 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2998 } else { 2999 struct ip6_hdr *ip6; 3000 3001 ip6 = (void *)(mtod(mp, char *) + offp + offset); 3002 tcp->th_sum = in6_cksum_phdr(&ip6->ip6_src, 3003 &ip6->ip6_dst, 0, htonl(IPPROTO_TCP)); 3004 } 3005 3006 /* 3007 * For TSO, the size of the protocol header is also 3008 * included in the descriptor header size. 3009 */ 3010 *start += (tcp->th_off << 2); 3011 } else 3012 txq->vxtxq_stats.vmtxs_csum++; 3013 3014 return (0); 3015 } 3016 3017 static int 3018 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 3019 bus_dmamap_t dmap) 3020 { 3021 struct mbuf *m; 3022 bus_dma_tag_t tag; 3023 int error; 3024 3025 m = *m0; 3026 tag = txq->vxtxq_sc->vmx_dmat; 3027 3028 error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT); 3029 if (error == 0 || error != EFBIG) 3030 return (error); 3031 3032 m = m_defrag(m, M_NOWAIT); 3033 if (m != NULL) { 3034 *m0 = m; 3035 error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT); 3036 } else 3037 error = ENOBUFS; 3038 3039 if (error) { 3040 m_freem(*m0); 3041 *m0 = NULL; 3042 txq->vxtxq_defrag_failed.ev_count++; 3043 } else 3044 txq->vxtxq_defragged.ev_count++; 3045 3046 return (error); 3047 } 3048 3049 static void 3050 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 3051 { 3052 3053 bus_dmamap_unload(txq->vxtxq_sc->vmx_dmat, dmap); 3054 } 3055 3056 static int 3057 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 3058 { 3059 struct vmxnet3_softc *sc; 3060 struct vmxnet3_txring *txr; 3061 struct vmxnet3_txdesc *txd, *sop; 3062 struct mbuf *m; 3063 bus_dmamap_t dmap; 3064 bus_dma_segment_t *segs; 3065 int i, gen, start, csum_start, nsegs, error; 3066 3067 sc = txq->vxtxq_sc; 3068 start = 0; 3069 txd = NULL; 3070 txr = &txq->vxtxq_cmd_ring; 3071 dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 3072 csum_start = 0; /* GCC */ 3073 3074 error = vmxnet3_txq_load_mbuf(txq, m0, dmap); 3075 if (error) 3076 return (error); 3077 3078 nsegs = dmap->dm_nsegs; 3079 segs = dmap->dm_segs; 3080 3081 m = *m0; 3082 KASSERT(m->m_flags & M_PKTHDR); 3083 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS); 3084 3085 if (vmxnet3_txring_avail(txr) < nsegs) { 3086 txq->vxtxq_stats.vmtxs_full++; 3087 vmxnet3_txq_unload_mbuf(txq, dmap); 3088 return (ENOSPC); 3089 } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 3090 error = vmxnet3_txq_offload_ctx(txq, m, &start, &csum_start); 3091 if (error) { 3092 /* m is already freed */ 3093 txq->vxtxq_stats.vmtxs_offload_failed++; 3094 vmxnet3_txq_unload_mbuf(txq, dmap); 3095 *m0 = NULL; 3096 return (error); 3097 } 3098 } 3099 3100 txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m; 3101 sop = &txr->vxtxr_txd[txr->vxtxr_head]; 3102 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 3103 3104 for (i = 0; i < nsegs; i++) { 3105 txd = &txr->vxtxr_txd[txr->vxtxr_head]; 3106 3107 txd->addr = segs[i].ds_addr; 3108 txd->len = segs[i].ds_len; 3109 txd->gen = gen; 3110 txd->dtype = 0; 3111 txd->offload_mode = VMXNET3_OM_NONE; 3112 txd->offload_pos = 0; 3113 txd->hlen = 0; 3114 txd->eop = 0; 3115 txd->compreq = 0; 3116 txd->vtag_mode = 0; 3117 txd->vtag = 0; 3118 3119 if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 3120 txr->vxtxr_head = 0; 3121 txr->vxtxr_gen ^= 1; 3122 } 3123 gen = txr->vxtxr_gen; 3124 } 3125 txd->eop = 1; 3126 txd->compreq = 1; 3127 3128 if (vlan_has_tag(m)) { 3129 sop->vtag_mode = 1; 3130 sop->vtag = vlan_get_tag(m); 3131 } 3132 3133 if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) { 3134 sop->offload_mode = VMXNET3_OM_TSO; 3135 sop->hlen = start; 3136 sop->offload_pos = m->m_pkthdr.segsz; 3137 } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 3138 VMXNET3_CSUM_OFFLOAD_IPV6)) { 3139 sop->offload_mode = VMXNET3_OM_CSUM; 3140 sop->hlen = start; 3141 sop->offload_pos = csum_start; 3142 } 3143 3144 /* Finally, change the ownership. */ 3145 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 3146 sop->gen ^= 1; 3147 3148 txq->vxtxq_ts->npending += nsegs; 3149 if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 3150 struct vmxnet3_queue *vmxq; 3151 vmxq = container_of(txq, struct vmxnet3_queue, vxq_txqueue); 3152 txq->vxtxq_ts->npending = 0; 3153 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(vmxq->vxq_id), 3154 txr->vxtxr_head); 3155 } 3156 3157 return (0); 3158 } 3159 3160 #define VMXNET3_TX_START 1 3161 #define VMXNET3_TX_TRANSMIT 2 3162 static inline void 3163 vmxnet3_tx_common_locked(struct ifnet *ifp, struct vmxnet3_txqueue *txq, int txtype) 3164 { 3165 struct vmxnet3_softc *sc; 3166 struct vmxnet3_txring *txr; 3167 struct mbuf *m_head; 3168 int tx; 3169 3170 sc = ifp->if_softc; 3171 txr = &txq->vxtxq_cmd_ring; 3172 tx = 0; 3173 3174 VMXNET3_TXQ_LOCK_ASSERT(txq); 3175 3176 if ((ifp->if_flags & IFF_RUNNING) == 0 || 3177 sc->vmx_link_active == 0) 3178 return; 3179 3180 for (;;) { 3181 if (txtype == VMXNET3_TX_START) 3182 IFQ_POLL(&ifp->if_snd, m_head); 3183 else 3184 m_head = pcq_peek(txq->vxtxq_interq); 3185 if (m_head == NULL) 3186 break; 3187 3188 if (vmxnet3_txring_avail(txr) < VMXNET3_TX_MAXSEGS) 3189 break; 3190 3191 if (txtype == VMXNET3_TX_START) 3192 IFQ_DEQUEUE(&ifp->if_snd, m_head); 3193 else 3194 m_head = pcq_get(txq->vxtxq_interq); 3195 if (m_head == NULL) 3196 break; 3197 3198 if (vmxnet3_txq_encap(txq, &m_head) != 0) { 3199 if (m_head != NULL) 3200 m_freem(m_head); 3201 break; 3202 } 3203 3204 tx++; 3205 bpf_mtap(ifp, m_head, BPF_D_OUT); 3206 } 3207 3208 if (tx > 0) 3209 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 3210 } 3211 3212 static void 3213 vmxnet3_start_locked(struct ifnet *ifp) 3214 { 3215 struct vmxnet3_softc *sc; 3216 struct vmxnet3_txqueue *txq; 3217 3218 sc = ifp->if_softc; 3219 txq = &sc->vmx_queue[0].vxq_txqueue; 3220 3221 vmxnet3_tx_common_locked(ifp, txq, VMXNET3_TX_START); 3222 } 3223 3224 void 3225 vmxnet3_start(struct ifnet *ifp) 3226 { 3227 struct vmxnet3_softc *sc; 3228 struct vmxnet3_txqueue *txq; 3229 3230 sc = ifp->if_softc; 3231 txq = &sc->vmx_queue[0].vxq_txqueue; 3232 3233 VMXNET3_TXQ_LOCK(txq); 3234 vmxnet3_start_locked(ifp); 3235 VMXNET3_TXQ_UNLOCK(txq); 3236 } 3237 3238 static int 3239 vmxnet3_select_txqueue(struct ifnet *ifp, struct mbuf *m __unused) 3240 { 3241 struct vmxnet3_softc *sc; 3242 u_int cpuid; 3243 3244 sc = ifp->if_softc; 3245 cpuid = cpu_index(curcpu()); 3246 /* 3247 * Furure work 3248 * We should select txqueue to even up the load even if ncpu is 3249 * different from sc->vmx_ntxqueues. Currently, the load is not 3250 * even, that is, when ncpu is six and ntxqueues is four, the load 3251 * of vmx_queue[0] and vmx_queue[1] is higher than vmx_queue[2] and 3252 * vmx_queue[3] because CPU#4 always uses vmx_queue[0] and CPU#5 always 3253 * uses vmx_queue[1]. 3254 * Furthermore, we should not use random value to select txqueue to 3255 * avoid reordering. We should use flow information of mbuf. 3256 */ 3257 return cpuid % sc->vmx_ntxqueues; 3258 } 3259 3260 static void 3261 vmxnet3_transmit_locked(struct ifnet *ifp, struct vmxnet3_txqueue *txq) 3262 { 3263 3264 vmxnet3_tx_common_locked(ifp, txq, VMXNET3_TX_TRANSMIT); 3265 } 3266 3267 static int 3268 vmxnet3_transmit(struct ifnet *ifp, struct mbuf *m) 3269 { 3270 struct vmxnet3_softc *sc; 3271 struct vmxnet3_txqueue *txq; 3272 int qid; 3273 3274 qid = vmxnet3_select_txqueue(ifp, m); 3275 sc = ifp->if_softc; 3276 txq = &sc->vmx_queue[qid].vxq_txqueue; 3277 3278 if (__predict_false(!pcq_put(txq->vxtxq_interq, m))) { 3279 VMXNET3_TXQ_LOCK(txq); 3280 txq->vxtxq_pcqdrop.ev_count++; 3281 VMXNET3_TXQ_UNLOCK(txq); 3282 m_freem(m); 3283 return ENOBUFS; 3284 } 3285 3286 if (VMXNET3_TXQ_TRYLOCK(txq)) { 3287 vmxnet3_transmit_locked(ifp, txq); 3288 VMXNET3_TXQ_UNLOCK(txq); 3289 } else { 3290 kpreempt_disable(); 3291 softint_schedule(txq->vxtxq_si); 3292 kpreempt_enable(); 3293 } 3294 3295 return 0; 3296 } 3297 3298 static void 3299 vmxnet3_deferred_transmit(void *arg) 3300 { 3301 struct vmxnet3_txqueue *txq = arg; 3302 struct vmxnet3_softc *sc = txq->vxtxq_sc; 3303 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 3304 3305 VMXNET3_TXQ_LOCK(txq); 3306 txq->vxtxq_transmitdef.ev_count++; 3307 if (pcq_peek(txq->vxtxq_interq) != NULL) 3308 vmxnet3_transmit_locked(ifp, txq); 3309 VMXNET3_TXQ_UNLOCK(txq); 3310 } 3311 3312 static void 3313 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 3314 { 3315 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 3316 struct ethercom *ec = &sc->vmx_ethercom; 3317 struct vmxnet3_driver_shared *ds = sc->vmx_ds; 3318 struct ether_multi *enm; 3319 struct ether_multistep step; 3320 u_int mode; 3321 uint8_t *p; 3322 3323 ds->mcast_tablelen = 0; 3324 ETHER_LOCK(ec); 3325 CLR(ec->ec_flags, ETHER_F_ALLMULTI); 3326 ETHER_UNLOCK(ec); 3327 3328 /* 3329 * Always accept broadcast frames. 3330 * Always accept frames destined to our station address. 3331 */ 3332 mode = VMXNET3_RXMODE_BCAST | VMXNET3_RXMODE_UCAST; 3333 3334 ETHER_LOCK(ec); 3335 if (ISSET(ifp->if_flags, IFF_PROMISC) || 3336 ec->ec_multicnt > VMXNET3_MULTICAST_MAX) 3337 goto allmulti; 3338 3339 p = sc->vmx_mcast; 3340 ETHER_FIRST_MULTI(step, ec, enm); 3341 while (enm != NULL) { 3342 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) { 3343 /* 3344 * We must listen to a range of multicast addresses. 3345 * For now, just accept all multicasts, rather than 3346 * trying to set only those filter bits needed to match 3347 * the range. (At this time, the only use of address 3348 * ranges is for IP multicast routing, for which the 3349 * range is big enough to require all bits set.) 3350 */ 3351 goto allmulti; 3352 } 3353 memcpy(p, enm->enm_addrlo, ETHER_ADDR_LEN); 3354 3355 p += ETHER_ADDR_LEN; 3356 3357 ETHER_NEXT_MULTI(step, enm); 3358 } 3359 3360 if (ec->ec_multicnt > 0) { 3361 SET(mode, VMXNET3_RXMODE_MCAST); 3362 ds->mcast_tablelen = p - sc->vmx_mcast; 3363 } 3364 ETHER_UNLOCK(ec); 3365 3366 goto setit; 3367 3368 allmulti: 3369 SET(ec->ec_flags, ETHER_F_ALLMULTI); 3370 ETHER_UNLOCK(ec); 3371 SET(mode, (VMXNET3_RXMODE_ALLMULTI | VMXNET3_RXMODE_MCAST)); 3372 if (ifp->if_flags & IFF_PROMISC) 3373 SET(mode, VMXNET3_RXMODE_PROMISC); 3374 3375 setit: 3376 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 3377 ds->rxmode = mode; 3378 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 3379 } 3380 3381 static int 3382 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, void *data) 3383 { 3384 struct vmxnet3_softc *sc = ifp->if_softc; 3385 struct ifreq *ifr = (struct ifreq *)data; 3386 int s, error = 0; 3387 3388 switch (cmd) { 3389 case SIOCSIFMTU: { 3390 int nmtu = ifr->ifr_mtu; 3391 3392 if (nmtu < VMXNET3_MIN_MTU || nmtu > VMXNET3_MAX_MTU) { 3393 error = EINVAL; 3394 break; 3395 } 3396 if (ifp->if_mtu != (uint64_t)nmtu) { 3397 s = splnet(); 3398 error = ether_ioctl(ifp, cmd, data); 3399 splx(s); 3400 if (error == ENETRESET) 3401 error = vmxnet3_init(ifp); 3402 } 3403 break; 3404 } 3405 3406 default: 3407 s = splnet(); 3408 error = ether_ioctl(ifp, cmd, data); 3409 splx(s); 3410 } 3411 3412 if (error == ENETRESET) { 3413 VMXNET3_CORE_LOCK(sc); 3414 if (ifp->if_flags & IFF_RUNNING) 3415 vmxnet3_set_rxfilter(sc); 3416 VMXNET3_CORE_UNLOCK(sc); 3417 error = 0; 3418 } 3419 3420 return error; 3421 } 3422 3423 static int 3424 vmxnet3_ifflags_cb(struct ethercom *ec) 3425 { 3426 struct vmxnet3_softc *sc; 3427 3428 sc = ec->ec_if.if_softc; 3429 3430 VMXNET3_CORE_LOCK(sc); 3431 vmxnet3_set_rxfilter(sc); 3432 VMXNET3_CORE_UNLOCK(sc); 3433 3434 vmxnet3_if_link_status(sc); 3435 3436 return 0; 3437 } 3438 3439 static int 3440 vmxnet3_watchdog(struct vmxnet3_txqueue *txq) 3441 { 3442 struct vmxnet3_softc *sc; 3443 struct vmxnet3_queue *vmxq; 3444 3445 sc = txq->vxtxq_sc; 3446 vmxq = container_of(txq, struct vmxnet3_queue, vxq_txqueue); 3447 3448 VMXNET3_TXQ_LOCK(txq); 3449 if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 3450 VMXNET3_TXQ_UNLOCK(txq); 3451 return (0); 3452 } 3453 txq->vxtxq_watchdogto.ev_count++; 3454 VMXNET3_TXQ_UNLOCK(txq); 3455 3456 device_printf(sc->vmx_dev, "watchdog timeout on queue %d\n", 3457 vmxq->vxq_id); 3458 return (1); 3459 } 3460 3461 static void 3462 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) 3463 { 3464 3465 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 3466 } 3467 3468 static void 3469 vmxnet3_tick(void *xsc) 3470 { 3471 struct vmxnet3_softc *sc; 3472 int i, timedout; 3473 3474 sc = xsc; 3475 timedout = 0; 3476 3477 VMXNET3_CORE_LOCK(sc); 3478 3479 vmxnet3_refresh_host_stats(sc); 3480 3481 for (i = 0; i < sc->vmx_ntxqueues; i++) 3482 timedout |= vmxnet3_watchdog(&sc->vmx_queue[i].vxq_txqueue); 3483 3484 if (timedout != 0) 3485 vmxnet3_init_locked(sc); 3486 else 3487 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 3488 3489 VMXNET3_CORE_UNLOCK(sc); 3490 } 3491 3492 /* 3493 * update link state of ifnet and softc 3494 */ 3495 static void 3496 vmxnet3_if_link_status(struct vmxnet3_softc *sc) 3497 { 3498 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 3499 u_int link; 3500 bool up; 3501 3502 up = vmxnet3_cmd_link_status(ifp); 3503 if (up) { 3504 sc->vmx_link_active = 1; 3505 link = LINK_STATE_UP; 3506 } else { 3507 sc->vmx_link_active = 0; 3508 link = LINK_STATE_DOWN; 3509 } 3510 3511 if_link_state_change(ifp, link); 3512 } 3513 3514 /* 3515 * check vmx(4) state by VMXNET3_CMD and update ifp->if_baudrate 3516 * returns 3517 * - true: link up 3518 * - flase: link down 3519 */ 3520 static bool 3521 vmxnet3_cmd_link_status(struct ifnet *ifp) 3522 { 3523 struct vmxnet3_softc *sc = ifp->if_softc; 3524 u_int x, speed; 3525 3526 x = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 3527 if ((x & 1) == 0) 3528 return false; 3529 3530 speed = x >> 16; 3531 ifp->if_baudrate = IF_Mbps(speed); 3532 return true; 3533 } 3534 3535 static void 3536 vmxnet3_ifmedia_status(struct ifnet *ifp, struct ifmediareq *ifmr) 3537 { 3538 bool up; 3539 3540 ifmr->ifm_status = IFM_AVALID; 3541 ifmr->ifm_active = IFM_ETHER; 3542 3543 up = vmxnet3_cmd_link_status(ifp); 3544 if (!up) 3545 return; 3546 3547 ifmr->ifm_status |= IFM_ACTIVE; 3548 3549 if (ifp->if_baudrate >= IF_Gbps(10ULL)) 3550 ifmr->ifm_active |= IFM_10G_T; 3551 } 3552 3553 static int 3554 vmxnet3_ifmedia_change(struct ifnet *ifp) 3555 { 3556 return 0; 3557 } 3558 3559 static void 3560 vmxnet3_set_lladdr(struct vmxnet3_softc *sc) 3561 { 3562 uint32_t ml, mh; 3563 3564 ml = sc->vmx_lladdr[0]; 3565 ml |= sc->vmx_lladdr[1] << 8; 3566 ml |= sc->vmx_lladdr[2] << 16; 3567 ml |= sc->vmx_lladdr[3] << 24; 3568 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 3569 3570 mh = sc->vmx_lladdr[4]; 3571 mh |= sc->vmx_lladdr[5] << 8; 3572 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 3573 } 3574 3575 static void 3576 vmxnet3_get_lladdr(struct vmxnet3_softc *sc) 3577 { 3578 uint32_t ml, mh; 3579 3580 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 3581 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 3582 3583 sc->vmx_lladdr[0] = ml; 3584 sc->vmx_lladdr[1] = ml >> 8; 3585 sc->vmx_lladdr[2] = ml >> 16; 3586 sc->vmx_lladdr[3] = ml >> 24; 3587 sc->vmx_lladdr[4] = mh; 3588 sc->vmx_lladdr[5] = mh >> 8; 3589 } 3590 3591 static void 3592 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3593 { 3594 int i; 3595 3596 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3597 for (i = 0; i < sc->vmx_nintrs; i++) 3598 vmxnet3_enable_intr(sc, i); 3599 } 3600 3601 static void 3602 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3603 { 3604 int i; 3605 3606 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3607 for (i = 0; i < sc->vmx_nintrs; i++) 3608 vmxnet3_disable_intr(sc, i); 3609 } 3610 3611 static int 3612 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3613 struct vmxnet3_dma_alloc *dma) 3614 { 3615 bus_dma_tag_t t = sc->vmx_dmat; 3616 bus_dma_segment_t *segs = dma->dma_segs; 3617 int n, error; 3618 3619 memset(dma, 0, sizeof(*dma)); 3620 3621 error = bus_dmamem_alloc(t, size, align, 0, segs, 1, &n, BUS_DMA_NOWAIT); 3622 if (error) { 3623 aprint_error_dev(sc->vmx_dev, "bus_dmamem_alloc failed: %d\n", error); 3624 goto fail1; 3625 } 3626 KASSERT(n == 1); 3627 3628 error = bus_dmamem_map(t, segs, 1, size, &dma->dma_vaddr, BUS_DMA_NOWAIT); 3629 if (error) { 3630 aprint_error_dev(sc->vmx_dev, "bus_dmamem_map failed: %d\n", error); 3631 goto fail2; 3632 } 3633 3634 error = bus_dmamap_create(t, size, 1, size, 0, BUS_DMA_NOWAIT, &dma->dma_map); 3635 if (error) { 3636 aprint_error_dev(sc->vmx_dev, "bus_dmamap_create failed: %d\n", error); 3637 goto fail3; 3638 } 3639 3640 error = bus_dmamap_load(t, dma->dma_map, dma->dma_vaddr, size, NULL, 3641 BUS_DMA_NOWAIT); 3642 if (error) { 3643 aprint_error_dev(sc->vmx_dev, "bus_dmamap_load failed: %d\n", error); 3644 goto fail4; 3645 } 3646 3647 memset(dma->dma_vaddr, 0, size); 3648 dma->dma_paddr = DMAADDR(dma->dma_map); 3649 dma->dma_size = size; 3650 3651 return (0); 3652 fail4: 3653 bus_dmamap_destroy(t, dma->dma_map); 3654 fail3: 3655 bus_dmamem_unmap(t, dma->dma_vaddr, size); 3656 fail2: 3657 bus_dmamem_free(t, segs, 1); 3658 fail1: 3659 return (error); 3660 } 3661 3662 static void 3663 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 3664 { 3665 bus_dma_tag_t t = sc->vmx_dmat; 3666 3667 bus_dmamap_unload(t, dma->dma_map); 3668 bus_dmamap_destroy(t, dma->dma_map); 3669 bus_dmamem_unmap(t, dma->dma_vaddr, dma->dma_size); 3670 bus_dmamem_free(t, dma->dma_segs, 1); 3671 3672 memset(dma, 0, sizeof(*dma)); 3673 } 3674 3675 MODULE(MODULE_CLASS_DRIVER, if_vmx, "pci"); 3676 3677 #ifdef _MODULE 3678 #include "ioconf.c" 3679 #endif 3680 3681 static int 3682 if_vmx_modcmd(modcmd_t cmd, void *opaque) 3683 { 3684 int error = 0; 3685 3686 switch (cmd) { 3687 case MODULE_CMD_INIT: 3688 #ifdef _MODULE 3689 error = config_init_component(cfdriver_ioconf_if_vmx, 3690 cfattach_ioconf_if_vmx, cfdata_ioconf_if_vmx); 3691 #endif 3692 return error; 3693 case MODULE_CMD_FINI: 3694 #ifdef _MODULE 3695 error = config_fini_component(cfdriver_ioconf_if_vmx, 3696 cfattach_ioconf_if_vmx, cfdata_ioconf_if_vmx); 3697 #endif 3698 return error; 3699 default: 3700 return ENOTTY; 3701 } 3702 } 3703 3704
Indexes created Wed Sep 24 05:09:52 GMT 2025