if_vmx.c revision 1.10
1 /* $NetBSD: if_vmx.c,v 1.10 2022/09/16 03:10:12 knakahara Exp $ */ 2 /* $OpenBSD: if_vmx.c,v 1.16 2014/01/22 06:04:17 brad Exp $ */ 3 4 /* 5 * Copyright (c) 2013 Tsubai Masanari 6 * Copyright (c) 2013 Bryan Venteicher <bryanv (at) FreeBSD.org> 7 * 8 * Permission to use, copy, modify, and distribute this software for any 9 * purpose with or without fee is hereby granted, provided that the above 10 * copyright notice and this permission notice appear in all copies. 11 * 12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 19 */ 20 21 #include <sys/cdefs.h> 22 __KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1.10 2022/09/16 03:10:12 knakahara Exp $"); 23 24 #ifdef _KERNEL_OPT 25 #include "opt_if_vmx.h" 26 #endif 27 28 #include <sys/param.h> 29 #include <sys/cpu.h> 30 #include <sys/kernel.h> 31 #include <sys/kmem.h> 32 #include <sys/bitops.h> 33 #include <sys/bus.h> 34 #include <sys/device.h> 35 #include <sys/mbuf.h> 36 #include <sys/module.h> 37 #include <sys/sockio.h> 38 #include <sys/pcq.h> 39 #include <sys/workqueue.h> 40 #include <sys/interrupt.h> 41 42 #include <net/bpf.h> 43 #include <net/if.h> 44 #include <net/if_ether.h> 45 #include <net/if_media.h> 46 47 #include <netinet/if_inarp.h> 48 #include <netinet/in_systm.h> /* for <netinet/ip.h> */ 49 #include <netinet/in.h> /* for <netinet/ip.h> */ 50 #include <netinet/ip.h> /* for struct ip */ 51 #include <netinet/ip6.h> /* for struct ip6_hdr */ 52 #include <netinet/tcp.h> /* for struct tcphdr */ 53 #include <netinet/udp.h> /* for struct udphdr */ 54 55 #include <dev/pci/pcivar.h> 56 #include <dev/pci/pcireg.h> 57 #include <dev/pci/pcidevs.h> 58 59 #include <dev/pci/if_vmxreg.h> 60 61 #define VMXNET3_DRIVER_VERSION 0x00010000 62 63 /* 64 * Max descriptors per Tx packet. We must limit the size of the 65 * any TSO packets based on the number of segments. 66 */ 67 #define VMXNET3_TX_MAXSEGS 32 68 #define VMXNET3_TX_MAXSIZE (VMXNET3_TX_MAXSEGS * MCLBYTES) 69 70 /* 71 * Maximum support Tx segments size. The length field in the 72 * Tx descriptor is 14 bits. 73 */ 74 #define VMXNET3_TX_MAXSEGSIZE (1 << 14) 75 76 /* 77 * The maximum number of Rx segments we accept. 78 */ 79 #define VMXNET3_MAX_RX_SEGS 0 /* no segments */ 80 81 /* 82 * Predetermined size of the multicast MACs filter table. If the 83 * number of multicast addresses exceeds this size, then the 84 * ALL_MULTI mode is use instead. 85 */ 86 #define VMXNET3_MULTICAST_MAX 32 87 88 /* 89 * Our Tx watchdog timeout. 90 */ 91 #define VMXNET3_WATCHDOG_TIMEOUT 5 92 93 /* 94 * Default value for vmx_intr_{rx,tx}_process_limit which is used for 95 * max number of packets to process for interrupt handler 96 */ 97 #define VMXNET3_RX_INTR_PROCESS_LIMIT 0U 98 #define VMXNET3_TX_INTR_PROCESS_LIMIT 256 99 100 /* 101 * Default value for vmx_{rx,tx}_process_limit which is used for 102 * max number of packets to process for deferred processing 103 */ 104 #define VMXNET3_RX_PROCESS_LIMIT 256 105 #define VMXNET3_TX_PROCESS_LIMIT 256 106 107 #define VMXNET3_WORKQUEUE_PRI PRI_SOFTNET 108 109 /* 110 * IP protocols that we can perform Tx checksum offloading of. 111 */ 112 #define VMXNET3_CSUM_OFFLOAD \ 113 (M_CSUM_TCPv4 | M_CSUM_UDPv4) 114 #define VMXNET3_CSUM_OFFLOAD_IPV6 \ 115 (M_CSUM_TCPv6 | M_CSUM_UDPv6) 116 117 #define VMXNET3_CSUM_ALL_OFFLOAD \ 118 (VMXNET3_CSUM_OFFLOAD | VMXNET3_CSUM_OFFLOAD_IPV6 | M_CSUM_TSOv4 | M_CSUM_TSOv6) 119 120 #define VMXNET3_RXRINGS_PERQ 2 121 122 #define VMXNET3_CORE_LOCK(_sc) mutex_enter((_sc)->vmx_mtx) 123 #define VMXNET3_CORE_UNLOCK(_sc) mutex_exit((_sc)->vmx_mtx) 124 #define VMXNET3_CORE_LOCK_ASSERT(_sc) mutex_owned((_sc)->vmx_mtx) 125 126 #define VMXNET3_RXQ_LOCK(_rxq) mutex_enter((_rxq)->vxrxq_mtx) 127 #define VMXNET3_RXQ_UNLOCK(_rxq) mutex_exit((_rxq)->vxrxq_mtx) 128 #define VMXNET3_RXQ_LOCK_ASSERT(_rxq) \ 129 mutex_owned((_rxq)->vxrxq_mtx) 130 131 #define VMXNET3_TXQ_LOCK(_txq) mutex_enter((_txq)->vxtxq_mtx) 132 #define VMXNET3_TXQ_TRYLOCK(_txq) mutex_tryenter((_txq)->vxtxq_mtx) 133 #define VMXNET3_TXQ_UNLOCK(_txq) mutex_exit((_txq)->vxtxq_mtx) 134 #define VMXNET3_TXQ_LOCK_ASSERT(_txq) \ 135 mutex_owned((_txq)->vxtxq_mtx) 136 137 struct vmxnet3_dma_alloc { 138 bus_addr_t dma_paddr; 139 void *dma_vaddr; 140 bus_dmamap_t dma_map; 141 bus_size_t dma_size; 142 bus_dma_segment_t dma_segs[1]; 143 }; 144 145 struct vmxnet3_txbuf { 146 bus_dmamap_t vtxb_dmamap; 147 struct mbuf *vtxb_m; 148 }; 149 150 struct vmxnet3_txring { 151 struct vmxnet3_txbuf *vxtxr_txbuf; 152 struct vmxnet3_txdesc *vxtxr_txd; 153 u_int vxtxr_head; 154 u_int vxtxr_next; 155 u_int vxtxr_ndesc; 156 int vxtxr_gen; 157 struct vmxnet3_dma_alloc vxtxr_dma; 158 }; 159 160 struct vmxnet3_rxbuf { 161 bus_dmamap_t vrxb_dmamap; 162 struct mbuf *vrxb_m; 163 }; 164 165 struct vmxnet3_rxring { 166 struct vmxnet3_rxbuf *vxrxr_rxbuf; 167 struct vmxnet3_rxdesc *vxrxr_rxd; 168 u_int vxrxr_fill; 169 u_int vxrxr_ndesc; 170 int vxrxr_gen; 171 int vxrxr_rid; 172 struct vmxnet3_dma_alloc vxrxr_dma; 173 bus_dmamap_t vxrxr_spare_dmap; 174 }; 175 176 struct vmxnet3_comp_ring { 177 union { 178 struct vmxnet3_txcompdesc *txcd; 179 struct vmxnet3_rxcompdesc *rxcd; 180 } vxcr_u; 181 u_int vxcr_next; 182 u_int vxcr_ndesc; 183 int vxcr_gen; 184 struct vmxnet3_dma_alloc vxcr_dma; 185 }; 186 187 struct vmxnet3_txq_stats { 188 #if 0 189 uint64_t vmtxs_opackets; /* if_opackets */ 190 uint64_t vmtxs_obytes; /* if_obytes */ 191 uint64_t vmtxs_omcasts; /* if_omcasts */ 192 #endif 193 uint64_t vmtxs_csum; 194 uint64_t vmtxs_tso; 195 uint64_t vmtxs_full; 196 uint64_t vmtxs_offload_failed; 197 }; 198 199 struct vmxnet3_txqueue { 200 kmutex_t *vxtxq_mtx; 201 struct vmxnet3_softc *vxtxq_sc; 202 int vxtxq_watchdog; 203 pcq_t *vxtxq_interq; 204 struct vmxnet3_txring vxtxq_cmd_ring; 205 struct vmxnet3_comp_ring vxtxq_comp_ring; 206 struct vmxnet3_txq_stats vxtxq_stats; 207 struct vmxnet3_txq_shared *vxtxq_ts; 208 char vxtxq_name[16]; 209 210 void *vxtxq_si; 211 212 struct evcnt vxtxq_intr; 213 struct evcnt vxtxq_defer; 214 struct evcnt vxtxq_deferreq; 215 struct evcnt vxtxq_pcqdrop; 216 struct evcnt vxtxq_transmitdef; 217 struct evcnt vxtxq_watchdogto; 218 struct evcnt vxtxq_defragged; 219 struct evcnt vxtxq_defrag_failed; 220 }; 221 222 #if 0 223 struct vmxnet3_rxq_stats { 224 uint64_t vmrxs_ipackets; /* if_ipackets */ 225 uint64_t vmrxs_ibytes; /* if_ibytes */ 226 uint64_t vmrxs_iqdrops; /* if_iqdrops */ 227 uint64_t vmrxs_ierrors; /* if_ierrors */ 228 }; 229 #endif 230 231 struct vmxnet3_rxqueue { 232 kmutex_t *vxrxq_mtx; 233 struct vmxnet3_softc *vxrxq_sc; 234 struct mbuf *vxrxq_mhead; 235 struct mbuf *vxrxq_mtail; 236 struct vmxnet3_rxring vxrxq_cmd_ring[VMXNET3_RXRINGS_PERQ]; 237 struct vmxnet3_comp_ring vxrxq_comp_ring; 238 #if 0 239 struct vmxnet3_rxq_stats vxrxq_stats; 240 #endif 241 struct vmxnet3_rxq_shared *vxrxq_rs; 242 char vxrxq_name[16]; 243 244 struct evcnt vxrxq_intr; 245 struct evcnt vxrxq_defer; 246 struct evcnt vxrxq_deferreq; 247 struct evcnt vxrxq_mgetcl_failed; 248 struct evcnt vxrxq_mbuf_load_failed; 249 }; 250 251 struct vmxnet3_queue { 252 int vxq_id; 253 int vxq_intr_idx; 254 255 struct vmxnet3_txqueue vxq_txqueue; 256 struct vmxnet3_rxqueue vxq_rxqueue; 257 258 void *vxq_si; 259 bool vxq_workqueue; 260 bool vxq_wq_enqueued; 261 struct work vxq_wq_cookie; 262 }; 263 264 struct vmxnet3_softc { 265 device_t vmx_dev; 266 struct ethercom vmx_ethercom; 267 struct ifmedia vmx_media; 268 struct vmxnet3_driver_shared *vmx_ds; 269 int vmx_flags; 270 #define VMXNET3_FLAG_NO_MSIX (1 << 0) 271 #define VMXNET3_FLAG_RSS (1 << 1) 272 #define VMXNET3_FLAG_ATTACHED (1 << 2) 273 274 struct vmxnet3_queue *vmx_queue; 275 276 struct pci_attach_args *vmx_pa; 277 pci_chipset_tag_t vmx_pc; 278 279 bus_space_tag_t vmx_iot0; 280 bus_space_tag_t vmx_iot1; 281 bus_space_handle_t vmx_ioh0; 282 bus_space_handle_t vmx_ioh1; 283 bus_size_t vmx_ios0; 284 bus_size_t vmx_ios1; 285 bus_dma_tag_t vmx_dmat; 286 287 int vmx_link_active; 288 int vmx_ntxqueues; 289 int vmx_nrxqueues; 290 int vmx_ntxdescs; 291 int vmx_nrxdescs; 292 int vmx_max_rxsegs; 293 294 struct evcnt vmx_event_intr; 295 struct evcnt vmx_event_link; 296 struct evcnt vmx_event_txqerror; 297 struct evcnt vmx_event_rxqerror; 298 struct evcnt vmx_event_dic; 299 struct evcnt vmx_event_debug; 300 301 int vmx_intr_type; 302 int vmx_intr_mask_mode; 303 int vmx_event_intr_idx; 304 int vmx_nintrs; 305 pci_intr_handle_t *vmx_intrs; /* legacy use vmx_intrs[0] */ 306 void *vmx_ihs[VMXNET3_MAX_INTRS]; 307 308 kmutex_t *vmx_mtx; 309 310 uint8_t *vmx_mcast; 311 void *vmx_qs; 312 struct vmxnet3_rss_shared *vmx_rss; 313 callout_t vmx_tick; 314 struct vmxnet3_dma_alloc vmx_ds_dma; 315 struct vmxnet3_dma_alloc vmx_qs_dma; 316 struct vmxnet3_dma_alloc vmx_mcast_dma; 317 struct vmxnet3_dma_alloc vmx_rss_dma; 318 int vmx_max_ntxqueues; 319 int vmx_max_nrxqueues; 320 uint8_t vmx_lladdr[ETHER_ADDR_LEN]; 321 322 u_int vmx_rx_intr_process_limit; 323 u_int vmx_tx_intr_process_limit; 324 u_int vmx_rx_process_limit; 325 u_int vmx_tx_process_limit; 326 struct sysctllog *vmx_sysctllog; 327 328 bool vmx_txrx_workqueue; 329 struct workqueue *vmx_queue_wq; 330 }; 331 332 #define VMXNET3_STAT 333 334 #ifdef VMXNET3_STAT 335 struct { 336 u_int txhead; 337 u_int txdone; 338 u_int maxtxlen; 339 u_int rxdone; 340 u_int rxfill; 341 u_int intr; 342 } vmxstat; 343 #endif 344 345 typedef enum { 346 VMXNET3_BARRIER_RD, 347 VMXNET3_BARRIER_WR, 348 } vmxnet3_barrier_t; 349 350 #define JUMBO_LEN (MCLBYTES - ETHER_ALIGN) /* XXX */ 351 #define DMAADDR(map) ((map)->dm_segs[0].ds_addr) 352 353 #define vtophys(va) 0 /* XXX ok? */ 354 355 static int vmxnet3_match(device_t, cfdata_t, void *); 356 static void vmxnet3_attach(device_t, device_t, void *); 357 static int vmxnet3_detach(device_t, int); 358 359 static int vmxnet3_alloc_pci_resources(struct vmxnet3_softc *); 360 static void vmxnet3_free_pci_resources(struct vmxnet3_softc *); 361 static int vmxnet3_check_version(struct vmxnet3_softc *); 362 static void vmxnet3_check_multiqueue(struct vmxnet3_softc *); 363 364 static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 365 static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 366 static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 367 static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 368 static void vmxnet3_free_interrupts(struct vmxnet3_softc *); 369 370 static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 371 static int vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *); 372 static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 373 static void vmxnet3_set_interrupt_idx(struct vmxnet3_softc *); 374 static int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 375 static int vmxnet3_setup_sysctl(struct vmxnet3_softc *); 376 377 static int vmxnet3_setup_stats(struct vmxnet3_softc *); 378 static void vmxnet3_teardown_stats(struct vmxnet3_softc *); 379 380 static int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 381 static int vmxnet3_init_txq(struct vmxnet3_softc *, int); 382 static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 383 static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 384 static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 385 static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 386 387 static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 388 static void vmxnet3_free_shared_data(struct vmxnet3_softc *); 389 static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 390 static void vmxnet3_free_txq_data(struct vmxnet3_softc *); 391 static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 392 static void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 393 static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 394 static void vmxnet3_free_queue_data(struct vmxnet3_softc *); 395 static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 396 static void vmxnet3_free_mcast_table(struct vmxnet3_softc *); 397 static void vmxnet3_init_shared_data(struct vmxnet3_softc *); 398 static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); 399 static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 400 static int vmxnet3_alloc_data(struct vmxnet3_softc *); 401 static void vmxnet3_free_data(struct vmxnet3_softc *); 402 static int vmxnet3_setup_interface(struct vmxnet3_softc *); 403 404 static void vmxnet3_evintr(struct vmxnet3_softc *); 405 static bool vmxnet3_txq_eof(struct vmxnet3_txqueue *, u_int); 406 static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxqueue *, 407 struct vmxnet3_rxring *); 408 static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 409 struct vmxnet3_rxring *, int); 410 static void vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *); 411 static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 412 static void vmxnet3_rxq_input(struct vmxnet3_rxqueue *, 413 struct vmxnet3_rxcompdesc *, struct mbuf *); 414 static bool vmxnet3_rxq_eof(struct vmxnet3_rxqueue *, u_int); 415 static int vmxnet3_legacy_intr(void *); 416 static int vmxnet3_txrxq_intr(void *); 417 static void vmxnet3_handle_queue(void *); 418 static void vmxnet3_handle_queue_work(struct work *, void *); 419 static int vmxnet3_event_intr(void *); 420 421 static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 422 static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 423 static void vmxnet3_stop_locked(struct vmxnet3_softc *); 424 static void vmxnet3_stop_rendezvous(struct vmxnet3_softc *); 425 static void vmxnet3_stop(struct ifnet *, int); 426 427 static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 428 static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 429 static int vmxnet3_reinit_queues(struct vmxnet3_softc *); 430 static int vmxnet3_enable_device(struct vmxnet3_softc *); 431 static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 432 static int vmxnet3_reinit(struct vmxnet3_softc *); 433 434 static int vmxnet3_init_locked(struct vmxnet3_softc *); 435 static int vmxnet3_init(struct ifnet *); 436 437 static int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *, struct mbuf *, int *, int *); 438 static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, bus_dmamap_t); 439 static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 440 static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 441 static void vmxnet3_start_locked(struct ifnet *); 442 static void vmxnet3_start(struct ifnet *); 443 static void vmxnet3_transmit_locked(struct ifnet *, struct vmxnet3_txqueue *); 444 static int vmxnet3_transmit(struct ifnet *, struct mbuf *); 445 static void vmxnet3_deferred_transmit(void *); 446 447 static void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 448 static int vmxnet3_ioctl(struct ifnet *, u_long, void *); 449 static int vmxnet3_ifflags_cb(struct ethercom *); 450 451 static int vmxnet3_watchdog(struct vmxnet3_txqueue *); 452 static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); 453 static void vmxnet3_tick(void *); 454 static void vmxnet3_if_link_status(struct vmxnet3_softc *); 455 static bool vmxnet3_cmd_link_status(struct ifnet *); 456 static void vmxnet3_ifmedia_status(struct ifnet *, struct ifmediareq *); 457 static int vmxnet3_ifmedia_change(struct ifnet *); 458 static void vmxnet3_set_lladdr(struct vmxnet3_softc *); 459 static void vmxnet3_get_lladdr(struct vmxnet3_softc *); 460 461 static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 462 static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 463 464 static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, bus_size_t, 465 struct vmxnet3_dma_alloc *); 466 static void vmxnet3_dma_free(struct vmxnet3_softc *, struct vmxnet3_dma_alloc *); 467 468 CFATTACH_DECL3_NEW(vmx, sizeof(struct vmxnet3_softc), 469 vmxnet3_match, vmxnet3_attach, vmxnet3_detach, NULL, NULL, NULL, 0); 470 471 /* round down to the nearest power of 2 */ 472 static int 473 vmxnet3_calc_queue_size(int n) 474 { 475 476 if (__predict_false(n <= 0)) 477 return 1; 478 479 return (1U << (fls32(n) - 1)); 480 } 481 482 static inline void 483 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 484 { 485 486 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 487 } 488 489 static inline uint32_t 490 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 491 { 492 493 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 494 } 495 496 static inline void 497 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 498 { 499 500 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 501 } 502 503 static inline void 504 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 505 { 506 507 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 508 } 509 510 static inline uint32_t 511 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 512 { 513 514 vmxnet3_write_cmd(sc, cmd); 515 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 516 } 517 518 static inline void 519 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 520 { 521 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 522 } 523 524 static inline void 525 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 526 { 527 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 528 } 529 530 static inline void 531 vmxnet3_rxr_increment_fill(struct vmxnet3_rxring *rxr) 532 { 533 534 if (++rxr->vxrxr_fill == rxr->vxrxr_ndesc) { 535 rxr->vxrxr_fill = 0; 536 rxr->vxrxr_gen ^= 1; 537 } 538 } 539 540 static inline int 541 vmxnet3_txring_avail(struct vmxnet3_txring *txr) 542 { 543 int avail = txr->vxtxr_next - txr->vxtxr_head - 1; 544 return (avail < 0 ? (int)txr->vxtxr_ndesc + avail : avail); 545 } 546 547 /* 548 * Since this is a purely paravirtualized device, we do not have 549 * to worry about DMA coherency. But at times, we must make sure 550 * both the compiler and CPU do not reorder memory operations. 551 */ 552 static inline void 553 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 554 { 555 556 switch (type) { 557 case VMXNET3_BARRIER_RD: 558 membar_consumer(); 559 break; 560 case VMXNET3_BARRIER_WR: 561 membar_producer(); 562 break; 563 default: 564 panic("%s: bad barrier type %d", __func__, type); 565 } 566 } 567 568 static int 569 vmxnet3_match(device_t parent, cfdata_t match, void *aux) 570 { 571 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 572 573 if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_VMWARE && 574 PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_VMWARE_VMXNET3) 575 return 1; 576 577 return 0; 578 } 579 580 static void 581 vmxnet3_attach(device_t parent, device_t self, void *aux) 582 { 583 struct vmxnet3_softc *sc = device_private(self); 584 struct pci_attach_args *pa = aux; 585 pcireg_t preg; 586 int error; 587 int candidate; 588 589 sc->vmx_dev = self; 590 sc->vmx_pa = pa; 591 sc->vmx_pc = pa->pa_pc; 592 if (pci_dma64_available(pa)) 593 sc->vmx_dmat = pa->pa_dmat64; 594 else 595 sc->vmx_dmat = pa->pa_dmat; 596 597 pci_aprint_devinfo_fancy(pa, "Ethernet controller", "vmxnet3", 1); 598 599 preg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 600 preg |= PCI_COMMAND_MASTER_ENABLE; 601 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, preg); 602 603 sc->vmx_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 604 callout_init(&sc->vmx_tick, CALLOUT_MPSAFE); 605 606 candidate = MIN(MIN(VMXNET3_MAX_TX_QUEUES, VMXNET3_MAX_RX_QUEUES), 607 ncpu); 608 sc->vmx_max_ntxqueues = sc->vmx_max_nrxqueues = 609 vmxnet3_calc_queue_size(candidate); 610 sc->vmx_ntxdescs = 512; 611 sc->vmx_nrxdescs = 256; 612 sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 613 614 error = vmxnet3_alloc_pci_resources(sc); 615 if (error) 616 return; 617 618 error = vmxnet3_check_version(sc); 619 if (error) 620 return; 621 622 error = vmxnet3_alloc_rxtx_queues(sc); 623 if (error) 624 return; 625 626 error = vmxnet3_alloc_interrupts(sc); 627 if (error) 628 return; 629 630 vmxnet3_check_multiqueue(sc); 631 632 error = vmxnet3_alloc_data(sc); 633 if (error) 634 return; 635 636 error = vmxnet3_setup_interface(sc); 637 if (error) 638 return; 639 640 error = vmxnet3_setup_interrupts(sc); 641 if (error) 642 return; 643 644 error = vmxnet3_setup_sysctl(sc); 645 if (error) 646 return; 647 648 error = vmxnet3_setup_stats(sc); 649 if (error) 650 return; 651 652 sc->vmx_flags |= VMXNET3_FLAG_ATTACHED; 653 } 654 655 static int 656 vmxnet3_detach(device_t self, int flags) 657 { 658 struct vmxnet3_softc *sc; 659 struct ifnet *ifp; 660 661 sc = device_private(self); 662 ifp = &sc->vmx_ethercom.ec_if; 663 664 if (sc->vmx_flags & VMXNET3_FLAG_ATTACHED) { 665 VMXNET3_CORE_LOCK(sc); 666 vmxnet3_stop_locked(sc); 667 callout_halt(&sc->vmx_tick, sc->vmx_mtx); 668 callout_destroy(&sc->vmx_tick); 669 VMXNET3_CORE_UNLOCK(sc); 670 671 ether_ifdetach(ifp); 672 if_detach(ifp); 673 ifmedia_fini(&sc->vmx_media); 674 } 675 676 vmxnet3_teardown_stats(sc); 677 sysctl_teardown(&sc->vmx_sysctllog); 678 679 vmxnet3_free_interrupts(sc); 680 681 vmxnet3_free_data(sc); 682 vmxnet3_free_pci_resources(sc); 683 vmxnet3_free_rxtx_queues(sc); 684 685 if (sc->vmx_mtx) 686 mutex_obj_free(sc->vmx_mtx); 687 688 return (0); 689 } 690 691 static int 692 vmxnet3_alloc_pci_resources(struct vmxnet3_softc *sc) 693 { 694 struct pci_attach_args *pa = sc->vmx_pa; 695 pcireg_t memtype; 696 697 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 698 if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->vmx_iot0, &sc->vmx_ioh0, 699 NULL, &sc->vmx_ios0)) { 700 aprint_error_dev(sc->vmx_dev, "failed to map BAR0\n"); 701 return (ENXIO); 702 } 703 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(1)); 704 if (pci_mapreg_map(pa, PCI_BAR(1), memtype, 0, &sc->vmx_iot1, &sc->vmx_ioh1, 705 NULL, &sc->vmx_ios1)) { 706 aprint_error_dev(sc->vmx_dev, "failed to map BAR1\n"); 707 return (ENXIO); 708 } 709 710 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, NULL, NULL)) { 711 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 712 return (0); 713 } 714 715 return (0); 716 } 717 718 static void 719 vmxnet3_free_pci_resources(struct vmxnet3_softc *sc) 720 { 721 722 if (sc->vmx_ios0) { 723 bus_space_unmap(sc->vmx_iot0, sc->vmx_ioh0, sc->vmx_ios0); 724 sc->vmx_ios0 = 0; 725 } 726 727 if (sc->vmx_ios1) { 728 bus_space_unmap(sc->vmx_iot1, sc->vmx_ioh1, sc->vmx_ios1); 729 sc->vmx_ios1 = 0; 730 } 731 } 732 733 static int 734 vmxnet3_check_version(struct vmxnet3_softc *sc) 735 { 736 u_int ver; 737 738 ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 739 if ((ver & 0x1) == 0) { 740 aprint_error_dev(sc->vmx_dev, 741 "unsupported hardware version 0x%x\n", ver); 742 return (ENOTSUP); 743 } 744 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 745 746 ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 747 if ((ver & 0x1) == 0) { 748 aprint_error_dev(sc->vmx_dev, 749 "incompatiable UPT version 0x%x\n", ver); 750 return (ENOTSUP); 751 } 752 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 753 754 return (0); 755 } 756 757 static void 758 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc) 759 { 760 761 if (sc->vmx_intr_type != VMXNET3_IT_MSIX) 762 goto out; 763 764 /* Just use the maximum configured for now. */ 765 sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; 766 sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; 767 768 if (sc->vmx_nrxqueues > 1) 769 sc->vmx_flags |= VMXNET3_FLAG_RSS; 770 771 return; 772 773 out: 774 sc->vmx_ntxqueues = 1; 775 sc->vmx_nrxqueues = 1; 776 } 777 778 static int 779 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 780 { 781 int required; 782 struct pci_attach_args *pa = sc->vmx_pa; 783 784 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 785 return (1); 786 787 /* Allocate an additional vector for the events interrupt. */ 788 required = MIN(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues) + 1; 789 790 if (pci_msix_count(pa->pa_pc, pa->pa_tag) < required) 791 return (1); 792 793 if (pci_msix_alloc_exact(pa, &sc->vmx_intrs, required) == 0) { 794 sc->vmx_nintrs = required; 795 return (0); 796 } 797 798 return (1); 799 } 800 801 static int 802 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 803 { 804 int nmsi, required; 805 struct pci_attach_args *pa = sc->vmx_pa; 806 807 required = 1; 808 809 nmsi = pci_msi_count(pa->pa_pc, pa->pa_tag); 810 if (nmsi < required) 811 return (1); 812 813 if (pci_msi_alloc_exact(pa, &sc->vmx_intrs, required) == 0) { 814 sc->vmx_nintrs = required; 815 return (0); 816 } 817 818 return (1); 819 } 820 821 static int 822 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 823 { 824 825 if (pci_intx_alloc(sc->vmx_pa, &sc->vmx_intrs) == 0) { 826 sc->vmx_nintrs = 1; 827 return (0); 828 } 829 830 return (1); 831 } 832 833 static int 834 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 835 { 836 u_int config; 837 int error; 838 839 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 840 841 sc->vmx_intr_type = config & 0x03; 842 sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 843 844 switch (sc->vmx_intr_type) { 845 case VMXNET3_IT_AUTO: 846 sc->vmx_intr_type = VMXNET3_IT_MSIX; 847 /* FALLTHROUGH */ 848 case VMXNET3_IT_MSIX: 849 error = vmxnet3_alloc_msix_interrupts(sc); 850 if (error == 0) 851 break; 852 sc->vmx_intr_type = VMXNET3_IT_MSI; 853 /* FALLTHROUGH */ 854 case VMXNET3_IT_MSI: 855 error = vmxnet3_alloc_msi_interrupts(sc); 856 if (error == 0) 857 break; 858 sc->vmx_intr_type = VMXNET3_IT_LEGACY; 859 /* FALLTHROUGH */ 860 case VMXNET3_IT_LEGACY: 861 error = vmxnet3_alloc_legacy_interrupts(sc); 862 if (error == 0) 863 break; 864 /* FALLTHROUGH */ 865 default: 866 sc->vmx_intr_type = -1; 867 aprint_error_dev(sc->vmx_dev, "cannot allocate any interrupt resources\n"); 868 return (ENXIO); 869 } 870 871 return (error); 872 } 873 874 static void 875 vmxnet3_free_interrupts(struct vmxnet3_softc *sc) 876 { 877 pci_chipset_tag_t pc = sc->vmx_pc; 878 int i; 879 880 workqueue_destroy(sc->vmx_queue_wq); 881 for (i = 0; i < sc->vmx_ntxqueues; i++) { 882 struct vmxnet3_queue *vmxq = &sc->vmx_queue[i]; 883 884 softint_disestablish(vmxq->vxq_si); 885 vmxq->vxq_si = NULL; 886 } 887 for (i = 0; i < sc->vmx_nintrs; i++) { 888 pci_intr_disestablish(pc, sc->vmx_ihs[i]); 889 } 890 pci_intr_release(pc, sc->vmx_intrs, sc->vmx_nintrs); 891 } 892 893 static int 894 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 895 { 896 pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; 897 struct vmxnet3_queue *vmxq; 898 pci_intr_handle_t *intr; 899 void **ihs; 900 int intr_idx, i, use_queues, error; 901 kcpuset_t *affinity; 902 const char *intrstr; 903 char intrbuf[PCI_INTRSTR_LEN]; 904 char xnamebuf[32]; 905 906 intr = sc->vmx_intrs; 907 intr_idx = 0; 908 ihs = sc->vmx_ihs; 909 910 /* See vmxnet3_alloc_msix_interrupts() */ 911 use_queues = MIN(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues); 912 for (i = 0; i < use_queues; i++, intr++, ihs++, intr_idx++) { 913 snprintf(xnamebuf, 32, "%s: txrx %d", device_xname(sc->vmx_dev), i); 914 915 vmxq = &sc->vmx_queue[i]; 916 917 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 918 919 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 920 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 921 vmxnet3_txrxq_intr, vmxq, xnamebuf); 922 if (*ihs == NULL) { 923 aprint_error_dev(sc->vmx_dev, 924 "unable to establish txrx interrupt at %s\n", intrstr); 925 return (-1); 926 } 927 aprint_normal_dev(sc->vmx_dev, "txrx interrupting at %s\n", intrstr); 928 929 kcpuset_create(&affinity, true); 930 kcpuset_set(affinity, intr_idx % ncpu); 931 error = interrupt_distribute(*ihs, affinity, NULL); 932 if (error) { 933 aprint_normal_dev(sc->vmx_dev, 934 "%s cannot be changed affinity, use default CPU\n", 935 intrstr); 936 } 937 kcpuset_destroy(affinity); 938 939 vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 940 vmxnet3_handle_queue, vmxq); 941 if (vmxq->vxq_si == NULL) { 942 aprint_error_dev(sc->vmx_dev, 943 "softint_establish for vxq_si failed\n"); 944 return (-1); 945 } 946 947 vmxq->vxq_intr_idx = intr_idx; 948 } 949 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(sc->vmx_dev)); 950 error = workqueue_create(&sc->vmx_queue_wq, xnamebuf, 951 vmxnet3_handle_queue_work, sc, VMXNET3_WORKQUEUE_PRI, IPL_NET, 952 WQ_PERCPU | WQ_MPSAFE); 953 if (error) { 954 aprint_error_dev(sc->vmx_dev, "workqueue_create failed\n"); 955 return (-1); 956 } 957 sc->vmx_txrx_workqueue = false; 958 959 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 960 961 snprintf(xnamebuf, 32, "%s: link", device_xname(sc->vmx_dev)); 962 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 963 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 964 vmxnet3_event_intr, sc, xnamebuf); 965 if (*ihs == NULL) { 966 aprint_error_dev(sc->vmx_dev, 967 "unable to establish event interrupt at %s\n", intrstr); 968 return (-1); 969 } 970 aprint_normal_dev(sc->vmx_dev, "event interrupting at %s\n", intrstr); 971 972 sc->vmx_event_intr_idx = intr_idx; 973 974 return (0); 975 } 976 977 static int 978 vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *sc) 979 { 980 pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; 981 pci_intr_handle_t *intr; 982 void **ihs; 983 struct vmxnet3_queue *vmxq; 984 int i; 985 const char *intrstr; 986 char intrbuf[PCI_INTRSTR_LEN]; 987 char xnamebuf[32]; 988 989 intr = &sc->vmx_intrs[0]; 990 ihs = sc->vmx_ihs; 991 vmxq = &sc->vmx_queue[0]; 992 993 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 994 995 snprintf(xnamebuf, 32, "%s: msi", device_xname(sc->vmx_dev)); 996 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 997 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 998 vmxnet3_legacy_intr, sc, xnamebuf); 999 if (*ihs == NULL) { 1000 aprint_error_dev(sc->vmx_dev, 1001 "unable to establish interrupt at %s\n", intrstr); 1002 return (-1); 1003 } 1004 aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr); 1005 1006 vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1007 vmxnet3_handle_queue, vmxq); 1008 if (vmxq->vxq_si == NULL) { 1009 aprint_error_dev(sc->vmx_dev, 1010 "softint_establish for vxq_si failed\n"); 1011 return (-1); 1012 } 1013 1014 for (i = 0; i < MIN(sc->vmx_nrxqueues, sc->vmx_nrxqueues); i++) 1015 sc->vmx_queue[i].vxq_intr_idx = 0; 1016 sc->vmx_event_intr_idx = 0; 1017 1018 return (0); 1019 } 1020 1021 static int 1022 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 1023 { 1024 pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; 1025 pci_intr_handle_t *intr; 1026 void **ihs; 1027 struct vmxnet3_queue *vmxq; 1028 int i; 1029 const char *intrstr; 1030 char intrbuf[PCI_INTRSTR_LEN]; 1031 char xnamebuf[32]; 1032 1033 intr = &sc->vmx_intrs[0]; 1034 ihs = sc->vmx_ihs; 1035 vmxq = &sc->vmx_queue[0]; 1036 1037 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1038 1039 snprintf(xnamebuf, 32, "%s:legacy", device_xname(sc->vmx_dev)); 1040 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1041 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1042 vmxnet3_legacy_intr, sc, xnamebuf); 1043 if (*ihs == NULL) { 1044 aprint_error_dev(sc->vmx_dev, 1045 "unable to establish interrupt at %s\n", intrstr); 1046 return (-1); 1047 } 1048 aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr); 1049 1050 vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1051 vmxnet3_handle_queue, vmxq); 1052 if (vmxq->vxq_si == NULL) { 1053 aprint_error_dev(sc->vmx_dev, 1054 "softint_establish for vxq_si failed\n"); 1055 return (-1); 1056 } 1057 1058 for (i = 0; i < MIN(sc->vmx_nrxqueues, sc->vmx_nrxqueues); i++) 1059 sc->vmx_queue[i].vxq_intr_idx = 0; 1060 sc->vmx_event_intr_idx = 0; 1061 1062 return (0); 1063 } 1064 1065 static void 1066 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 1067 { 1068 struct vmxnet3_queue *vmxq; 1069 struct vmxnet3_txqueue *txq; 1070 struct vmxnet3_txq_shared *txs; 1071 struct vmxnet3_rxqueue *rxq; 1072 struct vmxnet3_rxq_shared *rxs; 1073 int i; 1074 1075 sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 1076 1077 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1078 vmxq = &sc->vmx_queue[i]; 1079 txq = &vmxq->vxq_txqueue; 1080 txs = txq->vxtxq_ts; 1081 txs->intr_idx = vmxq->vxq_intr_idx; 1082 } 1083 1084 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1085 vmxq = &sc->vmx_queue[i]; 1086 rxq = &vmxq->vxq_rxqueue; 1087 rxs = rxq->vxrxq_rs; 1088 rxs->intr_idx = vmxq->vxq_intr_idx; 1089 } 1090 } 1091 1092 static int 1093 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 1094 { 1095 int error; 1096 1097 switch (sc->vmx_intr_type) { 1098 case VMXNET3_IT_MSIX: 1099 error = vmxnet3_setup_msix_interrupts(sc); 1100 break; 1101 case VMXNET3_IT_MSI: 1102 error = vmxnet3_setup_msi_interrupt(sc); 1103 break; 1104 case VMXNET3_IT_LEGACY: 1105 error = vmxnet3_setup_legacy_interrupt(sc); 1106 break; 1107 default: 1108 panic("%s: invalid interrupt type %d", __func__, 1109 sc->vmx_intr_type); 1110 } 1111 1112 if (error == 0) 1113 vmxnet3_set_interrupt_idx(sc); 1114 1115 return (error); 1116 } 1117 1118 static int 1119 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 1120 { 1121 struct vmxnet3_rxqueue *rxq; 1122 struct vmxnet3_rxring *rxr; 1123 int i; 1124 1125 rxq = &sc->vmx_queue[q].vxq_rxqueue; 1126 1127 snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 1128 device_xname(sc->vmx_dev), q); 1129 rxq->vxrxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */); 1130 1131 rxq->vxrxq_sc = sc; 1132 1133 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1134 rxr = &rxq->vxrxq_cmd_ring[i]; 1135 rxr->vxrxr_rid = i; 1136 rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 1137 rxr->vxrxr_rxbuf = kmem_zalloc(rxr->vxrxr_ndesc * 1138 sizeof(struct vmxnet3_rxbuf), KM_SLEEP); 1139 1140 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 1141 } 1142 1143 return (0); 1144 } 1145 1146 static int 1147 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 1148 { 1149 struct vmxnet3_txqueue *txq; 1150 struct vmxnet3_txring *txr; 1151 1152 txq = &sc->vmx_queue[q].vxq_txqueue; 1153 txr = &txq->vxtxq_cmd_ring; 1154 1155 snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 1156 device_xname(sc->vmx_dev), q); 1157 txq->vxtxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */); 1158 1159 txq->vxtxq_sc = sc; 1160 1161 txq->vxtxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1162 vmxnet3_deferred_transmit, txq); 1163 if (txq->vxtxq_si == NULL) { 1164 mutex_obj_free(txq->vxtxq_mtx); 1165 aprint_error_dev(sc->vmx_dev, 1166 "softint_establish for vxtxq_si failed\n"); 1167 return ENOMEM; 1168 } 1169 1170 txr->vxtxr_ndesc = sc->vmx_ntxdescs; 1171 txr->vxtxr_txbuf = kmem_zalloc(txr->vxtxr_ndesc * 1172 sizeof(struct vmxnet3_txbuf), KM_SLEEP); 1173 1174 txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 1175 1176 txq->vxtxq_interq = pcq_create(sc->vmx_ntxdescs, KM_SLEEP); 1177 1178 return (0); 1179 } 1180 1181 static int 1182 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 1183 { 1184 int i, error, max_nqueues; 1185 1186 KASSERT(!cpu_intr_p()); 1187 KASSERT(!cpu_softintr_p()); 1188 1189 /* 1190 * Only attempt to create multiple queues if MSIX is available. 1191 * This check prevents us from allocating queue structures that 1192 * we will not use. 1193 * 1194 * FreeBSD: 1195 * MSIX is disabled by default because its apparently broken for 1196 * devices passed through by at least ESXi 5.1. 1197 * The hw.pci.honor_msi_blacklist tunable must be set to zero for MSIX. 1198 */ 1199 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { 1200 sc->vmx_max_nrxqueues = 1; 1201 sc->vmx_max_ntxqueues = 1; 1202 } 1203 1204 max_nqueues = MAX(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues); 1205 sc->vmx_queue = kmem_zalloc(sizeof(struct vmxnet3_queue) * max_nqueues, 1206 KM_SLEEP); 1207 1208 for (i = 0; i < max_nqueues; i++) { 1209 struct vmxnet3_queue *vmxq = &sc->vmx_queue[i]; 1210 vmxq->vxq_id = i; 1211 } 1212 1213 for (i = 0; i < sc->vmx_max_nrxqueues; i++) { 1214 error = vmxnet3_init_rxq(sc, i); 1215 if (error) 1216 return (error); 1217 } 1218 1219 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1220 error = vmxnet3_init_txq(sc, i); 1221 if (error) 1222 return (error); 1223 } 1224 1225 return (0); 1226 } 1227 1228 static void 1229 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 1230 { 1231 struct vmxnet3_rxring *rxr; 1232 int i; 1233 1234 rxq->vxrxq_sc = NULL; 1235 1236 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1237 rxr = &rxq->vxrxq_cmd_ring[i]; 1238 1239 if (rxr->vxrxr_rxbuf != NULL) { 1240 kmem_free(rxr->vxrxr_rxbuf, 1241 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxbuf)); 1242 rxr->vxrxr_rxbuf = NULL; 1243 } 1244 } 1245 1246 if (rxq->vxrxq_mtx != NULL) 1247 mutex_obj_free(rxq->vxrxq_mtx); 1248 } 1249 1250 static void 1251 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 1252 { 1253 struct vmxnet3_txring *txr; 1254 struct mbuf *m; 1255 1256 txr = &txq->vxtxq_cmd_ring; 1257 1258 txq->vxtxq_sc = NULL; 1259 1260 softint_disestablish(txq->vxtxq_si); 1261 1262 while ((m = pcq_get(txq->vxtxq_interq)) != NULL) 1263 m_freem(m); 1264 pcq_destroy(txq->vxtxq_interq); 1265 1266 if (txr->vxtxr_txbuf != NULL) { 1267 kmem_free(txr->vxtxr_txbuf, 1268 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txbuf)); 1269 txr->vxtxr_txbuf = NULL; 1270 } 1271 1272 if (txq->vxtxq_mtx != NULL) 1273 mutex_obj_free(txq->vxtxq_mtx); 1274 } 1275 1276 static void 1277 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 1278 { 1279 int i; 1280 1281 if (sc->vmx_queue != NULL) { 1282 int max_nqueues; 1283 1284 for (i = 0; i < sc->vmx_max_nrxqueues; i++) 1285 vmxnet3_destroy_rxq(&sc->vmx_queue[i].vxq_rxqueue); 1286 1287 for (i = 0; i < sc->vmx_max_ntxqueues; i++) 1288 vmxnet3_destroy_txq(&sc->vmx_queue[i].vxq_txqueue); 1289 1290 max_nqueues = MAX(sc->vmx_max_nrxqueues, sc->vmx_max_ntxqueues); 1291 kmem_free(sc->vmx_queue, 1292 sizeof(struct vmxnet3_queue) * max_nqueues); 1293 } 1294 } 1295 1296 static int 1297 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 1298 { 1299 device_t dev; 1300 uint8_t *kva; 1301 size_t size; 1302 int i, error; 1303 1304 dev = sc->vmx_dev; 1305 1306 size = sizeof(struct vmxnet3_driver_shared); 1307 error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 1308 if (error) { 1309 device_printf(dev, "cannot alloc shared memory\n"); 1310 return (error); 1311 } 1312 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 1313 1314 size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 1315 sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 1316 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 1317 if (error) { 1318 device_printf(dev, "cannot alloc queue shared memory\n"); 1319 return (error); 1320 } 1321 sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 1322 kva = sc->vmx_qs; 1323 1324 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1325 sc->vmx_queue[i].vxq_txqueue.vxtxq_ts = 1326 (struct vmxnet3_txq_shared *) kva; 1327 kva += sizeof(struct vmxnet3_txq_shared); 1328 } 1329 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1330 sc->vmx_queue[i].vxq_rxqueue.vxrxq_rs = 1331 (struct vmxnet3_rxq_shared *) kva; 1332 kva += sizeof(struct vmxnet3_rxq_shared); 1333 } 1334 1335 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1336 size = sizeof(struct vmxnet3_rss_shared); 1337 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); 1338 if (error) { 1339 device_printf(dev, "cannot alloc rss shared memory\n"); 1340 return (error); 1341 } 1342 sc->vmx_rss = 1343 (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; 1344 } 1345 1346 return (0); 1347 } 1348 1349 static void 1350 vmxnet3_free_shared_data(struct vmxnet3_softc *sc) 1351 { 1352 1353 if (sc->vmx_rss != NULL) { 1354 vmxnet3_dma_free(sc, &sc->vmx_rss_dma); 1355 sc->vmx_rss = NULL; 1356 } 1357 1358 if (sc->vmx_qs != NULL) { 1359 vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 1360 sc->vmx_qs = NULL; 1361 } 1362 1363 if (sc->vmx_ds != NULL) { 1364 vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 1365 sc->vmx_ds = NULL; 1366 } 1367 } 1368 1369 static int 1370 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 1371 { 1372 device_t dev; 1373 struct vmxnet3_txqueue *txq; 1374 struct vmxnet3_txring *txr; 1375 struct vmxnet3_comp_ring *txc; 1376 size_t descsz, compsz; 1377 u_int i; 1378 int q, error; 1379 1380 dev = sc->vmx_dev; 1381 1382 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1383 txq = &sc->vmx_queue[q].vxq_txqueue; 1384 txr = &txq->vxtxq_cmd_ring; 1385 txc = &txq->vxtxq_comp_ring; 1386 1387 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 1388 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 1389 1390 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 1391 if (error) { 1392 device_printf(dev, "cannot alloc Tx descriptors for " 1393 "queue %d error %d\n", q, error); 1394 return (error); 1395 } 1396 txr->vxtxr_txd = 1397 (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1398 1399 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1400 if (error) { 1401 device_printf(dev, "cannot alloc Tx comp descriptors " 1402 "for queue %d error %d\n", q, error); 1403 return (error); 1404 } 1405 txc->vxcr_u.txcd = 1406 (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1407 1408 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1409 error = bus_dmamap_create(sc->vmx_dmat, VMXNET3_TX_MAXSIZE, 1410 VMXNET3_TX_MAXSEGS, VMXNET3_TX_MAXSEGSIZE, 0, BUS_DMA_NOWAIT, 1411 &txr->vxtxr_txbuf[i].vtxb_dmamap); 1412 if (error) { 1413 device_printf(dev, "unable to create Tx buf " 1414 "dmamap for queue %d idx %d\n", q, i); 1415 return (error); 1416 } 1417 } 1418 } 1419 1420 return (0); 1421 } 1422 1423 static void 1424 vmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1425 { 1426 struct vmxnet3_txqueue *txq; 1427 struct vmxnet3_txring *txr; 1428 struct vmxnet3_comp_ring *txc; 1429 struct vmxnet3_txbuf *txb; 1430 u_int i; 1431 int q; 1432 1433 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1434 txq = &sc->vmx_queue[q].vxq_txqueue; 1435 txr = &txq->vxtxq_cmd_ring; 1436 txc = &txq->vxtxq_comp_ring; 1437 1438 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1439 txb = &txr->vxtxr_txbuf[i]; 1440 if (txb->vtxb_dmamap != NULL) { 1441 bus_dmamap_destroy(sc->vmx_dmat, 1442 txb->vtxb_dmamap); 1443 txb->vtxb_dmamap = NULL; 1444 } 1445 } 1446 1447 if (txc->vxcr_u.txcd != NULL) { 1448 vmxnet3_dma_free(sc, &txc->vxcr_dma); 1449 txc->vxcr_u.txcd = NULL; 1450 } 1451 1452 if (txr->vxtxr_txd != NULL) { 1453 vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1454 txr->vxtxr_txd = NULL; 1455 } 1456 } 1457 } 1458 1459 static int 1460 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1461 { 1462 device_t dev; 1463 struct vmxnet3_rxqueue *rxq; 1464 struct vmxnet3_rxring *rxr; 1465 struct vmxnet3_comp_ring *rxc; 1466 int descsz, compsz; 1467 u_int i, j; 1468 int q, error; 1469 1470 dev = sc->vmx_dev; 1471 1472 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1473 rxq = &sc->vmx_queue[q].vxq_rxqueue; 1474 rxc = &rxq->vxrxq_comp_ring; 1475 compsz = 0; 1476 1477 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1478 rxr = &rxq->vxrxq_cmd_ring[i]; 1479 1480 descsz = rxr->vxrxr_ndesc * 1481 sizeof(struct vmxnet3_rxdesc); 1482 compsz += rxr->vxrxr_ndesc * 1483 sizeof(struct vmxnet3_rxcompdesc); 1484 1485 error = vmxnet3_dma_malloc(sc, descsz, 512, 1486 &rxr->vxrxr_dma); 1487 if (error) { 1488 device_printf(dev, "cannot allocate Rx " 1489 "descriptors for queue %d/%d error %d\n", 1490 i, q, error); 1491 return (error); 1492 } 1493 rxr->vxrxr_rxd = 1494 (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1495 } 1496 1497 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1498 if (error) { 1499 device_printf(dev, "cannot alloc Rx comp descriptors " 1500 "for queue %d error %d\n", q, error); 1501 return (error); 1502 } 1503 rxc->vxcr_u.rxcd = 1504 (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1505 1506 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1507 rxr = &rxq->vxrxq_cmd_ring[i]; 1508 1509 error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1, 1510 JUMBO_LEN, 0, BUS_DMA_NOWAIT, 1511 &rxr->vxrxr_spare_dmap); 1512 if (error) { 1513 device_printf(dev, "unable to create spare " 1514 "dmamap for queue %d/%d error %d\n", 1515 q, i, error); 1516 return (error); 1517 } 1518 1519 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1520 error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1, 1521 JUMBO_LEN, 0, BUS_DMA_NOWAIT, 1522 &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1523 if (error) { 1524 device_printf(dev, "unable to create " 1525 "dmamap for queue %d/%d slot %d " 1526 "error %d\n", 1527 q, i, j, error); 1528 return (error); 1529 } 1530 } 1531 } 1532 } 1533 1534 return (0); 1535 } 1536 1537 static void 1538 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1539 { 1540 struct vmxnet3_rxqueue *rxq; 1541 struct vmxnet3_rxring *rxr; 1542 struct vmxnet3_comp_ring *rxc; 1543 struct vmxnet3_rxbuf *rxb; 1544 u_int i, j; 1545 int q; 1546 1547 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1548 rxq = &sc->vmx_queue[q].vxq_rxqueue; 1549 rxc = &rxq->vxrxq_comp_ring; 1550 1551 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1552 rxr = &rxq->vxrxq_cmd_ring[i]; 1553 1554 if (rxr->vxrxr_spare_dmap != NULL) { 1555 bus_dmamap_destroy(sc->vmx_dmat, 1556 rxr->vxrxr_spare_dmap); 1557 rxr->vxrxr_spare_dmap = NULL; 1558 } 1559 1560 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1561 rxb = &rxr->vxrxr_rxbuf[j]; 1562 if (rxb->vrxb_dmamap != NULL) { 1563 bus_dmamap_destroy(sc->vmx_dmat, 1564 rxb->vrxb_dmamap); 1565 rxb->vrxb_dmamap = NULL; 1566 } 1567 } 1568 } 1569 1570 if (rxc->vxcr_u.rxcd != NULL) { 1571 vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1572 rxc->vxcr_u.rxcd = NULL; 1573 } 1574 1575 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1576 rxr = &rxq->vxrxq_cmd_ring[i]; 1577 1578 if (rxr->vxrxr_rxd != NULL) { 1579 vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1580 rxr->vxrxr_rxd = NULL; 1581 } 1582 } 1583 } 1584 } 1585 1586 static int 1587 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1588 { 1589 int error; 1590 1591 error = vmxnet3_alloc_txq_data(sc); 1592 if (error) 1593 return (error); 1594 1595 error = vmxnet3_alloc_rxq_data(sc); 1596 if (error) 1597 return (error); 1598 1599 return (0); 1600 } 1601 1602 static void 1603 vmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1604 { 1605 1606 if (sc->vmx_queue != NULL) { 1607 vmxnet3_free_rxq_data(sc); 1608 vmxnet3_free_txq_data(sc); 1609 } 1610 } 1611 1612 static int 1613 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1614 { 1615 int error; 1616 1617 error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1618 32, &sc->vmx_mcast_dma); 1619 if (error) 1620 device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1621 else 1622 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1623 1624 return (error); 1625 } 1626 1627 static void 1628 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1629 { 1630 1631 if (sc->vmx_mcast != NULL) { 1632 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1633 sc->vmx_mcast = NULL; 1634 } 1635 } 1636 1637 static void 1638 vmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1639 { 1640 struct vmxnet3_driver_shared *ds; 1641 struct vmxnet3_txqueue *txq; 1642 struct vmxnet3_txq_shared *txs; 1643 struct vmxnet3_rxqueue *rxq; 1644 struct vmxnet3_rxq_shared *rxs; 1645 int i; 1646 1647 ds = sc->vmx_ds; 1648 1649 /* 1650 * Initialize fields of the shared data that remains the same across 1651 * reinits. Note the shared data is zero'd when allocated. 1652 */ 1653 1654 ds->magic = VMXNET3_REV1_MAGIC; 1655 1656 /* DriverInfo */ 1657 ds->version = VMXNET3_DRIVER_VERSION; 1658 ds->guest = VMXNET3_GOS_FREEBSD | 1659 #ifdef __LP64__ 1660 VMXNET3_GOS_64BIT; 1661 #else 1662 VMXNET3_GOS_32BIT; 1663 #endif 1664 ds->vmxnet3_revision = 1; 1665 ds->upt_version = 1; 1666 1667 /* Misc. conf */ 1668 ds->driver_data = vtophys(sc); 1669 ds->driver_data_len = sizeof(struct vmxnet3_softc); 1670 ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1671 ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1672 ds->nrxsg_max = sc->vmx_max_rxsegs; 1673 1674 /* RSS conf */ 1675 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1676 ds->rss.version = 1; 1677 ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; 1678 ds->rss.len = sc->vmx_rss_dma.dma_size; 1679 } 1680 1681 /* Interrupt control. */ 1682 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1683 ds->nintr = sc->vmx_nintrs; 1684 ds->evintr = sc->vmx_event_intr_idx; 1685 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1686 1687 for (i = 0; i < sc->vmx_nintrs; i++) 1688 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1689 1690 /* Receive filter. */ 1691 ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1692 ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1693 1694 /* Tx queues */ 1695 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1696 txq = &sc->vmx_queue[i].vxq_txqueue; 1697 txs = txq->vxtxq_ts; 1698 1699 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1700 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1701 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1702 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1703 txs->driver_data = vtophys(txq); 1704 txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1705 } 1706 1707 /* Rx queues */ 1708 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1709 rxq = &sc->vmx_queue[i].vxq_rxqueue; 1710 rxs = rxq->vxrxq_rs; 1711 1712 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1713 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1714 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1715 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1716 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1717 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1718 rxs->driver_data = vtophys(rxq); 1719 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1720 } 1721 } 1722 1723 static void 1724 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) 1725 { 1726 /* 1727 * Use the same key as the Linux driver until FreeBSD can do 1728 * RSS (presumably Toeplitz) in software. 1729 */ 1730 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 1731 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 1732 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 1733 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 1734 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 1735 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, 1736 }; 1737 1738 struct vmxnet3_rss_shared *rss; 1739 int i; 1740 1741 rss = sc->vmx_rss; 1742 1743 rss->hash_type = 1744 UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | 1745 UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; 1746 rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 1747 rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; 1748 rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 1749 memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); 1750 1751 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) 1752 rss->ind_table[i] = i % sc->vmx_nrxqueues; 1753 } 1754 1755 static void 1756 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1757 { 1758 struct ifnet *ifp; 1759 struct vmxnet3_driver_shared *ds; 1760 1761 ifp = &sc->vmx_ethercom.ec_if; 1762 ds = sc->vmx_ds; 1763 1764 ds->mtu = ifp->if_mtu; 1765 ds->ntxqueue = sc->vmx_ntxqueues; 1766 ds->nrxqueue = sc->vmx_nrxqueues; 1767 1768 ds->upt_features = 0; 1769 if (ifp->if_capenable & 1770 (IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 1771 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) 1772 ds->upt_features |= UPT1_F_CSUM; 1773 if (sc->vmx_ethercom.ec_capenable & ETHERCAP_VLAN_HWTAGGING) 1774 ds->upt_features |= UPT1_F_VLAN; 1775 1776 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1777 ds->upt_features |= UPT1_F_RSS; 1778 vmxnet3_reinit_rss_shared_data(sc); 1779 } 1780 1781 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1782 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1783 (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1784 } 1785 1786 static int 1787 vmxnet3_alloc_data(struct vmxnet3_softc *sc) 1788 { 1789 int error; 1790 1791 error = vmxnet3_alloc_shared_data(sc); 1792 if (error) 1793 return (error); 1794 1795 error = vmxnet3_alloc_queue_data(sc); 1796 if (error) 1797 return (error); 1798 1799 error = vmxnet3_alloc_mcast_table(sc); 1800 if (error) 1801 return (error); 1802 1803 vmxnet3_init_shared_data(sc); 1804 1805 return (0); 1806 } 1807 1808 static void 1809 vmxnet3_free_data(struct vmxnet3_softc *sc) 1810 { 1811 1812 vmxnet3_free_mcast_table(sc); 1813 vmxnet3_free_queue_data(sc); 1814 vmxnet3_free_shared_data(sc); 1815 } 1816 1817 static int 1818 vmxnet3_setup_interface(struct vmxnet3_softc *sc) 1819 { 1820 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 1821 1822 vmxnet3_get_lladdr(sc); 1823 aprint_normal_dev(sc->vmx_dev, "Ethernet address %s\n", 1824 ether_sprintf(sc->vmx_lladdr)); 1825 vmxnet3_set_lladdr(sc); 1826 1827 strlcpy(ifp->if_xname, device_xname(sc->vmx_dev), IFNAMSIZ); 1828 ifp->if_softc = sc; 1829 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; 1830 ifp->if_extflags = IFEF_MPSAFE; 1831 ifp->if_ioctl = vmxnet3_ioctl; 1832 ifp->if_start = vmxnet3_start; 1833 ifp->if_transmit = vmxnet3_transmit; 1834 ifp->if_watchdog = NULL; 1835 ifp->if_init = vmxnet3_init; 1836 ifp->if_stop = vmxnet3_stop; 1837 sc->vmx_ethercom.ec_if.if_capabilities |=IFCAP_CSUM_IPv4_Rx | 1838 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1839 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 1840 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 1841 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 1842 1843 ifp->if_capenable = ifp->if_capabilities; 1844 1845 sc->vmx_ethercom.ec_if.if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1846 1847 sc->vmx_ethercom.ec_capabilities |= 1848 ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING | ETHERCAP_JUMBO_MTU; 1849 sc->vmx_ethercom.ec_capenable |= ETHERCAP_VLAN_HWTAGGING; 1850 1851 IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs); 1852 IFQ_SET_READY(&ifp->if_snd); 1853 1854 /* Initialize ifmedia structures. */ 1855 sc->vmx_ethercom.ec_ifmedia = &sc->vmx_media; 1856 ifmedia_init_with_lock(&sc->vmx_media, IFM_IMASK, vmxnet3_ifmedia_change, 1857 vmxnet3_ifmedia_status, sc->vmx_mtx); 1858 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1859 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL); 1860 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_10G_T, 0, NULL); 1861 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1862 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_1000_T, 0, NULL); 1863 ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1864 1865 if_attach(ifp); 1866 if_deferred_start_init(ifp, NULL); 1867 ether_ifattach(ifp, sc->vmx_lladdr); 1868 ether_set_ifflags_cb(&sc->vmx_ethercom, vmxnet3_ifflags_cb); 1869 vmxnet3_cmd_link_status(ifp); 1870 1871 /* should set before setting interrupts */ 1872 sc->vmx_rx_intr_process_limit = VMXNET3_RX_INTR_PROCESS_LIMIT; 1873 sc->vmx_rx_process_limit = VMXNET3_RX_PROCESS_LIMIT; 1874 sc->vmx_tx_intr_process_limit = VMXNET3_TX_INTR_PROCESS_LIMIT; 1875 sc->vmx_tx_process_limit = VMXNET3_TX_PROCESS_LIMIT; 1876 1877 return (0); 1878 } 1879 1880 static int 1881 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 1882 { 1883 const char *devname; 1884 struct sysctllog **log; 1885 const struct sysctlnode *rnode, *rxnode, *txnode; 1886 int error; 1887 1888 log = &sc->vmx_sysctllog; 1889 devname = device_xname(sc->vmx_dev); 1890 1891 error = sysctl_createv(log, 0, NULL, &rnode, 1892 0, CTLTYPE_NODE, devname, 1893 SYSCTL_DESCR("vmxnet3 information and settings"), 1894 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 1895 if (error) 1896 goto out; 1897 error = sysctl_createv(log, 0, &rnode, NULL, 1898 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 1899 SYSCTL_DESCR("Use workqueue for packet processing"), 1900 NULL, 0, &sc->vmx_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 1901 if (error) 1902 goto out; 1903 1904 error = sysctl_createv(log, 0, &rnode, &rxnode, 1905 0, CTLTYPE_NODE, "rx", 1906 SYSCTL_DESCR("vmxnet3 information and settings for Rx"), 1907 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 1908 if (error) 1909 goto out; 1910 error = sysctl_createv(log, 0, &rxnode, NULL, 1911 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 1912 SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), 1913 NULL, 0, &sc->vmx_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 1914 if (error) 1915 goto out; 1916 error = sysctl_createv(log, 0, &rxnode, NULL, 1917 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 1918 SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), 1919 NULL, 0, &sc->vmx_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 1920 if (error) 1921 goto out; 1922 1923 error = sysctl_createv(log, 0, &rnode, &txnode, 1924 0, CTLTYPE_NODE, "tx", 1925 SYSCTL_DESCR("vmxnet3 information and settings for Tx"), 1926 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 1927 if (error) 1928 goto out; 1929 error = sysctl_createv(log, 0, &txnode, NULL, 1930 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 1931 SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), 1932 NULL, 0, &sc->vmx_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 1933 if (error) 1934 goto out; 1935 error = sysctl_createv(log, 0, &txnode, NULL, 1936 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 1937 SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), 1938 NULL, 0, &sc->vmx_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 1939 1940 out: 1941 if (error) { 1942 aprint_error_dev(sc->vmx_dev, 1943 "unable to create sysctl node\n"); 1944 sysctl_teardown(log); 1945 } 1946 return error; 1947 } 1948 1949 static int 1950 vmxnet3_setup_stats(struct vmxnet3_softc *sc) 1951 { 1952 struct vmxnet3_queue *vmxq; 1953 struct vmxnet3_txqueue *txq; 1954 struct vmxnet3_rxqueue *rxq; 1955 int i; 1956 1957 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1958 vmxq = &sc->vmx_queue[i]; 1959 txq = &vmxq->vxq_txqueue; 1960 evcnt_attach_dynamic(&txq->vxtxq_intr, EVCNT_TYPE_INTR, 1961 NULL, txq->vxtxq_name, "Interrupt on queue"); 1962 evcnt_attach_dynamic(&txq->vxtxq_defer, EVCNT_TYPE_MISC, 1963 NULL, txq->vxtxq_name, "Handled queue in softint/workqueue"); 1964 evcnt_attach_dynamic(&txq->vxtxq_deferreq, EVCNT_TYPE_MISC, 1965 NULL, txq->vxtxq_name, "Requested in softint/workqueue"); 1966 evcnt_attach_dynamic(&txq->vxtxq_pcqdrop, EVCNT_TYPE_MISC, 1967 NULL, txq->vxtxq_name, "Dropped in pcq"); 1968 evcnt_attach_dynamic(&txq->vxtxq_transmitdef, EVCNT_TYPE_MISC, 1969 NULL, txq->vxtxq_name, "Deferred transmit"); 1970 evcnt_attach_dynamic(&txq->vxtxq_watchdogto, EVCNT_TYPE_MISC, 1971 NULL, txq->vxtxq_name, "Watchdog timeout"); 1972 evcnt_attach_dynamic(&txq->vxtxq_defragged, EVCNT_TYPE_MISC, 1973 NULL, txq->vxtxq_name, "m_defrag successed"); 1974 evcnt_attach_dynamic(&txq->vxtxq_defrag_failed, EVCNT_TYPE_MISC, 1975 NULL, txq->vxtxq_name, "m_defrag failed"); 1976 } 1977 1978 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1979 vmxq = &sc->vmx_queue[i]; 1980 rxq = &vmxq->vxq_rxqueue; 1981 evcnt_attach_dynamic(&rxq->vxrxq_intr, EVCNT_TYPE_INTR, 1982 NULL, rxq->vxrxq_name, "Interrupt on queue"); 1983 evcnt_attach_dynamic(&rxq->vxrxq_defer, EVCNT_TYPE_MISC, 1984 NULL, rxq->vxrxq_name, "Handled queue in softint/workqueue"); 1985 evcnt_attach_dynamic(&rxq->vxrxq_deferreq, EVCNT_TYPE_MISC, 1986 NULL, rxq->vxrxq_name, "Requested in softint/workqueue"); 1987 evcnt_attach_dynamic(&rxq->vxrxq_mgetcl_failed, EVCNT_TYPE_MISC, 1988 NULL, rxq->vxrxq_name, "MCLGET failed"); 1989 evcnt_attach_dynamic(&rxq->vxrxq_mbuf_load_failed, EVCNT_TYPE_MISC, 1990 NULL, rxq->vxrxq_name, "bus_dmamap_load_mbuf failed"); 1991 } 1992 1993 evcnt_attach_dynamic(&sc->vmx_event_intr, EVCNT_TYPE_INTR, 1994 NULL, device_xname(sc->vmx_dev), "Interrupt for other events"); 1995 evcnt_attach_dynamic(&sc->vmx_event_link, EVCNT_TYPE_MISC, 1996 NULL, device_xname(sc->vmx_dev), "Link status event"); 1997 evcnt_attach_dynamic(&sc->vmx_event_txqerror, EVCNT_TYPE_MISC, 1998 NULL, device_xname(sc->vmx_dev), "Tx queue error event"); 1999 evcnt_attach_dynamic(&sc->vmx_event_rxqerror, EVCNT_TYPE_MISC, 2000 NULL, device_xname(sc->vmx_dev), "Rx queue error event"); 2001 evcnt_attach_dynamic(&sc->vmx_event_dic, EVCNT_TYPE_MISC, 2002 NULL, device_xname(sc->vmx_dev), "Device impl change event"); 2003 evcnt_attach_dynamic(&sc->vmx_event_debug, EVCNT_TYPE_MISC, 2004 NULL, device_xname(sc->vmx_dev), "Debug event"); 2005 2006 return 0; 2007 } 2008 2009 static void 2010 vmxnet3_teardown_stats(struct vmxnet3_softc *sc) 2011 { 2012 struct vmxnet3_queue *vmxq; 2013 struct vmxnet3_txqueue *txq; 2014 struct vmxnet3_rxqueue *rxq; 2015 int i; 2016 2017 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2018 vmxq = &sc->vmx_queue[i]; 2019 txq = &vmxq->vxq_txqueue; 2020 evcnt_detach(&txq->vxtxq_intr); 2021 evcnt_detach(&txq->vxtxq_defer); 2022 evcnt_detach(&txq->vxtxq_deferreq); 2023 evcnt_detach(&txq->vxtxq_pcqdrop); 2024 evcnt_detach(&txq->vxtxq_transmitdef); 2025 evcnt_detach(&txq->vxtxq_watchdogto); 2026 evcnt_detach(&txq->vxtxq_defragged); 2027 evcnt_detach(&txq->vxtxq_defrag_failed); 2028 } 2029 2030 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2031 vmxq = &sc->vmx_queue[i]; 2032 rxq = &vmxq->vxq_rxqueue; 2033 evcnt_detach(&rxq->vxrxq_intr); 2034 evcnt_detach(&rxq->vxrxq_defer); 2035 evcnt_detach(&rxq->vxrxq_deferreq); 2036 evcnt_detach(&rxq->vxrxq_mgetcl_failed); 2037 evcnt_detach(&rxq->vxrxq_mbuf_load_failed); 2038 } 2039 2040 evcnt_detach(&sc->vmx_event_intr); 2041 evcnt_detach(&sc->vmx_event_link); 2042 evcnt_detach(&sc->vmx_event_txqerror); 2043 evcnt_detach(&sc->vmx_event_rxqerror); 2044 evcnt_detach(&sc->vmx_event_dic); 2045 evcnt_detach(&sc->vmx_event_debug); 2046 } 2047 2048 static void 2049 vmxnet3_evintr(struct vmxnet3_softc *sc) 2050 { 2051 device_t dev; 2052 struct vmxnet3_txq_shared *ts; 2053 struct vmxnet3_rxq_shared *rs; 2054 uint32_t event; 2055 int reset; 2056 2057 dev = sc->vmx_dev; 2058 reset = 0; 2059 2060 VMXNET3_CORE_LOCK(sc); 2061 2062 /* Clear events. */ 2063 event = sc->vmx_ds->event; 2064 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 2065 2066 if (event & VMXNET3_EVENT_LINK) { 2067 sc->vmx_event_link.ev_count++; 2068 vmxnet3_if_link_status(sc); 2069 if (sc->vmx_link_active != 0) 2070 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); 2071 } 2072 2073 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 2074 if (event & VMXNET3_EVENT_TQERROR) 2075 sc->vmx_event_txqerror.ev_count++; 2076 if (event & VMXNET3_EVENT_RQERROR) 2077 sc->vmx_event_rxqerror.ev_count++; 2078 2079 reset = 1; 2080 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 2081 ts = sc->vmx_queue[0].vxq_txqueue.vxtxq_ts; 2082 if (ts->stopped != 0) 2083 device_printf(dev, "Tx queue error %#x\n", ts->error); 2084 rs = sc->vmx_queue[0].vxq_rxqueue.vxrxq_rs; 2085 if (rs->stopped != 0) 2086 device_printf(dev, "Rx queue error %#x\n", rs->error); 2087 device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 2088 } 2089 2090 if (event & VMXNET3_EVENT_DIC) { 2091 sc->vmx_event_dic.ev_count++; 2092 device_printf(dev, "device implementation change event\n"); 2093 } 2094 if (event & VMXNET3_EVENT_DEBUG) { 2095 sc->vmx_event_debug.ev_count++; 2096 device_printf(dev, "debug event\n"); 2097 } 2098 2099 if (reset != 0) 2100 vmxnet3_init_locked(sc); 2101 2102 VMXNET3_CORE_UNLOCK(sc); 2103 } 2104 2105 static bool 2106 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq, u_int limit) 2107 { 2108 struct vmxnet3_softc *sc; 2109 struct vmxnet3_txring *txr; 2110 struct vmxnet3_comp_ring *txc; 2111 struct vmxnet3_txcompdesc *txcd; 2112 struct vmxnet3_txbuf *txb; 2113 struct ifnet *ifp; 2114 struct mbuf *m; 2115 u_int sop; 2116 bool more = false; 2117 2118 sc = txq->vxtxq_sc; 2119 txr = &txq->vxtxq_cmd_ring; 2120 txc = &txq->vxtxq_comp_ring; 2121 ifp = &sc->vmx_ethercom.ec_if; 2122 2123 VMXNET3_TXQ_LOCK_ASSERT(txq); 2124 2125 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2126 for (;;) { 2127 if (limit-- == 0) { 2128 more = true; 2129 break; 2130 } 2131 2132 txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 2133 if (txcd->gen != txc->vxcr_gen) 2134 break; 2135 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2136 2137 if (++txc->vxcr_next == txc->vxcr_ndesc) { 2138 txc->vxcr_next = 0; 2139 txc->vxcr_gen ^= 1; 2140 } 2141 2142 sop = txr->vxtxr_next; 2143 txb = &txr->vxtxr_txbuf[sop]; 2144 2145 if ((m = txb->vtxb_m) != NULL) { 2146 bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap, 2147 0, txb->vtxb_dmamap->dm_mapsize, 2148 BUS_DMASYNC_POSTWRITE); 2149 bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap); 2150 2151 if_statinc_ref(nsr, if_opackets); 2152 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 2153 if (m->m_flags & M_MCAST) 2154 if_statinc_ref(nsr, if_omcasts); 2155 2156 m_freem(m); 2157 txb->vtxb_m = NULL; 2158 } 2159 2160 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 2161 } 2162 IF_STAT_PUTREF(ifp); 2163 2164 if (txr->vxtxr_head == txr->vxtxr_next) 2165 txq->vxtxq_watchdog = 0; 2166 2167 return more; 2168 } 2169 2170 static int 2171 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq, 2172 struct vmxnet3_rxring *rxr) 2173 { 2174 struct mbuf *m; 2175 struct vmxnet3_rxdesc *rxd; 2176 struct vmxnet3_rxbuf *rxb; 2177 bus_dma_tag_t tag; 2178 bus_dmamap_t dmap; 2179 int idx, btype, error; 2180 2181 tag = sc->vmx_dmat; 2182 dmap = rxr->vxrxr_spare_dmap; 2183 idx = rxr->vxrxr_fill; 2184 rxd = &rxr->vxrxr_rxd[idx]; 2185 rxb = &rxr->vxrxr_rxbuf[idx]; 2186 2187 /* Don't allocate buffers for ring 2 for now. */ 2188 if (rxr->vxrxr_rid != 0) 2189 return -1; 2190 btype = VMXNET3_BTYPE_HEAD; 2191 2192 MGETHDR(m, M_DONTWAIT, MT_DATA); 2193 if (m == NULL) 2194 return (ENOBUFS); 2195 2196 MCLGET(m, M_DONTWAIT); 2197 if ((m->m_flags & M_EXT) == 0) { 2198 rxq->vxrxq_mgetcl_failed.ev_count++; 2199 m_freem(m); 2200 return (ENOBUFS); 2201 } 2202 2203 m->m_pkthdr.len = m->m_len = JUMBO_LEN; 2204 m_adj(m, ETHER_ALIGN); 2205 2206 error = bus_dmamap_load_mbuf(sc->vmx_dmat, dmap, m, BUS_DMA_NOWAIT); 2207 if (error) { 2208 m_freem(m); 2209 rxq->vxrxq_mbuf_load_failed.ev_count++; 2210 return (error); 2211 } 2212 2213 if (rxb->vrxb_m != NULL) { 2214 bus_dmamap_sync(tag, rxb->vrxb_dmamap, 2215 0, rxb->vrxb_dmamap->dm_mapsize, 2216 BUS_DMASYNC_POSTREAD); 2217 bus_dmamap_unload(tag, rxb->vrxb_dmamap); 2218 } 2219 2220 rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 2221 rxb->vrxb_dmamap = dmap; 2222 rxb->vrxb_m = m; 2223 2224 rxd->addr = DMAADDR(dmap); 2225 rxd->len = m->m_pkthdr.len; 2226 rxd->btype = btype; 2227 rxd->gen = rxr->vxrxr_gen; 2228 2229 vmxnet3_rxr_increment_fill(rxr); 2230 return (0); 2231 } 2232 2233 static void 2234 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 2235 struct vmxnet3_rxring *rxr, int idx) 2236 { 2237 struct vmxnet3_rxdesc *rxd; 2238 2239 rxd = &rxr->vxrxr_rxd[idx]; 2240 rxd->gen = rxr->vxrxr_gen; 2241 vmxnet3_rxr_increment_fill(rxr); 2242 } 2243 2244 static void 2245 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 2246 { 2247 struct vmxnet3_softc *sc; 2248 struct vmxnet3_rxring *rxr; 2249 struct vmxnet3_comp_ring *rxc; 2250 struct vmxnet3_rxcompdesc *rxcd; 2251 int idx, eof; 2252 2253 sc = rxq->vxrxq_sc; 2254 rxc = &rxq->vxrxq_comp_ring; 2255 2256 do { 2257 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2258 if (rxcd->gen != rxc->vxcr_gen) 2259 break; /* Not expected. */ 2260 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2261 2262 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2263 rxc->vxcr_next = 0; 2264 rxc->vxcr_gen ^= 1; 2265 } 2266 2267 idx = rxcd->rxd_idx; 2268 eof = rxcd->eop; 2269 if (rxcd->qid < sc->vmx_nrxqueues) 2270 rxr = &rxq->vxrxq_cmd_ring[0]; 2271 else 2272 rxr = &rxq->vxrxq_cmd_ring[1]; 2273 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2274 } while (!eof); 2275 } 2276 2277 static void 2278 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2279 { 2280 if (rxcd->no_csum) 2281 return; 2282 2283 if (rxcd->ipv4) { 2284 m->m_pkthdr.csum_flags |= M_CSUM_IPv4; 2285 if (rxcd->ipcsum_ok == 0) 2286 m->m_pkthdr.csum_flags |= M_CSUM_IPv4_BAD; 2287 } 2288 2289 if (rxcd->fragment) 2290 return; 2291 2292 if (rxcd->tcp) { 2293 m->m_pkthdr.csum_flags |= 2294 rxcd->ipv4 ? M_CSUM_TCPv4 : M_CSUM_TCPv6; 2295 if ((rxcd->csum_ok) == 0) 2296 m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; 2297 } 2298 2299 if (rxcd->udp) { 2300 m->m_pkthdr.csum_flags |= 2301 rxcd->ipv4 ? M_CSUM_UDPv4 : M_CSUM_UDPv6 ; 2302 if ((rxcd->csum_ok) == 0) 2303 m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; 2304 } 2305 } 2306 2307 static void 2308 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 2309 struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2310 { 2311 struct vmxnet3_softc *sc; 2312 struct ifnet *ifp; 2313 2314 sc = rxq->vxrxq_sc; 2315 ifp = &sc->vmx_ethercom.ec_if; 2316 2317 if (rxcd->error) { 2318 if_statinc(ifp, if_ierrors); 2319 m_freem(m); 2320 return; 2321 } 2322 2323 if (!rxcd->no_csum) 2324 vmxnet3_rx_csum(rxcd, m); 2325 if (rxcd->vlan) 2326 vlan_set_tag(m, rxcd->vtag); 2327 2328 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2329 if_statinc_ref(nsr, if_ipackets); 2330 if_statadd_ref(nsr, if_ibytes, m->m_pkthdr.len); 2331 IF_STAT_PUTREF(ifp); 2332 2333 if_percpuq_enqueue(ifp->if_percpuq, m); 2334 } 2335 2336 static bool 2337 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq, u_int limit) 2338 { 2339 struct vmxnet3_softc *sc; 2340 struct ifnet *ifp; 2341 struct vmxnet3_rxring *rxr; 2342 struct vmxnet3_comp_ring *rxc; 2343 struct vmxnet3_rxdesc *rxd __diagused; 2344 struct vmxnet3_rxcompdesc *rxcd; 2345 struct mbuf *m, *m_head, *m_tail; 2346 u_int idx, length; 2347 bool more = false; 2348 2349 sc = rxq->vxrxq_sc; 2350 ifp = &sc->vmx_ethercom.ec_if; 2351 rxc = &rxq->vxrxq_comp_ring; 2352 2353 VMXNET3_RXQ_LOCK_ASSERT(rxq); 2354 2355 if ((ifp->if_flags & IFF_RUNNING) == 0) 2356 return more; 2357 2358 m_head = rxq->vxrxq_mhead; 2359 rxq->vxrxq_mhead = NULL; 2360 m_tail = rxq->vxrxq_mtail; 2361 rxq->vxrxq_mtail = NULL; 2362 KASSERT(m_head == NULL || m_tail != NULL); 2363 2364 for (;;) { 2365 if (limit-- == 0) { 2366 more = true; 2367 break; 2368 } 2369 2370 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2371 if (rxcd->gen != rxc->vxcr_gen) { 2372 rxq->vxrxq_mhead = m_head; 2373 rxq->vxrxq_mtail = m_tail; 2374 break; 2375 } 2376 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2377 2378 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2379 rxc->vxcr_next = 0; 2380 rxc->vxcr_gen ^= 1; 2381 } 2382 2383 idx = rxcd->rxd_idx; 2384 length = rxcd->len; 2385 if (rxcd->qid < sc->vmx_nrxqueues) 2386 rxr = &rxq->vxrxq_cmd_ring[0]; 2387 else 2388 rxr = &rxq->vxrxq_cmd_ring[1]; 2389 rxd = &rxr->vxrxr_rxd[idx]; 2390 2391 m = rxr->vxrxr_rxbuf[idx].vrxb_m; 2392 KASSERT(m != NULL); 2393 2394 /* 2395 * The host may skip descriptors. We detect this when this 2396 * descriptor does not match the previous fill index. Catch 2397 * up with the host now. 2398 */ 2399 if (__predict_false(rxr->vxrxr_fill != idx)) { 2400 while (rxr->vxrxr_fill != idx) { 2401 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 2402 rxr->vxrxr_gen; 2403 vmxnet3_rxr_increment_fill(rxr); 2404 } 2405 } 2406 2407 if (rxcd->sop) { 2408 /* start of frame w/o head buffer */ 2409 KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD); 2410 /* start of frame not in ring 0 */ 2411 KASSERT(rxr == &rxq->vxrxq_cmd_ring[0]); 2412 /* duplicate start of frame? */ 2413 KASSERT(m_head == NULL); 2414 2415 if (length == 0) { 2416 /* Just ignore this descriptor. */ 2417 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2418 goto nextp; 2419 } 2420 2421 if (vmxnet3_newbuf(sc, rxq, rxr) != 0) { 2422 if_statinc(ifp, if_iqdrops); 2423 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2424 if (!rxcd->eop) 2425 vmxnet3_rxq_discard_chain(rxq); 2426 goto nextp; 2427 } 2428 2429 m_set_rcvif(m, ifp); 2430 m->m_pkthdr.len = m->m_len = length; 2431 m->m_pkthdr.csum_flags = 0; 2432 m_head = m_tail = m; 2433 2434 } else { 2435 /* non start of frame w/o body buffer */ 2436 KASSERT(rxd->btype == VMXNET3_BTYPE_BODY); 2437 /* frame not started? */ 2438 KASSERT(m_head != NULL); 2439 2440 if (vmxnet3_newbuf(sc, rxq, rxr) != 0) { 2441 if_statinc(ifp, if_iqdrops); 2442 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2443 if (!rxcd->eop) 2444 vmxnet3_rxq_discard_chain(rxq); 2445 m_freem(m_head); 2446 m_head = m_tail = NULL; 2447 goto nextp; 2448 } 2449 2450 m->m_len = length; 2451 m_head->m_pkthdr.len += length; 2452 m_tail->m_next = m; 2453 m_tail = m; 2454 } 2455 2456 if (rxcd->eop) { 2457 vmxnet3_rxq_input(rxq, rxcd, m_head); 2458 m_head = m_tail = NULL; 2459 2460 /* Must recheck after dropping the Rx lock. */ 2461 if ((ifp->if_flags & IFF_RUNNING) == 0) 2462 break; 2463 } 2464 2465 nextp: 2466 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 2467 int qid = rxcd->qid; 2468 bus_size_t r; 2469 2470 idx = (idx + 1) % rxr->vxrxr_ndesc; 2471 if (qid >= sc->vmx_nrxqueues) { 2472 qid -= sc->vmx_nrxqueues; 2473 r = VMXNET3_BAR0_RXH2(qid); 2474 } else 2475 r = VMXNET3_BAR0_RXH1(qid); 2476 vmxnet3_write_bar0(sc, r, idx); 2477 } 2478 } 2479 2480 return more; 2481 } 2482 2483 static inline void 2484 vmxnet3_sched_handle_queue(struct vmxnet3_softc *sc, struct vmxnet3_queue *vmxq) 2485 { 2486 2487 if (vmxq->vxq_workqueue) { 2488 /* 2489 * When this function is called, "vmxq" is owned by one CPU. 2490 * so, atomic operation is not required here. 2491 */ 2492 if (!vmxq->vxq_wq_enqueued) { 2493 vmxq->vxq_wq_enqueued = true; 2494 workqueue_enqueue(sc->vmx_queue_wq, 2495 &vmxq->vxq_wq_cookie, curcpu()); 2496 } 2497 } else { 2498 softint_schedule(vmxq->vxq_si); 2499 } 2500 } 2501 2502 static int 2503 vmxnet3_legacy_intr(void *xsc) 2504 { 2505 struct vmxnet3_softc *sc; 2506 struct vmxnet3_queue *vmxq; 2507 struct vmxnet3_txqueue *txq; 2508 struct vmxnet3_rxqueue *rxq; 2509 u_int txlimit, rxlimit; 2510 bool txmore, rxmore; 2511 2512 sc = xsc; 2513 vmxq = &sc->vmx_queue[0]; 2514 txq = &vmxq->vxq_txqueue; 2515 rxq = &vmxq->vxq_rxqueue; 2516 txlimit = sc->vmx_tx_intr_process_limit; 2517 rxlimit = sc->vmx_rx_intr_process_limit; 2518 2519 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 2520 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 2521 return (0); 2522 } 2523 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2524 vmxnet3_disable_all_intrs(sc); 2525 2526 if (sc->vmx_ds->event != 0) 2527 vmxnet3_evintr(sc); 2528 2529 VMXNET3_TXQ_LOCK(txq); 2530 txmore = vmxnet3_txq_eof(txq, txlimit); 2531 VMXNET3_TXQ_UNLOCK(txq); 2532 2533 VMXNET3_RXQ_LOCK(rxq); 2534 rxmore = vmxnet3_rxq_eof(rxq, rxlimit); 2535 VMXNET3_RXQ_UNLOCK(rxq); 2536 2537 if (txmore || rxmore) 2538 vmxnet3_sched_handle_queue(sc, vmxq); 2539 else { 2540 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); 2541 vmxnet3_enable_all_intrs(sc); 2542 } 2543 2544 return (1); 2545 } 2546 2547 static int 2548 vmxnet3_txrxq_intr(void *xvmxq) 2549 { 2550 struct vmxnet3_softc *sc; 2551 struct vmxnet3_queue *vmxq; 2552 struct vmxnet3_txqueue *txq; 2553 struct vmxnet3_rxqueue *rxq; 2554 u_int txlimit, rxlimit; 2555 bool txmore, rxmore; 2556 2557 vmxq = xvmxq; 2558 txq = &vmxq->vxq_txqueue; 2559 rxq = &vmxq->vxq_rxqueue; 2560 sc = txq->vxtxq_sc; 2561 txlimit = sc->vmx_tx_intr_process_limit; 2562 rxlimit = sc->vmx_rx_intr_process_limit; 2563 vmxq->vxq_workqueue = sc->vmx_txrx_workqueue; 2564 2565 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2566 vmxnet3_disable_intr(sc, vmxq->vxq_intr_idx); 2567 2568 VMXNET3_TXQ_LOCK(txq); 2569 txq->vxtxq_intr.ev_count++; 2570 txmore = vmxnet3_txq_eof(txq, txlimit); 2571 VMXNET3_TXQ_UNLOCK(txq); 2572 2573 VMXNET3_RXQ_LOCK(rxq); 2574 rxq->vxrxq_intr.ev_count++; 2575 rxmore = vmxnet3_rxq_eof(rxq, rxlimit); 2576 VMXNET3_RXQ_UNLOCK(rxq); 2577 2578 if (txmore || rxmore) 2579 vmxnet3_sched_handle_queue(sc, vmxq); 2580 else { 2581 /* for ALTQ */ 2582 if (vmxq->vxq_id == 0) 2583 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); 2584 softint_schedule(txq->vxtxq_si); 2585 2586 vmxnet3_enable_intr(sc, vmxq->vxq_intr_idx); 2587 } 2588 2589 return (1); 2590 } 2591 2592 static void 2593 vmxnet3_handle_queue(void *xvmxq) 2594 { 2595 struct vmxnet3_softc *sc; 2596 struct vmxnet3_queue *vmxq; 2597 struct vmxnet3_txqueue *txq; 2598 struct vmxnet3_rxqueue *rxq; 2599 u_int txlimit, rxlimit; 2600 bool txmore, rxmore; 2601 2602 vmxq = xvmxq; 2603 txq = &vmxq->vxq_txqueue; 2604 rxq = &vmxq->vxq_rxqueue; 2605 sc = txq->vxtxq_sc; 2606 txlimit = sc->vmx_tx_process_limit; 2607 rxlimit = sc->vmx_rx_process_limit; 2608 2609 VMXNET3_TXQ_LOCK(txq); 2610 txq->vxtxq_defer.ev_count++; 2611 txmore = vmxnet3_txq_eof(txq, txlimit); 2612 if (txmore) 2613 txq->vxtxq_deferreq.ev_count++; 2614 /* for ALTQ */ 2615 if (vmxq->vxq_id == 0) 2616 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); 2617 softint_schedule(txq->vxtxq_si); 2618 VMXNET3_TXQ_UNLOCK(txq); 2619 2620 VMXNET3_RXQ_LOCK(rxq); 2621 rxq->vxrxq_defer.ev_count++; 2622 rxmore = vmxnet3_rxq_eof(rxq, rxlimit); 2623 if (rxmore) 2624 rxq->vxrxq_deferreq.ev_count++; 2625 VMXNET3_RXQ_UNLOCK(rxq); 2626 2627 if (txmore || rxmore) 2628 vmxnet3_sched_handle_queue(sc, vmxq); 2629 else 2630 vmxnet3_enable_intr(sc, vmxq->vxq_intr_idx); 2631 } 2632 2633 static void 2634 vmxnet3_handle_queue_work(struct work *wk, void *context) 2635 { 2636 struct vmxnet3_queue *vmxq; 2637 2638 vmxq = container_of(wk, struct vmxnet3_queue, vxq_wq_cookie); 2639 vmxq->vxq_wq_enqueued = false; 2640 vmxnet3_handle_queue(vmxq); 2641 } 2642 2643 static int 2644 vmxnet3_event_intr(void *xsc) 2645 { 2646 struct vmxnet3_softc *sc; 2647 2648 sc = xsc; 2649 2650 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2651 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 2652 2653 sc->vmx_event_intr.ev_count++; 2654 2655 if (sc->vmx_ds->event != 0) 2656 vmxnet3_evintr(sc); 2657 2658 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2659 2660 return (1); 2661 } 2662 2663 static void 2664 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2665 { 2666 struct vmxnet3_txring *txr; 2667 struct vmxnet3_txbuf *txb; 2668 u_int i; 2669 2670 txr = &txq->vxtxq_cmd_ring; 2671 2672 for (i = 0; i < txr->vxtxr_ndesc; i++) { 2673 txb = &txr->vxtxr_txbuf[i]; 2674 2675 if (txb->vtxb_m == NULL) 2676 continue; 2677 2678 bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap, 2679 0, txb->vtxb_dmamap->dm_mapsize, 2680 BUS_DMASYNC_POSTWRITE); 2681 bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap); 2682 m_freem(txb->vtxb_m); 2683 txb->vtxb_m = NULL; 2684 } 2685 } 2686 2687 static void 2688 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2689 { 2690 struct vmxnet3_rxring *rxr; 2691 struct vmxnet3_rxbuf *rxb; 2692 u_int i, j; 2693 2694 if (rxq->vxrxq_mhead != NULL) { 2695 m_freem(rxq->vxrxq_mhead); 2696 rxq->vxrxq_mhead = NULL; 2697 rxq->vxrxq_mtail = NULL; 2698 } 2699 2700 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2701 rxr = &rxq->vxrxq_cmd_ring[i]; 2702 2703 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2704 rxb = &rxr->vxrxr_rxbuf[j]; 2705 2706 if (rxb->vrxb_m == NULL) 2707 continue; 2708 2709 bus_dmamap_sync(sc->vmx_dmat, rxb->vrxb_dmamap, 2710 0, rxb->vrxb_dmamap->dm_mapsize, 2711 BUS_DMASYNC_POSTREAD); 2712 bus_dmamap_unload(sc->vmx_dmat, rxb->vrxb_dmamap); 2713 m_freem(rxb->vrxb_m); 2714 rxb->vrxb_m = NULL; 2715 } 2716 } 2717 } 2718 2719 static void 2720 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2721 { 2722 struct vmxnet3_rxqueue *rxq; 2723 struct vmxnet3_txqueue *txq; 2724 struct vmxnet3_queue *vmxq; 2725 int i; 2726 2727 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2728 rxq = &sc->vmx_queue[i].vxq_rxqueue; 2729 VMXNET3_RXQ_LOCK(rxq); 2730 VMXNET3_RXQ_UNLOCK(rxq); 2731 } 2732 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2733 txq = &sc->vmx_queue[i].vxq_txqueue; 2734 VMXNET3_TXQ_LOCK(txq); 2735 VMXNET3_TXQ_UNLOCK(txq); 2736 } 2737 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2738 vmxq = &sc->vmx_queue[i]; 2739 workqueue_wait(sc->vmx_queue_wq, &vmxq->vxq_wq_cookie); 2740 } 2741 } 2742 2743 static void 2744 vmxnet3_stop_locked(struct vmxnet3_softc *sc) 2745 { 2746 struct ifnet *ifp; 2747 int q; 2748 2749 ifp = &sc->vmx_ethercom.ec_if; 2750 VMXNET3_CORE_LOCK_ASSERT(sc); 2751 2752 ifp->if_flags &= ~IFF_RUNNING; 2753 sc->vmx_link_active = 0; 2754 callout_stop(&sc->vmx_tick); 2755 2756 /* Disable interrupts. */ 2757 vmxnet3_disable_all_intrs(sc); 2758 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2759 2760 vmxnet3_stop_rendezvous(sc); 2761 2762 for (q = 0; q < sc->vmx_ntxqueues; q++) 2763 vmxnet3_txstop(sc, &sc->vmx_queue[q].vxq_txqueue); 2764 for (q = 0; q < sc->vmx_nrxqueues; q++) 2765 vmxnet3_rxstop(sc, &sc->vmx_queue[q].vxq_rxqueue); 2766 2767 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2768 } 2769 2770 static void 2771 vmxnet3_stop(struct ifnet *ifp, int disable) 2772 { 2773 struct vmxnet3_softc *sc = ifp->if_softc; 2774 2775 VMXNET3_CORE_LOCK(sc); 2776 vmxnet3_stop_locked(sc); 2777 VMXNET3_CORE_UNLOCK(sc); 2778 } 2779 2780 static void 2781 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2782 { 2783 struct vmxnet3_txring *txr; 2784 struct vmxnet3_comp_ring *txc; 2785 2786 txr = &txq->vxtxq_cmd_ring; 2787 txr->vxtxr_head = 0; 2788 txr->vxtxr_next = 0; 2789 txr->vxtxr_gen = VMXNET3_INIT_GEN; 2790 memset(txr->vxtxr_txd, 0, 2791 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2792 2793 txc = &txq->vxtxq_comp_ring; 2794 txc->vxcr_next = 0; 2795 txc->vxcr_gen = VMXNET3_INIT_GEN; 2796 memset(txc->vxcr_u.txcd, 0, 2797 txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2798 } 2799 2800 static int 2801 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2802 { 2803 struct vmxnet3_rxring *rxr; 2804 struct vmxnet3_comp_ring *rxc; 2805 u_int i, populate, idx; 2806 int error; 2807 2808 /* LRO and jumbo frame is not supported yet */ 2809 populate = 1; 2810 2811 for (i = 0; i < populate; i++) { 2812 rxr = &rxq->vxrxq_cmd_ring[i]; 2813 rxr->vxrxr_fill = 0; 2814 rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2815 memset(rxr->vxrxr_rxd, 0, 2816 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2817 2818 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2819 error = vmxnet3_newbuf(sc, rxq, rxr); 2820 if (error) 2821 return (error); 2822 } 2823 } 2824 2825 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2826 rxr = &rxq->vxrxq_cmd_ring[i]; 2827 rxr->vxrxr_fill = 0; 2828 rxr->vxrxr_gen = 0; 2829 memset(rxr->vxrxr_rxd, 0, 2830 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2831 } 2832 2833 rxc = &rxq->vxrxq_comp_ring; 2834 rxc->vxcr_next = 0; 2835 rxc->vxcr_gen = VMXNET3_INIT_GEN; 2836 memset(rxc->vxcr_u.rxcd, 0, 2837 rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2838 2839 return (0); 2840 } 2841 2842 static int 2843 vmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2844 { 2845 device_t dev; 2846 int q, error; 2847 dev = sc->vmx_dev; 2848 2849 for (q = 0; q < sc->vmx_ntxqueues; q++) 2850 vmxnet3_txinit(sc, &sc->vmx_queue[q].vxq_txqueue); 2851 2852 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2853 error = vmxnet3_rxinit(sc, &sc->vmx_queue[q].vxq_rxqueue); 2854 if (error) { 2855 device_printf(dev, "cannot populate Rx queue %d\n", q); 2856 return (error); 2857 } 2858 } 2859 2860 return (0); 2861 } 2862 2863 static int 2864 vmxnet3_enable_device(struct vmxnet3_softc *sc) 2865 { 2866 int q; 2867 2868 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2869 device_printf(sc->vmx_dev, "device enable command failed!\n"); 2870 return (1); 2871 } 2872 2873 /* Reset the Rx queue heads. */ 2874 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2875 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2876 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2877 } 2878 2879 return (0); 2880 } 2881 2882 static void 2883 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2884 { 2885 2886 vmxnet3_set_rxfilter(sc); 2887 2888 memset(sc->vmx_ds->vlan_filter, 0, sizeof(sc->vmx_ds->vlan_filter)); 2889 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2890 } 2891 2892 static int 2893 vmxnet3_reinit(struct vmxnet3_softc *sc) 2894 { 2895 2896 vmxnet3_set_lladdr(sc); 2897 vmxnet3_reinit_shared_data(sc); 2898 2899 if (vmxnet3_reinit_queues(sc) != 0) 2900 return (ENXIO); 2901 2902 if (vmxnet3_enable_device(sc) != 0) 2903 return (ENXIO); 2904 2905 vmxnet3_reinit_rxfilters(sc); 2906 2907 return (0); 2908 } 2909 2910 static int 2911 vmxnet3_init_locked(struct vmxnet3_softc *sc) 2912 { 2913 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 2914 int error; 2915 2916 vmxnet3_stop_locked(sc); 2917 2918 error = vmxnet3_reinit(sc); 2919 if (error) { 2920 vmxnet3_stop_locked(sc); 2921 return (error); 2922 } 2923 2924 ifp->if_flags |= IFF_RUNNING; 2925 vmxnet3_if_link_status(sc); 2926 2927 vmxnet3_enable_all_intrs(sc); 2928 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2929 2930 return (0); 2931 } 2932 2933 static int 2934 vmxnet3_init(struct ifnet *ifp) 2935 { 2936 struct vmxnet3_softc *sc = ifp->if_softc; 2937 int error; 2938 2939 VMXNET3_CORE_LOCK(sc); 2940 error = vmxnet3_init_locked(sc); 2941 VMXNET3_CORE_UNLOCK(sc); 2942 2943 return (error); 2944 } 2945 2946 static int 2947 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, 2948 int *start, int *csum_start) 2949 { 2950 struct ether_header *eh; 2951 struct mbuf *mp; 2952 int offset, csum_off, iphl, offp; 2953 bool v4; 2954 2955 eh = mtod(m, struct ether_header *); 2956 switch (htons(eh->ether_type)) { 2957 case ETHERTYPE_IP: 2958 case ETHERTYPE_IPV6: 2959 offset = ETHER_HDR_LEN; 2960 break; 2961 case ETHERTYPE_VLAN: 2962 offset = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2963 break; 2964 default: 2965 m_freem(m); 2966 return (EINVAL); 2967 } 2968 2969 if ((m->m_pkthdr.csum_flags & 2970 (M_CSUM_TSOv4 | M_CSUM_UDPv4 | M_CSUM_TCPv4)) != 0) { 2971 iphl = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); 2972 v4 = true; 2973 } else { 2974 iphl = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); 2975 v4 = false; 2976 } 2977 *start = offset + iphl; 2978 2979 if (m->m_pkthdr.csum_flags & 2980 (M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_TSOv4 | M_CSUM_TSOv6)) { 2981 csum_off = offsetof(struct tcphdr, th_sum); 2982 } else { 2983 csum_off = offsetof(struct udphdr, uh_sum); 2984 } 2985 2986 *csum_start = *start + csum_off; 2987 mp = m_pulldown(m, 0, *csum_start + 2, &offp); 2988 if (!mp) { 2989 /* m is already freed */ 2990 return ENOBUFS; 2991 } 2992 2993 if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) { 2994 struct tcphdr *tcp; 2995 2996 txq->vxtxq_stats.vmtxs_tso++; 2997 tcp = (void *)(mtod(mp, char *) + offp + *start); 2998 2999 if (v4) { 3000 struct ip *ip; 3001 3002 ip = (void *)(mtod(mp, char *) + offp + offset); 3003 tcp->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 3004 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3005 } else { 3006 struct ip6_hdr *ip6; 3007 3008 ip6 = (void *)(mtod(mp, char *) + offp + offset); 3009 tcp->th_sum = in6_cksum_phdr(&ip6->ip6_src, 3010 &ip6->ip6_dst, 0, htonl(IPPROTO_TCP)); 3011 } 3012 3013 /* 3014 * For TSO, the size of the protocol header is also 3015 * included in the descriptor header size. 3016 */ 3017 *start += (tcp->th_off << 2); 3018 } else 3019 txq->vxtxq_stats.vmtxs_csum++; 3020 3021 return (0); 3022 } 3023 3024 static int 3025 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 3026 bus_dmamap_t dmap) 3027 { 3028 struct mbuf *m; 3029 bus_dma_tag_t tag; 3030 int error; 3031 3032 m = *m0; 3033 tag = txq->vxtxq_sc->vmx_dmat; 3034 3035 error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT); 3036 if (error == 0 || error != EFBIG) 3037 return (error); 3038 3039 m = m_defrag(m, M_NOWAIT); 3040 if (m != NULL) { 3041 *m0 = m; 3042 error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT); 3043 } else 3044 error = ENOBUFS; 3045 3046 if (error) { 3047 m_freem(*m0); 3048 *m0 = NULL; 3049 txq->vxtxq_defrag_failed.ev_count++; 3050 } else 3051 txq->vxtxq_defragged.ev_count++; 3052 3053 return (error); 3054 } 3055 3056 static void 3057 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 3058 { 3059 3060 bus_dmamap_unload(txq->vxtxq_sc->vmx_dmat, dmap); 3061 } 3062 3063 static int 3064 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 3065 { 3066 struct vmxnet3_softc *sc; 3067 struct vmxnet3_txring *txr; 3068 struct vmxnet3_txdesc *txd, *sop; 3069 struct mbuf *m; 3070 bus_dmamap_t dmap; 3071 bus_dma_segment_t *segs; 3072 int i, gen, start, csum_start, nsegs, error; 3073 3074 sc = txq->vxtxq_sc; 3075 start = 0; 3076 txd = NULL; 3077 txr = &txq->vxtxq_cmd_ring; 3078 dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 3079 csum_start = 0; /* GCC */ 3080 3081 error = vmxnet3_txq_load_mbuf(txq, m0, dmap); 3082 if (error) 3083 return (error); 3084 3085 nsegs = dmap->dm_nsegs; 3086 segs = dmap->dm_segs; 3087 3088 m = *m0; 3089 KASSERT(m->m_flags & M_PKTHDR); 3090 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS); 3091 3092 if (vmxnet3_txring_avail(txr) < nsegs) { 3093 txq->vxtxq_stats.vmtxs_full++; 3094 vmxnet3_txq_unload_mbuf(txq, dmap); 3095 return (ENOSPC); 3096 } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 3097 error = vmxnet3_txq_offload_ctx(txq, m, &start, &csum_start); 3098 if (error) { 3099 /* m is already freed */ 3100 txq->vxtxq_stats.vmtxs_offload_failed++; 3101 vmxnet3_txq_unload_mbuf(txq, dmap); 3102 *m0 = NULL; 3103 return (error); 3104 } 3105 } 3106 3107 txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m; 3108 sop = &txr->vxtxr_txd[txr->vxtxr_head]; 3109 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 3110 3111 for (i = 0; i < nsegs; i++) { 3112 txd = &txr->vxtxr_txd[txr->vxtxr_head]; 3113 3114 txd->addr = segs[i].ds_addr; 3115 txd->len = segs[i].ds_len; 3116 txd->gen = gen; 3117 txd->dtype = 0; 3118 txd->offload_mode = VMXNET3_OM_NONE; 3119 txd->offload_pos = 0; 3120 txd->hlen = 0; 3121 txd->eop = 0; 3122 txd->compreq = 0; 3123 txd->vtag_mode = 0; 3124 txd->vtag = 0; 3125 3126 if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 3127 txr->vxtxr_head = 0; 3128 txr->vxtxr_gen ^= 1; 3129 } 3130 gen = txr->vxtxr_gen; 3131 } 3132 txd->eop = 1; 3133 txd->compreq = 1; 3134 3135 if (vlan_has_tag(m)) { 3136 sop->vtag_mode = 1; 3137 sop->vtag = vlan_get_tag(m); 3138 } 3139 3140 if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) { 3141 sop->offload_mode = VMXNET3_OM_TSO; 3142 sop->hlen = start; 3143 sop->offload_pos = m->m_pkthdr.segsz; 3144 } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 3145 VMXNET3_CSUM_OFFLOAD_IPV6)) { 3146 sop->offload_mode = VMXNET3_OM_CSUM; 3147 sop->hlen = start; 3148 sop->offload_pos = csum_start; 3149 } 3150 3151 /* Finally, change the ownership. */ 3152 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 3153 sop->gen ^= 1; 3154 3155 txq->vxtxq_ts->npending += nsegs; 3156 if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 3157 struct vmxnet3_queue *vmxq; 3158 vmxq = container_of(txq, struct vmxnet3_queue, vxq_txqueue); 3159 txq->vxtxq_ts->npending = 0; 3160 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(vmxq->vxq_id), 3161 txr->vxtxr_head); 3162 } 3163 3164 return (0); 3165 } 3166 3167 #define VMXNET3_TX_START 1 3168 #define VMXNET3_TX_TRANSMIT 2 3169 static inline void 3170 vmxnet3_tx_common_locked(struct ifnet *ifp, struct vmxnet3_txqueue *txq, int txtype) 3171 { 3172 struct vmxnet3_softc *sc; 3173 struct vmxnet3_txring *txr; 3174 struct mbuf *m_head; 3175 int tx; 3176 3177 sc = ifp->if_softc; 3178 txr = &txq->vxtxq_cmd_ring; 3179 tx = 0; 3180 3181 VMXNET3_TXQ_LOCK_ASSERT(txq); 3182 3183 if ((ifp->if_flags & IFF_RUNNING) == 0 || 3184 sc->vmx_link_active == 0) 3185 return; 3186 3187 for (;;) { 3188 if (txtype == VMXNET3_TX_START) 3189 IFQ_POLL(&ifp->if_snd, m_head); 3190 else 3191 m_head = pcq_peek(txq->vxtxq_interq); 3192 if (m_head == NULL) 3193 break; 3194 3195 if (vmxnet3_txring_avail(txr) < VMXNET3_TX_MAXSEGS) 3196 break; 3197 3198 if (txtype == VMXNET3_TX_START) 3199 IFQ_DEQUEUE(&ifp->if_snd, m_head); 3200 else 3201 m_head = pcq_get(txq->vxtxq_interq); 3202 if (m_head == NULL) 3203 break; 3204 3205 if (vmxnet3_txq_encap(txq, &m_head) != 0) { 3206 if (m_head != NULL) 3207 m_freem(m_head); 3208 break; 3209 } 3210 3211 tx++; 3212 bpf_mtap(ifp, m_head, BPF_D_OUT); 3213 } 3214 3215 if (tx > 0) 3216 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 3217 } 3218 3219 static void 3220 vmxnet3_start_locked(struct ifnet *ifp) 3221 { 3222 struct vmxnet3_softc *sc; 3223 struct vmxnet3_txqueue *txq; 3224 3225 sc = ifp->if_softc; 3226 txq = &sc->vmx_queue[0].vxq_txqueue; 3227 3228 vmxnet3_tx_common_locked(ifp, txq, VMXNET3_TX_START); 3229 } 3230 3231 void 3232 vmxnet3_start(struct ifnet *ifp) 3233 { 3234 struct vmxnet3_softc *sc; 3235 struct vmxnet3_txqueue *txq; 3236 3237 sc = ifp->if_softc; 3238 txq = &sc->vmx_queue[0].vxq_txqueue; 3239 3240 VMXNET3_TXQ_LOCK(txq); 3241 vmxnet3_start_locked(ifp); 3242 VMXNET3_TXQ_UNLOCK(txq); 3243 } 3244 3245 static int 3246 vmxnet3_select_txqueue(struct ifnet *ifp, struct mbuf *m __unused) 3247 { 3248 struct vmxnet3_softc *sc; 3249 u_int cpuid; 3250 3251 sc = ifp->if_softc; 3252 cpuid = cpu_index(curcpu()); 3253 /* 3254 * Furure work 3255 * We should select txqueue to even up the load even if ncpu is 3256 * different from sc->vmx_ntxqueues. Currently, the load is not 3257 * even, that is, when ncpu is six and ntxqueues is four, the load 3258 * of vmx_queue[0] and vmx_queue[1] is higher than vmx_queue[2] and 3259 * vmx_queue[3] because CPU#4 always uses vmx_queue[0] and CPU#5 always 3260 * uses vmx_queue[1]. 3261 * Furthermore, we should not use random value to select txqueue to 3262 * avoid reordering. We should use flow information of mbuf. 3263 */ 3264 return cpuid % sc->vmx_ntxqueues; 3265 } 3266 3267 static void 3268 vmxnet3_transmit_locked(struct ifnet *ifp, struct vmxnet3_txqueue *txq) 3269 { 3270 3271 vmxnet3_tx_common_locked(ifp, txq, VMXNET3_TX_TRANSMIT); 3272 } 3273 3274 static int 3275 vmxnet3_transmit(struct ifnet *ifp, struct mbuf *m) 3276 { 3277 struct vmxnet3_softc *sc; 3278 struct vmxnet3_txqueue *txq; 3279 int qid; 3280 3281 qid = vmxnet3_select_txqueue(ifp, m); 3282 sc = ifp->if_softc; 3283 txq = &sc->vmx_queue[qid].vxq_txqueue; 3284 3285 if (__predict_false(!pcq_put(txq->vxtxq_interq, m))) { 3286 VMXNET3_TXQ_LOCK(txq); 3287 txq->vxtxq_pcqdrop.ev_count++; 3288 VMXNET3_TXQ_UNLOCK(txq); 3289 m_freem(m); 3290 return ENOBUFS; 3291 } 3292 3293 #ifdef VMXNET3_ALWAYS_TXDEFER 3294 kpreempt_disable(); 3295 softint_schedule(txq->vxtxq_si); 3296 kpreempt_enable(); 3297 #else 3298 if (VMXNET3_TXQ_TRYLOCK(txq)) { 3299 vmxnet3_transmit_locked(ifp, txq); 3300 VMXNET3_TXQ_UNLOCK(txq); 3301 } else { 3302 kpreempt_disable(); 3303 softint_schedule(txq->vxtxq_si); 3304 kpreempt_enable(); 3305 } 3306 #endif 3307 3308 return 0; 3309 } 3310 3311 static void 3312 vmxnet3_deferred_transmit(void *arg) 3313 { 3314 struct vmxnet3_txqueue *txq = arg; 3315 struct vmxnet3_softc *sc = txq->vxtxq_sc; 3316 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 3317 3318 VMXNET3_TXQ_LOCK(txq); 3319 txq->vxtxq_transmitdef.ev_count++; 3320 if (pcq_peek(txq->vxtxq_interq) != NULL) 3321 vmxnet3_transmit_locked(ifp, txq); 3322 VMXNET3_TXQ_UNLOCK(txq); 3323 } 3324 3325 static void 3326 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 3327 { 3328 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 3329 struct ethercom *ec = &sc->vmx_ethercom; 3330 struct vmxnet3_driver_shared *ds = sc->vmx_ds; 3331 struct ether_multi *enm; 3332 struct ether_multistep step; 3333 u_int mode; 3334 uint8_t *p; 3335 3336 ds->mcast_tablelen = 0; 3337 ETHER_LOCK(ec); 3338 CLR(ec->ec_flags, ETHER_F_ALLMULTI); 3339 ETHER_UNLOCK(ec); 3340 3341 /* 3342 * Always accept broadcast frames. 3343 * Always accept frames destined to our station address. 3344 */ 3345 mode = VMXNET3_RXMODE_BCAST | VMXNET3_RXMODE_UCAST; 3346 3347 ETHER_LOCK(ec); 3348 if (ISSET(ifp->if_flags, IFF_PROMISC) || 3349 ec->ec_multicnt > VMXNET3_MULTICAST_MAX) 3350 goto allmulti; 3351 3352 p = sc->vmx_mcast; 3353 ETHER_FIRST_MULTI(step, ec, enm); 3354 while (enm != NULL) { 3355 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) { 3356 /* 3357 * We must listen to a range of multicast addresses. 3358 * For now, just accept all multicasts, rather than 3359 * trying to set only those filter bits needed to match 3360 * the range. (At this time, the only use of address 3361 * ranges is for IP multicast routing, for which the 3362 * range is big enough to require all bits set.) 3363 */ 3364 goto allmulti; 3365 } 3366 memcpy(p, enm->enm_addrlo, ETHER_ADDR_LEN); 3367 3368 p += ETHER_ADDR_LEN; 3369 3370 ETHER_NEXT_MULTI(step, enm); 3371 } 3372 3373 if (ec->ec_multicnt > 0) { 3374 SET(mode, VMXNET3_RXMODE_MCAST); 3375 ds->mcast_tablelen = p - sc->vmx_mcast; 3376 } 3377 ETHER_UNLOCK(ec); 3378 3379 goto setit; 3380 3381 allmulti: 3382 SET(ec->ec_flags, ETHER_F_ALLMULTI); 3383 ETHER_UNLOCK(ec); 3384 SET(mode, (VMXNET3_RXMODE_ALLMULTI | VMXNET3_RXMODE_MCAST)); 3385 if (ifp->if_flags & IFF_PROMISC) 3386 SET(mode, VMXNET3_RXMODE_PROMISC); 3387 3388 setit: 3389 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 3390 ds->rxmode = mode; 3391 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 3392 } 3393 3394 static int 3395 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, void *data) 3396 { 3397 struct vmxnet3_softc *sc = ifp->if_softc; 3398 struct ifreq *ifr = (struct ifreq *)data; 3399 int s, error = 0; 3400 3401 switch (cmd) { 3402 case SIOCSIFMTU: { 3403 int nmtu = ifr->ifr_mtu; 3404 3405 if (nmtu < VMXNET3_MIN_MTU || nmtu > VMXNET3_MAX_MTU) { 3406 error = EINVAL; 3407 break; 3408 } 3409 if (ifp->if_mtu != (uint64_t)nmtu) { 3410 s = splnet(); 3411 error = ether_ioctl(ifp, cmd, data); 3412 splx(s); 3413 if (error == ENETRESET) 3414 error = vmxnet3_init(ifp); 3415 } 3416 break; 3417 } 3418 3419 default: 3420 s = splnet(); 3421 error = ether_ioctl(ifp, cmd, data); 3422 splx(s); 3423 } 3424 3425 if (error == ENETRESET) { 3426 VMXNET3_CORE_LOCK(sc); 3427 if (ifp->if_flags & IFF_RUNNING) 3428 vmxnet3_set_rxfilter(sc); 3429 VMXNET3_CORE_UNLOCK(sc); 3430 error = 0; 3431 } 3432 3433 return error; 3434 } 3435 3436 static int 3437 vmxnet3_ifflags_cb(struct ethercom *ec) 3438 { 3439 struct vmxnet3_softc *sc; 3440 3441 sc = ec->ec_if.if_softc; 3442 3443 VMXNET3_CORE_LOCK(sc); 3444 vmxnet3_set_rxfilter(sc); 3445 VMXNET3_CORE_UNLOCK(sc); 3446 3447 vmxnet3_if_link_status(sc); 3448 3449 return 0; 3450 } 3451 3452 static int 3453 vmxnet3_watchdog(struct vmxnet3_txqueue *txq) 3454 { 3455 struct vmxnet3_softc *sc; 3456 struct vmxnet3_queue *vmxq; 3457 3458 sc = txq->vxtxq_sc; 3459 vmxq = container_of(txq, struct vmxnet3_queue, vxq_txqueue); 3460 3461 VMXNET3_TXQ_LOCK(txq); 3462 if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 3463 VMXNET3_TXQ_UNLOCK(txq); 3464 return (0); 3465 } 3466 txq->vxtxq_watchdogto.ev_count++; 3467 VMXNET3_TXQ_UNLOCK(txq); 3468 3469 device_printf(sc->vmx_dev, "watchdog timeout on queue %d\n", 3470 vmxq->vxq_id); 3471 return (1); 3472 } 3473 3474 static void 3475 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) 3476 { 3477 3478 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 3479 } 3480 3481 static void 3482 vmxnet3_tick(void *xsc) 3483 { 3484 struct vmxnet3_softc *sc; 3485 int i, timedout; 3486 3487 sc = xsc; 3488 timedout = 0; 3489 3490 VMXNET3_CORE_LOCK(sc); 3491 3492 vmxnet3_refresh_host_stats(sc); 3493 3494 for (i = 0; i < sc->vmx_ntxqueues; i++) 3495 timedout |= vmxnet3_watchdog(&sc->vmx_queue[i].vxq_txqueue); 3496 3497 if (timedout != 0) 3498 vmxnet3_init_locked(sc); 3499 else 3500 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 3501 3502 VMXNET3_CORE_UNLOCK(sc); 3503 } 3504 3505 /* 3506 * update link state of ifnet and softc 3507 */ 3508 static void 3509 vmxnet3_if_link_status(struct vmxnet3_softc *sc) 3510 { 3511 struct ifnet *ifp = &sc->vmx_ethercom.ec_if; 3512 u_int link; 3513 bool up; 3514 3515 up = vmxnet3_cmd_link_status(ifp); 3516 if (up) { 3517 sc->vmx_link_active = 1; 3518 link = LINK_STATE_UP; 3519 } else { 3520 sc->vmx_link_active = 0; 3521 link = LINK_STATE_DOWN; 3522 } 3523 3524 if_link_state_change(ifp, link); 3525 } 3526 3527 /* 3528 * check vmx(4) state by VMXNET3_CMD and update ifp->if_baudrate 3529 * returns 3530 * - true: link up 3531 * - flase: link down 3532 */ 3533 static bool 3534 vmxnet3_cmd_link_status(struct ifnet *ifp) 3535 { 3536 struct vmxnet3_softc *sc = ifp->if_softc; 3537 u_int x, speed; 3538 3539 x = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 3540 if ((x & 1) == 0) 3541 return false; 3542 3543 speed = x >> 16; 3544 ifp->if_baudrate = IF_Mbps(speed); 3545 return true; 3546 } 3547 3548 static void 3549 vmxnet3_ifmedia_status(struct ifnet *ifp, struct ifmediareq *ifmr) 3550 { 3551 bool up; 3552 3553 ifmr->ifm_status = IFM_AVALID; 3554 ifmr->ifm_active = IFM_ETHER; 3555 3556 up = vmxnet3_cmd_link_status(ifp); 3557 if (!up) 3558 return; 3559 3560 ifmr->ifm_status |= IFM_ACTIVE; 3561 3562 if (ifp->if_baudrate >= IF_Gbps(10ULL)) 3563 ifmr->ifm_active |= IFM_10G_T; 3564 } 3565 3566 static int 3567 vmxnet3_ifmedia_change(struct ifnet *ifp) 3568 { 3569 return 0; 3570 } 3571 3572 static void 3573 vmxnet3_set_lladdr(struct vmxnet3_softc *sc) 3574 { 3575 uint32_t ml, mh; 3576 3577 ml = sc->vmx_lladdr[0]; 3578 ml |= sc->vmx_lladdr[1] << 8; 3579 ml |= sc->vmx_lladdr[2] << 16; 3580 ml |= sc->vmx_lladdr[3] << 24; 3581 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 3582 3583 mh = sc->vmx_lladdr[4]; 3584 mh |= sc->vmx_lladdr[5] << 8; 3585 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 3586 } 3587 3588 static void 3589 vmxnet3_get_lladdr(struct vmxnet3_softc *sc) 3590 { 3591 uint32_t ml, mh; 3592 3593 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 3594 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 3595 3596 sc->vmx_lladdr[0] = ml; 3597 sc->vmx_lladdr[1] = ml >> 8; 3598 sc->vmx_lladdr[2] = ml >> 16; 3599 sc->vmx_lladdr[3] = ml >> 24; 3600 sc->vmx_lladdr[4] = mh; 3601 sc->vmx_lladdr[5] = mh >> 8; 3602 } 3603 3604 static void 3605 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3606 { 3607 int i; 3608 3609 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3610 for (i = 0; i < sc->vmx_nintrs; i++) 3611 vmxnet3_enable_intr(sc, i); 3612 } 3613 3614 static void 3615 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3616 { 3617 int i; 3618 3619 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3620 for (i = 0; i < sc->vmx_nintrs; i++) 3621 vmxnet3_disable_intr(sc, i); 3622 } 3623 3624 static int 3625 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3626 struct vmxnet3_dma_alloc *dma) 3627 { 3628 bus_dma_tag_t t = sc->vmx_dmat; 3629 bus_dma_segment_t *segs = dma->dma_segs; 3630 int n, error; 3631 3632 memset(dma, 0, sizeof(*dma)); 3633 3634 error = bus_dmamem_alloc(t, size, align, 0, segs, 1, &n, BUS_DMA_NOWAIT); 3635 if (error) { 3636 aprint_error_dev(sc->vmx_dev, "bus_dmamem_alloc failed: %d\n", error); 3637 goto fail1; 3638 } 3639 KASSERT(n == 1); 3640 3641 error = bus_dmamem_map(t, segs, 1, size, &dma->dma_vaddr, BUS_DMA_NOWAIT); 3642 if (error) { 3643 aprint_error_dev(sc->vmx_dev, "bus_dmamem_map failed: %d\n", error); 3644 goto fail2; 3645 } 3646 3647 error = bus_dmamap_create(t, size, 1, size, 0, BUS_DMA_NOWAIT, &dma->dma_map); 3648 if (error) { 3649 aprint_error_dev(sc->vmx_dev, "bus_dmamap_create failed: %d\n", error); 3650 goto fail3; 3651 } 3652 3653 error = bus_dmamap_load(t, dma->dma_map, dma->dma_vaddr, size, NULL, 3654 BUS_DMA_NOWAIT); 3655 if (error) { 3656 aprint_error_dev(sc->vmx_dev, "bus_dmamap_load failed: %d\n", error); 3657 goto fail4; 3658 } 3659 3660 memset(dma->dma_vaddr, 0, size); 3661 dma->dma_paddr = DMAADDR(dma->dma_map); 3662 dma->dma_size = size; 3663 3664 return (0); 3665 fail4: 3666 bus_dmamap_destroy(t, dma->dma_map); 3667 fail3: 3668 bus_dmamem_unmap(t, dma->dma_vaddr, size); 3669 fail2: 3670 bus_dmamem_free(t, segs, 1); 3671 fail1: 3672 return (error); 3673 } 3674 3675 static void 3676 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 3677 { 3678 bus_dma_tag_t t = sc->vmx_dmat; 3679 3680 bus_dmamap_unload(t, dma->dma_map); 3681 bus_dmamap_destroy(t, dma->dma_map); 3682 bus_dmamem_unmap(t, dma->dma_vaddr, dma->dma_size); 3683 bus_dmamem_free(t, dma->dma_segs, 1); 3684 3685 memset(dma, 0, sizeof(*dma)); 3686 } 3687 3688 MODULE(MODULE_CLASS_DRIVER, if_vmx, "pci"); 3689 3690 #ifdef _MODULE 3691 #include "ioconf.c" 3692 #endif 3693 3694 static int 3695 if_vmx_modcmd(modcmd_t cmd, void *opaque) 3696 { 3697 int error = 0; 3698 3699 switch (cmd) { 3700 case MODULE_CMD_INIT: 3701 #ifdef _MODULE 3702 error = config_init_component(cfdriver_ioconf_if_vmx, 3703 cfattach_ioconf_if_vmx, cfdata_ioconf_if_vmx); 3704 #endif 3705 return error; 3706 case MODULE_CMD_FINI: 3707 #ifdef _MODULE 3708 error = config_fini_component(cfdriver_ioconf_if_vmx, 3709 cfattach_ioconf_if_vmx, cfdata_ioconf_if_vmx); 3710 #endif 3711 return error; 3712 default: 3713 return ENOTTY; 3714 } 3715 } 3716 3717
Indexes created Tue Sep 23 14:10:03 GMT 2025