if_vioif.c revision 1.115
1 /* $NetBSD: if_vioif.c,v 1.115 2025/07/15 05:09:28 ozaki-r Exp $ */ 2 3 /* 4 * Copyright (c) 2020 The NetBSD Foundation, Inc. 5 * Copyright (c) 2010 Minoura Makoto. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.115 2025/07/15 05:09:28 ozaki-r Exp $"); 31 32 #ifdef _KERNEL_OPT 33 #include "opt_net_mpsafe.h" 34 #endif 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/kernel.h> 39 #include <sys/atomic.h> 40 #include <sys/bus.h> 41 #include <sys/condvar.h> 42 #include <sys/device.h> 43 #include <sys/evcnt.h> 44 #include <sys/intr.h> 45 #include <sys/kmem.h> 46 #include <sys/mbuf.h> 47 #include <sys/mutex.h> 48 #include <sys/sockio.h> 49 #include <sys/syslog.h> 50 #include <sys/cpu.h> 51 #include <sys/module.h> 52 #include <sys/pcq.h> 53 #include <sys/workqueue.h> 54 #include <sys/xcall.h> 55 56 #include <dev/pci/virtioreg.h> 57 #include <dev/pci/virtiovar.h> 58 59 #include <net/if.h> 60 #include <net/if_dl.h> 61 #include <net/if_media.h> 62 #include <net/if_ether.h> 63 64 #include <net/bpf.h> 65 66 #include "ioconf.h" 67 68 #ifdef NET_MPSAFE 69 #define VIOIF_MPSAFE 1 70 #define VIOIF_MULTIQ 1 71 #endif 72 73 /* 74 * if_vioifreg.h: 75 */ 76 /* Configuration registers */ 77 #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */ 78 #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */ 79 #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */ 80 #define VIRTIO_NET_CONFIG_MTU 10 /* 16bit */ 81 82 /* Feature bits */ 83 #define VIRTIO_NET_F_CSUM __BIT(0) 84 #define VIRTIO_NET_F_GUEST_CSUM __BIT(1) 85 #define VIRTIO_NET_F_MAC __BIT(5) 86 #define VIRTIO_NET_F_GSO __BIT(6) 87 #define VIRTIO_NET_F_GUEST_TSO4 __BIT(7) 88 #define VIRTIO_NET_F_GUEST_TSO6 __BIT(8) 89 #define VIRTIO_NET_F_GUEST_ECN __BIT(9) 90 #define VIRTIO_NET_F_GUEST_UFO __BIT(10) 91 #define VIRTIO_NET_F_HOST_TSO4 __BIT(11) 92 #define VIRTIO_NET_F_HOST_TSO6 __BIT(12) 93 #define VIRTIO_NET_F_HOST_ECN __BIT(13) 94 #define VIRTIO_NET_F_HOST_UFO __BIT(14) 95 #define VIRTIO_NET_F_MRG_RXBUF __BIT(15) 96 #define VIRTIO_NET_F_STATUS __BIT(16) 97 #define VIRTIO_NET_F_CTRL_VQ __BIT(17) 98 #define VIRTIO_NET_F_CTRL_RX __BIT(18) 99 #define VIRTIO_NET_F_CTRL_VLAN __BIT(19) 100 #define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20) 101 #define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21) 102 #define VIRTIO_NET_F_MQ __BIT(22) 103 #define VIRTIO_NET_F_CTRL_MAC_ADDR __BIT(23) 104 105 #define VIRTIO_NET_FLAG_BITS \ 106 VIRTIO_COMMON_FLAG_BITS \ 107 "b\x17" "CTRL_MAC\0" \ 108 "b\x16" "MQ\0" \ 109 "b\x15" "GUEST_ANNOUNCE\0" \ 110 "b\x14" "CTRL_RX_EXTRA\0" \ 111 "b\x13" "CTRL_VLAN\0" \ 112 "b\x12" "CTRL_RX\0" \ 113 "b\x11" "CTRL_VQ\0" \ 114 "b\x10" "STATUS\0" \ 115 "b\x0f" "MRG_RXBUF\0" \ 116 "b\x0e" "HOST_UFO\0" \ 117 "b\x0d" "HOST_ECN\0" \ 118 "b\x0c" "HOST_TSO6\0" \ 119 "b\x0b" "HOST_TSO4\0" \ 120 "b\x0a" "GUEST_UFO\0" \ 121 "b\x09" "GUEST_ECN\0" \ 122 "b\x08" "GUEST_TSO6\0" \ 123 "b\x07" "GUEST_TSO4\0" \ 124 "b\x06" "GSO\0" \ 125 "b\x05" "MAC\0" \ 126 "b\x01" "GUEST_CSUM\0" \ 127 "b\x00" "CSUM\0" 128 129 /* Status */ 130 #define VIRTIO_NET_S_LINK_UP 1 131 132 /* Packet header structure */ 133 struct virtio_net_hdr { 134 uint8_t flags; 135 uint8_t gso_type; 136 uint16_t hdr_len; 137 uint16_t gso_size; 138 uint16_t csum_start; 139 uint16_t csum_offset; 140 141 uint16_t num_buffers; /* VIRTIO_NET_F_MRG_RXBUF enabled or v1 */ 142 } __packed; 143 144 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */ 145 #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */ 146 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */ 147 #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */ 148 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */ 149 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */ 150 151 #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN) 152 153 /* Control virtqueue */ 154 struct virtio_net_ctrl_cmd { 155 uint8_t class; 156 uint8_t command; 157 } __packed; 158 #define VIRTIO_NET_CTRL_RX 0 159 # define VIRTIO_NET_CTRL_RX_PROMISC 0 160 # define VIRTIO_NET_CTRL_RX_ALLMULTI 1 161 162 #define VIRTIO_NET_CTRL_MAC 1 163 # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 164 # define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 165 166 #define VIRTIO_NET_CTRL_VLAN 2 167 # define VIRTIO_NET_CTRL_VLAN_ADD 0 168 # define VIRTIO_NET_CTRL_VLAN_DEL 1 169 170 #define VIRTIO_NET_CTRL_MQ 4 171 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 172 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 173 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 174 175 struct virtio_net_ctrl_status { 176 uint8_t ack; 177 } __packed; 178 #define VIRTIO_NET_OK 0 179 #define VIRTIO_NET_ERR 1 180 181 struct virtio_net_ctrl_rx { 182 uint8_t onoff; 183 } __packed; 184 185 struct virtio_net_ctrl_mac_tbl { 186 uint32_t nentries; 187 uint8_t macs[][ETHER_ADDR_LEN]; 188 } __packed; 189 190 struct virtio_net_ctrl_mac_addr { 191 uint8_t mac[ETHER_ADDR_LEN]; 192 } __packed; 193 194 struct virtio_net_ctrl_vlan { 195 uint16_t id; 196 } __packed; 197 198 struct virtio_net_ctrl_mq { 199 uint16_t virtqueue_pairs; 200 } __packed; 201 202 /* 203 * if_vioifvar.h: 204 */ 205 206 /* 207 * Locking notes: 208 * + a field in vioif_netqueue is protected by netq_lock (a spin mutex) 209 * - more than one lock cannot be held at onece 210 * + a field in vioif_tx_context and vioif_rx_context is also protected 211 * by netq_lock. 212 * + ctrlq_inuse is protected by ctrlq_wait_lock. 213 * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse 214 * - netq_lock cannot be held along with ctrlq_wait_lock 215 * + fields in vioif_softc except queues are protected by 216 * sc->sc_lock(an adaptive mutex) 217 * - the lock is held before acquisition of other locks 218 */ 219 220 struct vioif_ctrl_cmdspec { 221 bus_dmamap_t dmamap; 222 void *buf; 223 bus_size_t bufsize; 224 }; 225 226 struct vioif_work { 227 struct work cookie; 228 void (*func)(void *); 229 void *arg; 230 unsigned int added; 231 }; 232 233 struct vioif_net_map { 234 struct virtio_net_hdr *vnm_hdr; 235 bus_dmamap_t vnm_hdr_map; 236 struct mbuf *vnm_mbuf; 237 bus_dmamap_t vnm_mbuf_map; 238 }; 239 240 #define VIOIF_NETQ_RX 0 241 #define VIOIF_NETQ_TX 1 242 #define VIOIF_NETQ_IDX 2 243 #define VIOIF_NETQ_DIR(n) ((n) % VIOIF_NETQ_IDX) 244 #define VIOIF_NETQ_PAIRIDX(n) ((n) / VIOIF_NETQ_IDX) 245 #define VIOIF_NETQ_RXQID(n) ((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_RX) 246 #define VIOIF_NETQ_TXQID(n) ((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_TX) 247 248 struct vioif_netqueue { 249 kmutex_t netq_lock; 250 struct virtqueue *netq_vq; 251 bool netq_stopping; 252 bool netq_running_handle; 253 void *netq_maps_kva; 254 struct vioif_net_map *netq_maps; 255 256 void *netq_softint; 257 struct vioif_work netq_work; 258 bool netq_workqueue; 259 260 char netq_evgroup[32]; 261 struct evcnt netq_mbuf_load_failed; 262 struct evcnt netq_enqueue_failed; 263 264 void *netq_ctx; 265 }; 266 267 struct vioif_tx_context { 268 bool txc_link_active; 269 bool txc_no_free_slots; 270 pcq_t *txc_intrq; 271 void *txc_deferred_transmit; 272 273 struct evcnt txc_defrag_failed; 274 struct evcnt txc_pcq_full; 275 }; 276 277 struct vioif_rx_context { 278 struct evcnt rxc_mbuf_enobufs; 279 }; 280 struct vioif_ctrlqueue { 281 struct virtqueue *ctrlq_vq; 282 enum { 283 FREE, INUSE, DONE 284 } ctrlq_inuse; 285 kcondvar_t ctrlq_wait; 286 kmutex_t ctrlq_wait_lock; 287 struct lwp *ctrlq_owner; 288 289 struct virtio_net_ctrl_cmd *ctrlq_cmd; 290 struct virtio_net_ctrl_status *ctrlq_status; 291 struct virtio_net_ctrl_rx *ctrlq_rx; 292 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc; 293 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc; 294 struct virtio_net_ctrl_mac_addr *ctrlq_mac_addr; 295 struct virtio_net_ctrl_mq *ctrlq_mq; 296 297 bus_dmamap_t ctrlq_cmd_dmamap; 298 bus_dmamap_t ctrlq_status_dmamap; 299 bus_dmamap_t ctrlq_rx_dmamap; 300 bus_dmamap_t ctrlq_tbl_uc_dmamap; 301 bus_dmamap_t ctrlq_tbl_mc_dmamap; 302 bus_dmamap_t ctrlq_mac_addr_dmamap; 303 bus_dmamap_t ctrlq_mq_dmamap; 304 305 struct evcnt ctrlq_cmd_load_failed; 306 struct evcnt ctrlq_cmd_failed; 307 }; 308 309 struct vioif_softc { 310 device_t sc_dev; 311 kmutex_t sc_lock; 312 struct sysctllog *sc_sysctllog; 313 314 struct virtio_softc *sc_virtio; 315 struct virtqueue *sc_vqs; 316 u_int sc_hdr_size; 317 318 int sc_max_nvq_pairs; 319 int sc_req_nvq_pairs; 320 int sc_act_nvq_pairs; 321 322 uint8_t sc_mac[ETHER_ADDR_LEN]; 323 struct ethercom sc_ethercom; 324 int sc_link_state; 325 326 struct vioif_netqueue *sc_netqs; 327 328 bool sc_has_ctrl; 329 struct vioif_ctrlqueue sc_ctrlq; 330 331 bus_dma_segment_t sc_segs[1]; 332 void *sc_dmamem; 333 void *sc_kmem; 334 335 void *sc_cfg_softint; 336 337 struct workqueue *sc_txrx_workqueue; 338 bool sc_txrx_workqueue_sysctl; 339 u_int sc_tx_intr_process_limit; 340 u_int sc_tx_process_limit; 341 u_int sc_rx_intr_process_limit; 342 u_int sc_rx_process_limit; 343 }; 344 #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */ 345 #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */ 346 347 #define VIOIF_TX_INTR_PROCESS_LIMIT 256 348 #define VIOIF_TX_PROCESS_LIMIT 256 349 #define VIOIF_RX_INTR_PROCESS_LIMIT 0U 350 #define VIOIF_RX_PROCESS_LIMIT 256 351 352 #define VIOIF_WORKQUEUE_PRI PRI_SOFTNET 353 #define VIOIF_IS_LINK_ACTIVE(_sc) ((_sc)->sc_link_state == LINK_STATE_UP ? \ 354 true : false) 355 356 /* cfattach interface functions */ 357 static int vioif_match(device_t, cfdata_t, void *); 358 static void vioif_attach(device_t, device_t, void *); 359 static int vioif_finalize_teardown(device_t); 360 361 /* ifnet interface functions */ 362 static int vioif_init(struct ifnet *); 363 static void vioif_stop(struct ifnet *, int); 364 static void vioif_start(struct ifnet *); 365 static int vioif_transmit(struct ifnet *, struct mbuf *); 366 static int vioif_ioctl(struct ifnet *, u_long, void *); 367 static void vioif_watchdog(struct ifnet *); 368 static int vioif_ifflags(struct vioif_softc *); 369 static int vioif_ifflags_cb(struct ethercom *); 370 371 /* tx & rx */ 372 static int vioif_netqueue_init(struct vioif_softc *, 373 struct virtio_softc *, size_t, u_int); 374 static void vioif_netqueue_teardown(struct vioif_softc *, 375 struct virtio_softc *, size_t); 376 static void vioif_net_intr_enable(struct vioif_softc *, 377 struct virtio_softc *); 378 static void vioif_net_intr_disable(struct vioif_softc *, 379 struct virtio_softc *); 380 static void vioif_net_sched_handle(struct vioif_softc *, 381 struct vioif_netqueue *); 382 383 /* rx */ 384 static void vioif_populate_rx_mbufs_locked(struct vioif_softc *, 385 struct vioif_netqueue *); 386 static int vioif_rx_intr(void *); 387 static void vioif_rx_handle(void *); 388 static void vioif_rx_queue_clear(struct vioif_softc *, 389 struct virtio_softc *, struct vioif_netqueue *); 390 391 /* tx */ 392 static void vioif_start_locked(struct ifnet *, struct vioif_netqueue *); 393 static void vioif_transmit_locked(struct ifnet *, struct vioif_netqueue *); 394 static void vioif_deferred_transmit(void *); 395 static int vioif_tx_intr(void *); 396 static void vioif_tx_handle(void *); 397 static void vioif_tx_queue_clear(struct vioif_softc *, struct virtio_softc *, 398 struct vioif_netqueue *); 399 400 /* controls */ 401 static int vioif_ctrl_intr(void *); 402 static int vioif_ctrl_rx(struct vioif_softc *, int, bool); 403 static int vioif_set_promisc(struct vioif_softc *, bool); 404 static int vioif_set_allmulti(struct vioif_softc *, bool); 405 static int vioif_set_rx_filter(struct vioif_softc *); 406 static int vioif_rx_filter(struct vioif_softc *); 407 static int vioif_set_mac_addr(struct vioif_softc *); 408 static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int); 409 410 /* config interrupt */ 411 static int vioif_config_change(struct virtio_softc *); 412 static void vioif_cfg_softint(void *); 413 static void vioif_update_link_status(struct vioif_softc *); 414 415 /* others */ 416 static void vioif_alloc_queues(struct vioif_softc *); 417 static void vioif_free_queues(struct vioif_softc *); 418 static int vioif_alloc_mems(struct vioif_softc *); 419 static struct workqueue* 420 vioif_workq_create(const char *, pri_t, int, int); 421 static void vioif_workq_destroy(struct workqueue *); 422 static void vioif_work_set(struct vioif_work *, void(*)(void *), void *); 423 static void vioif_work_add(struct workqueue *, struct vioif_work *); 424 static void vioif_work_wait(struct workqueue *, struct vioif_work *); 425 static int vioif_setup_sysctl(struct vioif_softc *); 426 static void vioif_setup_stats(struct vioif_softc *); 427 428 CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc), 429 vioif_match, vioif_attach, NULL, NULL); 430 431 static void 432 vioif_intr_barrier(void) 433 { 434 435 /* wait for finish all interrupt handler */ 436 xc_barrier(0); 437 } 438 439 static void 440 vioif_notify(struct virtio_softc *vsc, struct virtqueue *vq) 441 { 442 443 virtio_enqueue_commit(vsc, vq, -1, true); 444 } 445 446 static int 447 vioif_match(device_t parent, cfdata_t match, void *aux) 448 { 449 struct virtio_attach_args *va = aux; 450 451 if (va->sc_childdevid == VIRTIO_DEVICE_ID_NETWORK) 452 return 1; 453 454 return 0; 455 } 456 457 static void 458 vioif_attach(device_t parent, device_t self, void *aux) 459 { 460 struct vioif_softc *sc = device_private(self); 461 struct virtio_softc *vsc = device_private(parent); 462 struct vioif_netqueue *txq0; 463 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 464 uint64_t features, req_features; 465 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 466 u_int softint_flags; 467 int r, i, req_flags; 468 char xnamebuf[MAXCOMLEN]; 469 size_t nvqs; 470 471 if (virtio_child(vsc) != NULL) { 472 aprint_normal(": child already attached for %s; " 473 "something wrong...\n", device_xname(parent)); 474 return; 475 } 476 477 sc->sc_dev = self; 478 sc->sc_virtio = vsc; 479 sc->sc_link_state = LINK_STATE_UNKNOWN; 480 481 sc->sc_max_nvq_pairs = 1; 482 sc->sc_req_nvq_pairs = 1; 483 sc->sc_act_nvq_pairs = 1; 484 sc->sc_txrx_workqueue_sysctl = true; 485 sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT; 486 sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT; 487 sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT; 488 sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT; 489 490 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE); 491 492 snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self)); 493 sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI, 494 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 495 if (sc->sc_txrx_workqueue == NULL) 496 goto err; 497 498 req_flags = 0; 499 500 #ifdef VIOIF_MPSAFE 501 req_flags |= VIRTIO_F_INTR_MPSAFE; 502 #endif 503 req_flags |= VIRTIO_F_INTR_MSIX; 504 505 req_features = 506 VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | 507 VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY; 508 req_features |= VIRTIO_F_RING_EVENT_IDX; 509 req_features |= VIRTIO_NET_F_CTRL_MAC_ADDR; 510 #ifdef VIOIF_MULTIQ 511 req_features |= VIRTIO_NET_F_MQ; 512 #endif 513 514 virtio_child_attach_start(vsc, self, IPL_NET, 515 req_features, VIRTIO_NET_FLAG_BITS); 516 features = virtio_features(vsc); 517 518 if (features == 0) 519 goto err; 520 521 if (features & VIRTIO_NET_F_MAC) { 522 for (i = 0; i < __arraycount(sc->sc_mac); i++) { 523 sc->sc_mac[i] = virtio_read_device_config_1(vsc, 524 VIRTIO_NET_CONFIG_MAC + i); 525 } 526 } else { 527 /* code stolen from sys/net/if_tap.c */ 528 struct timeval tv; 529 uint32_t ui; 530 getmicrouptime(&tv); 531 ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; 532 memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); 533 for (i = 0; i < __arraycount(sc->sc_mac); i++) { 534 virtio_write_device_config_1(vsc, 535 VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]); 536 } 537 } 538 539 /* 'Ethernet' with capital follows other ethernet driver attachment */ 540 aprint_normal_dev(self, "Ethernet address %s\n", 541 ether_sprintf(sc->sc_mac)); 542 543 if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) { 544 sc->sc_hdr_size = sizeof(struct virtio_net_hdr); 545 } else { 546 sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers); 547 } 548 549 if ((features & VIRTIO_NET_F_CTRL_VQ) && 550 (features & VIRTIO_NET_F_CTRL_RX)) { 551 sc->sc_has_ctrl = true; 552 553 cv_init(&ctrlq->ctrlq_wait, "ctrl_vq"); 554 mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET); 555 ctrlq->ctrlq_inuse = FREE; 556 } else { 557 sc->sc_has_ctrl = false; 558 } 559 560 if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) { 561 sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc, 562 VIRTIO_NET_CONFIG_MAX_VQ_PAIRS); 563 564 if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) 565 goto err; 566 567 /* Limit the number of queue pairs to use */ 568 sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu); 569 570 if (sc->sc_max_nvq_pairs > 1) 571 req_flags |= VIRTIO_F_INTR_PERVQ; 572 } 573 574 vioif_alloc_queues(sc); 575 576 #ifdef VIOIF_MPSAFE 577 softint_flags = SOFTINT_NET | SOFTINT_MPSAFE; 578 #else 579 softint_flags = SOFTINT_NET; 580 #endif 581 582 /* 583 * Initialize network queues 584 */ 585 nvqs = sc->sc_max_nvq_pairs * 2; 586 for (i = 0; i < nvqs; i++) { 587 r = vioif_netqueue_init(sc, vsc, i, softint_flags); 588 if (r != 0) 589 goto err; 590 } 591 592 if (sc->sc_has_ctrl) { 593 int ctrlq_idx = nvqs; 594 595 nvqs++; 596 /* 597 * Allocating a virtqueue for control channel 598 */ 599 sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[ctrlq_idx]; 600 virtio_init_vq(vsc, ctrlq->ctrlq_vq, ctrlq_idx, 601 vioif_ctrl_intr, ctrlq); 602 603 r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, NBPG, 1, "control"); 604 if (r != 0) { 605 aprint_error_dev(self, "failed to allocate " 606 "a virtqueue for control channel, error code %d\n", 607 r); 608 609 sc->sc_has_ctrl = false; 610 cv_destroy(&ctrlq->ctrlq_wait); 611 mutex_destroy(&ctrlq->ctrlq_wait_lock); 612 } 613 } 614 615 sc->sc_cfg_softint = softint_establish(softint_flags, 616 vioif_cfg_softint, sc); 617 if (sc->sc_cfg_softint == NULL) { 618 aprint_error_dev(self, "cannot establish ctl softint\n"); 619 goto err; 620 } 621 622 if (vioif_alloc_mems(sc) < 0) 623 goto err; 624 625 r = virtio_child_attach_finish(vsc, sc->sc_vqs, nvqs, 626 vioif_config_change, req_flags); 627 if (r != 0) 628 goto err; 629 630 if (vioif_setup_sysctl(sc) != 0) { 631 aprint_error_dev(self, "unable to create sysctl node\n"); 632 /* continue */ 633 } 634 635 vioif_setup_stats(sc); 636 637 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 638 ifp->if_softc = sc; 639 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 640 #ifdef VIOIF_MPSAFE 641 ifp->if_extflags = IFEF_MPSAFE; 642 #endif 643 ifp->if_start = vioif_start; 644 if (sc->sc_req_nvq_pairs > 1) 645 ifp->if_transmit = vioif_transmit; 646 ifp->if_ioctl = vioif_ioctl; 647 ifp->if_init = vioif_init; 648 ifp->if_stop = vioif_stop; 649 ifp->if_capabilities = 0; 650 ifp->if_watchdog = vioif_watchdog; 651 txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]; 652 IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq0->netq_vq->vq_num, IFQ_MAXLEN)); 653 IFQ_SET_READY(&ifp->if_snd); 654 655 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; 656 657 if_attach(ifp); 658 if_deferred_start_init(ifp, NULL); 659 ether_ifattach(ifp, sc->sc_mac); 660 ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb); 661 662 return; 663 664 err: 665 nvqs = sc->sc_max_nvq_pairs * 2; 666 for (i = 0; i < nvqs; i++) { 667 vioif_netqueue_teardown(sc, vsc, i); 668 } 669 670 if (sc->sc_has_ctrl) { 671 cv_destroy(&ctrlq->ctrlq_wait); 672 mutex_destroy(&ctrlq->ctrlq_wait_lock); 673 virtio_free_vq(vsc, ctrlq->ctrlq_vq); 674 ctrlq->ctrlq_vq = NULL; 675 } 676 677 vioif_free_queues(sc); 678 mutex_destroy(&sc->sc_lock); 679 virtio_child_attach_failed(vsc); 680 config_finalize_register(self, vioif_finalize_teardown); 681 682 return; 683 } 684 685 static int 686 vioif_finalize_teardown(device_t self) 687 { 688 struct vioif_softc *sc = device_private(self); 689 690 if (sc->sc_txrx_workqueue != NULL) { 691 vioif_workq_destroy(sc->sc_txrx_workqueue); 692 sc->sc_txrx_workqueue = NULL; 693 } 694 695 return 0; 696 } 697 698 /* 699 * Interface functions for ifnet 700 */ 701 static int 702 vioif_init(struct ifnet *ifp) 703 { 704 struct vioif_softc *sc = ifp->if_softc; 705 struct virtio_softc *vsc = sc->sc_virtio; 706 struct vioif_netqueue *netq; 707 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 708 int r, i; 709 710 vioif_stop(ifp, 0); 711 712 r = virtio_reinit_start(vsc); 713 if (r != 0) { 714 log(LOG_ERR, "%s: reset failed\n", ifp->if_xname); 715 return EIO; 716 } 717 718 virtio_negotiate_features(vsc, virtio_features(vsc)); 719 720 for (i = 0; i < sc->sc_req_nvq_pairs; i++) { 721 netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)]; 722 723 mutex_enter(&netq->netq_lock); 724 vioif_populate_rx_mbufs_locked(sc, netq); 725 mutex_exit(&netq->netq_lock); 726 } 727 728 virtio_reinit_end(vsc); 729 730 if (sc->sc_has_ctrl) 731 virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); 732 733 r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs); 734 if (r == 0) 735 sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs; 736 else 737 sc->sc_act_nvq_pairs = 1; 738 739 SET(ifp->if_flags, IFF_RUNNING); 740 741 vioif_net_intr_enable(sc, vsc); 742 743 vioif_update_link_status(sc); 744 r = vioif_rx_filter(sc); 745 746 return r; 747 } 748 749 static void 750 vioif_stop(struct ifnet *ifp, int disable) 751 { 752 struct vioif_softc *sc = ifp->if_softc; 753 struct virtio_softc *vsc = sc->sc_virtio; 754 struct vioif_netqueue *netq; 755 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 756 size_t i, act_qnum; 757 758 act_qnum = sc->sc_act_nvq_pairs * 2; 759 760 CLR(ifp->if_flags, IFF_RUNNING); 761 for (i = 0; i < act_qnum; i++) { 762 netq = &sc->sc_netqs[i]; 763 764 mutex_enter(&netq->netq_lock); 765 netq->netq_stopping = true; 766 mutex_exit(&netq->netq_lock); 767 } 768 769 /* disable interrupts */ 770 vioif_net_intr_disable(sc, vsc); 771 if (sc->sc_has_ctrl) 772 virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq); 773 774 /* 775 * only way to stop interrupt, I/O and DMA is resetting... 776 * 777 * NOTE: Devices based on VirtIO draft specification can not 778 * stop interrupt completely even if virtio_stop_vq_intr() is called. 779 */ 780 virtio_reset(vsc); 781 782 vioif_intr_barrier(); 783 784 for (i = 0; i < act_qnum; i++) { 785 netq = &sc->sc_netqs[i]; 786 vioif_work_wait(sc->sc_txrx_workqueue, &netq->netq_work); 787 } 788 789 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 790 netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)]; 791 vioif_rx_queue_clear(sc, vsc, netq); 792 793 netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)]; 794 vioif_tx_queue_clear(sc, vsc, netq); 795 } 796 797 /* all packet processing is stopped */ 798 for (i = 0; i < act_qnum; i++) { 799 netq = &sc->sc_netqs[i]; 800 801 mutex_enter(&netq->netq_lock); 802 netq->netq_stopping = false; 803 mutex_exit(&netq->netq_lock); 804 } 805 } 806 807 static void 808 vioif_start(struct ifnet *ifp) 809 { 810 struct vioif_softc *sc = ifp->if_softc; 811 struct vioif_netqueue *txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]; 812 813 #ifdef VIOIF_MPSAFE 814 KASSERT(if_is_mpsafe(ifp)); 815 #endif 816 817 mutex_enter(&txq0->netq_lock); 818 vioif_start_locked(ifp, txq0); 819 mutex_exit(&txq0->netq_lock); 820 } 821 822 static inline int 823 vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m) 824 { 825 struct vioif_softc *sc = ifp->if_softc; 826 u_int cpuid = cpu_index(curcpu()); 827 828 return VIOIF_NETQ_TXQID(cpuid % sc->sc_act_nvq_pairs); 829 } 830 831 static int 832 vioif_transmit(struct ifnet *ifp, struct mbuf *m) 833 { 834 struct vioif_softc *sc = ifp->if_softc; 835 struct vioif_netqueue *netq; 836 struct vioif_tx_context *txc; 837 int qid; 838 839 qid = vioif_select_txqueue(ifp, m); 840 netq = &sc->sc_netqs[qid]; 841 txc = netq->netq_ctx; 842 843 if (__predict_false(!pcq_put(txc->txc_intrq, m))) { 844 txc->txc_pcq_full.ev_count++; 845 m_freem(m); 846 return ENOBUFS; 847 } 848 849 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 850 if_statadd_ref(ifp, nsr, if_obytes, m->m_pkthdr.len); 851 if (m->m_flags & M_MCAST) 852 if_statinc_ref(ifp, nsr, if_omcasts); 853 IF_STAT_PUTREF(ifp); 854 855 if (mutex_tryenter(&netq->netq_lock)) { 856 vioif_transmit_locked(ifp, netq); 857 mutex_exit(&netq->netq_lock); 858 } 859 860 return 0; 861 } 862 863 void 864 vioif_watchdog(struct ifnet *ifp) 865 { 866 struct vioif_softc *sc = ifp->if_softc; 867 struct vioif_netqueue *netq; 868 int i; 869 870 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 871 if (ISSET(ifp->if_flags, IFF_DEBUG)) { 872 log(LOG_DEBUG, "%s: watchdog timed out\n", 873 ifp->if_xname); 874 } 875 876 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 877 netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)]; 878 879 mutex_enter(&netq->netq_lock); 880 if (!netq->netq_running_handle) { 881 netq->netq_running_handle = true; 882 vioif_net_sched_handle(sc, netq); 883 } 884 mutex_exit(&netq->netq_lock); 885 } 886 } 887 } 888 889 static int 890 vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) 891 { 892 int s, r; 893 894 s = splnet(); 895 896 r = ether_ioctl(ifp, cmd, data); 897 if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) { 898 if (ifp->if_flags & IFF_RUNNING) { 899 r = vioif_rx_filter(ifp->if_softc); 900 } else { 901 r = 0; 902 } 903 } 904 905 splx(s); 906 907 return r; 908 } 909 910 static int 911 vioif_ifflags(struct vioif_softc *sc) 912 { 913 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 914 bool onoff; 915 int r; 916 917 if (!sc->sc_has_ctrl) { 918 /* no ctrl vq; always promisc and allmulti */ 919 ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI); 920 return 0; 921 } 922 923 onoff = ifp->if_flags & IFF_ALLMULTI ? true : false; 924 r = vioif_set_allmulti(sc, onoff); 925 if (r != 0) { 926 log(LOG_WARNING, 927 "%s: couldn't %sable ALLMULTI\n", 928 ifp->if_xname, onoff ? "en" : "dis"); 929 if (onoff) { 930 CLR(ifp->if_flags, IFF_ALLMULTI); 931 } else { 932 SET(ifp->if_flags, IFF_ALLMULTI); 933 } 934 } 935 936 onoff = ifp->if_flags & IFF_PROMISC ? true : false; 937 r = vioif_set_promisc(sc, onoff); 938 if (r != 0) { 939 log(LOG_WARNING, 940 "%s: couldn't %sable PROMISC\n", 941 ifp->if_xname, onoff ? "en" : "dis"); 942 if (onoff) { 943 CLR(ifp->if_flags, IFF_PROMISC); 944 } else { 945 SET(ifp->if_flags, IFF_PROMISC); 946 } 947 } 948 949 return 0; 950 } 951 952 static int 953 vioif_ifflags_cb(struct ethercom *ec) 954 { 955 struct ifnet *ifp = &ec->ec_if; 956 struct vioif_softc *sc = ifp->if_softc; 957 958 return vioif_ifflags(sc); 959 } 960 961 static int 962 vioif_setup_sysctl(struct vioif_softc *sc) 963 { 964 const char *devname; 965 struct sysctllog **log; 966 const struct sysctlnode *rnode, *rxnode, *txnode; 967 int error; 968 969 log = &sc->sc_sysctllog; 970 devname = device_xname(sc->sc_dev); 971 972 error = sysctl_createv(log, 0, NULL, &rnode, 973 0, CTLTYPE_NODE, devname, 974 SYSCTL_DESCR("virtio-net information and settings"), 975 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 976 if (error) 977 goto out; 978 979 error = sysctl_createv(log, 0, &rnode, NULL, 980 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 981 SYSCTL_DESCR("Use workqueue for packet processing"), 982 NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL); 983 if (error) 984 goto out; 985 986 error = sysctl_createv(log, 0, &rnode, &rxnode, 987 0, CTLTYPE_NODE, "rx", 988 SYSCTL_DESCR("virtio-net information and settings for Rx"), 989 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 990 if (error) 991 goto out; 992 993 error = sysctl_createv(log, 0, &rxnode, NULL, 994 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 995 SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), 996 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 997 if (error) 998 goto out; 999 1000 error = sysctl_createv(log, 0, &rxnode, NULL, 1001 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 1002 SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), 1003 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 1004 if (error) 1005 goto out; 1006 1007 error = sysctl_createv(log, 0, &rnode, &txnode, 1008 0, CTLTYPE_NODE, "tx", 1009 SYSCTL_DESCR("virtio-net information and settings for Tx"), 1010 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 1011 if (error) 1012 goto out; 1013 1014 error = sysctl_createv(log, 0, &txnode, NULL, 1015 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 1016 SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), 1017 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 1018 if (error) 1019 goto out; 1020 1021 error = sysctl_createv(log, 0, &txnode, NULL, 1022 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 1023 SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), 1024 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 1025 1026 out: 1027 if (error) 1028 sysctl_teardown(log); 1029 1030 return error; 1031 } 1032 1033 static void 1034 vioif_setup_stats(struct vioif_softc *sc) 1035 { 1036 struct vioif_netqueue *netq; 1037 struct vioif_tx_context *txc; 1038 struct vioif_rx_context *rxc; 1039 size_t i, netq_num; 1040 1041 netq_num = sc->sc_max_nvq_pairs * 2; 1042 for (i = 0; i < netq_num; i++) { 1043 netq = &sc->sc_netqs[i]; 1044 evcnt_attach_dynamic(&netq->netq_mbuf_load_failed, EVCNT_TYPE_MISC, 1045 NULL, netq->netq_evgroup, "failed to load mbuf to DMA"); 1046 evcnt_attach_dynamic(&netq->netq_enqueue_failed, 1047 EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, 1048 "virtqueue enqueue failed failed"); 1049 1050 switch (VIOIF_NETQ_DIR(i)) { 1051 case VIOIF_NETQ_RX: 1052 rxc = netq->netq_ctx; 1053 evcnt_attach_dynamic(&rxc->rxc_mbuf_enobufs, 1054 EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, 1055 "no receive buffer"); 1056 break; 1057 case VIOIF_NETQ_TX: 1058 txc = netq->netq_ctx; 1059 evcnt_attach_dynamic(&txc->txc_defrag_failed, 1060 EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, 1061 "m_defrag() failed"); 1062 evcnt_attach_dynamic(&txc->txc_pcq_full, 1063 EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, 1064 "pcq full"); 1065 break; 1066 } 1067 } 1068 1069 evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC, 1070 NULL, device_xname(sc->sc_dev), "control command dmamap load failed"); 1071 evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC, 1072 NULL, device_xname(sc->sc_dev), "control command failed"); 1073 } 1074 1075 /* 1076 * allocate memory 1077 */ 1078 static int 1079 vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map, 1080 bus_size_t size, int nsegs, const char *usage) 1081 { 1082 int r; 1083 1084 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size, 1085 nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map); 1086 1087 if (r != 0) { 1088 aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, " 1089 "error code %d\n", usage, r); 1090 } 1091 1092 return r; 1093 } 1094 1095 static void 1096 vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map) 1097 { 1098 1099 if (*map) { 1100 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map); 1101 *map = NULL; 1102 } 1103 } 1104 1105 static int 1106 vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map, 1107 void *buf, bus_size_t size, int nsegs, int rw, const char *usage) 1108 { 1109 int r; 1110 1111 r = vioif_dmamap_create(sc, map, size, nsegs, usage); 1112 if (r != 0) 1113 return 1; 1114 1115 r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf, 1116 size, NULL, rw | BUS_DMA_NOWAIT); 1117 if (r != 0) { 1118 vioif_dmamap_destroy(sc, map); 1119 aprint_error_dev(sc->sc_dev, "%s dmamap load failed. " 1120 "error code %d\n", usage, r); 1121 } 1122 1123 return r; 1124 } 1125 1126 static void * 1127 vioif_assign_mem(intptr_t *p, size_t size) 1128 { 1129 intptr_t rv; 1130 1131 rv = *p; 1132 *p += size; 1133 1134 return (void *)rv; 1135 } 1136 1137 /* 1138 * dma memory is used for: 1139 * netq_maps_kva: metadata array for received frames (READ) and 1140 * sent frames (WRITE) 1141 * ctrlq_cmd: command to be sent via ctrl vq (WRITE) 1142 * ctrlq_status: return value for a command via ctrl vq (READ) 1143 * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command 1144 * (WRITE) 1145 * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC 1146 * class command (WRITE) 1147 * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC 1148 * class command (WRITE) 1149 * ctrlq_* structures are allocated only one each; they are protected by 1150 * ctrlq_inuse variable and ctrlq_wait condvar. 1151 */ 1152 static int 1153 vioif_alloc_mems(struct vioif_softc *sc) 1154 { 1155 struct virtio_softc *vsc = sc->sc_virtio; 1156 struct vioif_netqueue *netq; 1157 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1158 struct vioif_net_map *maps; 1159 unsigned int vq_num; 1160 int r, rsegs; 1161 bus_size_t dmamemsize; 1162 size_t qid, i, netq_num, kmemsize; 1163 void *vaddr; 1164 intptr_t p; 1165 1166 netq_num = sc->sc_max_nvq_pairs * 2; 1167 1168 /* allocate DMA memory */ 1169 dmamemsize = 0; 1170 1171 for (qid = 0; qid < netq_num; qid++) { 1172 maps = sc->sc_netqs[qid].netq_maps; 1173 vq_num = sc->sc_netqs[qid].netq_vq->vq_num; 1174 dmamemsize += sizeof(*maps[0].vnm_hdr) * vq_num; 1175 } 1176 1177 if (sc->sc_has_ctrl) { 1178 dmamemsize += sizeof(struct virtio_net_ctrl_cmd); 1179 dmamemsize += sizeof(struct virtio_net_ctrl_status); 1180 dmamemsize += sizeof(struct virtio_net_ctrl_rx); 1181 dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl) 1182 + ETHER_ADDR_LEN; 1183 dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl) 1184 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; 1185 dmamemsize += sizeof(struct virtio_net_ctrl_mac_addr); 1186 dmamemsize += sizeof(struct virtio_net_ctrl_mq); 1187 } 1188 1189 r = bus_dmamem_alloc(virtio_dmat(vsc), dmamemsize, 0, 0, 1190 &sc->sc_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 1191 if (r != 0) { 1192 aprint_error_dev(sc->sc_dev, 1193 "DMA memory allocation failed, size %" PRIuBUSSIZE ", " 1194 "error code %d\n", dmamemsize, r); 1195 goto err_none; 1196 } 1197 r = bus_dmamem_map(virtio_dmat(vsc), &sc->sc_segs[0], 1, 1198 dmamemsize, &vaddr, BUS_DMA_NOWAIT); 1199 if (r != 0) { 1200 aprint_error_dev(sc->sc_dev, 1201 "DMA memory map failed, error code %d\n", r); 1202 goto err_dmamem_alloc; 1203 } 1204 1205 /* assign DMA memory */ 1206 memset(vaddr, 0, dmamemsize); 1207 sc->sc_dmamem = vaddr; 1208 p = (intptr_t) vaddr; 1209 1210 for (qid = 0; qid < netq_num; qid++) { 1211 netq = &sc->sc_netqs[qid]; 1212 maps = netq->netq_maps; 1213 vq_num = netq->netq_vq->vq_num; 1214 1215 netq->netq_maps_kva = vioif_assign_mem(&p, 1216 sizeof(*maps[0].vnm_hdr) * vq_num); 1217 } 1218 1219 if (sc->sc_has_ctrl) { 1220 ctrlq->ctrlq_cmd = vioif_assign_mem(&p, 1221 sizeof(*ctrlq->ctrlq_cmd)); 1222 ctrlq->ctrlq_status = vioif_assign_mem(&p, 1223 sizeof(*ctrlq->ctrlq_status)); 1224 ctrlq->ctrlq_rx = vioif_assign_mem(&p, 1225 sizeof(*ctrlq->ctrlq_rx)); 1226 ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p, 1227 sizeof(*ctrlq->ctrlq_mac_tbl_uc) 1228 + ETHER_ADDR_LEN); 1229 ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p, 1230 sizeof(*ctrlq->ctrlq_mac_tbl_mc) 1231 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES); 1232 ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p, 1233 sizeof(*ctrlq->ctrlq_mac_addr)); 1234 ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq)); 1235 } 1236 1237 /* allocate kmem */ 1238 kmemsize = 0; 1239 1240 for (qid = 0; qid < netq_num; qid++) { 1241 netq = &sc->sc_netqs[qid]; 1242 vq_num = netq->netq_vq->vq_num; 1243 1244 kmemsize += sizeof(netq->netq_maps[0]) * vq_num; 1245 } 1246 1247 vaddr = kmem_zalloc(kmemsize, KM_SLEEP); 1248 sc->sc_kmem = vaddr; 1249 1250 /* assign allocated kmem */ 1251 p = (intptr_t) vaddr; 1252 1253 for (qid = 0; qid < netq_num; qid++) { 1254 netq = &sc->sc_netqs[qid]; 1255 vq_num = netq->netq_vq->vq_num; 1256 1257 netq->netq_maps = vioif_assign_mem(&p, 1258 sizeof(netq->netq_maps[0]) * vq_num); 1259 } 1260 1261 /* prepare dmamaps */ 1262 for (qid = 0; qid < netq_num; qid++) { 1263 static const struct { 1264 const char *msg_hdr; 1265 const char *msg_payload; 1266 int dma_flag; 1267 bus_size_t dma_size; 1268 int dma_nsegs; 1269 } dmaparams[VIOIF_NETQ_IDX] = { 1270 [VIOIF_NETQ_RX] = { 1271 .msg_hdr = "rx header", 1272 .msg_payload = "rx payload", 1273 .dma_flag = BUS_DMA_READ, 1274 .dma_size = MCLBYTES - ETHER_ALIGN, 1275 .dma_nsegs = 1, 1276 }, 1277 [VIOIF_NETQ_TX] = { 1278 .msg_hdr = "tx header", 1279 .msg_payload = "tx payload", 1280 .dma_flag = BUS_DMA_WRITE, 1281 .dma_size = ETHER_MAX_LEN, 1282 .dma_nsegs = VIRTIO_NET_TX_MAXNSEGS, 1283 } 1284 }; 1285 1286 struct virtio_net_hdr *hdrs; 1287 int dir; 1288 int nsegs; 1289 1290 dir = VIOIF_NETQ_DIR(qid); 1291 netq = &sc->sc_netqs[qid]; 1292 vq_num = netq->netq_vq->vq_num; 1293 maps = netq->netq_maps; 1294 hdrs = netq->netq_maps_kva; 1295 nsegs = uimin(dmaparams[dir].dma_nsegs, vq_num - 1/*hdr*/); 1296 1297 for (i = 0; i < vq_num; i++) { 1298 maps[i].vnm_hdr = &hdrs[i]; 1299 1300 r = vioif_dmamap_create_load(sc, &maps[i].vnm_hdr_map, 1301 maps[i].vnm_hdr, sc->sc_hdr_size, 1, 1302 dmaparams[dir].dma_flag, dmaparams[dir].msg_hdr); 1303 if (r != 0) 1304 goto err_reqs; 1305 1306 r = vioif_dmamap_create(sc, &maps[i].vnm_mbuf_map, 1307 dmaparams[dir].dma_size, nsegs, 1308 dmaparams[dir].msg_payload); 1309 if (r != 0) 1310 goto err_reqs; 1311 } 1312 } 1313 1314 if (sc->sc_has_ctrl) { 1315 /* control vq class & command */ 1316 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap, 1317 ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1, 1318 BUS_DMA_WRITE, "control command"); 1319 if (r != 0) 1320 goto err_reqs; 1321 1322 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap, 1323 ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1, 1324 BUS_DMA_READ, "control status"); 1325 if (r != 0) 1326 goto err_reqs; 1327 1328 /* control vq rx mode command parameter */ 1329 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap, 1330 ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1, 1331 BUS_DMA_WRITE, "rx mode control command"); 1332 if (r != 0) 1333 goto err_reqs; 1334 1335 /* multiqueue set command */ 1336 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap, 1337 ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1, 1338 BUS_DMA_WRITE, "multiqueue set command"); 1339 if (r != 0) 1340 goto err_reqs; 1341 1342 /* control vq MAC filter table for unicast */ 1343 /* do not load now since its length is variable */ 1344 r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap, 1345 sizeof(*ctrlq->ctrlq_mac_tbl_uc) 1346 + ETHER_ADDR_LEN, 1, 1347 "unicast MAC address filter command"); 1348 if (r != 0) 1349 goto err_reqs; 1350 1351 /* control vq MAC filter table for multicast */ 1352 r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap, 1353 sizeof(*ctrlq->ctrlq_mac_tbl_mc) 1354 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1, 1355 "multicast MAC address filter command"); 1356 if (r != 0) 1357 goto err_reqs; 1358 1359 /* control vq MAC address set command */ 1360 r = vioif_dmamap_create_load(sc, 1361 &ctrlq->ctrlq_mac_addr_dmamap, 1362 ctrlq->ctrlq_mac_addr, 1363 sizeof(*ctrlq->ctrlq_mac_addr), 1, 1364 BUS_DMA_WRITE, "mac addr set command"); 1365 if (r != 0) 1366 goto err_reqs; 1367 } 1368 1369 return 0; 1370 1371 err_reqs: 1372 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap); 1373 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap); 1374 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap); 1375 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap); 1376 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap); 1377 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap); 1378 for (qid = 0; qid < netq_num; qid++) { 1379 vq_num = sc->sc_netqs[qid].netq_vq->vq_num; 1380 maps = sc->sc_netqs[qid].netq_maps; 1381 1382 for (i = 0; i < vq_num; i++) { 1383 vioif_dmamap_destroy(sc, &maps[i].vnm_mbuf_map); 1384 vioif_dmamap_destroy(sc, &maps[i].vnm_hdr_map); 1385 } 1386 } 1387 if (sc->sc_kmem) { 1388 kmem_free(sc->sc_kmem, kmemsize); 1389 sc->sc_kmem = NULL; 1390 } 1391 bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, dmamemsize); 1392 err_dmamem_alloc: 1393 bus_dmamem_free(virtio_dmat(vsc), &sc->sc_segs[0], 1); 1394 err_none: 1395 return -1; 1396 } 1397 1398 static void 1399 vioif_alloc_queues(struct vioif_softc *sc) 1400 { 1401 int nvq_pairs = sc->sc_max_nvq_pairs; 1402 size_t nvqs, netq_num; 1403 1404 KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX); 1405 1406 nvqs = netq_num = sc->sc_max_nvq_pairs * 2; 1407 if (sc->sc_has_ctrl) 1408 nvqs++; 1409 1410 sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP); 1411 sc->sc_netqs = kmem_zalloc(sizeof(sc->sc_netqs[0]) * netq_num, 1412 KM_SLEEP); 1413 } 1414 1415 static void 1416 vioif_free_queues(struct vioif_softc *sc) 1417 { 1418 size_t nvqs, netq_num; 1419 1420 nvqs = netq_num = sc->sc_max_nvq_pairs * 2; 1421 if (sc->sc_ctrlq.ctrlq_vq) 1422 nvqs++; 1423 1424 kmem_free(sc->sc_netqs, sizeof(sc->sc_netqs[0]) * netq_num); 1425 kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs); 1426 sc->sc_netqs = NULL; 1427 sc->sc_vqs = NULL; 1428 } 1429 1430 /* 1431 * Network queues 1432 */ 1433 static int 1434 vioif_netqueue_init(struct vioif_softc *sc, struct virtio_softc *vsc, 1435 size_t qid, u_int softint_flags) 1436 { 1437 static const struct { 1438 const char *dirname; 1439 int segsize; 1440 int nsegs; 1441 int (*intrhand)(void *); 1442 void (*sihand)(void *); 1443 } params[VIOIF_NETQ_IDX] = { 1444 [VIOIF_NETQ_RX] = { 1445 .dirname = "rx", 1446 .segsize = MCLBYTES, 1447 .nsegs = 2, 1448 .intrhand = vioif_rx_intr, 1449 .sihand = vioif_rx_handle, 1450 }, 1451 [VIOIF_NETQ_TX] = { 1452 .dirname = "tx", 1453 .segsize = ETHER_MAX_LEN - ETHER_HDR_LEN, 1454 .nsegs = 2, 1455 .intrhand = vioif_tx_intr, 1456 .sihand = vioif_tx_handle, 1457 } 1458 }; 1459 1460 struct virtqueue *vq; 1461 struct vioif_netqueue *netq; 1462 struct vioif_tx_context *txc; 1463 struct vioif_rx_context *rxc; 1464 char qname[32]; 1465 int r, dir; 1466 1467 txc = NULL; 1468 rxc = NULL; 1469 netq = &sc->sc_netqs[qid]; 1470 vq = &sc->sc_vqs[qid]; 1471 dir = VIOIF_NETQ_DIR(qid); 1472 1473 netq->netq_vq = &sc->sc_vqs[qid]; 1474 netq->netq_stopping = false; 1475 netq->netq_running_handle = false; 1476 1477 snprintf(qname, sizeof(qname), "%s%zu", 1478 params[dir].dirname, VIOIF_NETQ_PAIRIDX(qid)); 1479 snprintf(netq->netq_evgroup, sizeof(netq->netq_evgroup), 1480 "%s-%s", device_xname(sc->sc_dev), qname); 1481 1482 mutex_init(&netq->netq_lock, MUTEX_DEFAULT, IPL_NET); 1483 virtio_init_vq(vsc, vq, qid, params[dir].intrhand, netq); 1484 1485 r = virtio_alloc_vq(vsc, vq, 1486 params[dir].segsize + sc->sc_hdr_size, 1487 params[dir].nsegs, qname); 1488 if (r != 0) 1489 goto err; 1490 netq->netq_vq = vq; 1491 1492 netq->netq_softint = softint_establish(softint_flags, 1493 params[dir].sihand, netq); 1494 if (netq->netq_softint == NULL) { 1495 aprint_error_dev(sc->sc_dev, 1496 "couldn't establish %s softint\n", 1497 params[dir].dirname); 1498 goto err; 1499 } 1500 vioif_work_set(&netq->netq_work, params[dir].sihand, netq); 1501 1502 switch (dir) { 1503 case VIOIF_NETQ_RX: 1504 rxc = kmem_zalloc(sizeof(*rxc), KM_SLEEP); 1505 netq->netq_ctx = rxc; 1506 /* nothing to do */ 1507 break; 1508 case VIOIF_NETQ_TX: 1509 txc = kmem_zalloc(sizeof(*txc), KM_SLEEP); 1510 netq->netq_ctx = (void *)txc; 1511 txc->txc_deferred_transmit = softint_establish(softint_flags, 1512 vioif_deferred_transmit, netq); 1513 if (txc->txc_deferred_transmit == NULL) { 1514 aprint_error_dev(sc->sc_dev, 1515 "couldn't establish softint for " 1516 "tx deferred transmit\n"); 1517 goto err; 1518 } 1519 txc->txc_link_active = VIOIF_IS_LINK_ACTIVE(sc); 1520 txc->txc_no_free_slots = false; 1521 txc->txc_intrq = pcq_create(vq->vq_num, KM_SLEEP); 1522 break; 1523 } 1524 1525 return 0; 1526 1527 err: 1528 netq->netq_ctx = NULL; 1529 1530 if (rxc != NULL) { 1531 kmem_free(rxc, sizeof(*rxc)); 1532 } 1533 1534 if (txc != NULL) { 1535 if (txc->txc_deferred_transmit != NULL) 1536 softint_disestablish(txc->txc_deferred_transmit); 1537 if (txc->txc_intrq != NULL) 1538 pcq_destroy(txc->txc_intrq); 1539 kmem_free(txc, sizeof(*txc)); 1540 } 1541 1542 vioif_work_set(&netq->netq_work, NULL, NULL); 1543 if (netq->netq_softint != NULL) { 1544 softint_disestablish(netq->netq_softint); 1545 netq->netq_softint = NULL; 1546 } 1547 1548 virtio_free_vq(vsc, vq); 1549 mutex_destroy(&netq->netq_lock); 1550 netq->netq_vq = NULL; 1551 1552 return -1; 1553 } 1554 1555 static void 1556 vioif_netqueue_teardown(struct vioif_softc *sc, struct virtio_softc *vsc, 1557 size_t qid) 1558 { 1559 struct vioif_netqueue *netq; 1560 struct vioif_rx_context *rxc; 1561 struct vioif_tx_context *txc; 1562 int dir; 1563 1564 netq = &sc->sc_netqs[qid]; 1565 1566 if (netq->netq_vq == NULL) 1567 return; 1568 1569 netq = &sc->sc_netqs[qid]; 1570 dir = VIOIF_NETQ_DIR(qid); 1571 switch (dir) { 1572 case VIOIF_NETQ_RX: 1573 rxc = netq->netq_ctx; 1574 netq->netq_ctx = NULL; 1575 kmem_free(rxc, sizeof(*rxc)); 1576 break; 1577 case VIOIF_NETQ_TX: 1578 txc = netq->netq_ctx; 1579 netq->netq_ctx = NULL; 1580 softint_disestablish(txc->txc_deferred_transmit); 1581 pcq_destroy(txc->txc_intrq); 1582 kmem_free(txc, sizeof(*txc)); 1583 break; 1584 } 1585 1586 softint_disestablish(netq->netq_softint); 1587 virtio_free_vq(vsc, netq->netq_vq); 1588 mutex_destroy(&netq->netq_lock); 1589 netq->netq_vq = NULL; 1590 } 1591 1592 static void 1593 vioif_net_sched_handle(struct vioif_softc *sc, struct vioif_netqueue *netq) 1594 { 1595 1596 KASSERT(mutex_owned(&netq->netq_lock)); 1597 KASSERT(!netq->netq_stopping); 1598 1599 if (netq->netq_workqueue) { 1600 vioif_work_add(sc->sc_txrx_workqueue, &netq->netq_work); 1601 } else { 1602 softint_schedule(netq->netq_softint); 1603 } 1604 } 1605 1606 static int 1607 vioif_net_load_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map, 1608 struct mbuf *m, int dma_flags) 1609 { 1610 int r; 1611 1612 KASSERT(map->vnm_mbuf == NULL); 1613 1614 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 1615 map->vnm_mbuf_map, m, dma_flags | BUS_DMA_NOWAIT); 1616 if (r == 0) { 1617 map->vnm_mbuf = m; 1618 } 1619 1620 return r; 1621 } 1622 1623 static void 1624 vioif_net_unload_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map) 1625 { 1626 1627 KASSERT(map->vnm_mbuf != NULL); 1628 bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map); 1629 map->vnm_mbuf = NULL; 1630 } 1631 1632 static int 1633 vioif_net_enqueue(struct virtio_softc *vsc, struct virtqueue *vq, 1634 int slot, struct vioif_net_map *map, int dma_ops, bool is_write) 1635 { 1636 int r; 1637 1638 KASSERT(map->vnm_mbuf != NULL); 1639 1640 /* This should actually never fail */ 1641 r = virtio_enqueue_reserve(vsc, vq, slot, 1642 map->vnm_mbuf_map->dm_nsegs + 1); 1643 if (r != 0) { 1644 /* slot already freed by virtio_enqueue_reserve */ 1645 return r; 1646 } 1647 1648 bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map, 1649 0, map->vnm_mbuf_map->dm_mapsize, dma_ops); 1650 bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map, 1651 0, map->vnm_hdr_map->dm_mapsize, dma_ops); 1652 1653 virtio_enqueue(vsc, vq, slot, map->vnm_hdr_map, is_write); 1654 virtio_enqueue(vsc, vq, slot, map->vnm_mbuf_map, is_write); 1655 virtio_enqueue_commit(vsc, vq, slot, false); 1656 1657 return 0; 1658 } 1659 1660 static int 1661 vioif_net_enqueue_tx(struct virtio_softc *vsc, struct virtqueue *vq, 1662 int slot, struct vioif_net_map *map) 1663 { 1664 1665 return vioif_net_enqueue(vsc, vq, slot, map, 1666 BUS_DMASYNC_PREWRITE, true); 1667 } 1668 1669 static int 1670 vioif_net_enqueue_rx(struct virtio_softc *vsc, struct virtqueue *vq, 1671 int slot, struct vioif_net_map *map) 1672 { 1673 1674 return vioif_net_enqueue(vsc, vq, slot, map, 1675 BUS_DMASYNC_PREREAD, false); 1676 } 1677 1678 static struct mbuf * 1679 vioif_net_dequeue_commit(struct virtio_softc *vsc, struct virtqueue *vq, 1680 int slot, struct vioif_net_map *map, int dma_flags) 1681 { 1682 struct mbuf *m; 1683 1684 m = map->vnm_mbuf; 1685 KASSERT(m != NULL); 1686 map->vnm_mbuf = NULL; 1687 1688 bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map, 1689 0, map->vnm_hdr_map->dm_mapsize, dma_flags); 1690 bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map, 1691 0, map->vnm_mbuf_map->dm_mapsize, dma_flags); 1692 1693 bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map); 1694 virtio_dequeue_commit(vsc, vq, slot); 1695 1696 return m; 1697 } 1698 1699 static void 1700 vioif_net_intr_enable(struct vioif_softc *sc, struct virtio_softc *vsc) 1701 { 1702 struct vioif_netqueue *netq; 1703 size_t i, act_qnum; 1704 int enqueued; 1705 1706 act_qnum = sc->sc_act_nvq_pairs * 2; 1707 for (i = 0; i < act_qnum; i++) { 1708 netq = &sc->sc_netqs[i]; 1709 1710 KASSERT(!netq->netq_stopping); 1711 KASSERT(!netq->netq_running_handle); 1712 1713 enqueued = virtio_start_vq_intr(vsc, netq->netq_vq); 1714 if (enqueued != 0) { 1715 virtio_stop_vq_intr(vsc, netq->netq_vq); 1716 1717 mutex_enter(&netq->netq_lock); 1718 netq->netq_running_handle = true; 1719 vioif_net_sched_handle(sc, netq); 1720 mutex_exit(&netq->netq_lock); 1721 } 1722 } 1723 } 1724 1725 static void 1726 vioif_net_intr_disable(struct vioif_softc *sc, struct virtio_softc *vsc) 1727 { 1728 struct vioif_netqueue *netq; 1729 size_t i, act_qnum; 1730 1731 act_qnum = sc->sc_act_nvq_pairs * 2; 1732 for (i = 0; i < act_qnum; i++) { 1733 netq = &sc->sc_netqs[i]; 1734 1735 virtio_stop_vq_intr(vsc, netq->netq_vq); 1736 } 1737 } 1738 1739 /* 1740 * Receive implementation 1741 */ 1742 /* enqueue mbufs to receive slots */ 1743 static void 1744 vioif_populate_rx_mbufs_locked(struct vioif_softc *sc, struct vioif_netqueue *netq) 1745 { 1746 struct virtqueue *vq = netq->netq_vq; 1747 struct virtio_softc *vsc = vq->vq_owner; 1748 struct vioif_rx_context *rxc; 1749 struct vioif_net_map *map; 1750 struct mbuf *m; 1751 int i, r, ndone = 0; 1752 1753 KASSERT(mutex_owned(&netq->netq_lock)); 1754 1755 rxc = netq->netq_ctx; 1756 1757 for (i = 0; i < vq->vq_num; i++) { 1758 int slot; 1759 r = virtio_enqueue_prep(vsc, vq, &slot); 1760 if (r == EAGAIN) 1761 break; 1762 if (__predict_false(r != 0)) 1763 panic("enqueue_prep for rx buffers"); 1764 1765 MGETHDR(m, M_DONTWAIT, MT_DATA); 1766 if (m == NULL) { 1767 virtio_enqueue_abort(vsc, vq, slot); 1768 rxc->rxc_mbuf_enobufs.ev_count++; 1769 break; 1770 } 1771 MCLAIM(m, &sc->sc_ethercom.ec_rx_mowner); 1772 MCLGET(m, M_DONTWAIT); 1773 if ((m->m_flags & M_EXT) == 0) { 1774 virtio_enqueue_abort(vsc, vq, slot); 1775 m_freem(m); 1776 rxc->rxc_mbuf_enobufs.ev_count++; 1777 break; 1778 } 1779 1780 m->m_len = m->m_pkthdr.len = MCLBYTES; 1781 m_adj(m, ETHER_ALIGN); 1782 1783 map = &netq->netq_maps[slot]; 1784 r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_READ); 1785 if (r != 0) { 1786 virtio_enqueue_abort(vsc, vq, slot); 1787 m_freem(m); 1788 netq->netq_mbuf_load_failed.ev_count++; 1789 break; 1790 } 1791 1792 r = vioif_net_enqueue_rx(vsc, vq, slot, map); 1793 if (r != 0) { 1794 vioif_net_unload_mbuf(vsc, map); 1795 netq->netq_enqueue_failed.ev_count++; 1796 m_freem(m); 1797 /* slot already freed by vioif_net_enqueue_rx */ 1798 break; 1799 } 1800 1801 ndone++; 1802 } 1803 1804 if (ndone > 0) 1805 vioif_notify(vsc, vq); 1806 } 1807 1808 /* dequeue received packets */ 1809 static bool 1810 vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, 1811 struct vioif_netqueue *netq, u_int limit, size_t *ndeqp) 1812 { 1813 struct virtqueue *vq = netq->netq_vq; 1814 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1815 struct vioif_net_map *map; 1816 struct mbuf *m; 1817 int slot, len; 1818 bool more; 1819 size_t ndeq; 1820 1821 KASSERT(mutex_owned(&netq->netq_lock)); 1822 1823 more = false; 1824 ndeq = 0; 1825 1826 if (virtio_vq_is_enqueued(vsc, vq) == false) 1827 goto done; 1828 1829 for (;;ndeq++) { 1830 if (ndeq >= limit) { 1831 more = true; 1832 break; 1833 } 1834 1835 if (virtio_dequeue(vsc, vq, &slot, &len) != 0) 1836 break; 1837 1838 map = &netq->netq_maps[slot]; 1839 KASSERT(map->vnm_mbuf != NULL); 1840 m = vioif_net_dequeue_commit(vsc, vq, slot, 1841 map, BUS_DMASYNC_POSTREAD); 1842 KASSERT(m != NULL); 1843 1844 m->m_len = m->m_pkthdr.len = len - sc->sc_hdr_size; 1845 m_set_rcvif(m, ifp); 1846 if_percpuq_enqueue(ifp->if_percpuq, m); 1847 } 1848 1849 done: 1850 if (ndeqp != NULL) 1851 *ndeqp = ndeq; 1852 1853 return more; 1854 } 1855 1856 static void 1857 vioif_rx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc, 1858 struct vioif_netqueue *netq) 1859 { 1860 struct vioif_net_map *map; 1861 struct mbuf *m; 1862 unsigned int i, vq_num; 1863 bool more; 1864 1865 mutex_enter(&netq->netq_lock); 1866 1867 vq_num = netq->netq_vq->vq_num; 1868 for (;;) { 1869 more = vioif_rx_deq_locked(sc, vsc, netq, vq_num, NULL); 1870 if (more == false) 1871 break; 1872 } 1873 1874 for (i = 0; i < vq_num; i++) { 1875 map = &netq->netq_maps[i]; 1876 1877 m = map->vnm_mbuf; 1878 if (m == NULL) 1879 continue; 1880 1881 vioif_net_unload_mbuf(vsc, map); 1882 m_freem(m); 1883 } 1884 mutex_exit(&netq->netq_lock); 1885 } 1886 1887 static void 1888 vioif_rx_handle_locked(void *xnetq, u_int limit) 1889 { 1890 struct vioif_netqueue *netq = xnetq; 1891 struct virtqueue *vq = netq->netq_vq; 1892 struct virtio_softc *vsc = vq->vq_owner; 1893 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1894 bool more; 1895 int enqueued; 1896 size_t ndeq; 1897 1898 KASSERT(mutex_owned(&netq->netq_lock)); 1899 KASSERT(!netq->netq_stopping); 1900 1901 more = vioif_rx_deq_locked(sc, vsc, netq, limit, &ndeq); 1902 if (ndeq > 0) 1903 vioif_populate_rx_mbufs_locked(sc, netq); 1904 1905 if (more) { 1906 vioif_net_sched_handle(sc, netq); 1907 return; 1908 } 1909 1910 enqueued = virtio_start_vq_intr(vsc, netq->netq_vq); 1911 if (enqueued != 0) { 1912 virtio_stop_vq_intr(vsc, netq->netq_vq); 1913 vioif_net_sched_handle(sc, netq); 1914 return; 1915 } 1916 1917 netq->netq_running_handle = false; 1918 } 1919 1920 static int 1921 vioif_rx_intr(void *arg) 1922 { 1923 struct vioif_netqueue *netq = arg; 1924 struct virtqueue *vq = netq->netq_vq; 1925 struct virtio_softc *vsc = vq->vq_owner; 1926 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1927 u_int limit; 1928 1929 mutex_enter(&netq->netq_lock); 1930 1931 /* handler is already running in softint/workqueue */ 1932 if (netq->netq_running_handle) 1933 goto done; 1934 1935 if (netq->netq_stopping) 1936 goto done; 1937 1938 netq->netq_running_handle = true; 1939 1940 limit = sc->sc_rx_intr_process_limit; 1941 virtio_stop_vq_intr(vsc, vq); 1942 vioif_rx_handle_locked(netq, limit); 1943 1944 done: 1945 mutex_exit(&netq->netq_lock); 1946 return 1; 1947 } 1948 1949 static void 1950 vioif_rx_handle(void *xnetq) 1951 { 1952 struct vioif_netqueue *netq = xnetq; 1953 struct virtqueue *vq = netq->netq_vq; 1954 struct virtio_softc *vsc = vq->vq_owner; 1955 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1956 u_int limit; 1957 1958 mutex_enter(&netq->netq_lock); 1959 1960 KASSERT(netq->netq_running_handle); 1961 1962 if (netq->netq_stopping) { 1963 netq->netq_running_handle = false; 1964 goto done; 1965 } 1966 1967 limit = sc->sc_rx_process_limit; 1968 vioif_rx_handle_locked(netq, limit); 1969 1970 done: 1971 mutex_exit(&netq->netq_lock); 1972 } 1973 1974 /* 1975 * Transmission implementation 1976 */ 1977 /* enqueue mbufs to send */ 1978 static void 1979 vioif_send_common_locked(struct ifnet *ifp, struct vioif_netqueue *netq, 1980 bool is_transmit) 1981 { 1982 struct vioif_softc *sc = ifp->if_softc; 1983 struct virtio_softc *vsc = sc->sc_virtio; 1984 struct virtqueue *vq = netq->netq_vq; 1985 struct vioif_tx_context *txc; 1986 struct vioif_net_map *map; 1987 struct mbuf *m; 1988 int queued = 0; 1989 1990 KASSERT(mutex_owned(&netq->netq_lock)); 1991 1992 if (netq->netq_stopping || 1993 !ISSET(ifp->if_flags, IFF_RUNNING)) 1994 return; 1995 1996 txc = netq->netq_ctx; 1997 1998 if (!txc->txc_link_active || 1999 txc->txc_no_free_slots) 2000 return; 2001 2002 for (;;) { 2003 int slot, r; 2004 r = virtio_enqueue_prep(vsc, vq, &slot); 2005 if (r == EAGAIN) { 2006 txc->txc_no_free_slots = true; 2007 break; 2008 } 2009 if (__predict_false(r != 0)) 2010 panic("enqueue_prep for tx buffers"); 2011 2012 if (is_transmit) 2013 m = pcq_get(txc->txc_intrq); 2014 else 2015 IFQ_DEQUEUE(&ifp->if_snd, m); 2016 2017 if (m == NULL) { 2018 virtio_enqueue_abort(vsc, vq, slot); 2019 break; 2020 } 2021 2022 map = &netq->netq_maps[slot]; 2023 KASSERT(map->vnm_mbuf == NULL); 2024 2025 r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_WRITE); 2026 if (r != 0) { 2027 /* maybe just too fragmented */ 2028 struct mbuf *newm; 2029 2030 newm = m_defrag(m, M_NOWAIT); 2031 if (newm != NULL) { 2032 m = newm; 2033 r = vioif_net_load_mbuf(vsc, map, m, 2034 BUS_DMA_WRITE); 2035 } else { 2036 txc->txc_defrag_failed.ev_count++; 2037 r = -1; 2038 } 2039 2040 if (r != 0) { 2041 netq->netq_mbuf_load_failed.ev_count++; 2042 m_freem(m); 2043 if_statinc(ifp, if_oerrors); 2044 virtio_enqueue_abort(vsc, vq, slot); 2045 continue; 2046 } 2047 } 2048 2049 memset(map->vnm_hdr, 0, sc->sc_hdr_size); 2050 2051 r = vioif_net_enqueue_tx(vsc, vq, slot, map); 2052 if (r != 0) { 2053 netq->netq_enqueue_failed.ev_count++; 2054 vioif_net_unload_mbuf(vsc, map); 2055 m_freem(m); 2056 /* slot already freed by vioif_net_enqueue_tx */ 2057 2058 if_statinc(ifp, if_oerrors); 2059 continue; 2060 } 2061 2062 queued++; 2063 bpf_mtap(ifp, m, BPF_D_OUT); 2064 } 2065 2066 if (queued > 0) { 2067 vioif_notify(vsc, vq); 2068 ifp->if_timer = 5; 2069 } 2070 } 2071 2072 /* dequeue sent mbufs */ 2073 static bool 2074 vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, 2075 struct vioif_netqueue *netq, u_int limit, size_t *ndeqp) 2076 { 2077 struct virtqueue *vq = netq->netq_vq; 2078 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2079 struct vioif_net_map *map; 2080 struct mbuf *m; 2081 int slot, len; 2082 bool more; 2083 size_t ndeq; 2084 2085 KASSERT(mutex_owned(&netq->netq_lock)); 2086 2087 more = false; 2088 ndeq = 0; 2089 2090 if (virtio_vq_is_enqueued(vsc, vq) == false) 2091 goto done; 2092 2093 for (;;ndeq++) { 2094 if (limit-- == 0) { 2095 more = true; 2096 break; 2097 } 2098 2099 if (virtio_dequeue(vsc, vq, &slot, &len) != 0) 2100 break; 2101 2102 map = &netq->netq_maps[slot]; 2103 KASSERT(map->vnm_mbuf != NULL); 2104 m = vioif_net_dequeue_commit(vsc, vq, slot, 2105 map, BUS_DMASYNC_POSTWRITE); 2106 KASSERT(m != NULL); 2107 2108 if_statinc(ifp, if_opackets); 2109 m_freem(m); 2110 } 2111 2112 done: 2113 if (ndeqp != NULL) 2114 *ndeqp = ndeq; 2115 return more; 2116 } 2117 2118 static void 2119 vioif_tx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc, 2120 struct vioif_netqueue *netq) 2121 { 2122 struct vioif_tx_context *txc; 2123 struct vioif_net_map *map; 2124 struct mbuf *m; 2125 unsigned int i, vq_num; 2126 bool more; 2127 2128 mutex_enter(&netq->netq_lock); 2129 2130 txc = netq->netq_ctx; 2131 vq_num = netq->netq_vq->vq_num; 2132 2133 for (;;) { 2134 more = vioif_tx_deq_locked(sc, vsc, netq, vq_num, NULL); 2135 if (more == false) 2136 break; 2137 } 2138 2139 for (i = 0; i < vq_num; i++) { 2140 map = &netq->netq_maps[i]; 2141 2142 m = map->vnm_mbuf; 2143 if (m == NULL) 2144 continue; 2145 2146 vioif_net_unload_mbuf(vsc, map); 2147 m_freem(m); 2148 } 2149 2150 txc->txc_no_free_slots = false; 2151 2152 mutex_exit(&netq->netq_lock); 2153 } 2154 2155 static void 2156 vioif_start_locked(struct ifnet *ifp, struct vioif_netqueue *netq) 2157 { 2158 2159 /* 2160 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c. 2161 */ 2162 vioif_send_common_locked(ifp, netq, false); 2163 2164 } 2165 2166 static void 2167 vioif_transmit_locked(struct ifnet *ifp, struct vioif_netqueue *netq) 2168 { 2169 2170 vioif_send_common_locked(ifp, netq, true); 2171 } 2172 2173 static void 2174 vioif_deferred_transmit(void *arg) 2175 { 2176 struct vioif_netqueue *netq = arg; 2177 struct virtio_softc *vsc = netq->netq_vq->vq_owner; 2178 struct vioif_softc *sc = device_private(virtio_child(vsc)); 2179 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2180 2181 mutex_enter(&netq->netq_lock); 2182 vioif_send_common_locked(ifp, netq, true); 2183 mutex_exit(&netq->netq_lock); 2184 } 2185 2186 static void 2187 vioif_tx_handle_locked(struct vioif_netqueue *netq, u_int limit) 2188 { 2189 struct virtqueue *vq = netq->netq_vq; 2190 struct vioif_tx_context *txc = netq->netq_ctx; 2191 struct virtio_softc *vsc = vq->vq_owner; 2192 struct vioif_softc *sc = device_private(virtio_child(vsc)); 2193 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2194 bool more; 2195 int enqueued; 2196 size_t ndeq; 2197 2198 KASSERT(mutex_owned(&netq->netq_lock)); 2199 KASSERT(!netq->netq_stopping); 2200 2201 more = vioif_tx_deq_locked(sc, vsc, netq, limit, &ndeq); 2202 if (txc->txc_no_free_slots && ndeq > 0) { 2203 txc->txc_no_free_slots = false; 2204 softint_schedule(txc->txc_deferred_transmit); 2205 } 2206 2207 if (more) { 2208 vioif_net_sched_handle(sc, netq); 2209 return; 2210 } 2211 2212 enqueued = (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) ? 2213 virtio_postpone_intr_smart(vsc, vq): 2214 virtio_start_vq_intr(vsc, vq); 2215 if (enqueued != 0) { 2216 virtio_stop_vq_intr(vsc, vq); 2217 vioif_net_sched_handle(sc, netq); 2218 return; 2219 } 2220 2221 netq->netq_running_handle = false; 2222 2223 /* for ALTQ */ 2224 if (netq == &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]) 2225 if_schedule_deferred_start(ifp); 2226 2227 softint_schedule(txc->txc_deferred_transmit); 2228 } 2229 2230 static int 2231 vioif_tx_intr(void *arg) 2232 { 2233 struct vioif_netqueue *netq = arg; 2234 struct virtqueue *vq = netq->netq_vq; 2235 struct virtio_softc *vsc = vq->vq_owner; 2236 struct vioif_softc *sc = device_private(virtio_child(vsc)); 2237 u_int limit; 2238 2239 mutex_enter(&netq->netq_lock); 2240 2241 /* tx handler is already running in softint/workqueue */ 2242 if (netq->netq_running_handle) 2243 goto done; 2244 2245 if (netq->netq_stopping) 2246 goto done; 2247 2248 netq->netq_running_handle = true; 2249 2250 virtio_stop_vq_intr(vsc, vq); 2251 netq->netq_workqueue = sc->sc_txrx_workqueue_sysctl; 2252 limit = sc->sc_tx_intr_process_limit; 2253 vioif_tx_handle_locked(netq, limit); 2254 2255 done: 2256 mutex_exit(&netq->netq_lock); 2257 return 1; 2258 } 2259 2260 static void 2261 vioif_tx_handle(void *xnetq) 2262 { 2263 struct vioif_netqueue *netq = xnetq; 2264 struct virtqueue *vq = netq->netq_vq; 2265 struct virtio_softc *vsc = vq->vq_owner; 2266 struct vioif_softc *sc = device_private(virtio_child(vsc)); 2267 u_int limit; 2268 2269 mutex_enter(&netq->netq_lock); 2270 2271 KASSERT(netq->netq_running_handle); 2272 2273 if (netq->netq_stopping) { 2274 netq->netq_running_handle = false; 2275 goto done; 2276 } 2277 2278 limit = sc->sc_tx_process_limit; 2279 vioif_tx_handle_locked(netq, limit); 2280 2281 done: 2282 mutex_exit(&netq->netq_lock); 2283 } 2284 2285 /* 2286 * Control vq 2287 */ 2288 /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */ 2289 static void 2290 vioif_ctrl_acquire(struct vioif_softc *sc) 2291 { 2292 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 2293 2294 mutex_enter(&ctrlq->ctrlq_wait_lock); 2295 while (ctrlq->ctrlq_inuse != FREE) 2296 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); 2297 ctrlq->ctrlq_inuse = INUSE; 2298 ctrlq->ctrlq_owner = curlwp; 2299 mutex_exit(&ctrlq->ctrlq_wait_lock); 2300 } 2301 2302 static void 2303 vioif_ctrl_release(struct vioif_softc *sc) 2304 { 2305 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 2306 2307 KASSERT(ctrlq->ctrlq_inuse != FREE); 2308 KASSERT(ctrlq->ctrlq_owner == curlwp); 2309 2310 mutex_enter(&ctrlq->ctrlq_wait_lock); 2311 ctrlq->ctrlq_inuse = FREE; 2312 ctrlq->ctrlq_owner = NULL; 2313 cv_signal(&ctrlq->ctrlq_wait); 2314 mutex_exit(&ctrlq->ctrlq_wait_lock); 2315 } 2316 2317 static int 2318 vioif_ctrl_load_cmdspec(struct vioif_softc *sc, 2319 struct vioif_ctrl_cmdspec *specs, int nspecs) 2320 { 2321 struct virtio_softc *vsc = sc->sc_virtio; 2322 int i, r, loaded; 2323 2324 loaded = 0; 2325 for (i = 0; i < nspecs; i++) { 2326 r = bus_dmamap_load(virtio_dmat(vsc), 2327 specs[i].dmamap, specs[i].buf, specs[i].bufsize, 2328 NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2329 if (r) { 2330 sc->sc_ctrlq.ctrlq_cmd_load_failed.ev_count++; 2331 goto err; 2332 } 2333 loaded++; 2334 2335 } 2336 2337 return r; 2338 2339 err: 2340 for (i = 0; i < loaded; i++) { 2341 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); 2342 } 2343 2344 return r; 2345 } 2346 2347 static void 2348 vioif_ctrl_unload_cmdspec(struct vioif_softc *sc, 2349 struct vioif_ctrl_cmdspec *specs, int nspecs) 2350 { 2351 struct virtio_softc *vsc = sc->sc_virtio; 2352 int i; 2353 2354 for (i = 0; i < nspecs; i++) { 2355 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); 2356 } 2357 } 2358 2359 static int 2360 vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd, 2361 struct vioif_ctrl_cmdspec *specs, int nspecs) 2362 { 2363 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 2364 struct virtqueue *vq = ctrlq->ctrlq_vq; 2365 struct virtio_softc *vsc = sc->sc_virtio; 2366 int i, r, slot; 2367 2368 ctrlq->ctrlq_cmd->class = class; 2369 ctrlq->ctrlq_cmd->command = cmd; 2370 2371 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 2372 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE); 2373 for (i = 0; i < nspecs; i++) { 2374 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 2375 0, specs[i].bufsize, BUS_DMASYNC_PREWRITE); 2376 } 2377 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 2378 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD); 2379 2380 /* we need to explicitly (re)start vq intr when using RING EVENT IDX */ 2381 if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) 2382 virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); 2383 2384 r = virtio_enqueue_prep(vsc, vq, &slot); 2385 if (r != 0) 2386 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 2387 r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2); 2388 if (r != 0) 2389 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 2390 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true); 2391 for (i = 0; i < nspecs; i++) { 2392 virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true); 2393 } 2394 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false); 2395 virtio_enqueue_commit(vsc, vq, slot, true); 2396 2397 /* wait for done */ 2398 mutex_enter(&ctrlq->ctrlq_wait_lock); 2399 while (ctrlq->ctrlq_inuse != DONE) 2400 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); 2401 mutex_exit(&ctrlq->ctrlq_wait_lock); 2402 /* already dequeued */ 2403 2404 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0, 2405 sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE); 2406 for (i = 0; i < nspecs; i++) { 2407 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0, 2408 specs[i].bufsize, BUS_DMASYNC_POSTWRITE); 2409 } 2410 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0, 2411 sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD); 2412 2413 if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK) 2414 r = 0; 2415 else { 2416 device_printf(sc->sc_dev, "failed setting rx mode\n"); 2417 sc->sc_ctrlq.ctrlq_cmd_failed.ev_count++; 2418 r = EIO; 2419 } 2420 2421 return r; 2422 } 2423 2424 /* ctrl vq interrupt; wake up the command issuer */ 2425 static int 2426 vioif_ctrl_intr(void *arg) 2427 { 2428 struct vioif_ctrlqueue *ctrlq = arg; 2429 struct virtqueue *vq = ctrlq->ctrlq_vq; 2430 struct virtio_softc *vsc = vq->vq_owner; 2431 int r, slot; 2432 2433 if (virtio_vq_is_enqueued(vsc, vq) == false) 2434 return 0; 2435 2436 r = virtio_dequeue(vsc, vq, &slot, NULL); 2437 if (r == ENOENT) 2438 return 0; 2439 virtio_dequeue_commit(vsc, vq, slot); 2440 2441 mutex_enter(&ctrlq->ctrlq_wait_lock); 2442 ctrlq->ctrlq_inuse = DONE; 2443 cv_signal(&ctrlq->ctrlq_wait); 2444 mutex_exit(&ctrlq->ctrlq_wait_lock); 2445 2446 return 1; 2447 } 2448 2449 static int 2450 vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) 2451 { 2452 struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx; 2453 struct vioif_ctrl_cmdspec specs[1]; 2454 int r; 2455 2456 if (!sc->sc_has_ctrl) 2457 return ENOTSUP; 2458 2459 vioif_ctrl_acquire(sc); 2460 2461 rx->onoff = onoff; 2462 specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap; 2463 specs[0].buf = rx; 2464 specs[0].bufsize = sizeof(*rx); 2465 2466 r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd, 2467 specs, __arraycount(specs)); 2468 2469 vioif_ctrl_release(sc); 2470 return r; 2471 } 2472 2473 static int 2474 vioif_set_promisc(struct vioif_softc *sc, bool onoff) 2475 { 2476 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff); 2477 } 2478 2479 static int 2480 vioif_set_allmulti(struct vioif_softc *sc, bool onoff) 2481 { 2482 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff); 2483 } 2484 2485 static int 2486 vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs) 2487 { 2488 struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq; 2489 struct vioif_ctrl_cmdspec specs[1]; 2490 int r; 2491 2492 if (!sc->sc_has_ctrl) 2493 return ENOTSUP; 2494 2495 if (nvq_pairs <= 1) 2496 return EINVAL; 2497 2498 vioif_ctrl_acquire(sc); 2499 2500 mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs); 2501 specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap; 2502 specs[0].buf = mq; 2503 specs[0].bufsize = sizeof(*mq); 2504 2505 r = vioif_ctrl_send_command(sc, 2506 VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, 2507 specs, __arraycount(specs)); 2508 2509 vioif_ctrl_release(sc); 2510 2511 return r; 2512 } 2513 2514 static int 2515 vioif_set_mac_addr(struct vioif_softc *sc) 2516 { 2517 struct virtio_net_ctrl_mac_addr *ma = 2518 sc->sc_ctrlq.ctrlq_mac_addr; 2519 struct vioif_ctrl_cmdspec specs[1]; 2520 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2521 int nspecs = __arraycount(specs); 2522 uint64_t features; 2523 int r; 2524 size_t i; 2525 2526 if (!sc->sc_has_ctrl) 2527 return ENOTSUP; 2528 2529 if (memcmp(CLLADDR(ifp->if_sadl), sc->sc_mac, 2530 ETHER_ADDR_LEN) == 0) { 2531 return 0; 2532 } 2533 2534 memcpy(sc->sc_mac, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2535 2536 features = virtio_features(sc->sc_virtio); 2537 if (features & VIRTIO_NET_F_CTRL_MAC_ADDR) { 2538 vioif_ctrl_acquire(sc); 2539 2540 memcpy(ma->mac, sc->sc_mac, ETHER_ADDR_LEN); 2541 specs[0].dmamap = sc->sc_ctrlq.ctrlq_mac_addr_dmamap; 2542 specs[0].buf = ma; 2543 specs[0].bufsize = sizeof(*ma); 2544 2545 r = vioif_ctrl_send_command(sc, 2546 VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_ADDR_SET, 2547 specs, nspecs); 2548 2549 vioif_ctrl_release(sc); 2550 } else { 2551 for (i = 0; i < __arraycount(sc->sc_mac); i++) { 2552 virtio_write_device_config_1(sc->sc_virtio, 2553 VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]); 2554 } 2555 r = 0; 2556 } 2557 2558 return r; 2559 } 2560 2561 static int 2562 vioif_set_rx_filter(struct vioif_softc *sc) 2563 { 2564 /* filter already set in ctrlq->ctrlq_mac_tbl */ 2565 struct virtio_softc *vsc = sc->sc_virtio; 2566 struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc; 2567 struct vioif_ctrl_cmdspec specs[2]; 2568 int nspecs = __arraycount(specs); 2569 int r; 2570 2571 mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc; 2572 mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc; 2573 2574 if (!sc->sc_has_ctrl) 2575 return ENOTSUP; 2576 2577 vioif_ctrl_acquire(sc); 2578 2579 specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap; 2580 specs[0].buf = mac_tbl_uc; 2581 specs[0].bufsize = sizeof(*mac_tbl_uc) 2582 + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries)); 2583 2584 specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap; 2585 specs[1].buf = mac_tbl_mc; 2586 specs[1].bufsize = sizeof(*mac_tbl_mc) 2587 + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries)); 2588 2589 r = vioif_ctrl_load_cmdspec(sc, specs, nspecs); 2590 if (r != 0) 2591 goto out; 2592 2593 r = vioif_ctrl_send_command(sc, 2594 VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET, 2595 specs, nspecs); 2596 2597 vioif_ctrl_unload_cmdspec(sc, specs, nspecs); 2598 2599 out: 2600 vioif_ctrl_release(sc); 2601 2602 return r; 2603 } 2604 2605 /* 2606 * If multicast filter small enough (<=MAXENTRIES) set rx filter 2607 * If large multicast filter exist use ALLMULTI 2608 * If setting rx filter fails fall back to ALLMULTI 2609 */ 2610 static int 2611 vioif_rx_filter(struct vioif_softc *sc) 2612 { 2613 struct virtio_softc *vsc = sc->sc_virtio; 2614 struct ethercom *ec = &sc->sc_ethercom; 2615 struct ifnet *ifp = &ec->ec_if; 2616 struct ether_multi *enm; 2617 struct ether_multistep step; 2618 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 2619 int nentries; 2620 bool allmulti = 0; 2621 int r; 2622 2623 if (!sc->sc_has_ctrl) { 2624 goto set_ifflags; 2625 } 2626 2627 memcpy(ctrlq->ctrlq_mac_tbl_uc->macs[0], 2628 CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2629 2630 nentries = 0; 2631 allmulti = false; 2632 2633 ETHER_LOCK(ec); 2634 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 2635 ETHER_NEXT_MULTI(step, enm)) { 2636 if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) { 2637 allmulti = true; 2638 break; 2639 } 2640 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) { 2641 allmulti = true; 2642 break; 2643 } 2644 2645 memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries], 2646 enm->enm_addrlo, ETHER_ADDR_LEN); 2647 nentries++; 2648 } 2649 ETHER_UNLOCK(ec); 2650 2651 r = vioif_set_mac_addr(sc); 2652 if (r != 0) { 2653 log(LOG_WARNING, "%s: couldn't set MAC address\n", 2654 ifp->if_xname); 2655 } 2656 2657 if (!allmulti) { 2658 ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 1); 2659 ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, nentries); 2660 r = vioif_set_rx_filter(sc); 2661 if (r != 0) { 2662 allmulti = true; /* fallback */ 2663 } 2664 } 2665 2666 if (allmulti) { 2667 ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 0); 2668 ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, 0); 2669 r = vioif_set_rx_filter(sc); 2670 if (r != 0) { 2671 log(LOG_DEBUG, "%s: couldn't clear RX filter\n", 2672 ifp->if_xname); 2673 /* what to do on failure? */ 2674 } 2675 2676 ifp->if_flags |= IFF_ALLMULTI; 2677 } 2678 2679 set_ifflags: 2680 r = vioif_ifflags(sc); 2681 2682 return r; 2683 } 2684 2685 /* 2686 * VM configuration changes 2687 */ 2688 static int 2689 vioif_config_change(struct virtio_softc *vsc) 2690 { 2691 struct vioif_softc *sc = device_private(virtio_child(vsc)); 2692 2693 softint_schedule(sc->sc_cfg_softint); 2694 return 0; 2695 } 2696 2697 static void 2698 vioif_cfg_softint(void *arg) 2699 { 2700 struct vioif_softc *sc = arg; 2701 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2702 2703 vioif_update_link_status(sc); 2704 vioif_start(ifp); 2705 } 2706 2707 static int 2708 vioif_get_link_status(struct vioif_softc *sc) 2709 { 2710 struct virtio_softc *vsc = sc->sc_virtio; 2711 uint16_t status; 2712 2713 if (virtio_features(vsc) & VIRTIO_NET_F_STATUS) 2714 status = virtio_read_device_config_2(vsc, 2715 VIRTIO_NET_CONFIG_STATUS); 2716 else 2717 status = VIRTIO_NET_S_LINK_UP; 2718 2719 if ((status & VIRTIO_NET_S_LINK_UP) != 0) 2720 return LINK_STATE_UP; 2721 2722 return LINK_STATE_DOWN; 2723 } 2724 2725 static void 2726 vioif_update_link_status(struct vioif_softc *sc) 2727 { 2728 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2729 struct vioif_netqueue *netq; 2730 struct vioif_tx_context *txc; 2731 bool active; 2732 int link, i; 2733 2734 mutex_enter(&sc->sc_lock); 2735 2736 link = vioif_get_link_status(sc); 2737 2738 if (link == sc->sc_link_state) 2739 goto done; 2740 2741 sc->sc_link_state = link; 2742 2743 active = VIOIF_IS_LINK_ACTIVE(sc); 2744 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 2745 netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)]; 2746 2747 mutex_enter(&netq->netq_lock); 2748 txc = netq->netq_ctx; 2749 txc->txc_link_active = active; 2750 mutex_exit(&netq->netq_lock); 2751 } 2752 2753 if_link_state_change(ifp, sc->sc_link_state); 2754 2755 done: 2756 mutex_exit(&sc->sc_lock); 2757 } 2758 2759 static void 2760 vioif_workq_work(struct work *wk, void *context) 2761 { 2762 struct vioif_work *work; 2763 2764 work = container_of(wk, struct vioif_work, cookie); 2765 2766 atomic_store_relaxed(&work->added, 0); 2767 work->func(work->arg); 2768 } 2769 2770 static struct workqueue * 2771 vioif_workq_create(const char *name, pri_t prio, int ipl, int flags) 2772 { 2773 struct workqueue *wq; 2774 int error; 2775 2776 error = workqueue_create(&wq, name, vioif_workq_work, NULL, 2777 prio, ipl, flags); 2778 2779 if (error) 2780 return NULL; 2781 2782 return wq; 2783 } 2784 2785 static void 2786 vioif_workq_destroy(struct workqueue *wq) 2787 { 2788 2789 workqueue_destroy(wq); 2790 } 2791 2792 static void 2793 vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg) 2794 { 2795 2796 memset(work, 0, sizeof(*work)); 2797 work->func = func; 2798 work->arg = arg; 2799 } 2800 2801 static void 2802 vioif_work_add(struct workqueue *wq, struct vioif_work *work) 2803 { 2804 2805 if (atomic_load_relaxed(&work->added) != 0) 2806 return; 2807 2808 atomic_store_relaxed(&work->added, 1); 2809 kpreempt_disable(); 2810 workqueue_enqueue(wq, &work->cookie, NULL); 2811 kpreempt_enable(); 2812 } 2813 2814 static void 2815 vioif_work_wait(struct workqueue *wq, struct vioif_work *work) 2816 { 2817 2818 workqueue_wait(wq, &work->cookie); 2819 } 2820 2821 MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio"); 2822 2823 #ifdef _MODULE 2824 #include "ioconf.c" 2825 #endif 2826 2827 static int 2828 if_vioif_modcmd(modcmd_t cmd, void *opaque) 2829 { 2830 int error = 0; 2831 2832 #ifdef _MODULE 2833 switch (cmd) { 2834 case MODULE_CMD_INIT: 2835 error = config_init_component(cfdriver_ioconf_if_vioif, 2836 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 2837 break; 2838 case MODULE_CMD_FINI: 2839 error = config_fini_component(cfdriver_ioconf_if_vioif, 2840 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 2841 break; 2842 default: 2843 error = ENOTTY; 2844 break; 2845 } 2846 #endif 2847 2848 return error; 2849 } 2850
Indexes created Tue Sep 23 07:09:52 GMT 2025