if_vioif.c revision 1.82.4.1
1 /* $NetBSD: if_vioif.c,v 1.82.4.1 2023/03/30 11:36:26 martin Exp $ */ 2 3 /* 4 * Copyright (c) 2020 The NetBSD Foundation, Inc. 5 * Copyright (c) 2010 Minoura Makoto. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.82.4.1 2023/03/30 11:36:26 martin Exp $"); 31 32 #ifdef _KERNEL_OPT 33 #include "opt_net_mpsafe.h" 34 #endif 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/kernel.h> 39 #include <sys/atomic.h> 40 #include <sys/bus.h> 41 #include <sys/condvar.h> 42 #include <sys/device.h> 43 #include <sys/evcnt.h> 44 #include <sys/intr.h> 45 #include <sys/kmem.h> 46 #include <sys/mbuf.h> 47 #include <sys/mutex.h> 48 #include <sys/sockio.h> 49 #include <sys/syslog.h> 50 #include <sys/cpu.h> 51 #include <sys/module.h> 52 #include <sys/pcq.h> 53 #include <sys/workqueue.h> 54 #include <sys/xcall.h> 55 56 #include <dev/pci/virtioreg.h> 57 #include <dev/pci/virtiovar.h> 58 59 #include <net/if.h> 60 #include <net/if_dl.h> 61 #include <net/if_media.h> 62 #include <net/if_ether.h> 63 64 #include <net/bpf.h> 65 66 #include "ioconf.h" 67 68 #ifdef NET_MPSAFE 69 #define VIOIF_MPSAFE 1 70 #define VIOIF_MULTIQ 1 71 #endif 72 73 /* 74 * if_vioifreg.h: 75 */ 76 /* Configuration registers */ 77 #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */ 78 #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */ 79 #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */ 80 #define VIRTIO_NET_CONFIG_MTU 10 /* 16bit */ 81 82 /* Feature bits */ 83 #define VIRTIO_NET_F_CSUM __BIT(0) 84 #define VIRTIO_NET_F_GUEST_CSUM __BIT(1) 85 #define VIRTIO_NET_F_MAC __BIT(5) 86 #define VIRTIO_NET_F_GSO __BIT(6) 87 #define VIRTIO_NET_F_GUEST_TSO4 __BIT(7) 88 #define VIRTIO_NET_F_GUEST_TSO6 __BIT(8) 89 #define VIRTIO_NET_F_GUEST_ECN __BIT(9) 90 #define VIRTIO_NET_F_GUEST_UFO __BIT(10) 91 #define VIRTIO_NET_F_HOST_TSO4 __BIT(11) 92 #define VIRTIO_NET_F_HOST_TSO6 __BIT(12) 93 #define VIRTIO_NET_F_HOST_ECN __BIT(13) 94 #define VIRTIO_NET_F_HOST_UFO __BIT(14) 95 #define VIRTIO_NET_F_MRG_RXBUF __BIT(15) 96 #define VIRTIO_NET_F_STATUS __BIT(16) 97 #define VIRTIO_NET_F_CTRL_VQ __BIT(17) 98 #define VIRTIO_NET_F_CTRL_RX __BIT(18) 99 #define VIRTIO_NET_F_CTRL_VLAN __BIT(19) 100 #define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20) 101 #define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21) 102 #define VIRTIO_NET_F_MQ __BIT(22) 103 #define VIRTIO_NET_F_CTRL_MAC_ADDR __BIT(23) 104 105 #define VIRTIO_NET_FLAG_BITS \ 106 VIRTIO_COMMON_FLAG_BITS \ 107 "b\x17" "CTRL_MAC\0" \ 108 "b\x16" "MQ\0" \ 109 "b\x15" "GUEST_ANNOUNCE\0" \ 110 "b\x14" "CTRL_RX_EXTRA\0" \ 111 "b\x13" "CTRL_VLAN\0" \ 112 "b\x12" "CTRL_RX\0" \ 113 "b\x11" "CTRL_VQ\0" \ 114 "b\x10" "STATUS\0" \ 115 "b\x0f" "MRG_RXBUF\0" \ 116 "b\x0e" "HOST_UFO\0" \ 117 "b\x0d" "HOST_ECN\0" \ 118 "b\x0c" "HOST_TSO6\0" \ 119 "b\x0b" "HOST_TSO4\0" \ 120 "b\x0a" "GUEST_UFO\0" \ 121 "b\x09" "GUEST_ECN\0" \ 122 "b\x08" "GUEST_TSO6\0" \ 123 "b\x07" "GUEST_TSO4\0" \ 124 "b\x06" "GSO\0" \ 125 "b\x05" "MAC\0" \ 126 "b\x01" "GUEST_CSUM\0" \ 127 "b\x00" "CSUM\0" 128 129 /* Status */ 130 #define VIRTIO_NET_S_LINK_UP 1 131 132 /* Packet header structure */ 133 struct virtio_net_hdr { 134 uint8_t flags; 135 uint8_t gso_type; 136 uint16_t hdr_len; 137 uint16_t gso_size; 138 uint16_t csum_start; 139 uint16_t csum_offset; 140 141 uint16_t num_buffers; /* VIRTIO_NET_F_MRG_RXBUF enabled or v1 */ 142 } __packed; 143 144 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */ 145 #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */ 146 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */ 147 #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */ 148 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */ 149 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */ 150 151 #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN) 152 153 /* Control virtqueue */ 154 struct virtio_net_ctrl_cmd { 155 uint8_t class; 156 uint8_t command; 157 } __packed; 158 #define VIRTIO_NET_CTRL_RX 0 159 # define VIRTIO_NET_CTRL_RX_PROMISC 0 160 # define VIRTIO_NET_CTRL_RX_ALLMULTI 1 161 162 #define VIRTIO_NET_CTRL_MAC 1 163 # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 164 # define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 165 166 #define VIRTIO_NET_CTRL_VLAN 2 167 # define VIRTIO_NET_CTRL_VLAN_ADD 0 168 # define VIRTIO_NET_CTRL_VLAN_DEL 1 169 170 #define VIRTIO_NET_CTRL_MQ 4 171 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 172 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 173 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 174 175 struct virtio_net_ctrl_status { 176 uint8_t ack; 177 } __packed; 178 #define VIRTIO_NET_OK 0 179 #define VIRTIO_NET_ERR 1 180 181 struct virtio_net_ctrl_rx { 182 uint8_t onoff; 183 } __packed; 184 185 struct virtio_net_ctrl_mac_tbl { 186 uint32_t nentries; 187 uint8_t macs[][ETHER_ADDR_LEN]; 188 } __packed; 189 190 struct virtio_net_ctrl_mac_addr { 191 uint8_t mac[ETHER_ADDR_LEN]; 192 } __packed; 193 194 struct virtio_net_ctrl_vlan { 195 uint16_t id; 196 } __packed; 197 198 struct virtio_net_ctrl_mq { 199 uint16_t virtqueue_pairs; 200 } __packed; 201 202 /* 203 * if_vioifvar.h: 204 */ 205 206 /* 207 * Locking notes: 208 * + a field in vioif_netueue is protected by netq_lock (a spin mutex) 209 * - more than one lock cannot be held at onece 210 * + a field in vioif_tx_context and vioif_rx_context is also protected 211 * by netq_lock. 212 * + ctrlq_inuse is protected by ctrlq_wait_lock. 213 * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse 214 * - netq_lock cannot be held along with ctrlq_wait_lock 215 * + fields in vioif_softc except queues are protected by 216 * sc->sc_lock(an adaptive mutex) 217 * - the lock is held before acquisition of other locks 218 */ 219 220 struct vioif_ctrl_cmdspec { 221 bus_dmamap_t dmamap; 222 void *buf; 223 bus_size_t bufsize; 224 }; 225 226 struct vioif_work { 227 struct work cookie; 228 void (*func)(void *); 229 void *arg; 230 unsigned int added; 231 }; 232 233 struct vioif_net_map { 234 struct virtio_net_hdr *vnm_hdr; 235 bus_dmamap_t vnm_hdr_map; 236 struct mbuf *vnm_mbuf; 237 bus_dmamap_t vnm_mbuf_map; 238 }; 239 240 #define VIOIF_NETQ_RX 0 241 #define VIOIF_NETQ_TX 1 242 #define VIOIF_NETQ_IDX 2 243 #define VIOIF_NETQ_DIR(n) ((n) % VIOIF_NETQ_IDX) 244 #define VIOIF_NETQ_PAIRIDX(n) ((n) / VIOIF_NETQ_IDX) 245 #define VIOIF_NETQ_RXQID(n) ((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_RX) 246 #define VIOIF_NETQ_TXQID(n) ((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_TX) 247 248 struct vioif_netqueue { 249 kmutex_t netq_lock; 250 struct virtqueue *netq_vq; 251 bool netq_stopping; 252 bool netq_running_handle; 253 void *netq_maps_kva; 254 struct vioif_net_map *netq_maps; 255 256 void *netq_softint; 257 struct vioif_work netq_work; 258 bool netq_workqueue; 259 260 char netq_evgroup[32]; 261 struct evcnt netq_mbuf_load_failed; 262 struct evcnt netq_enqueue_failed; 263 264 void *netq_ctx; 265 }; 266 267 struct vioif_tx_context { 268 bool txc_link_active; 269 bool txc_no_free_slots; 270 pcq_t *txc_intrq; 271 void *txc_deferred_transmit; 272 273 struct evcnt txc_defrag_failed; 274 }; 275 276 struct vioif_rx_context { 277 struct evcnt rxc_mbuf_enobufs; 278 }; 279 struct vioif_ctrlqueue { 280 struct virtqueue *ctrlq_vq; 281 enum { 282 FREE, INUSE, DONE 283 } ctrlq_inuse; 284 kcondvar_t ctrlq_wait; 285 kmutex_t ctrlq_wait_lock; 286 struct lwp *ctrlq_owner; 287 288 struct virtio_net_ctrl_cmd *ctrlq_cmd; 289 struct virtio_net_ctrl_status *ctrlq_status; 290 struct virtio_net_ctrl_rx *ctrlq_rx; 291 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc; 292 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc; 293 struct virtio_net_ctrl_mac_addr *ctrlq_mac_addr; 294 struct virtio_net_ctrl_mq *ctrlq_mq; 295 296 bus_dmamap_t ctrlq_cmd_dmamap; 297 bus_dmamap_t ctrlq_status_dmamap; 298 bus_dmamap_t ctrlq_rx_dmamap; 299 bus_dmamap_t ctrlq_tbl_uc_dmamap; 300 bus_dmamap_t ctrlq_tbl_mc_dmamap; 301 bus_dmamap_t ctrlq_mac_addr_dmamap; 302 bus_dmamap_t ctrlq_mq_dmamap; 303 304 struct evcnt ctrlq_cmd_load_failed; 305 struct evcnt ctrlq_cmd_failed; 306 }; 307 308 struct vioif_softc { 309 device_t sc_dev; 310 kmutex_t sc_lock; 311 struct sysctllog *sc_sysctllog; 312 313 struct virtio_softc *sc_virtio; 314 struct virtqueue *sc_vqs; 315 u_int sc_hdr_size; 316 317 int sc_max_nvq_pairs; 318 int sc_req_nvq_pairs; 319 int sc_act_nvq_pairs; 320 321 uint8_t sc_mac[ETHER_ADDR_LEN]; 322 struct ethercom sc_ethercom; 323 int sc_link_state; 324 325 struct vioif_netqueue *sc_netqs; 326 327 bool sc_has_ctrl; 328 struct vioif_ctrlqueue sc_ctrlq; 329 330 bus_dma_segment_t sc_segs[1]; 331 void *sc_dmamem; 332 void *sc_kmem; 333 334 void *sc_cfg_softint; 335 336 struct workqueue *sc_txrx_workqueue; 337 bool sc_txrx_workqueue_sysctl; 338 u_int sc_tx_intr_process_limit; 339 u_int sc_tx_process_limit; 340 u_int sc_rx_intr_process_limit; 341 u_int sc_rx_process_limit; 342 }; 343 #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */ 344 #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */ 345 346 #define VIOIF_TX_INTR_PROCESS_LIMIT 256 347 #define VIOIF_TX_PROCESS_LIMIT 256 348 #define VIOIF_RX_INTR_PROCESS_LIMIT 0U 349 #define VIOIF_RX_PROCESS_LIMIT 256 350 351 #define VIOIF_WORKQUEUE_PRI PRI_SOFTNET 352 #define VIOIF_IS_LINK_ACTIVE(_sc) ((_sc)->sc_link_state == LINK_STATE_UP ? \ 353 true : false) 354 355 /* cfattach interface functions */ 356 static int vioif_match(device_t, cfdata_t, void *); 357 static void vioif_attach(device_t, device_t, void *); 358 static int vioif_finalize_teardown(device_t); 359 360 /* ifnet interface functions */ 361 static int vioif_init(struct ifnet *); 362 static void vioif_stop(struct ifnet *, int); 363 static void vioif_start(struct ifnet *); 364 static int vioif_transmit(struct ifnet *, struct mbuf *); 365 static int vioif_ioctl(struct ifnet *, u_long, void *); 366 static void vioif_watchdog(struct ifnet *); 367 static int vioif_ifflags(struct vioif_softc *); 368 static int vioif_ifflags_cb(struct ethercom *); 369 370 /* tx & rx */ 371 static int vioif_netqueue_init(struct vioif_softc *, 372 struct virtio_softc *, size_t, u_int); 373 static void vioif_netqueue_teardown(struct vioif_softc *, 374 struct virtio_softc *, size_t); 375 static void vioif_net_intr_enable(struct vioif_softc *, 376 struct virtio_softc *); 377 static void vioif_net_intr_disable(struct vioif_softc *, 378 struct virtio_softc *); 379 static void vioif_net_sched_handle(struct vioif_softc *, 380 struct vioif_netqueue *); 381 382 /* rx */ 383 static void vioif_populate_rx_mbufs_locked(struct vioif_softc *, 384 struct vioif_netqueue *); 385 static int vioif_rx_intr(void *); 386 static void vioif_rx_handle(void *); 387 static void vioif_rx_queue_clear(struct vioif_softc *, 388 struct virtio_softc *, struct vioif_netqueue *); 389 390 /* tx */ 391 static void vioif_start_locked(struct ifnet *, struct vioif_netqueue *); 392 static void vioif_transmit_locked(struct ifnet *, struct vioif_netqueue *); 393 static void vioif_deferred_transmit(void *); 394 static int vioif_tx_intr(void *); 395 static void vioif_tx_handle(void *); 396 static void vioif_tx_queue_clear(struct vioif_softc *, struct virtio_softc *, 397 struct vioif_netqueue *); 398 399 /* controls */ 400 static int vioif_ctrl_intr(void *); 401 static int vioif_ctrl_rx(struct vioif_softc *, int, bool); 402 static int vioif_set_promisc(struct vioif_softc *, bool); 403 static int vioif_set_allmulti(struct vioif_softc *, bool); 404 static int vioif_set_rx_filter(struct vioif_softc *); 405 static int vioif_rx_filter(struct vioif_softc *); 406 static int vioif_set_mac_addr(struct vioif_softc *); 407 static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int); 408 409 /* config interrupt */ 410 static int vioif_config_change(struct virtio_softc *); 411 static void vioif_cfg_softint(void *); 412 static void vioif_update_link_status(struct vioif_softc *); 413 414 /* others */ 415 static void vioif_alloc_queues(struct vioif_softc *); 416 static void vioif_free_queues(struct vioif_softc *); 417 static int vioif_alloc_mems(struct vioif_softc *); 418 static struct workqueue* 419 vioif_workq_create(const char *, pri_t, int, int); 420 static void vioif_workq_destroy(struct workqueue *); 421 static void vioif_work_set(struct vioif_work *, void(*)(void *), void *); 422 static void vioif_work_add(struct workqueue *, struct vioif_work *); 423 static void vioif_work_wait(struct workqueue *, struct vioif_work *); 424 static int vioif_setup_sysctl(struct vioif_softc *); 425 static void vioif_setup_stats(struct vioif_softc *); 426 427 CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc), 428 vioif_match, vioif_attach, NULL, NULL); 429 430 static void 431 vioif_intr_barrier(void) 432 { 433 434 /* wait for finish all interrupt handler */ 435 xc_barrier(0); 436 } 437 438 static void 439 vioif_notify(struct virtio_softc *vsc, struct virtqueue *vq) 440 { 441 442 virtio_enqueue_commit(vsc, vq, -1, true); 443 } 444 445 static int 446 vioif_match(device_t parent, cfdata_t match, void *aux) 447 { 448 struct virtio_attach_args *va = aux; 449 450 if (va->sc_childdevid == VIRTIO_DEVICE_ID_NETWORK) 451 return 1; 452 453 return 0; 454 } 455 456 static void 457 vioif_attach(device_t parent, device_t self, void *aux) 458 { 459 struct vioif_softc *sc = device_private(self); 460 struct virtio_softc *vsc = device_private(parent); 461 struct vioif_netqueue *txq0; 462 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 463 uint64_t features, req_features; 464 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 465 u_int softint_flags; 466 int r, i, req_flags; 467 char xnamebuf[MAXCOMLEN]; 468 size_t netq_num; 469 470 if (virtio_child(vsc) != NULL) { 471 aprint_normal(": child already attached for %s; " 472 "something wrong...\n", device_xname(parent)); 473 return; 474 } 475 476 sc->sc_dev = self; 477 sc->sc_virtio = vsc; 478 sc->sc_link_state = LINK_STATE_UNKNOWN; 479 480 sc->sc_max_nvq_pairs = 1; 481 sc->sc_req_nvq_pairs = 1; 482 sc->sc_act_nvq_pairs = 1; 483 sc->sc_txrx_workqueue_sysctl = true; 484 sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT; 485 sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT; 486 sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT; 487 sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT; 488 489 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE); 490 491 snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self)); 492 sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI, 493 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 494 if (sc->sc_txrx_workqueue == NULL) 495 goto err; 496 497 req_flags = 0; 498 499 #ifdef VIOIF_MPSAFE 500 req_flags |= VIRTIO_F_INTR_MPSAFE; 501 #endif 502 req_flags |= VIRTIO_F_INTR_MSIX; 503 504 req_features = 505 VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | 506 VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY; 507 req_features |= VIRTIO_F_RING_EVENT_IDX; 508 req_features |= VIRTIO_NET_F_CTRL_MAC_ADDR; 509 #ifdef VIOIF_MULTIQ 510 req_features |= VIRTIO_NET_F_MQ; 511 #endif 512 virtio_child_attach_start(vsc, self, IPL_NET, NULL, 513 vioif_config_change, virtio_vq_intrhand, req_flags, 514 req_features, VIRTIO_NET_FLAG_BITS); 515 516 features = virtio_features(vsc); 517 if (features == 0) 518 goto err; 519 520 if (features & VIRTIO_NET_F_MAC) { 521 for (i = 0; i < __arraycount(sc->sc_mac); i++) { 522 sc->sc_mac[i] = virtio_read_device_config_1(vsc, 523 VIRTIO_NET_CONFIG_MAC + i); 524 } 525 } else { 526 /* code stolen from sys/net/if_tap.c */ 527 struct timeval tv; 528 uint32_t ui; 529 getmicrouptime(&tv); 530 ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; 531 memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); 532 for (i = 0; i < __arraycount(sc->sc_mac); i++) { 533 virtio_write_device_config_1(vsc, 534 VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]); 535 } 536 } 537 538 /* 'Ethernet' with capital follows other ethernet driver attachment */ 539 aprint_normal_dev(self, "Ethernet address %s\n", 540 ether_sprintf(sc->sc_mac)); 541 542 if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) { 543 sc->sc_hdr_size = sizeof(struct virtio_net_hdr); 544 } else { 545 sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers); 546 } 547 548 if ((features & VIRTIO_NET_F_CTRL_VQ) && 549 (features & VIRTIO_NET_F_CTRL_RX)) { 550 sc->sc_has_ctrl = true; 551 552 cv_init(&ctrlq->ctrlq_wait, "ctrl_vq"); 553 mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET); 554 ctrlq->ctrlq_inuse = FREE; 555 } else { 556 sc->sc_has_ctrl = false; 557 } 558 559 if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) { 560 sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc, 561 VIRTIO_NET_CONFIG_MAX_VQ_PAIRS); 562 563 if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) 564 goto err; 565 566 /* Limit the number of queue pairs to use */ 567 sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu); 568 } 569 570 vioif_alloc_queues(sc); 571 virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs); 572 573 #ifdef VIOIF_MPSAFE 574 softint_flags = SOFTINT_NET | SOFTINT_MPSAFE; 575 #else 576 softint_flags = SOFTINT_NET; 577 #endif 578 579 /* 580 * Initialize network queues 581 */ 582 netq_num = sc->sc_max_nvq_pairs * 2; 583 for (i = 0; i < netq_num; i++) { 584 r = vioif_netqueue_init(sc, vsc, i, softint_flags); 585 if (r != 0) 586 goto err; 587 } 588 589 if (sc->sc_has_ctrl) { 590 int ctrlq_idx = sc->sc_max_nvq_pairs * 2; 591 /* 592 * Allocating a virtqueue for control channel 593 */ 594 sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[ctrlq_idx]; 595 r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, ctrlq_idx, 596 NBPG, 1, "control"); 597 if (r != 0) { 598 aprint_error_dev(self, "failed to allocate " 599 "a virtqueue for control channel, error code %d\n", 600 r); 601 602 sc->sc_has_ctrl = false; 603 cv_destroy(&ctrlq->ctrlq_wait); 604 mutex_destroy(&ctrlq->ctrlq_wait_lock); 605 } else { 606 ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr; 607 ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq; 608 } 609 } 610 611 sc->sc_cfg_softint = softint_establish(softint_flags, 612 vioif_cfg_softint, sc); 613 if (sc->sc_cfg_softint == NULL) { 614 aprint_error_dev(self, "cannot establish ctl softint\n"); 615 goto err; 616 } 617 618 if (vioif_alloc_mems(sc) < 0) 619 goto err; 620 621 if (virtio_child_attach_finish(vsc) != 0) 622 goto err; 623 624 if (vioif_setup_sysctl(sc) != 0) { 625 aprint_error_dev(self, "unable to create sysctl node\n"); 626 /* continue */ 627 } 628 629 vioif_setup_stats(sc); 630 631 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 632 ifp->if_softc = sc; 633 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 634 #ifdef VIOIF_MPSAFE 635 ifp->if_extflags = IFEF_MPSAFE; 636 #endif 637 ifp->if_start = vioif_start; 638 if (sc->sc_req_nvq_pairs > 1) 639 ifp->if_transmit = vioif_transmit; 640 ifp->if_ioctl = vioif_ioctl; 641 ifp->if_init = vioif_init; 642 ifp->if_stop = vioif_stop; 643 ifp->if_capabilities = 0; 644 ifp->if_watchdog = vioif_watchdog; 645 txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]; 646 IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq0->netq_vq->vq_num, IFQ_MAXLEN)); 647 IFQ_SET_READY(&ifp->if_snd); 648 649 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; 650 651 if_attach(ifp); 652 if_deferred_start_init(ifp, NULL); 653 ether_ifattach(ifp, sc->sc_mac); 654 ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb); 655 656 return; 657 658 err: 659 netq_num = sc->sc_max_nvq_pairs * 2; 660 for (i = 0; i < netq_num; i++) { 661 vioif_netqueue_teardown(sc, vsc, i); 662 } 663 664 if (sc->sc_has_ctrl) { 665 cv_destroy(&ctrlq->ctrlq_wait); 666 mutex_destroy(&ctrlq->ctrlq_wait_lock); 667 virtio_free_vq(vsc, ctrlq->ctrlq_vq); 668 ctrlq->ctrlq_vq = NULL; 669 } 670 671 vioif_free_queues(sc); 672 mutex_destroy(&sc->sc_lock); 673 virtio_child_attach_failed(vsc); 674 config_finalize_register(self, vioif_finalize_teardown); 675 676 return; 677 } 678 679 static int 680 vioif_finalize_teardown(device_t self) 681 { 682 struct vioif_softc *sc = device_private(self); 683 684 if (sc->sc_txrx_workqueue != NULL) { 685 vioif_workq_destroy(sc->sc_txrx_workqueue); 686 sc->sc_txrx_workqueue = NULL; 687 } 688 689 return 0; 690 } 691 692 /* 693 * Interface functions for ifnet 694 */ 695 static int 696 vioif_init(struct ifnet *ifp) 697 { 698 struct vioif_softc *sc = ifp->if_softc; 699 struct virtio_softc *vsc = sc->sc_virtio; 700 struct vioif_netqueue *netq; 701 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 702 int r, i; 703 704 vioif_stop(ifp, 0); 705 706 r = virtio_reinit_start(vsc); 707 if (r != 0) { 708 log(LOG_ERR, "%s: reset failed\n", ifp->if_xname); 709 return EIO; 710 } 711 712 virtio_negotiate_features(vsc, virtio_features(vsc)); 713 714 for (i = 0; i < sc->sc_req_nvq_pairs; i++) { 715 netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)]; 716 717 mutex_enter(&netq->netq_lock); 718 vioif_populate_rx_mbufs_locked(sc, netq); 719 mutex_exit(&netq->netq_lock); 720 } 721 722 virtio_reinit_end(vsc); 723 724 if (sc->sc_has_ctrl) 725 virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); 726 727 r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs); 728 if (r == 0) 729 sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs; 730 else 731 sc->sc_act_nvq_pairs = 1; 732 733 SET(ifp->if_flags, IFF_RUNNING); 734 735 vioif_net_intr_enable(sc, vsc); 736 737 vioif_update_link_status(sc); 738 r = vioif_rx_filter(sc); 739 740 return r; 741 } 742 743 static void 744 vioif_stop(struct ifnet *ifp, int disable) 745 { 746 struct vioif_softc *sc = ifp->if_softc; 747 struct virtio_softc *vsc = sc->sc_virtio; 748 struct vioif_netqueue *netq; 749 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 750 size_t i, act_qnum; 751 752 act_qnum = sc->sc_act_nvq_pairs * 2; 753 754 CLR(ifp->if_flags, IFF_RUNNING); 755 for (i = 0; i < act_qnum; i++) { 756 netq = &sc->sc_netqs[i]; 757 758 mutex_enter(&netq->netq_lock); 759 netq->netq_stopping = true; 760 mutex_exit(&netq->netq_lock); 761 } 762 763 /* disable interrupts */ 764 vioif_net_intr_disable(sc, vsc); 765 if (sc->sc_has_ctrl) 766 virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq); 767 768 /* 769 * only way to stop interrupt, I/O and DMA is resetting... 770 * 771 * NOTE: Devices based on VirtIO draft specification can not 772 * stop interrupt completely even if virtio_stop_vq_intr() is called. 773 */ 774 virtio_reset(vsc); 775 776 vioif_intr_barrier(); 777 778 for (i = 0; i < act_qnum; i++) { 779 netq = &sc->sc_netqs[i]; 780 vioif_work_wait(sc->sc_txrx_workqueue, &netq->netq_work); 781 } 782 783 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 784 netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)]; 785 vioif_rx_queue_clear(sc, vsc, netq); 786 787 netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)]; 788 vioif_tx_queue_clear(sc, vsc, netq); 789 } 790 791 /* all packet processing is stopped */ 792 for (i = 0; i < act_qnum; i++) { 793 netq = &sc->sc_netqs[i]; 794 795 mutex_enter(&netq->netq_lock); 796 netq->netq_stopping = false; 797 mutex_exit(&netq->netq_lock); 798 } 799 } 800 801 static void 802 vioif_start(struct ifnet *ifp) 803 { 804 struct vioif_softc *sc = ifp->if_softc; 805 struct vioif_netqueue *txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]; 806 807 #ifdef VIOIF_MPSAFE 808 KASSERT(if_is_mpsafe(ifp)); 809 #endif 810 811 mutex_enter(&txq0->netq_lock); 812 vioif_start_locked(ifp, txq0); 813 mutex_exit(&txq0->netq_lock); 814 } 815 816 static inline int 817 vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m) 818 { 819 struct vioif_softc *sc = ifp->if_softc; 820 u_int cpuid = cpu_index(curcpu()); 821 822 return VIOIF_NETQ_TXQID(cpuid % sc->sc_act_nvq_pairs); 823 } 824 825 static int 826 vioif_transmit(struct ifnet *ifp, struct mbuf *m) 827 { 828 struct vioif_softc *sc = ifp->if_softc; 829 struct vioif_netqueue *netq; 830 struct vioif_tx_context *txc; 831 int qid; 832 833 qid = vioif_select_txqueue(ifp, m); 834 netq = &sc->sc_netqs[qid]; 835 txc = netq->netq_ctx; 836 837 if (__predict_false(!pcq_put(txc->txc_intrq, m))) { 838 m_freem(m); 839 return ENOBUFS; 840 } 841 842 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 843 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 844 if (m->m_flags & M_MCAST) 845 if_statinc_ref(nsr, if_omcasts); 846 IF_STAT_PUTREF(ifp); 847 848 if (mutex_tryenter(&netq->netq_lock)) { 849 vioif_transmit_locked(ifp, netq); 850 mutex_exit(&netq->netq_lock); 851 } 852 853 return 0; 854 } 855 856 void 857 vioif_watchdog(struct ifnet *ifp) 858 { 859 struct vioif_softc *sc = ifp->if_softc; 860 struct vioif_netqueue *netq; 861 int i; 862 863 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 864 if (ISSET(ifp->if_flags, IFF_DEBUG)) { 865 log(LOG_DEBUG, "%s: watchdog timed out\n", 866 ifp->if_xname); 867 } 868 869 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 870 netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)]; 871 872 mutex_enter(&netq->netq_lock); 873 if (!netq->netq_running_handle) { 874 netq->netq_running_handle = true; 875 vioif_net_sched_handle(sc, netq); 876 } 877 mutex_exit(&netq->netq_lock); 878 } 879 } 880 } 881 882 static int 883 vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) 884 { 885 int s, r; 886 887 s = splnet(); 888 889 r = ether_ioctl(ifp, cmd, data); 890 if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) { 891 if (ifp->if_flags & IFF_RUNNING) { 892 r = vioif_rx_filter(ifp->if_softc); 893 } else { 894 r = 0; 895 } 896 } 897 898 splx(s); 899 900 return r; 901 } 902 903 static int 904 vioif_ifflags(struct vioif_softc *sc) 905 { 906 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 907 bool onoff; 908 int r; 909 910 if (!sc->sc_has_ctrl) { 911 /* no ctrl vq; always promisc and allmulti */ 912 ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI); 913 return 0; 914 } 915 916 onoff = ifp->if_flags & IFF_ALLMULTI ? true : false; 917 r = vioif_set_allmulti(sc, onoff); 918 if (r != 0) { 919 log(LOG_WARNING, 920 "%s: couldn't %sable ALLMULTI\n", 921 ifp->if_xname, onoff ? "en" : "dis"); 922 if (onoff) { 923 CLR(ifp->if_flags, IFF_ALLMULTI); 924 } else { 925 SET(ifp->if_flags, IFF_ALLMULTI); 926 } 927 } 928 929 onoff = ifp->if_flags & IFF_PROMISC ? true : false; 930 r = vioif_set_promisc(sc, onoff); 931 if (r != 0) { 932 log(LOG_WARNING, 933 "%s: couldn't %sable PROMISC\n", 934 ifp->if_xname, onoff ? "en" : "dis"); 935 if (onoff) { 936 CLR(ifp->if_flags, IFF_PROMISC); 937 } else { 938 SET(ifp->if_flags, IFF_PROMISC); 939 } 940 } 941 942 return 0; 943 } 944 945 static int 946 vioif_ifflags_cb(struct ethercom *ec) 947 { 948 struct ifnet *ifp = &ec->ec_if; 949 struct vioif_softc *sc = ifp->if_softc; 950 951 return vioif_ifflags(sc); 952 } 953 954 static int 955 vioif_setup_sysctl(struct vioif_softc *sc) 956 { 957 const char *devname; 958 struct sysctllog **log; 959 const struct sysctlnode *rnode, *rxnode, *txnode; 960 int error; 961 962 log = &sc->sc_sysctllog; 963 devname = device_xname(sc->sc_dev); 964 965 error = sysctl_createv(log, 0, NULL, &rnode, 966 0, CTLTYPE_NODE, devname, 967 SYSCTL_DESCR("virtio-net information and settings"), 968 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 969 if (error) 970 goto out; 971 972 error = sysctl_createv(log, 0, &rnode, NULL, 973 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 974 SYSCTL_DESCR("Use workqueue for packet processing"), 975 NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL); 976 if (error) 977 goto out; 978 979 error = sysctl_createv(log, 0, &rnode, &rxnode, 980 0, CTLTYPE_NODE, "rx", 981 SYSCTL_DESCR("virtio-net information and settings for Rx"), 982 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 983 if (error) 984 goto out; 985 986 error = sysctl_createv(log, 0, &rxnode, NULL, 987 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 988 SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), 989 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 990 if (error) 991 goto out; 992 993 error = sysctl_createv(log, 0, &rxnode, NULL, 994 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 995 SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), 996 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 997 if (error) 998 goto out; 999 1000 error = sysctl_createv(log, 0, &rnode, &txnode, 1001 0, CTLTYPE_NODE, "tx", 1002 SYSCTL_DESCR("virtio-net information and settings for Tx"), 1003 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 1004 if (error) 1005 goto out; 1006 1007 error = sysctl_createv(log, 0, &txnode, NULL, 1008 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 1009 SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), 1010 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 1011 if (error) 1012 goto out; 1013 1014 error = sysctl_createv(log, 0, &txnode, NULL, 1015 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 1016 SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), 1017 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 1018 1019 out: 1020 if (error) 1021 sysctl_teardown(log); 1022 1023 return error; 1024 } 1025 1026 static void 1027 vioif_setup_stats(struct vioif_softc *sc) 1028 { 1029 struct vioif_netqueue *netq; 1030 struct vioif_tx_context *txc; 1031 struct vioif_rx_context *rxc; 1032 size_t i, netq_num; 1033 1034 netq_num = sc->sc_max_nvq_pairs * 2; 1035 for (i = 0; i < netq_num; i++) { 1036 netq = &sc->sc_netqs[i]; 1037 evcnt_attach_dynamic(&netq->netq_mbuf_load_failed, EVCNT_TYPE_MISC, 1038 NULL, netq->netq_evgroup, "failed to load mbuf to DMA"); 1039 evcnt_attach_dynamic(&netq->netq_enqueue_failed, 1040 EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, 1041 "virtqueue enqueue failed failed"); 1042 1043 switch (VIOIF_NETQ_DIR(i)) { 1044 case VIOIF_NETQ_RX: 1045 rxc = netq->netq_ctx; 1046 evcnt_attach_dynamic(&rxc->rxc_mbuf_enobufs, 1047 EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, 1048 "no receive buffer"); 1049 break; 1050 case VIOIF_NETQ_TX: 1051 txc = netq->netq_ctx; 1052 evcnt_attach_dynamic(&txc->txc_defrag_failed, 1053 EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, 1054 "m_defrag() failed"); 1055 break; 1056 } 1057 } 1058 1059 evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC, 1060 NULL, device_xname(sc->sc_dev), "control command dmamap load failed"); 1061 evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC, 1062 NULL, device_xname(sc->sc_dev), "control command failed"); 1063 } 1064 1065 /* 1066 * allocate memory 1067 */ 1068 static int 1069 vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map, 1070 bus_size_t size, int nsegs, const char *usage) 1071 { 1072 int r; 1073 1074 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size, 1075 nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map); 1076 1077 if (r != 0) { 1078 aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, " 1079 "error code %d\n", usage, r); 1080 } 1081 1082 return r; 1083 } 1084 1085 static void 1086 vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map) 1087 { 1088 1089 if (*map) { 1090 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map); 1091 *map = NULL; 1092 } 1093 } 1094 1095 static int 1096 vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map, 1097 void *buf, bus_size_t size, int nsegs, int rw, const char *usage) 1098 { 1099 int r; 1100 1101 r = vioif_dmamap_create(sc, map, size, nsegs, usage); 1102 if (r != 0) 1103 return 1; 1104 1105 r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf, 1106 size, NULL, rw | BUS_DMA_NOWAIT); 1107 if (r != 0) { 1108 vioif_dmamap_destroy(sc, map); 1109 aprint_error_dev(sc->sc_dev, "%s dmamap load failed. " 1110 "error code %d\n", usage, r); 1111 } 1112 1113 return r; 1114 } 1115 1116 static void * 1117 vioif_assign_mem(intptr_t *p, size_t size) 1118 { 1119 intptr_t rv; 1120 1121 rv = *p; 1122 *p += size; 1123 1124 return (void *)rv; 1125 } 1126 1127 /* 1128 * dma memory is used for: 1129 * netq_maps_kva: metadata array for received frames (READ) and 1130 * sent frames (WRITE) 1131 * ctrlq_cmd: command to be sent via ctrl vq (WRITE) 1132 * ctrlq_status: return value for a command via ctrl vq (READ) 1133 * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command 1134 * (WRITE) 1135 * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC 1136 * class command (WRITE) 1137 * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC 1138 * class command (WRITE) 1139 * ctrlq_* structures are allocated only one each; they are protected by 1140 * ctrlq_inuse variable and ctrlq_wait condvar. 1141 */ 1142 static int 1143 vioif_alloc_mems(struct vioif_softc *sc) 1144 { 1145 struct virtio_softc *vsc = sc->sc_virtio; 1146 struct vioif_netqueue *netq; 1147 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1148 struct vioif_net_map *maps; 1149 unsigned int vq_num; 1150 int r, rsegs; 1151 bus_size_t dmamemsize; 1152 size_t qid, i, netq_num, kmemsize; 1153 void *vaddr; 1154 intptr_t p; 1155 1156 netq_num = sc->sc_max_nvq_pairs * 2; 1157 1158 /* allocate DMA memory */ 1159 dmamemsize = 0; 1160 1161 for (qid = 0; qid < netq_num; qid++) { 1162 maps = sc->sc_netqs[qid].netq_maps; 1163 vq_num = sc->sc_netqs[qid].netq_vq->vq_num; 1164 dmamemsize += sizeof(*maps[0].vnm_hdr) * vq_num; 1165 } 1166 1167 if (sc->sc_has_ctrl) { 1168 dmamemsize += sizeof(struct virtio_net_ctrl_cmd); 1169 dmamemsize += sizeof(struct virtio_net_ctrl_status); 1170 dmamemsize += sizeof(struct virtio_net_ctrl_rx); 1171 dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl) 1172 + ETHER_ADDR_LEN; 1173 dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl) 1174 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; 1175 dmamemsize += sizeof(struct virtio_net_ctrl_mac_addr); 1176 dmamemsize += sizeof(struct virtio_net_ctrl_mq); 1177 } 1178 1179 r = bus_dmamem_alloc(virtio_dmat(vsc), dmamemsize, 0, 0, 1180 &sc->sc_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 1181 if (r != 0) { 1182 aprint_error_dev(sc->sc_dev, 1183 "DMA memory allocation failed, size %zu, " 1184 "error code %d\n", dmamemsize, r); 1185 goto err_none; 1186 } 1187 r = bus_dmamem_map(virtio_dmat(vsc), &sc->sc_segs[0], 1, 1188 dmamemsize, &vaddr, BUS_DMA_NOWAIT); 1189 if (r != 0) { 1190 aprint_error_dev(sc->sc_dev, 1191 "DMA memory map failed, error code %d\n", r); 1192 goto err_dmamem_alloc; 1193 } 1194 1195 /* assign DMA memory */ 1196 memset(vaddr, 0, dmamemsize); 1197 sc->sc_dmamem = vaddr; 1198 p = (intptr_t) vaddr; 1199 1200 for (qid = 0; qid < netq_num; qid++) { 1201 netq = &sc->sc_netqs[qid]; 1202 maps = netq->netq_maps; 1203 vq_num = netq->netq_vq->vq_num; 1204 1205 netq->netq_maps_kva = vioif_assign_mem(&p, 1206 sizeof(*maps[0].vnm_hdr) * vq_num); 1207 } 1208 1209 if (sc->sc_has_ctrl) { 1210 ctrlq->ctrlq_cmd = vioif_assign_mem(&p, 1211 sizeof(*ctrlq->ctrlq_cmd)); 1212 ctrlq->ctrlq_status = vioif_assign_mem(&p, 1213 sizeof(*ctrlq->ctrlq_status)); 1214 ctrlq->ctrlq_rx = vioif_assign_mem(&p, 1215 sizeof(*ctrlq->ctrlq_rx)); 1216 ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p, 1217 sizeof(*ctrlq->ctrlq_mac_tbl_uc) 1218 + ETHER_ADDR_LEN); 1219 ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p, 1220 sizeof(*ctrlq->ctrlq_mac_tbl_mc) 1221 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES); 1222 ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p, 1223 sizeof(*ctrlq->ctrlq_mac_addr)); 1224 ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq)); 1225 } 1226 1227 /* allocate kmem */ 1228 kmemsize = 0; 1229 1230 for (qid = 0; qid < netq_num; qid++) { 1231 netq = &sc->sc_netqs[qid]; 1232 vq_num = netq->netq_vq->vq_num; 1233 1234 kmemsize += sizeof(netq->netq_maps[0]) * vq_num; 1235 } 1236 1237 vaddr = kmem_zalloc(kmemsize, KM_SLEEP); 1238 sc->sc_kmem = vaddr; 1239 1240 /* assign allocated kmem */ 1241 p = (intptr_t) vaddr; 1242 1243 for (qid = 0; qid < netq_num; qid++) { 1244 netq = &sc->sc_netqs[qid]; 1245 vq_num = netq->netq_vq->vq_num; 1246 1247 netq->netq_maps = vioif_assign_mem(&p, 1248 sizeof(netq->netq_maps[0]) * vq_num); 1249 } 1250 1251 /* prepare dmamaps */ 1252 for (qid = 0; qid < netq_num; qid++) { 1253 static const struct { 1254 const char *msg_hdr; 1255 const char *msg_payload; 1256 int dma_flag; 1257 bus_size_t dma_size; 1258 int dma_nsegs; 1259 } dmaparams[VIOIF_NETQ_IDX] = { 1260 [VIOIF_NETQ_RX] = { 1261 .msg_hdr = "rx header", 1262 .msg_payload = "rx payload", 1263 .dma_flag = BUS_DMA_READ, 1264 .dma_size = MCLBYTES - ETHER_ALIGN, 1265 .dma_nsegs = 1, 1266 }, 1267 [VIOIF_NETQ_TX] = { 1268 .msg_hdr = "tx header", 1269 .msg_payload = "tx payload", 1270 .dma_flag = BUS_DMA_WRITE, 1271 .dma_size = ETHER_MAX_LEN, 1272 .dma_nsegs = VIRTIO_NET_TX_MAXNSEGS, 1273 } 1274 }; 1275 1276 struct virtio_net_hdr *hdrs; 1277 int dir; 1278 1279 dir = VIOIF_NETQ_DIR(qid); 1280 netq = &sc->sc_netqs[qid]; 1281 vq_num = netq->netq_vq->vq_num; 1282 maps = netq->netq_maps; 1283 hdrs = netq->netq_maps_kva; 1284 1285 for (i = 0; i < vq_num; i++) { 1286 maps[i].vnm_hdr = &hdrs[i]; 1287 1288 r = vioif_dmamap_create_load(sc, &maps[i].vnm_hdr_map, 1289 maps[i].vnm_hdr, sc->sc_hdr_size, 1, 1290 dmaparams[dir].dma_flag, dmaparams[dir].msg_hdr); 1291 if (r != 0) 1292 goto err_reqs; 1293 1294 r = vioif_dmamap_create(sc, &maps[i].vnm_mbuf_map, 1295 dmaparams[dir].dma_size, dmaparams[dir].dma_nsegs, 1296 dmaparams[dir].msg_payload); 1297 if (r != 0) 1298 goto err_reqs; 1299 } 1300 } 1301 1302 if (sc->sc_has_ctrl) { 1303 /* control vq class & command */ 1304 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap, 1305 ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1, 1306 BUS_DMA_WRITE, "control command"); 1307 if (r != 0) 1308 goto err_reqs; 1309 1310 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap, 1311 ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1, 1312 BUS_DMA_READ, "control status"); 1313 if (r != 0) 1314 goto err_reqs; 1315 1316 /* control vq rx mode command parameter */ 1317 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap, 1318 ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1, 1319 BUS_DMA_WRITE, "rx mode control command"); 1320 if (r != 0) 1321 goto err_reqs; 1322 1323 /* multiqueue set command */ 1324 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap, 1325 ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1, 1326 BUS_DMA_WRITE, "multiqueue set command"); 1327 if (r != 0) 1328 goto err_reqs; 1329 1330 /* control vq MAC filter table for unicast */ 1331 /* do not load now since its length is variable */ 1332 r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap, 1333 sizeof(*ctrlq->ctrlq_mac_tbl_uc) 1334 + ETHER_ADDR_LEN, 1, 1335 "unicast MAC address filter command"); 1336 if (r != 0) 1337 goto err_reqs; 1338 1339 /* control vq MAC filter table for multicast */ 1340 r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap, 1341 sizeof(*ctrlq->ctrlq_mac_tbl_mc) 1342 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1, 1343 "multicast MAC address filter command"); 1344 if (r != 0) 1345 goto err_reqs; 1346 1347 /* control vq MAC address set command */ 1348 r = vioif_dmamap_create_load(sc, 1349 &ctrlq->ctrlq_mac_addr_dmamap, 1350 ctrlq->ctrlq_mac_addr, 1351 sizeof(*ctrlq->ctrlq_mac_addr), 1, 1352 BUS_DMA_WRITE, "mac addr set command"); 1353 if (r != 0) 1354 goto err_reqs; 1355 } 1356 1357 return 0; 1358 1359 err_reqs: 1360 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap); 1361 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap); 1362 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap); 1363 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap); 1364 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap); 1365 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap); 1366 for (qid = 0; qid < netq_num; qid++) { 1367 vq_num = sc->sc_netqs[qid].netq_vq->vq_num; 1368 maps = sc->sc_netqs[qid].netq_maps; 1369 1370 for (i = 0; i < vq_num; i++) { 1371 vioif_dmamap_destroy(sc, &maps[i].vnm_mbuf_map); 1372 vioif_dmamap_destroy(sc, &maps[i].vnm_hdr_map); 1373 } 1374 } 1375 if (sc->sc_kmem) { 1376 kmem_free(sc->sc_kmem, kmemsize); 1377 sc->sc_kmem = NULL; 1378 } 1379 bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, dmamemsize); 1380 err_dmamem_alloc: 1381 bus_dmamem_free(virtio_dmat(vsc), &sc->sc_segs[0], 1); 1382 err_none: 1383 return -1; 1384 } 1385 1386 static void 1387 vioif_alloc_queues(struct vioif_softc *sc) 1388 { 1389 int nvq_pairs = sc->sc_max_nvq_pairs; 1390 size_t nvqs, netq_num; 1391 1392 KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX); 1393 1394 nvqs = netq_num = sc->sc_max_nvq_pairs * 2; 1395 if (sc->sc_has_ctrl) 1396 nvqs++; 1397 1398 sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP); 1399 sc->sc_netqs = kmem_zalloc(sizeof(sc->sc_netqs[0]) * netq_num, 1400 KM_SLEEP); 1401 } 1402 1403 static void 1404 vioif_free_queues(struct vioif_softc *sc) 1405 { 1406 size_t nvqs, netq_num; 1407 1408 nvqs = netq_num = sc->sc_max_nvq_pairs * 2; 1409 if (sc->sc_ctrlq.ctrlq_vq) 1410 nvqs++; 1411 1412 kmem_free(sc->sc_netqs, sizeof(sc->sc_netqs[0]) * netq_num); 1413 kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs); 1414 sc->sc_netqs = NULL; 1415 sc->sc_vqs = NULL; 1416 } 1417 1418 /* 1419 * Network queues 1420 */ 1421 static int 1422 vioif_netqueue_init(struct vioif_softc *sc, struct virtio_softc *vsc, 1423 size_t qid, u_int softint_flags) 1424 { 1425 static const struct { 1426 const char *dirname; 1427 int segsize; 1428 int nsegs; 1429 int (*intrhand)(void *); 1430 void (*sihand)(void *); 1431 } params[VIOIF_NETQ_IDX] = { 1432 [VIOIF_NETQ_RX] = { 1433 .dirname = "rx", 1434 .segsize = MCLBYTES, 1435 .nsegs = 2, 1436 .intrhand = vioif_rx_intr, 1437 .sihand = vioif_rx_handle, 1438 }, 1439 [VIOIF_NETQ_TX] = { 1440 .dirname = "tx", 1441 .segsize = ETHER_MAX_LEN - ETHER_HDR_LEN, 1442 .nsegs = 2, 1443 .intrhand = vioif_tx_intr, 1444 .sihand = vioif_tx_handle, 1445 } 1446 }; 1447 1448 struct virtqueue *vq; 1449 struct vioif_netqueue *netq; 1450 struct vioif_tx_context *txc; 1451 struct vioif_rx_context *rxc; 1452 char qname[32]; 1453 int r, dir; 1454 1455 txc = NULL; 1456 rxc = NULL; 1457 netq = &sc->sc_netqs[qid]; 1458 vq = &sc->sc_vqs[qid]; 1459 dir = VIOIF_NETQ_DIR(qid); 1460 1461 netq->netq_vq = &sc->sc_vqs[qid]; 1462 netq->netq_stopping = false; 1463 netq->netq_running_handle = false; 1464 1465 snprintf(qname, sizeof(qname), "%s%zu", 1466 params[dir].dirname, VIOIF_NETQ_PAIRIDX(qid)); 1467 snprintf(netq->netq_evgroup, sizeof(netq->netq_evgroup), 1468 "%s-%s", device_xname(sc->sc_dev), qname); 1469 1470 mutex_init(&netq->netq_lock, MUTEX_DEFAULT, IPL_NET); 1471 r = virtio_alloc_vq(vsc, vq, qid, 1472 params[dir].segsize + sc->sc_hdr_size, 1473 params[dir].nsegs, qname); 1474 if (r != 0) 1475 goto err; 1476 netq->netq_vq = vq; 1477 1478 netq->netq_vq->vq_intrhand = params[dir].intrhand; 1479 netq->netq_vq->vq_intrhand_arg = netq; 1480 netq->netq_softint = softint_establish(softint_flags, 1481 params[dir].sihand, netq); 1482 if (netq->netq_softint == NULL) { 1483 aprint_error_dev(sc->sc_dev, 1484 "couldn't establish %s softint\n", 1485 params[dir].dirname); 1486 goto err; 1487 } 1488 vioif_work_set(&netq->netq_work, params[dir].sihand, netq); 1489 1490 switch (dir) { 1491 case VIOIF_NETQ_RX: 1492 rxc = kmem_zalloc(sizeof(*rxc), KM_SLEEP); 1493 netq->netq_ctx = rxc; 1494 /* nothing to do */ 1495 break; 1496 case VIOIF_NETQ_TX: 1497 txc = kmem_zalloc(sizeof(*txc), KM_SLEEP); 1498 netq->netq_ctx = (void *)txc; 1499 txc->txc_deferred_transmit = softint_establish(softint_flags, 1500 vioif_deferred_transmit, netq); 1501 if (txc->txc_deferred_transmit == NULL) { 1502 aprint_error_dev(sc->sc_dev, 1503 "couldn't establish softint for " 1504 "tx deferred transmit\n"); 1505 goto err; 1506 } 1507 txc->txc_link_active = VIOIF_IS_LINK_ACTIVE(sc); 1508 txc->txc_no_free_slots = false; 1509 txc->txc_intrq = pcq_create(vq->vq_num, KM_SLEEP); 1510 break; 1511 } 1512 1513 return 0; 1514 1515 err: 1516 netq->netq_ctx = NULL; 1517 1518 if (rxc != NULL) { 1519 kmem_free(rxc, sizeof(*rxc)); 1520 } 1521 1522 if (txc != NULL) { 1523 if (txc->txc_deferred_transmit != NULL) 1524 softint_disestablish(txc->txc_deferred_transmit); 1525 if (txc->txc_intrq != NULL) 1526 pcq_destroy(txc->txc_intrq); 1527 kmem_free(txc, sizeof(txc)); 1528 } 1529 1530 vioif_work_set(&netq->netq_work, NULL, NULL); 1531 if (netq->netq_softint != NULL) { 1532 softint_disestablish(netq->netq_softint); 1533 netq->netq_softint = NULL; 1534 } 1535 netq->netq_vq->vq_intrhand = NULL; 1536 netq->netq_vq->vq_intrhand_arg = NULL; 1537 1538 virtio_free_vq(vsc, vq); 1539 mutex_destroy(&netq->netq_lock); 1540 netq->netq_vq = NULL; 1541 1542 return -1; 1543 } 1544 1545 static void 1546 vioif_netqueue_teardown(struct vioif_softc *sc, struct virtio_softc *vsc, 1547 size_t qid) 1548 { 1549 struct vioif_netqueue *netq; 1550 struct vioif_rx_context *rxc; 1551 struct vioif_tx_context *txc; 1552 int dir; 1553 1554 netq = &sc->sc_netqs[qid]; 1555 1556 if (netq->netq_vq == NULL) 1557 return; 1558 1559 netq = &sc->sc_netqs[qid]; 1560 dir = VIOIF_NETQ_DIR(qid); 1561 switch (dir) { 1562 case VIOIF_NETQ_RX: 1563 rxc = netq->netq_ctx; 1564 netq->netq_ctx = NULL; 1565 kmem_free(rxc, sizeof(*rxc)); 1566 break; 1567 case VIOIF_NETQ_TX: 1568 txc = netq->netq_ctx; 1569 netq->netq_ctx = NULL; 1570 softint_disestablish(txc->txc_deferred_transmit); 1571 pcq_destroy(txc->txc_intrq); 1572 kmem_free(txc, sizeof(*txc)); 1573 break; 1574 } 1575 1576 softint_disestablish(netq->netq_softint); 1577 virtio_free_vq(vsc, netq->netq_vq); 1578 mutex_destroy(&netq->netq_lock); 1579 netq->netq_vq = NULL; 1580 } 1581 1582 static void 1583 vioif_net_sched_handle(struct vioif_softc *sc, struct vioif_netqueue *netq) 1584 { 1585 1586 KASSERT(mutex_owned(&netq->netq_lock)); 1587 KASSERT(!netq->netq_stopping); 1588 1589 if (netq->netq_workqueue) { 1590 vioif_work_add(sc->sc_txrx_workqueue, &netq->netq_work); 1591 } else { 1592 softint_schedule(netq->netq_softint); 1593 } 1594 } 1595 1596 static int 1597 vioif_net_load_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map, 1598 struct mbuf *m, int dma_flags) 1599 { 1600 int r; 1601 1602 KASSERT(map->vnm_mbuf == NULL); 1603 1604 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 1605 map->vnm_mbuf_map, m, dma_flags | BUS_DMA_NOWAIT); 1606 if (r == 0) { 1607 map->vnm_mbuf = m; 1608 } 1609 1610 return r; 1611 } 1612 1613 static void 1614 vioif_net_unload_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map) 1615 { 1616 1617 KASSERT(map->vnm_mbuf != NULL); 1618 bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map); 1619 map->vnm_mbuf = NULL; 1620 } 1621 1622 static int 1623 vioif_net_enqueue(struct virtio_softc *vsc, struct virtqueue *vq, 1624 int slot, struct vioif_net_map *map, int dma_ops, bool is_write) 1625 { 1626 int r; 1627 1628 KASSERT(map->vnm_mbuf != NULL); 1629 1630 /* This should actually never fail */ 1631 r = virtio_enqueue_reserve(vsc, vq, slot, 1632 map->vnm_mbuf_map->dm_nsegs + 1); 1633 if (r != 0) { 1634 /* slot already freed by virtio_enqueue_reserve */ 1635 return r; 1636 } 1637 1638 bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map, 1639 0, map->vnm_mbuf_map->dm_mapsize, dma_ops); 1640 bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map, 1641 0, map->vnm_hdr_map->dm_mapsize, dma_ops); 1642 1643 virtio_enqueue(vsc, vq, slot, map->vnm_hdr_map, is_write); 1644 virtio_enqueue(vsc, vq, slot, map->vnm_mbuf_map, is_write); 1645 virtio_enqueue_commit(vsc, vq, slot, false); 1646 1647 return 0; 1648 } 1649 1650 static int 1651 vioif_net_enqueue_tx(struct virtio_softc *vsc, struct virtqueue *vq, 1652 int slot, struct vioif_net_map *map) 1653 { 1654 1655 return vioif_net_enqueue(vsc, vq, slot, map, 1656 BUS_DMASYNC_PREWRITE, true); 1657 } 1658 1659 static int 1660 vioif_net_enqueue_rx(struct virtio_softc *vsc, struct virtqueue *vq, 1661 int slot, struct vioif_net_map *map) 1662 { 1663 1664 return vioif_net_enqueue(vsc, vq, slot, map, 1665 BUS_DMASYNC_PREREAD, false); 1666 } 1667 1668 static struct mbuf * 1669 vioif_net_dequeue_commit(struct virtio_softc *vsc, struct virtqueue *vq, 1670 int slot, struct vioif_net_map *map, int dma_flags) 1671 { 1672 struct mbuf *m; 1673 1674 m = map->vnm_mbuf; 1675 KASSERT(m != NULL); 1676 map->vnm_mbuf = NULL; 1677 1678 bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map, 1679 0, map->vnm_hdr_map->dm_mapsize, dma_flags); 1680 bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map, 1681 0, map->vnm_mbuf_map->dm_mapsize, dma_flags); 1682 1683 bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map); 1684 virtio_dequeue_commit(vsc, vq, slot); 1685 1686 return m; 1687 } 1688 1689 static void 1690 vioif_net_intr_enable(struct vioif_softc *sc, struct virtio_softc *vsc) 1691 { 1692 struct vioif_netqueue *netq; 1693 size_t i, act_qnum; 1694 int enqueued; 1695 1696 act_qnum = sc->sc_act_nvq_pairs * 2; 1697 for (i = 0; i < act_qnum; i++) { 1698 netq = &sc->sc_netqs[i]; 1699 1700 KASSERT(!netq->netq_stopping); 1701 KASSERT(!netq->netq_running_handle); 1702 1703 enqueued = virtio_start_vq_intr(vsc, netq->netq_vq); 1704 if (enqueued != 0) { 1705 virtio_stop_vq_intr(vsc, netq->netq_vq); 1706 1707 mutex_enter(&netq->netq_lock); 1708 netq->netq_running_handle = true; 1709 vioif_net_sched_handle(sc, netq); 1710 mutex_exit(&netq->netq_lock); 1711 } 1712 } 1713 } 1714 1715 static void 1716 vioif_net_intr_disable(struct vioif_softc *sc, struct virtio_softc *vsc) 1717 { 1718 struct vioif_netqueue *netq; 1719 size_t i, act_qnum; 1720 1721 act_qnum = sc->sc_act_nvq_pairs * 2; 1722 for (i = 0; i < act_qnum; i++) { 1723 netq = &sc->sc_netqs[i]; 1724 1725 virtio_stop_vq_intr(vsc, netq->netq_vq); 1726 } 1727 } 1728 1729 /* 1730 * Receive implementation 1731 */ 1732 /* enqueue mbufs to receive slots */ 1733 static void 1734 vioif_populate_rx_mbufs_locked(struct vioif_softc *sc, struct vioif_netqueue *netq) 1735 { 1736 struct virtqueue *vq = netq->netq_vq; 1737 struct virtio_softc *vsc = vq->vq_owner; 1738 struct vioif_rx_context *rxc; 1739 struct vioif_net_map *map; 1740 struct mbuf *m; 1741 int i, r, ndone = 0; 1742 1743 KASSERT(mutex_owned(&netq->netq_lock)); 1744 1745 rxc = netq->netq_ctx; 1746 1747 for (i = 0; i < vq->vq_num; i++) { 1748 int slot; 1749 r = virtio_enqueue_prep(vsc, vq, &slot); 1750 if (r == EAGAIN) 1751 break; 1752 if (__predict_false(r != 0)) 1753 panic("enqueue_prep for rx buffers"); 1754 1755 MGETHDR(m, M_DONTWAIT, MT_DATA); 1756 if (m == NULL) { 1757 virtio_enqueue_abort(vsc, vq, slot); 1758 rxc->rxc_mbuf_enobufs.ev_count++; 1759 break; 1760 } 1761 MCLGET(m, M_DONTWAIT); 1762 if ((m->m_flags & M_EXT) == 0) { 1763 virtio_enqueue_abort(vsc, vq, slot); 1764 m_freem(m); 1765 rxc->rxc_mbuf_enobufs.ev_count++; 1766 break; 1767 } 1768 1769 m->m_len = m->m_pkthdr.len = MCLBYTES; 1770 m_adj(m, ETHER_ALIGN); 1771 1772 map = &netq->netq_maps[slot]; 1773 r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_READ); 1774 if (r != 0) { 1775 virtio_enqueue_abort(vsc, vq, slot); 1776 m_freem(m); 1777 netq->netq_mbuf_load_failed.ev_count++; 1778 break; 1779 } 1780 1781 r = vioif_net_enqueue_rx(vsc, vq, slot, map); 1782 if (r != 0) { 1783 vioif_net_unload_mbuf(vsc, map); 1784 netq->netq_enqueue_failed.ev_count++; 1785 m_freem(m); 1786 /* slot already freed by vioif_net_enqueue_rx */ 1787 break; 1788 } 1789 1790 ndone++; 1791 } 1792 1793 if (ndone > 0) 1794 vioif_notify(vsc, vq); 1795 } 1796 1797 /* dequeue received packets */ 1798 static bool 1799 vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, 1800 struct vioif_netqueue *netq, u_int limit, size_t *ndeqp) 1801 { 1802 struct virtqueue *vq = netq->netq_vq; 1803 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1804 struct vioif_net_map *map; 1805 struct mbuf *m; 1806 int slot, len; 1807 bool more; 1808 size_t ndeq; 1809 1810 KASSERT(mutex_owned(&netq->netq_lock)); 1811 1812 more = false; 1813 ndeq = 0; 1814 1815 if (virtio_vq_is_enqueued(vsc, vq) == false) 1816 goto done; 1817 1818 for (;;ndeq++) { 1819 if (ndeq >= limit) { 1820 more = true; 1821 break; 1822 } 1823 1824 if (virtio_dequeue(vsc, vq, &slot, &len) != 0) 1825 break; 1826 1827 map = &netq->netq_maps[slot]; 1828 KASSERT(map->vnm_mbuf != NULL); 1829 m = vioif_net_dequeue_commit(vsc, vq, slot, 1830 map, BUS_DMASYNC_POSTREAD); 1831 KASSERT(m != NULL); 1832 1833 m->m_len = m->m_pkthdr.len = len - sc->sc_hdr_size; 1834 m_set_rcvif(m, ifp); 1835 if_percpuq_enqueue(ifp->if_percpuq, m); 1836 } 1837 1838 done: 1839 if (ndeqp != NULL) 1840 *ndeqp = ndeq; 1841 1842 return more; 1843 } 1844 1845 static void 1846 vioif_rx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc, 1847 struct vioif_netqueue *netq) 1848 { 1849 struct vioif_net_map *map; 1850 struct mbuf *m; 1851 unsigned int i, vq_num; 1852 bool more; 1853 1854 mutex_enter(&netq->netq_lock); 1855 1856 vq_num = netq->netq_vq->vq_num; 1857 for (;;) { 1858 more = vioif_rx_deq_locked(sc, vsc, netq, vq_num, NULL); 1859 if (more == false) 1860 break; 1861 } 1862 1863 for (i = 0; i < vq_num; i++) { 1864 map = &netq->netq_maps[i]; 1865 1866 m = map->vnm_mbuf; 1867 if (m == NULL) 1868 continue; 1869 1870 vioif_net_unload_mbuf(vsc, map); 1871 m_freem(m); 1872 } 1873 mutex_exit(&netq->netq_lock); 1874 } 1875 1876 static void 1877 vioif_rx_handle_locked(void *xnetq, u_int limit) 1878 { 1879 struct vioif_netqueue *netq = xnetq; 1880 struct virtqueue *vq = netq->netq_vq; 1881 struct virtio_softc *vsc = vq->vq_owner; 1882 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1883 bool more; 1884 int enqueued; 1885 size_t ndeq; 1886 1887 KASSERT(mutex_owned(&netq->netq_lock)); 1888 KASSERT(!netq->netq_stopping); 1889 1890 more = vioif_rx_deq_locked(sc, vsc, netq, limit, &ndeq); 1891 if (ndeq > 0) 1892 vioif_populate_rx_mbufs_locked(sc, netq); 1893 1894 if (more) { 1895 vioif_net_sched_handle(sc, netq); 1896 return; 1897 } 1898 1899 enqueued = virtio_start_vq_intr(vsc, netq->netq_vq); 1900 if (enqueued != 0) { 1901 virtio_stop_vq_intr(vsc, netq->netq_vq); 1902 vioif_net_sched_handle(sc, netq); 1903 return; 1904 } 1905 1906 netq->netq_running_handle = false; 1907 } 1908 1909 static int 1910 vioif_rx_intr(void *arg) 1911 { 1912 struct vioif_netqueue *netq = arg; 1913 struct virtqueue *vq = netq->netq_vq; 1914 struct virtio_softc *vsc = vq->vq_owner; 1915 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1916 u_int limit; 1917 1918 mutex_enter(&netq->netq_lock); 1919 1920 /* handler is already running in softint/workqueue */ 1921 if (netq->netq_running_handle) 1922 goto done; 1923 1924 netq->netq_running_handle = true; 1925 1926 limit = sc->sc_rx_intr_process_limit; 1927 virtio_stop_vq_intr(vsc, vq); 1928 vioif_rx_handle_locked(netq, limit); 1929 1930 done: 1931 mutex_exit(&netq->netq_lock); 1932 return 1; 1933 } 1934 1935 static void 1936 vioif_rx_handle(void *xnetq) 1937 { 1938 struct vioif_netqueue *netq = xnetq; 1939 struct virtqueue *vq = netq->netq_vq; 1940 struct virtio_softc *vsc = vq->vq_owner; 1941 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1942 u_int limit; 1943 1944 mutex_enter(&netq->netq_lock); 1945 1946 KASSERT(netq->netq_running_handle); 1947 1948 if (netq->netq_stopping) { 1949 netq->netq_running_handle = false; 1950 goto done; 1951 } 1952 1953 limit = sc->sc_rx_process_limit; 1954 vioif_rx_handle_locked(netq, limit); 1955 1956 done: 1957 mutex_exit(&netq->netq_lock); 1958 } 1959 1960 /* 1961 * Transmition implementation 1962 */ 1963 /* enqueue mbufs to send */ 1964 static void 1965 vioif_send_common_locked(struct ifnet *ifp, struct vioif_netqueue *netq, 1966 bool is_transmit) 1967 { 1968 struct vioif_softc *sc = ifp->if_softc; 1969 struct virtio_softc *vsc = sc->sc_virtio; 1970 struct virtqueue *vq = netq->netq_vq; 1971 struct vioif_tx_context *txc; 1972 struct vioif_net_map *map; 1973 struct mbuf *m; 1974 int queued = 0; 1975 1976 KASSERT(mutex_owned(&netq->netq_lock)); 1977 1978 if (netq->netq_stopping || 1979 !ISSET(ifp->if_flags, IFF_RUNNING)) 1980 return; 1981 1982 txc = netq->netq_ctx; 1983 1984 if (!txc->txc_link_active || 1985 txc->txc_no_free_slots) 1986 return; 1987 1988 for (;;) { 1989 int slot, r; 1990 r = virtio_enqueue_prep(vsc, vq, &slot); 1991 if (r == EAGAIN) { 1992 txc->txc_no_free_slots = true; 1993 break; 1994 } 1995 if (__predict_false(r != 0)) 1996 panic("enqueue_prep for tx buffers"); 1997 1998 if (is_transmit) 1999 m = pcq_get(txc->txc_intrq); 2000 else 2001 IFQ_DEQUEUE(&ifp->if_snd, m); 2002 2003 if (m == NULL) { 2004 virtio_enqueue_abort(vsc, vq, slot); 2005 break; 2006 } 2007 2008 map = &netq->netq_maps[slot]; 2009 KASSERT(map->vnm_mbuf == NULL); 2010 2011 r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_WRITE); 2012 if (r != 0) { 2013 /* maybe just too fragmented */ 2014 struct mbuf *newm; 2015 2016 newm = m_defrag(m, M_NOWAIT); 2017 if (newm != NULL) { 2018 m = newm; 2019 r = vioif_net_load_mbuf(vsc, map, m, 2020 BUS_DMA_WRITE); 2021 } else { 2022 txc->txc_defrag_failed.ev_count++; 2023 r = -1; 2024 } 2025 2026 if (r != 0) { 2027 netq->netq_mbuf_load_failed.ev_count++; 2028 m_freem(m); 2029 if_statinc(ifp, if_oerrors); 2030 virtio_enqueue_abort(vsc, vq, slot); 2031 continue; 2032 } 2033 } 2034 2035 memset(map->vnm_hdr, 0, sc->sc_hdr_size); 2036 2037 r = vioif_net_enqueue_tx(vsc, vq, slot, map); 2038 if (r != 0) { 2039 netq->netq_enqueue_failed.ev_count++; 2040 vioif_net_unload_mbuf(vsc, map); 2041 m_freem(m); 2042 /* slot already freed by vioif_net_enqueue_tx */ 2043 2044 if_statinc(ifp, if_oerrors); 2045 continue; 2046 } 2047 2048 queued++; 2049 bpf_mtap(ifp, m, BPF_D_OUT); 2050 } 2051 2052 if (queued > 0) { 2053 vioif_notify(vsc, vq); 2054 ifp->if_timer = 5; 2055 } 2056 } 2057 2058 /* dequeue sent mbufs */ 2059 static bool 2060 vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, 2061 struct vioif_netqueue *netq, u_int limit, size_t *ndeqp) 2062 { 2063 struct virtqueue *vq = netq->netq_vq; 2064 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2065 struct vioif_net_map *map; 2066 struct mbuf *m; 2067 int slot, len; 2068 bool more; 2069 size_t ndeq; 2070 2071 KASSERT(mutex_owned(&netq->netq_lock)); 2072 2073 more = false; 2074 ndeq = 0; 2075 2076 if (virtio_vq_is_enqueued(vsc, vq) == false) 2077 goto done; 2078 2079 for (;;ndeq++) { 2080 if (limit-- == 0) { 2081 more = true; 2082 break; 2083 } 2084 2085 if (virtio_dequeue(vsc, vq, &slot, &len) != 0) 2086 break; 2087 2088 map = &netq->netq_maps[slot]; 2089 KASSERT(map->vnm_mbuf != NULL); 2090 m = vioif_net_dequeue_commit(vsc, vq, slot, 2091 map, BUS_DMASYNC_POSTWRITE); 2092 KASSERT(m != NULL); 2093 2094 if_statinc(ifp, if_opackets); 2095 m_freem(m); 2096 } 2097 2098 done: 2099 if (ndeqp != NULL) 2100 *ndeqp = ndeq; 2101 return more; 2102 } 2103 2104 static void 2105 vioif_tx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc, 2106 struct vioif_netqueue *netq) 2107 { 2108 struct vioif_tx_context *txc; 2109 struct vioif_net_map *map; 2110 struct mbuf *m; 2111 unsigned int i, vq_num; 2112 bool more; 2113 2114 mutex_enter(&netq->netq_lock); 2115 2116 txc = netq->netq_ctx; 2117 vq_num = netq->netq_vq->vq_num; 2118 2119 for (;;) { 2120 more = vioif_tx_deq_locked(sc, vsc, netq, vq_num, NULL); 2121 if (more == false) 2122 break; 2123 } 2124 2125 for (i = 0; i < vq_num; i++) { 2126 map = &netq->netq_maps[i]; 2127 2128 m = map->vnm_mbuf; 2129 if (m == NULL) 2130 continue; 2131 2132 vioif_net_unload_mbuf(vsc, map); 2133 m_freem(m); 2134 } 2135 2136 txc->txc_no_free_slots = false; 2137 2138 mutex_exit(&netq->netq_lock); 2139 } 2140 2141 static void 2142 vioif_start_locked(struct ifnet *ifp, struct vioif_netqueue *netq) 2143 { 2144 2145 /* 2146 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c. 2147 */ 2148 vioif_send_common_locked(ifp, netq, false); 2149 2150 } 2151 2152 static void 2153 vioif_transmit_locked(struct ifnet *ifp, struct vioif_netqueue *netq) 2154 { 2155 2156 vioif_send_common_locked(ifp, netq, true); 2157 } 2158 2159 static void 2160 vioif_deferred_transmit(void *arg) 2161 { 2162 struct vioif_netqueue *netq = arg; 2163 struct virtio_softc *vsc = netq->netq_vq->vq_owner; 2164 struct vioif_softc *sc = device_private(virtio_child(vsc)); 2165 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2166 2167 mutex_enter(&netq->netq_lock); 2168 vioif_send_common_locked(ifp, netq, true); 2169 mutex_exit(&netq->netq_lock); 2170 } 2171 2172 static void 2173 vioif_tx_handle_locked(struct vioif_netqueue *netq, u_int limit) 2174 { 2175 struct virtqueue *vq = netq->netq_vq; 2176 struct vioif_tx_context *txc = netq->netq_ctx; 2177 struct virtio_softc *vsc = vq->vq_owner; 2178 struct vioif_softc *sc = device_private(virtio_child(vsc)); 2179 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2180 bool more; 2181 int enqueued; 2182 size_t ndeq; 2183 2184 KASSERT(mutex_owned(&netq->netq_lock)); 2185 KASSERT(!netq->netq_stopping); 2186 2187 more = vioif_tx_deq_locked(sc, vsc, netq, limit, &ndeq); 2188 if (txc->txc_no_free_slots && ndeq > 0) { 2189 txc->txc_no_free_slots = false; 2190 softint_schedule(txc->txc_deferred_transmit); 2191 } 2192 2193 if (more) { 2194 vioif_net_sched_handle(sc, netq); 2195 return; 2196 } 2197 2198 enqueued = (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) ? 2199 virtio_postpone_intr_smart(vsc, vq): 2200 virtio_start_vq_intr(vsc, vq); 2201 if (enqueued != 0) { 2202 virtio_stop_vq_intr(vsc, vq); 2203 vioif_net_sched_handle(sc, netq); 2204 return; 2205 } 2206 2207 netq->netq_running_handle = false; 2208 2209 /* for ALTQ */ 2210 if (netq == &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]) 2211 if_schedule_deferred_start(ifp); 2212 2213 softint_schedule(txc->txc_deferred_transmit); 2214 } 2215 2216 static int 2217 vioif_tx_intr(void *arg) 2218 { 2219 struct vioif_netqueue *netq = arg; 2220 struct virtqueue *vq = netq->netq_vq; 2221 struct virtio_softc *vsc = vq->vq_owner; 2222 struct vioif_softc *sc = device_private(virtio_child(vsc)); 2223 u_int limit; 2224 2225 mutex_enter(&netq->netq_lock); 2226 2227 /* tx handler is already running in softint/workqueue */ 2228 if (netq->netq_running_handle) 2229 goto done; 2230 2231 if (netq->netq_stopping) 2232 goto done; 2233 2234 netq->netq_running_handle = true; 2235 2236 virtio_stop_vq_intr(vsc, vq); 2237 netq->netq_workqueue = sc->sc_txrx_workqueue_sysctl; 2238 limit = sc->sc_tx_intr_process_limit; 2239 vioif_tx_handle_locked(netq, limit); 2240 2241 done: 2242 mutex_exit(&netq->netq_lock); 2243 return 1; 2244 } 2245 2246 static void 2247 vioif_tx_handle(void *xnetq) 2248 { 2249 struct vioif_netqueue *netq = xnetq; 2250 struct virtqueue *vq = netq->netq_vq; 2251 struct virtio_softc *vsc = vq->vq_owner; 2252 struct vioif_softc *sc = device_private(virtio_child(vsc)); 2253 u_int limit; 2254 2255 mutex_enter(&netq->netq_lock); 2256 2257 KASSERT(netq->netq_running_handle); 2258 2259 if (netq->netq_stopping) { 2260 netq->netq_running_handle = false; 2261 goto done; 2262 } 2263 2264 limit = sc->sc_tx_process_limit; 2265 vioif_tx_handle_locked(netq, limit); 2266 2267 done: 2268 mutex_exit(&netq->netq_lock); 2269 } 2270 2271 /* 2272 * Control vq 2273 */ 2274 /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */ 2275 static void 2276 vioif_ctrl_acquire(struct vioif_softc *sc) 2277 { 2278 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 2279 2280 mutex_enter(&ctrlq->ctrlq_wait_lock); 2281 while (ctrlq->ctrlq_inuse != FREE) 2282 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); 2283 ctrlq->ctrlq_inuse = INUSE; 2284 ctrlq->ctrlq_owner = curlwp; 2285 mutex_exit(&ctrlq->ctrlq_wait_lock); 2286 } 2287 2288 static void 2289 vioif_ctrl_release(struct vioif_softc *sc) 2290 { 2291 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 2292 2293 KASSERT(ctrlq->ctrlq_inuse != FREE); 2294 KASSERT(ctrlq->ctrlq_owner == curlwp); 2295 2296 mutex_enter(&ctrlq->ctrlq_wait_lock); 2297 ctrlq->ctrlq_inuse = FREE; 2298 ctrlq->ctrlq_owner = NULL; 2299 cv_signal(&ctrlq->ctrlq_wait); 2300 mutex_exit(&ctrlq->ctrlq_wait_lock); 2301 } 2302 2303 static int 2304 vioif_ctrl_load_cmdspec(struct vioif_softc *sc, 2305 struct vioif_ctrl_cmdspec *specs, int nspecs) 2306 { 2307 struct virtio_softc *vsc = sc->sc_virtio; 2308 int i, r, loaded; 2309 2310 loaded = 0; 2311 for (i = 0; i < nspecs; i++) { 2312 r = bus_dmamap_load(virtio_dmat(vsc), 2313 specs[i].dmamap, specs[i].buf, specs[i].bufsize, 2314 NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2315 if (r) { 2316 sc->sc_ctrlq.ctrlq_cmd_load_failed.ev_count++; 2317 goto err; 2318 } 2319 loaded++; 2320 2321 } 2322 2323 return r; 2324 2325 err: 2326 for (i = 0; i < loaded; i++) { 2327 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); 2328 } 2329 2330 return r; 2331 } 2332 2333 static void 2334 vioif_ctrl_unload_cmdspec(struct vioif_softc *sc, 2335 struct vioif_ctrl_cmdspec *specs, int nspecs) 2336 { 2337 struct virtio_softc *vsc = sc->sc_virtio; 2338 int i; 2339 2340 for (i = 0; i < nspecs; i++) { 2341 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); 2342 } 2343 } 2344 2345 static int 2346 vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd, 2347 struct vioif_ctrl_cmdspec *specs, int nspecs) 2348 { 2349 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 2350 struct virtqueue *vq = ctrlq->ctrlq_vq; 2351 struct virtio_softc *vsc = sc->sc_virtio; 2352 int i, r, slot; 2353 2354 ctrlq->ctrlq_cmd->class = class; 2355 ctrlq->ctrlq_cmd->command = cmd; 2356 2357 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 2358 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE); 2359 for (i = 0; i < nspecs; i++) { 2360 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 2361 0, specs[i].bufsize, BUS_DMASYNC_PREWRITE); 2362 } 2363 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 2364 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD); 2365 2366 /* we need to explicitly (re)start vq intr when using RING EVENT IDX */ 2367 if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) 2368 virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); 2369 2370 r = virtio_enqueue_prep(vsc, vq, &slot); 2371 if (r != 0) 2372 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 2373 r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2); 2374 if (r != 0) 2375 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 2376 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true); 2377 for (i = 0; i < nspecs; i++) { 2378 virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true); 2379 } 2380 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false); 2381 virtio_enqueue_commit(vsc, vq, slot, true); 2382 2383 /* wait for done */ 2384 mutex_enter(&ctrlq->ctrlq_wait_lock); 2385 while (ctrlq->ctrlq_inuse != DONE) 2386 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); 2387 mutex_exit(&ctrlq->ctrlq_wait_lock); 2388 /* already dequeueued */ 2389 2390 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0, 2391 sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE); 2392 for (i = 0; i < nspecs; i++) { 2393 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0, 2394 specs[i].bufsize, BUS_DMASYNC_POSTWRITE); 2395 } 2396 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0, 2397 sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD); 2398 2399 if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK) 2400 r = 0; 2401 else { 2402 device_printf(sc->sc_dev, "failed setting rx mode\n"); 2403 sc->sc_ctrlq.ctrlq_cmd_failed.ev_count++; 2404 r = EIO; 2405 } 2406 2407 return r; 2408 } 2409 2410 /* ctrl vq interrupt; wake up the command issuer */ 2411 static int 2412 vioif_ctrl_intr(void *arg) 2413 { 2414 struct vioif_ctrlqueue *ctrlq = arg; 2415 struct virtqueue *vq = ctrlq->ctrlq_vq; 2416 struct virtio_softc *vsc = vq->vq_owner; 2417 int r, slot; 2418 2419 if (virtio_vq_is_enqueued(vsc, vq) == false) 2420 return 0; 2421 2422 r = virtio_dequeue(vsc, vq, &slot, NULL); 2423 if (r == ENOENT) 2424 return 0; 2425 virtio_dequeue_commit(vsc, vq, slot); 2426 2427 mutex_enter(&ctrlq->ctrlq_wait_lock); 2428 ctrlq->ctrlq_inuse = DONE; 2429 cv_signal(&ctrlq->ctrlq_wait); 2430 mutex_exit(&ctrlq->ctrlq_wait_lock); 2431 2432 return 1; 2433 } 2434 2435 static int 2436 vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) 2437 { 2438 struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx; 2439 struct vioif_ctrl_cmdspec specs[1]; 2440 int r; 2441 2442 if (!sc->sc_has_ctrl) 2443 return ENOTSUP; 2444 2445 vioif_ctrl_acquire(sc); 2446 2447 rx->onoff = onoff; 2448 specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap; 2449 specs[0].buf = rx; 2450 specs[0].bufsize = sizeof(*rx); 2451 2452 r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd, 2453 specs, __arraycount(specs)); 2454 2455 vioif_ctrl_release(sc); 2456 return r; 2457 } 2458 2459 static int 2460 vioif_set_promisc(struct vioif_softc *sc, bool onoff) 2461 { 2462 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff); 2463 } 2464 2465 static int 2466 vioif_set_allmulti(struct vioif_softc *sc, bool onoff) 2467 { 2468 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff); 2469 } 2470 2471 static int 2472 vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs) 2473 { 2474 struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq; 2475 struct vioif_ctrl_cmdspec specs[1]; 2476 int r; 2477 2478 if (!sc->sc_has_ctrl) 2479 return ENOTSUP; 2480 2481 if (nvq_pairs <= 1) 2482 return EINVAL; 2483 2484 vioif_ctrl_acquire(sc); 2485 2486 mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs); 2487 specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap; 2488 specs[0].buf = mq; 2489 specs[0].bufsize = sizeof(*mq); 2490 2491 r = vioif_ctrl_send_command(sc, 2492 VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, 2493 specs, __arraycount(specs)); 2494 2495 vioif_ctrl_release(sc); 2496 2497 return r; 2498 } 2499 2500 static int 2501 vioif_set_mac_addr(struct vioif_softc *sc) 2502 { 2503 struct virtio_net_ctrl_mac_addr *ma = 2504 sc->sc_ctrlq.ctrlq_mac_addr; 2505 struct vioif_ctrl_cmdspec specs[1]; 2506 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2507 int nspecs = __arraycount(specs); 2508 uint64_t features; 2509 int r; 2510 size_t i; 2511 2512 if (!sc->sc_has_ctrl) 2513 return ENOTSUP; 2514 2515 if (memcmp(CLLADDR(ifp->if_sadl), sc->sc_mac, 2516 ETHER_ADDR_LEN) == 0) { 2517 return 0; 2518 } 2519 2520 memcpy(sc->sc_mac, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2521 2522 features = virtio_features(sc->sc_virtio); 2523 if (features & VIRTIO_NET_F_CTRL_MAC_ADDR) { 2524 vioif_ctrl_acquire(sc); 2525 2526 memcpy(ma->mac, sc->sc_mac, ETHER_ADDR_LEN); 2527 specs[0].dmamap = sc->sc_ctrlq.ctrlq_mac_addr_dmamap; 2528 specs[0].buf = ma; 2529 specs[0].bufsize = sizeof(*ma); 2530 2531 r = vioif_ctrl_send_command(sc, 2532 VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_ADDR_SET, 2533 specs, nspecs); 2534 2535 vioif_ctrl_release(sc); 2536 } else { 2537 for (i = 0; i < __arraycount(sc->sc_mac); i++) { 2538 virtio_write_device_config_1(sc->sc_virtio, 2539 VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]); 2540 } 2541 r = 0; 2542 } 2543 2544 return r; 2545 } 2546 2547 static int 2548 vioif_set_rx_filter(struct vioif_softc *sc) 2549 { 2550 /* filter already set in ctrlq->ctrlq_mac_tbl */ 2551 struct virtio_softc *vsc = sc->sc_virtio; 2552 struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc; 2553 struct vioif_ctrl_cmdspec specs[2]; 2554 int nspecs = __arraycount(specs); 2555 int r; 2556 2557 mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc; 2558 mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc; 2559 2560 if (!sc->sc_has_ctrl) 2561 return ENOTSUP; 2562 2563 vioif_ctrl_acquire(sc); 2564 2565 specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap; 2566 specs[0].buf = mac_tbl_uc; 2567 specs[0].bufsize = sizeof(*mac_tbl_uc) 2568 + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries)); 2569 2570 specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap; 2571 specs[1].buf = mac_tbl_mc; 2572 specs[1].bufsize = sizeof(*mac_tbl_mc) 2573 + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries)); 2574 2575 r = vioif_ctrl_load_cmdspec(sc, specs, nspecs); 2576 if (r != 0) 2577 goto out; 2578 2579 r = vioif_ctrl_send_command(sc, 2580 VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET, 2581 specs, nspecs); 2582 2583 vioif_ctrl_unload_cmdspec(sc, specs, nspecs); 2584 2585 out: 2586 vioif_ctrl_release(sc); 2587 2588 return r; 2589 } 2590 2591 /* 2592 * If multicast filter small enough (<=MAXENTRIES) set rx filter 2593 * If large multicast filter exist use ALLMULTI 2594 * If setting rx filter fails fall back to ALLMULTI 2595 */ 2596 static int 2597 vioif_rx_filter(struct vioif_softc *sc) 2598 { 2599 struct virtio_softc *vsc = sc->sc_virtio; 2600 struct ethercom *ec = &sc->sc_ethercom; 2601 struct ifnet *ifp = &ec->ec_if; 2602 struct ether_multi *enm; 2603 struct ether_multistep step; 2604 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 2605 int nentries; 2606 bool allmulti = 0; 2607 int r; 2608 2609 if (!sc->sc_has_ctrl) { 2610 goto set_ifflags; 2611 } 2612 2613 memcpy(ctrlq->ctrlq_mac_tbl_uc->macs[0], 2614 CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2615 2616 nentries = 0; 2617 allmulti = false; 2618 2619 ETHER_LOCK(ec); 2620 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 2621 ETHER_NEXT_MULTI(step, enm)) { 2622 if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) { 2623 allmulti = true; 2624 break; 2625 } 2626 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) { 2627 allmulti = true; 2628 break; 2629 } 2630 2631 memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries], 2632 enm->enm_addrlo, ETHER_ADDR_LEN); 2633 nentries++; 2634 } 2635 ETHER_UNLOCK(ec); 2636 2637 r = vioif_set_mac_addr(sc); 2638 if (r != 0) { 2639 log(LOG_WARNING, "%s: couldn't set MAC address\n", 2640 ifp->if_xname); 2641 } 2642 2643 if (!allmulti) { 2644 ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 1); 2645 ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, nentries); 2646 r = vioif_set_rx_filter(sc); 2647 if (r != 0) { 2648 allmulti = true; /* fallback */ 2649 } 2650 } 2651 2652 if (allmulti) { 2653 ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 0); 2654 ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, 0); 2655 r = vioif_set_rx_filter(sc); 2656 if (r != 0) { 2657 log(LOG_DEBUG, "%s: couldn't clear RX filter\n", 2658 ifp->if_xname); 2659 /* what to do on failure? */ 2660 } 2661 2662 ifp->if_flags |= IFF_ALLMULTI; 2663 } 2664 2665 set_ifflags: 2666 r = vioif_ifflags(sc); 2667 2668 return r; 2669 } 2670 2671 /* 2672 * VM configuration changes 2673 */ 2674 static int 2675 vioif_config_change(struct virtio_softc *vsc) 2676 { 2677 struct vioif_softc *sc = device_private(virtio_child(vsc)); 2678 2679 softint_schedule(sc->sc_cfg_softint); 2680 return 0; 2681 } 2682 2683 static void 2684 vioif_cfg_softint(void *arg) 2685 { 2686 struct vioif_softc *sc = arg; 2687 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2688 2689 vioif_update_link_status(sc); 2690 vioif_start(ifp); 2691 } 2692 2693 static int 2694 vioif_get_link_status(struct vioif_softc *sc) 2695 { 2696 struct virtio_softc *vsc = sc->sc_virtio; 2697 uint16_t status; 2698 2699 if (virtio_features(vsc) & VIRTIO_NET_F_STATUS) 2700 status = virtio_read_device_config_2(vsc, 2701 VIRTIO_NET_CONFIG_STATUS); 2702 else 2703 status = VIRTIO_NET_S_LINK_UP; 2704 2705 if ((status & VIRTIO_NET_S_LINK_UP) != 0) 2706 return LINK_STATE_UP; 2707 2708 return LINK_STATE_DOWN; 2709 } 2710 2711 static void 2712 vioif_update_link_status(struct vioif_softc *sc) 2713 { 2714 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2715 struct vioif_netqueue *netq; 2716 struct vioif_tx_context *txc; 2717 bool active; 2718 int link, i; 2719 2720 mutex_enter(&sc->sc_lock); 2721 2722 link = vioif_get_link_status(sc); 2723 2724 if (link == sc->sc_link_state) 2725 goto done; 2726 2727 sc->sc_link_state = link; 2728 2729 active = VIOIF_IS_LINK_ACTIVE(sc); 2730 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 2731 netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)]; 2732 2733 mutex_enter(&netq->netq_lock); 2734 txc = netq->netq_ctx; 2735 txc->txc_link_active = active; 2736 mutex_exit(&netq->netq_lock); 2737 } 2738 2739 if_link_state_change(ifp, sc->sc_link_state); 2740 2741 done: 2742 mutex_exit(&sc->sc_lock); 2743 } 2744 2745 static void 2746 vioif_workq_work(struct work *wk, void *context) 2747 { 2748 struct vioif_work *work; 2749 2750 work = container_of(wk, struct vioif_work, cookie); 2751 2752 atomic_store_relaxed(&work->added, 0); 2753 work->func(work->arg); 2754 } 2755 2756 static struct workqueue * 2757 vioif_workq_create(const char *name, pri_t prio, int ipl, int flags) 2758 { 2759 struct workqueue *wq; 2760 int error; 2761 2762 error = workqueue_create(&wq, name, vioif_workq_work, NULL, 2763 prio, ipl, flags); 2764 2765 if (error) 2766 return NULL; 2767 2768 return wq; 2769 } 2770 2771 static void 2772 vioif_workq_destroy(struct workqueue *wq) 2773 { 2774 2775 workqueue_destroy(wq); 2776 } 2777 2778 static void 2779 vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg) 2780 { 2781 2782 memset(work, 0, sizeof(*work)); 2783 work->func = func; 2784 work->arg = arg; 2785 } 2786 2787 static void 2788 vioif_work_add(struct workqueue *wq, struct vioif_work *work) 2789 { 2790 2791 if (atomic_load_relaxed(&work->added) != 0) 2792 return; 2793 2794 atomic_store_relaxed(&work->added, 1); 2795 kpreempt_disable(); 2796 workqueue_enqueue(wq, &work->cookie, NULL); 2797 kpreempt_enable(); 2798 } 2799 2800 static void 2801 vioif_work_wait(struct workqueue *wq, struct vioif_work *work) 2802 { 2803 2804 workqueue_wait(wq, &work->cookie); 2805 } 2806 2807 MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio"); 2808 2809 #ifdef _MODULE 2810 #include "ioconf.c" 2811 #endif 2812 2813 static int 2814 if_vioif_modcmd(modcmd_t cmd, void *opaque) 2815 { 2816 int error = 0; 2817 2818 #ifdef _MODULE 2819 switch (cmd) { 2820 case MODULE_CMD_INIT: 2821 error = config_init_component(cfdriver_ioconf_if_vioif, 2822 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 2823 break; 2824 case MODULE_CMD_FINI: 2825 error = config_fini_component(cfdriver_ioconf_if_vioif, 2826 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 2827 break; 2828 default: 2829 error = ENOTTY; 2830 break; 2831 } 2832 #endif 2833 2834 return error; 2835 } 2836
Indexes created Tue Oct 14 15:09:51 GMT 2025