virtio.c revision 1.28.2.2
1 /* $NetBSD: virtio.c,v 1.28.2.2 2020/09/20 10:14:20 martin Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.28.2.2 2020/09/20 10:14:20 martin Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/atomic.h> 35 #include <sys/bus.h> 36 #include <sys/device.h> 37 #include <sys/kmem.h> 38 #include <sys/module.h> 39 40 #include <dev/pci/pcidevs.h> 41 #include <dev/pci/pcireg.h> 42 #include <dev/pci/pcivar.h> 43 44 #define VIRTIO_PRIVATE 45 46 #include <dev/pci/virtioreg.h> 47 #include <dev/pci/virtiovar.h> 48 49 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 50 51 static int virtio_match(device_t, cfdata_t, void *); 52 static void virtio_attach(device_t, device_t, void *); 53 static int virtio_rescan(device_t, const char *, const int *); 54 static int virtio_detach(device_t, int); 55 static int virtio_intr(void *arg); 56 static int virtio_msix_queue_intr(void *); 57 static int virtio_msix_config_intr(void *); 58 static int virtio_setup_msix_vectors(struct virtio_softc *); 59 static int virtio_setup_msix_interrupts(struct virtio_softc *, 60 struct pci_attach_args *); 61 static int virtio_setup_intx_interrupt(struct virtio_softc *, 62 struct pci_attach_args *); 63 static int virtio_setup_interrupts(struct virtio_softc *); 64 static void virtio_free_interrupts(struct virtio_softc *); 65 static void virtio_soft_intr(void *arg); 66 static void virtio_init_vq(struct virtio_softc *, 67 struct virtqueue *, const bool); 68 69 CFATTACH_DECL3_NEW(virtio, sizeof(struct virtio_softc), 70 virtio_match, virtio_attach, virtio_detach, NULL, virtio_rescan, NULL, 71 DVF_DETACH_SHUTDOWN); 72 73 /* we use the legacy virtio spec, so the pci registers are host native 74 * byte order, not pci (i.e. LE) byte order */ 75 static inline uint16_t 76 nbo_bus_space_read_2(bus_space_tag_t space, bus_space_handle_t handle, 77 bus_size_t offset) 78 { 79 return le16toh(bus_space_read_2(space, handle, offset)); 80 } 81 82 static inline uint32_t 83 nbo_bus_space_read_4(bus_space_tag_t space, bus_space_handle_t handle, 84 bus_size_t offset) 85 { 86 return le32toh(bus_space_read_4(space, handle, offset)); 87 } 88 89 static void 90 nbo_bus_space_write_2(bus_space_tag_t space, bus_space_handle_t handle, 91 bus_size_t offset, uint16_t value) 92 { 93 bus_space_write_2(space, handle, offset, htole16(value)); 94 } 95 96 static void 97 nbo_bus_space_write_4(bus_space_tag_t space, bus_space_handle_t handle, 98 bus_size_t offset, uint32_t value) 99 { 100 bus_space_write_4(space, handle, offset, htole32(value)); 101 } 102 103 /* some functions access registers at 4 byte offset for little/high halves */ 104 #if BYTE_ORDER == BIG_ENDIAN 105 #define REG_HI_OFF 0 106 #define REG_LO_OFF 4 107 #else 108 #define REG_HI_OFF 4 109 #define REG_LO_OFF 0 110 #endif 111 112 static void 113 virtio_set_status(struct virtio_softc *sc, int status) 114 { 115 int old = 0; 116 117 if (status != 0) 118 old = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 119 VIRTIO_CONFIG_DEVICE_STATUS); 120 bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS, 121 status|old); 122 } 123 124 #define virtio_device_reset(sc) virtio_set_status((sc), 0) 125 126 static int 127 virtio_match(device_t parent, cfdata_t match, void *aux) 128 { 129 struct pci_attach_args *pa; 130 131 pa = (struct pci_attach_args *)aux; 132 switch (PCI_VENDOR(pa->pa_id)) { 133 case PCI_VENDOR_QUMRANET: 134 if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 135 PCI_PRODUCT(pa->pa_id)) && 136 (PCI_PRODUCT(pa->pa_id) <= 137 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) 138 return 1; 139 break; 140 } 141 142 return 0; 143 } 144 145 static const char *virtio_device_name[] = { 146 "Unknown (0)", /* 0 */ 147 "Network", /* 1 */ 148 "Block", /* 2 */ 149 "Console", /* 3 */ 150 "Entropy", /* 4 */ 151 "Memory Balloon", /* 5 */ 152 "I/O Memory", /* 6 */ 153 "Remote Processor Messaging", /* 7 */ 154 "SCSI", /* 8 */ 155 "9P Transport", /* 9 */ 156 "mac80211 wlan", /* 10 */ 157 }; 158 #define NDEVNAMES __arraycount(virtio_device_name) 159 160 #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0 161 #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1 162 163 static int 164 virtio_setup_msix_vectors(struct virtio_softc *sc) 165 { 166 int offset, vector, ret, qid; 167 168 offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR; 169 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 170 171 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); 172 ret = nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); 173 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 174 vector, ret); 175 if (ret != vector) 176 return -1; 177 178 for (qid = 0; qid < sc->sc_nvqs; qid++) { 179 offset = VIRTIO_CONFIG_QUEUE_SELECT; 180 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, qid); 181 182 offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR; 183 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 184 185 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); 186 ret = nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); 187 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 188 vector, ret); 189 if (ret != vector) 190 return -1; 191 } 192 193 return 0; 194 } 195 196 static int 197 virtio_setup_msix_interrupts(struct virtio_softc *sc, 198 struct pci_attach_args *pa) 199 { 200 device_t self = sc->sc_dev; 201 pci_chipset_tag_t pc = pa->pa_pc; 202 char intrbuf[PCI_INTRSTR_LEN]; 203 char const *intrstr; 204 int idx; 205 206 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 207 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 208 pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 209 210 sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], 211 sc->sc_ipl, virtio_msix_config_intr, sc, device_xname(sc->sc_dev)); 212 if (sc->sc_ihs[idx] == NULL) { 213 aprint_error_dev(self, "couldn't establish MSI-X for config\n"); 214 goto error; 215 } 216 217 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 218 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 219 pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 220 221 sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], 222 sc->sc_ipl, virtio_msix_queue_intr, sc, device_xname(sc->sc_dev)); 223 if (sc->sc_ihs[idx] == NULL) { 224 aprint_error_dev(self, "couldn't establish MSI-X for queues\n"); 225 goto error; 226 } 227 228 if (virtio_setup_msix_vectors(sc) != 0) { 229 aprint_error_dev(self, "couldn't setup MSI-X vectors\n"); 230 goto error; 231 } 232 233 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 234 intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 235 aprint_normal_dev(self, "config interrupting at %s\n", intrstr); 236 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 237 intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 238 aprint_normal_dev(self, "queues interrupting at %s\n", intrstr); 239 240 return 0; 241 242 error: 243 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 244 if (sc->sc_ihs[idx] != NULL) 245 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); 246 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 247 if (sc->sc_ihs[idx] != NULL) 248 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); 249 250 return -1; 251 } 252 253 static int 254 virtio_setup_intx_interrupt(struct virtio_softc *sc, 255 struct pci_attach_args *pa) 256 { 257 device_t self = sc->sc_dev; 258 pci_chipset_tag_t pc = pa->pa_pc; 259 char intrbuf[PCI_INTRSTR_LEN]; 260 char const *intrstr; 261 262 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 263 pci_intr_setattr(pc, &sc->sc_ihp[0], PCI_INTR_MPSAFE, true); 264 265 sc->sc_ihs[0] = pci_intr_establish_xname(pc, sc->sc_ihp[0], 266 sc->sc_ipl, virtio_intr, sc, device_xname(sc->sc_dev)); 267 if (sc->sc_ihs[0] == NULL) { 268 aprint_error_dev(self, "couldn't establish INTx\n"); 269 return -1; 270 } 271 272 intrstr = pci_intr_string(pc, sc->sc_ihp[0], intrbuf, sizeof(intrbuf)); 273 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 274 275 return 0; 276 } 277 278 static int 279 virtio_setup_interrupts(struct virtio_softc *sc) 280 { 281 device_t self = sc->sc_dev; 282 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 283 int error; 284 int nmsix; 285 int counts[PCI_INTR_TYPE_SIZE]; 286 pci_intr_type_t max_type; 287 288 nmsix = pci_msix_count(sc->sc_pa.pa_pc, sc->sc_pa.pa_tag); 289 aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix); 290 291 /* We need at least two: one for config and the other for queues */ 292 if ((sc->sc_flags & VIRTIO_F_PCI_INTR_MSIX) == 0 || nmsix < 2) { 293 /* Try INTx only */ 294 max_type = PCI_INTR_TYPE_INTX; 295 counts[PCI_INTR_TYPE_INTX] = 1; 296 } else { 297 /* Try MSI-X first and INTx second */ 298 max_type = PCI_INTR_TYPE_MSIX; 299 counts[PCI_INTR_TYPE_MSIX] = 2; 300 counts[PCI_INTR_TYPE_MSI] = 0; 301 counts[PCI_INTR_TYPE_INTX] = 1; 302 } 303 304 retry: 305 error = pci_intr_alloc(&sc->sc_pa, &sc->sc_ihp, counts, max_type); 306 if (error != 0) { 307 aprint_error_dev(self, "couldn't map interrupt\n"); 308 return -1; 309 } 310 311 if (pci_intr_type(pc, sc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) { 312 sc->sc_ihs = kmem_zalloc(sizeof(*sc->sc_ihs) * 2, 313 KM_SLEEP); 314 315 error = virtio_setup_msix_interrupts(sc, &sc->sc_pa); 316 if (error != 0) { 317 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 2); 318 pci_intr_release(pc, sc->sc_ihp, 2); 319 320 /* Retry INTx */ 321 max_type = PCI_INTR_TYPE_INTX; 322 counts[PCI_INTR_TYPE_INTX] = 1; 323 goto retry; 324 } 325 326 sc->sc_ihs_num = 2; 327 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI; 328 } else if (pci_intr_type(pc, sc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) { 329 sc->sc_ihs = kmem_zalloc(sizeof(*sc->sc_ihs) * 1, 330 KM_SLEEP); 331 332 error = virtio_setup_intx_interrupt(sc, &sc->sc_pa); 333 if (error != 0) { 334 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 1); 335 pci_intr_release(pc, sc->sc_ihp, 1); 336 return -1; 337 } 338 339 sc->sc_ihs_num = 1; 340 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 341 } 342 343 KASSERT(sc->sc_soft_ih == NULL); 344 if (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) { 345 u_int flags = SOFTINT_NET; 346 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 347 flags |= SOFTINT_MPSAFE; 348 349 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, sc); 350 if (sc->sc_soft_ih == NULL) { 351 virtio_free_interrupts(sc); 352 aprint_error_dev(sc->sc_dev, 353 "failed to establish soft interrupt\n"); 354 return -1; 355 } 356 } 357 358 return 0; 359 } 360 361 static void 362 virtio_free_interrupts(struct virtio_softc *sc) 363 { 364 for (int i = 0; i < sc->sc_ihs_num; i++) { 365 if (sc->sc_ihs[i] == NULL) 366 continue; 367 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]); 368 sc->sc_ihs[i] = NULL; 369 } 370 371 if (sc->sc_ihs_num > 0) 372 pci_intr_release(sc->sc_pc, sc->sc_ihp, sc->sc_ihs_num); 373 374 if (sc->sc_soft_ih) { 375 softint_disestablish(sc->sc_soft_ih); 376 sc->sc_soft_ih = NULL; 377 } 378 379 if (sc->sc_ihs != NULL) { 380 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * sc->sc_ihs_num); 381 sc->sc_ihs = NULL; 382 } 383 sc->sc_ihs_num = 0; 384 } 385 386 static void 387 virtio_attach(device_t parent, device_t self, void *aux) 388 { 389 struct virtio_softc *sc = device_private(self); 390 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 391 pci_chipset_tag_t pc = pa->pa_pc; 392 pcitag_t tag = pa->pa_tag; 393 int revision; 394 pcireg_t id; 395 pcireg_t csr; 396 397 revision = PCI_REVISION(pa->pa_class); 398 if (revision != 0) { 399 aprint_normal(": unknown revision 0x%02x; giving up\n", 400 revision); 401 return; 402 } 403 aprint_normal("\n"); 404 aprint_naive("\n"); 405 406 /* subsystem ID shows what I am */ 407 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 408 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 409 (PCI_SUBSYS_ID(id) < NDEVNAMES? 410 virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"), 411 revision); 412 413 csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 414 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE; 415 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr); 416 417 sc->sc_dev = self; 418 sc->sc_pc = pc; 419 sc->sc_tag = tag; 420 sc->sc_iot = pa->pa_iot; 421 if (pci_dma64_available(pa)) 422 sc->sc_dmat = pa->pa_dmat64; 423 else 424 sc->sc_dmat = pa->pa_dmat; 425 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 426 427 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 428 &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) { 429 aprint_error_dev(self, "can't map i/o space\n"); 430 return; 431 } 432 433 virtio_device_reset(sc); 434 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 435 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 436 437 sc->sc_childdevid = PCI_SUBSYS_ID(id); 438 sc->sc_child = NULL; 439 sc->sc_pa = *pa; 440 virtio_rescan(self, "virtio", 0); 441 return; 442 } 443 444 /* ARGSUSED */ 445 static int 446 virtio_rescan(device_t self, const char *attr, const int *scan_flags) 447 { 448 struct virtio_softc *sc; 449 struct virtio_attach_args va; 450 451 sc = device_private(self); 452 if (sc->sc_child) /* Child already attached? */ 453 return 0; 454 455 memset(&va, 0, sizeof(va)); 456 va.sc_childdevid = sc->sc_childdevid; 457 458 config_found_ia(self, attr, &va, NULL); 459 460 if (sc->sc_child == NULL) { 461 aprint_error_dev(self, 462 "no matching child driver; not configured\n"); 463 return 0; 464 } 465 466 if (sc->sc_child == VIRTIO_CHILD_FAILED) { 467 aprint_error_dev(self, 468 "virtio configuration failed\n"); 469 return 0; 470 } 471 472 /* 473 * Make sure child drivers initialize interrupts via call 474 * to virtio_child_attach_finish(). 475 */ 476 KASSERT(sc->sc_ihs_num != 0); 477 478 return 0; 479 } 480 481 static int 482 virtio_detach(device_t self, int flags) 483 { 484 struct virtio_softc *sc = device_private(self); 485 int r; 486 487 if (sc->sc_child != NULL) { 488 r = config_detach(sc->sc_child, flags); 489 if (r) 490 return r; 491 } 492 493 /* Check that child detached properly */ 494 KASSERT(sc->sc_child == NULL); 495 KASSERT(sc->sc_vqs == NULL); 496 KASSERT(sc->sc_ihs_num == 0); 497 498 if (sc->sc_iosize) 499 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 500 sc->sc_iosize = 0; 501 502 return 0; 503 } 504 505 /* 506 * Reset the device. 507 */ 508 /* 509 * To reset the device to a known state, do following: 510 * virtio_reset(sc); // this will stop the device activity 511 * <dequeue finished requests>; // virtio_dequeue() still can be called 512 * <revoke pending requests in the vqs if any>; 513 * virtio_reinit_begin(sc); // dequeue prohibitted 514 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 515 * <some other initialization>; 516 * virtio_reinit_end(sc); // device activated; enqueue allowed 517 * Once attached, feature negotiation can only be allowed after virtio_reset. 518 */ 519 void 520 virtio_reset(struct virtio_softc *sc) 521 { 522 virtio_device_reset(sc); 523 } 524 525 void 526 virtio_reinit_start(struct virtio_softc *sc) 527 { 528 int i; 529 530 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 531 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 532 for (i = 0; i < sc->sc_nvqs; i++) { 533 int n; 534 struct virtqueue *vq = &sc->sc_vqs[i]; 535 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 536 VIRTIO_CONFIG_QUEUE_SELECT, 537 vq->vq_index); 538 n = nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, 539 VIRTIO_CONFIG_QUEUE_SIZE); 540 if (n == 0) /* vq disappeared */ 541 continue; 542 if (n != vq->vq_num) { 543 panic("%s: virtqueue size changed, vq index %d\n", 544 device_xname(sc->sc_dev), 545 vq->vq_index); 546 } 547 virtio_init_vq(sc, vq, true); 548 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 549 VIRTIO_CONFIG_QUEUE_ADDRESS, 550 (vq->vq_dmamap->dm_segs[0].ds_addr 551 / VIRTIO_PAGE_SIZE)); 552 } 553 554 /* MSI-X should have more than one handles where INTx has just one */ 555 if (sc->sc_ihs_num > 1) { 556 if (virtio_setup_msix_vectors(sc) != 0) { 557 aprint_error_dev(sc->sc_dev, 558 "couldn't setup MSI-X vectors\n"); 559 return; 560 } 561 } 562 } 563 564 void 565 virtio_reinit_end(struct virtio_softc *sc) 566 { 567 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 568 } 569 570 /* 571 * Feature negotiation. 572 */ 573 uint32_t 574 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 575 { 576 uint32_t r; 577 578 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 579 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 580 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 581 r = nbo_bus_space_read_4(sc->sc_iot, sc->sc_ioh, 582 VIRTIO_CONFIG_DEVICE_FEATURES); 583 r &= guest_features; 584 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 585 VIRTIO_CONFIG_GUEST_FEATURES, r); 586 sc->sc_features = r; 587 if (r & VIRTIO_F_RING_INDIRECT_DESC) 588 sc->sc_indirect = true; 589 else 590 sc->sc_indirect = false; 591 592 return r; 593 } 594 595 /* 596 * Device configuration registers. 597 */ 598 uint8_t 599 virtio_read_device_config_1(struct virtio_softc *sc, int index) 600 { 601 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 602 sc->sc_config_offset + index); 603 } 604 605 uint16_t 606 virtio_read_device_config_2(struct virtio_softc *sc, int index) 607 { 608 return nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, 609 sc->sc_config_offset + index); 610 } 611 612 uint32_t 613 virtio_read_device_config_4(struct virtio_softc *sc, int index) 614 { 615 return nbo_bus_space_read_4(sc->sc_iot, sc->sc_ioh, 616 sc->sc_config_offset + index); 617 } 618 619 uint64_t 620 virtio_read_device_config_8(struct virtio_softc *sc, int index) 621 { 622 uint64_t r; 623 624 r = nbo_bus_space_read_4(sc->sc_iot, sc->sc_ioh, 625 sc->sc_config_offset + index + REG_HI_OFF); 626 r <<= 32; 627 r |= nbo_bus_space_read_4(sc->sc_iot, sc->sc_ioh, 628 sc->sc_config_offset + index + REG_LO_OFF); 629 630 return r; 631 } 632 633 void 634 virtio_write_device_config_1(struct virtio_softc *sc, 635 int index, uint8_t value) 636 { 637 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 638 sc->sc_config_offset + index, value); 639 } 640 641 void 642 virtio_write_device_config_2(struct virtio_softc *sc, 643 int index, uint16_t value) 644 { 645 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 646 sc->sc_config_offset + index, value); 647 } 648 649 void 650 virtio_write_device_config_4(struct virtio_softc *sc, 651 int index, uint32_t value) 652 { 653 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 654 sc->sc_config_offset + index, value); 655 } 656 657 void 658 virtio_write_device_config_8(struct virtio_softc *sc, 659 int index, uint64_t value) 660 { 661 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 662 sc->sc_config_offset + index + REG_LO_OFF, 663 value & 0xffffffff); 664 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 665 sc->sc_config_offset + index + REG_HI_OFF, 666 value >> 32); 667 } 668 669 /* 670 * Interrupt handler. 671 */ 672 static int 673 virtio_intr(void *arg) 674 { 675 struct virtio_softc *sc = arg; 676 int isr, r = 0; 677 678 /* check and ack the interrupt */ 679 isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 680 VIRTIO_CONFIG_ISR_STATUS); 681 if (isr == 0) 682 return 0; 683 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 684 (sc->sc_config_change != NULL)) 685 r = (sc->sc_config_change)(sc); 686 if (sc->sc_intrhand != NULL) { 687 if (sc->sc_soft_ih != NULL) 688 softint_schedule(sc->sc_soft_ih); 689 else 690 r |= (sc->sc_intrhand)(sc); 691 } 692 693 return r; 694 } 695 696 static int 697 virtio_msix_queue_intr(void *arg) 698 { 699 struct virtio_softc *sc = arg; 700 int r = 0; 701 702 if (sc->sc_intrhand != NULL) { 703 if (sc->sc_soft_ih != NULL) 704 softint_schedule(sc->sc_soft_ih); 705 else 706 r |= (sc->sc_intrhand)(sc); 707 } 708 709 return r; 710 } 711 712 static int 713 virtio_msix_config_intr(void *arg) 714 { 715 struct virtio_softc *sc = arg; 716 int r = 0; 717 718 if (sc->sc_config_change != NULL) 719 r = (sc->sc_config_change)(sc); 720 return r; 721 } 722 723 static void 724 virtio_soft_intr(void *arg) 725 { 726 struct virtio_softc *sc = arg; 727 728 KASSERT(sc->sc_intrhand != NULL); 729 730 (sc->sc_intrhand)(sc); 731 } 732 733 /* 734 * dmamap sync operations for a virtqueue. 735 */ 736 static inline void 737 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 738 { 739 /* availoffset == sizeof(vring_desc)*vq_num */ 740 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 741 ops); 742 } 743 744 static inline void 745 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 746 { 747 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 748 vq->vq_availoffset, 749 offsetof(struct vring_avail, ring) 750 + vq->vq_num * sizeof(uint16_t), 751 ops); 752 } 753 754 static inline void 755 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 756 { 757 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 758 vq->vq_usedoffset, 759 offsetof(struct vring_used, ring) 760 + vq->vq_num * sizeof(struct vring_used_elem), 761 ops); 762 } 763 764 static inline void 765 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 766 int ops) 767 { 768 int offset = vq->vq_indirectoffset 769 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 770 771 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 772 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 773 ops); 774 } 775 776 /* 777 * Can be used as sc_intrhand. 778 */ 779 /* 780 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 781 * and calls (*vq_done)() if some entries are consumed. 782 */ 783 int 784 virtio_vq_intr(struct virtio_softc *sc) 785 { 786 struct virtqueue *vq; 787 int i, r = 0; 788 789 for (i = 0; i < sc->sc_nvqs; i++) { 790 vq = &sc->sc_vqs[i]; 791 if (vq->vq_queued) { 792 vq->vq_queued = 0; 793 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 794 } 795 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 796 membar_consumer(); 797 if (vq->vq_used_idx != vq->vq_used->idx) { 798 if (vq->vq_done) 799 r |= (vq->vq_done)(vq); 800 } 801 } 802 803 return r; 804 } 805 806 /* 807 * Start/stop vq interrupt. No guarantee. 808 */ 809 void 810 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 811 { 812 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 813 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 814 vq->vq_queued++; 815 } 816 817 void 818 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 819 { 820 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 821 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 822 vq->vq_queued++; 823 } 824 825 /* 826 * Initialize vq structure. 827 */ 828 static void 829 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, 830 const bool reinit) 831 { 832 int i, j; 833 int vq_size = vq->vq_num; 834 835 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 836 837 /* build the indirect descriptor chain */ 838 if (vq->vq_indirect != NULL) { 839 struct vring_desc *vd; 840 841 for (i = 0; i < vq_size; i++) { 842 vd = vq->vq_indirect; 843 vd += vq->vq_maxnsegs * i; 844 for (j = 0; j < vq->vq_maxnsegs-1; j++) { 845 vd[j].next = j + 1; 846 } 847 } 848 } 849 850 /* free slot management */ 851 SIMPLEQ_INIT(&vq->vq_freelist); 852 for (i = 0; i < vq_size; i++) { 853 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 854 &vq->vq_entries[i], qe_list); 855 vq->vq_entries[i].qe_index = i; 856 } 857 if (!reinit) 858 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 859 860 /* enqueue/dequeue status */ 861 vq->vq_avail_idx = 0; 862 vq->vq_used_idx = 0; 863 vq->vq_queued = 0; 864 if (!reinit) { 865 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 866 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 867 } 868 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 869 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 870 vq->vq_queued++; 871 } 872 873 /* 874 * Allocate/free a vq. 875 */ 876 int 877 virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, 878 int maxsegsize, int maxnsegs, const char *name) 879 { 880 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 881 int rsegs, r; 882 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 883 ~(VIRTIO_PAGE_SIZE-1)) 884 885 /* Make sure callers allocate vqs in order */ 886 KASSERT(sc->sc_nvqs == index); 887 888 memset(vq, 0, sizeof(*vq)); 889 890 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 891 VIRTIO_CONFIG_QUEUE_SELECT, index); 892 vq_size = nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, 893 VIRTIO_CONFIG_QUEUE_SIZE); 894 if (vq_size == 0) { 895 aprint_error_dev(sc->sc_dev, 896 "virtqueue not exist, index %d for %s\n", 897 index, name); 898 goto err; 899 } 900 /* allocsize1: descriptor table + avail ring + pad */ 901 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 902 + sizeof(uint16_t)*(2+vq_size)); 903 /* allocsize2: used ring + pad */ 904 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 905 + sizeof(struct vring_used_elem)*vq_size); 906 /* allocsize3: indirect table */ 907 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 908 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 909 else 910 allocsize3 = 0; 911 allocsize = allocsize1 + allocsize2 + allocsize3; 912 913 /* alloc and map the memory */ 914 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 915 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 916 if (r != 0) { 917 aprint_error_dev(sc->sc_dev, 918 "virtqueue %d for %s allocation failed, " 919 "error code %d\n", index, name, r); 920 goto err; 921 } 922 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 923 &vq->vq_vaddr, BUS_DMA_NOWAIT); 924 if (r != 0) { 925 aprint_error_dev(sc->sc_dev, 926 "virtqueue %d for %s map failed, " 927 "error code %d\n", index, name, r); 928 goto err; 929 } 930 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 931 BUS_DMA_NOWAIT, &vq->vq_dmamap); 932 if (r != 0) { 933 aprint_error_dev(sc->sc_dev, 934 "virtqueue %d for %s dmamap creation failed, " 935 "error code %d\n", index, name, r); 936 goto err; 937 } 938 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 939 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 940 if (r != 0) { 941 aprint_error_dev(sc->sc_dev, 942 "virtqueue %d for %s dmamap load failed, " 943 "error code %d\n", index, name, r); 944 goto err; 945 } 946 947 /* set the vq address */ 948 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 949 VIRTIO_CONFIG_QUEUE_ADDRESS, 950 (vq->vq_dmamap->dm_segs[0].ds_addr 951 / VIRTIO_PAGE_SIZE)); 952 953 /* remember addresses and offsets for later use */ 954 vq->vq_owner = sc; 955 vq->vq_num = vq_size; 956 vq->vq_index = index; 957 vq->vq_desc = vq->vq_vaddr; 958 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 959 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 960 vq->vq_usedoffset = allocsize1; 961 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 962 if (allocsize3 > 0) { 963 vq->vq_indirectoffset = allocsize1 + allocsize2; 964 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 965 + vq->vq_indirectoffset); 966 } 967 vq->vq_bytesize = allocsize; 968 vq->vq_maxsegsize = maxsegsize; 969 vq->vq_maxnsegs = maxnsegs; 970 971 /* free slot management */ 972 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 973 KM_NOSLEEP); 974 if (vq->vq_entries == NULL) { 975 r = ENOMEM; 976 goto err; 977 } 978 979 virtio_init_vq(sc, vq, false); 980 981 aprint_verbose_dev(sc->sc_dev, 982 "allocated %u byte for virtqueue %d for %s, " 983 "size %d\n", allocsize, index, name, vq_size); 984 if (allocsize3 > 0) 985 aprint_verbose_dev(sc->sc_dev, 986 "using %d byte (%d entries) " 987 "indirect descriptors\n", 988 allocsize3, maxnsegs * vq_size); 989 990 sc->sc_nvqs++; 991 992 return 0; 993 994 err: 995 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 996 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 997 if (vq->vq_dmamap) 998 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 999 if (vq->vq_vaddr) 1000 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 1001 if (vq->vq_segs[0].ds_addr) 1002 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 1003 memset(vq, 0, sizeof(*vq)); 1004 1005 return -1; 1006 } 1007 1008 int 1009 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 1010 { 1011 struct vq_entry *qe; 1012 int i = 0; 1013 1014 /* device must be already deactivated */ 1015 /* confirm the vq is empty */ 1016 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 1017 i++; 1018 } 1019 if (i != vq->vq_num) { 1020 printf("%s: freeing non-empty vq, index %d\n", 1021 device_xname(sc->sc_dev), vq->vq_index); 1022 return EBUSY; 1023 } 1024 1025 /* tell device that there's no virtqueue any longer */ 1026 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 1027 VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index); 1028 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 1029 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 1030 1031 kmem_free(vq->vq_entries, sizeof(*vq->vq_entries) * vq->vq_num); 1032 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 1033 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 1034 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 1035 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 1036 mutex_destroy(&vq->vq_freelist_lock); 1037 mutex_destroy(&vq->vq_uring_lock); 1038 mutex_destroy(&vq->vq_aring_lock); 1039 memset(vq, 0, sizeof(*vq)); 1040 1041 sc->sc_nvqs--; 1042 1043 return 0; 1044 } 1045 1046 /* 1047 * Free descriptor management. 1048 */ 1049 static struct vq_entry * 1050 vq_alloc_entry(struct virtqueue *vq) 1051 { 1052 struct vq_entry *qe; 1053 1054 mutex_enter(&vq->vq_freelist_lock); 1055 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 1056 mutex_exit(&vq->vq_freelist_lock); 1057 return NULL; 1058 } 1059 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 1060 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 1061 mutex_exit(&vq->vq_freelist_lock); 1062 1063 return qe; 1064 } 1065 1066 static void 1067 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 1068 { 1069 mutex_enter(&vq->vq_freelist_lock); 1070 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 1071 mutex_exit(&vq->vq_freelist_lock); 1072 1073 return; 1074 } 1075 1076 /* 1077 * Enqueue several dmamaps as a single request. 1078 */ 1079 /* 1080 * Typical usage: 1081 * <queue size> number of followings are stored in arrays 1082 * - command blocks (in dmamem) should be pre-allocated and mapped 1083 * - dmamaps for command blocks should be pre-allocated and loaded 1084 * - dmamaps for payload should be pre-allocated 1085 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 1086 * if (r) // currently 0 or EAGAIN 1087 * return r; 1088 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 1089 * if (r) { 1090 * virtio_enqueue_abort(sc, vq, slot); 1091 * return r; 1092 * } 1093 * r = virtio_enqueue_reserve(sc, vq, slot, 1094 * dmamap_payload[slot]->dm_nsegs+1); 1095 * // ^ +1 for command 1096 * if (r) { // currently 0 or EAGAIN 1097 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 1098 * return r; // do not call abort() 1099 * } 1100 * <setup and prepare commands> 1101 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 1102 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 1103 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 1104 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 1105 * virtio_enqueue_commit(sc, vq, slot, true); 1106 */ 1107 1108 /* 1109 * enqueue_prep: allocate a slot number 1110 */ 1111 int 1112 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 1113 { 1114 struct vq_entry *qe1; 1115 1116 KASSERT(slotp != NULL); 1117 1118 qe1 = vq_alloc_entry(vq); 1119 if (qe1 == NULL) 1120 return EAGAIN; 1121 /* next slot is not allocated yet */ 1122 qe1->qe_next = -1; 1123 *slotp = qe1->qe_index; 1124 1125 return 0; 1126 } 1127 1128 /* 1129 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 1130 */ 1131 int 1132 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 1133 int slot, int nsegs) 1134 { 1135 int indirect; 1136 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1137 1138 KASSERT(qe1->qe_next == -1); 1139 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 1140 1141 if ((vq->vq_indirect != NULL) && 1142 (nsegs >= MINSEG_INDIRECT) && 1143 (nsegs <= vq->vq_maxnsegs)) 1144 indirect = 1; 1145 else 1146 indirect = 0; 1147 qe1->qe_indirect = indirect; 1148 1149 if (indirect) { 1150 struct vring_desc *vd; 1151 int i; 1152 1153 vd = &vq->vq_desc[qe1->qe_index]; 1154 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 1155 + vq->vq_indirectoffset; 1156 vd->addr += sizeof(struct vring_desc) 1157 * vq->vq_maxnsegs * qe1->qe_index; 1158 vd->len = sizeof(struct vring_desc) * nsegs; 1159 vd->flags = VRING_DESC_F_INDIRECT; 1160 1161 vd = vq->vq_indirect; 1162 vd += vq->vq_maxnsegs * qe1->qe_index; 1163 qe1->qe_desc_base = vd; 1164 1165 for (i = 0; i < nsegs-1; i++) { 1166 vd[i].flags = VRING_DESC_F_NEXT; 1167 } 1168 vd[i].flags = 0; 1169 qe1->qe_next = 0; 1170 1171 return 0; 1172 } else { 1173 struct vring_desc *vd; 1174 struct vq_entry *qe; 1175 int i, s; 1176 1177 vd = &vq->vq_desc[0]; 1178 qe1->qe_desc_base = vd; 1179 qe1->qe_next = qe1->qe_index; 1180 s = slot; 1181 for (i = 0; i < nsegs - 1; i++) { 1182 qe = vq_alloc_entry(vq); 1183 if (qe == NULL) { 1184 vd[s].flags = 0; 1185 virtio_enqueue_abort(sc, vq, slot); 1186 return EAGAIN; 1187 } 1188 vd[s].flags = VRING_DESC_F_NEXT; 1189 vd[s].next = qe->qe_index; 1190 s = qe->qe_index; 1191 } 1192 vd[s].flags = 0; 1193 1194 return 0; 1195 } 1196 } 1197 1198 /* 1199 * enqueue: enqueue a single dmamap. 1200 */ 1201 int 1202 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1203 bus_dmamap_t dmamap, bool write) 1204 { 1205 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1206 struct vring_desc *vd = qe1->qe_desc_base; 1207 int i; 1208 int s = qe1->qe_next; 1209 1210 KASSERT(s >= 0); 1211 KASSERT(dmamap->dm_nsegs > 0); 1212 1213 for (i = 0; i < dmamap->dm_nsegs; i++) { 1214 vd[s].addr = dmamap->dm_segs[i].ds_addr; 1215 vd[s].len = dmamap->dm_segs[i].ds_len; 1216 if (!write) 1217 vd[s].flags |= VRING_DESC_F_WRITE; 1218 s = vd[s].next; 1219 } 1220 qe1->qe_next = s; 1221 1222 return 0; 1223 } 1224 1225 int 1226 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1227 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 1228 bool write) 1229 { 1230 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1231 struct vring_desc *vd = qe1->qe_desc_base; 1232 int s = qe1->qe_next; 1233 1234 KASSERT(s >= 0); 1235 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 1236 KASSERT((dmamap->dm_segs[0].ds_len > start) && 1237 (dmamap->dm_segs[0].ds_len >= start + len)); 1238 1239 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 1240 vd[s].len = len; 1241 if (!write) 1242 vd[s].flags |= VRING_DESC_F_WRITE; 1243 qe1->qe_next = vd[s].next; 1244 1245 return 0; 1246 } 1247 1248 /* 1249 * enqueue_commit: add it to the aring. 1250 */ 1251 int 1252 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1253 bool notifynow) 1254 { 1255 struct vq_entry *qe1; 1256 1257 if (slot < 0) { 1258 mutex_enter(&vq->vq_aring_lock); 1259 goto notify; 1260 } 1261 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 1262 qe1 = &vq->vq_entries[slot]; 1263 if (qe1->qe_indirect) 1264 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 1265 mutex_enter(&vq->vq_aring_lock); 1266 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 1267 1268 notify: 1269 if (notifynow) { 1270 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 1271 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 1272 membar_producer(); 1273 vq->vq_avail->idx = vq->vq_avail_idx; 1274 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 1275 membar_producer(); 1276 vq->vq_queued++; 1277 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 1278 membar_consumer(); 1279 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 1280 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 1281 VIRTIO_CONFIG_QUEUE_NOTIFY, 1282 vq->vq_index); 1283 } 1284 mutex_exit(&vq->vq_aring_lock); 1285 1286 return 0; 1287 } 1288 1289 /* 1290 * enqueue_abort: rollback. 1291 */ 1292 int 1293 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1294 { 1295 struct vq_entry *qe = &vq->vq_entries[slot]; 1296 struct vring_desc *vd; 1297 int s; 1298 1299 if (qe->qe_next < 0) { 1300 vq_free_entry(vq, qe); 1301 return 0; 1302 } 1303 1304 s = slot; 1305 vd = &vq->vq_desc[0]; 1306 while (vd[s].flags & VRING_DESC_F_NEXT) { 1307 s = vd[s].next; 1308 vq_free_entry(vq, qe); 1309 qe = &vq->vq_entries[s]; 1310 } 1311 vq_free_entry(vq, qe); 1312 return 0; 1313 } 1314 1315 /* 1316 * Dequeue a request. 1317 */ 1318 /* 1319 * dequeue: dequeue a request from uring; dmamap_sync for uring is 1320 * already done in the interrupt handler. 1321 */ 1322 int 1323 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 1324 int *slotp, int *lenp) 1325 { 1326 uint16_t slot, usedidx; 1327 struct vq_entry *qe; 1328 1329 if (vq->vq_used_idx == vq->vq_used->idx) 1330 return ENOENT; 1331 mutex_enter(&vq->vq_uring_lock); 1332 usedidx = vq->vq_used_idx++; 1333 mutex_exit(&vq->vq_uring_lock); 1334 usedidx %= vq->vq_num; 1335 slot = vq->vq_used->ring[usedidx].id; 1336 qe = &vq->vq_entries[slot]; 1337 1338 if (qe->qe_indirect) 1339 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 1340 1341 if (slotp) 1342 *slotp = slot; 1343 if (lenp) 1344 *lenp = vq->vq_used->ring[usedidx].len; 1345 1346 return 0; 1347 } 1348 1349 /* 1350 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1351 * if you forget to call this the slot will be leaked. 1352 */ 1353 int 1354 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1355 { 1356 struct vq_entry *qe = &vq->vq_entries[slot]; 1357 struct vring_desc *vd = &vq->vq_desc[0]; 1358 int s = slot; 1359 1360 while (vd[s].flags & VRING_DESC_F_NEXT) { 1361 s = vd[s].next; 1362 vq_free_entry(vq, qe); 1363 qe = &vq->vq_entries[s]; 1364 } 1365 vq_free_entry(vq, qe); 1366 1367 return 0; 1368 } 1369 1370 /* 1371 * Attach a child, fill all the members. 1372 */ 1373 void 1374 virtio_child_attach_start(struct virtio_softc *sc, device_t child, int ipl, 1375 struct virtqueue *vqs, 1376 virtio_callback config_change, 1377 virtio_callback intr_hand, 1378 int req_flags, int req_features, const char *feat_bits) 1379 { 1380 char buf[256]; 1381 int features; 1382 1383 sc->sc_child = child; 1384 sc->sc_ipl = ipl; 1385 sc->sc_vqs = vqs; 1386 sc->sc_config_change = config_change; 1387 sc->sc_intrhand = intr_hand; 1388 sc->sc_flags = req_flags; 1389 1390 features = virtio_negotiate_features(sc, req_features); 1391 snprintb(buf, sizeof(buf), feat_bits, features); 1392 aprint_normal(": Features: %s\n", buf); 1393 aprint_naive("\n"); 1394 } 1395 1396 int 1397 virtio_child_attach_finish(struct virtio_softc *sc) 1398 { 1399 int r; 1400 1401 r = virtio_setup_interrupts(sc); 1402 if (r != 0) { 1403 aprint_error_dev(sc->sc_dev, "failed to setup interrupts\n"); 1404 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 1405 return 1; 1406 } 1407 1408 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 1409 1410 return 0; 1411 } 1412 1413 void 1414 virtio_child_detach(struct virtio_softc *sc) 1415 { 1416 sc->sc_child = NULL; 1417 sc->sc_vqs = NULL; 1418 1419 virtio_device_reset(sc); 1420 1421 virtio_free_interrupts(sc); 1422 } 1423 1424 void 1425 virtio_child_attach_failed(struct virtio_softc *sc) 1426 { 1427 virtio_child_detach(sc); 1428 1429 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 1430 1431 sc->sc_child = VIRTIO_CHILD_FAILED; 1432 } 1433 1434 bus_dma_tag_t 1435 virtio_dmat(struct virtio_softc *sc) 1436 { 1437 return sc->sc_dmat; 1438 } 1439 1440 device_t 1441 virtio_child(struct virtio_softc *sc) 1442 { 1443 return sc->sc_child; 1444 } 1445 1446 int 1447 virtio_intrhand(struct virtio_softc *sc) 1448 { 1449 return (sc->sc_intrhand)(sc); 1450 } 1451 1452 uint32_t 1453 virtio_features(struct virtio_softc *sc) 1454 { 1455 return sc->sc_features; 1456 } 1457 1458 MODULE(MODULE_CLASS_DRIVER, virtio, "pci"); 1459 1460 #ifdef _MODULE 1461 #include "ioconf.c" 1462 #endif 1463 1464 static int 1465 virtio_modcmd(modcmd_t cmd, void *opaque) 1466 { 1467 int error = 0; 1468 1469 #ifdef _MODULE 1470 switch (cmd) { 1471 case MODULE_CMD_INIT: 1472 error = config_init_component(cfdriver_ioconf_virtio, 1473 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1474 break; 1475 case MODULE_CMD_FINI: 1476 error = config_fini_component(cfdriver_ioconf_virtio, 1477 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1478 break; 1479 default: 1480 error = ENOTTY; 1481 break; 1482 } 1483 #endif 1484 1485 return error; 1486 } 1487
Indexes created Thu Oct 02 14:10:14 GMT 2025