virtio.c revision 1.26
1 /* $NetBSD: virtio.c,v 1.26 2017/03/26 16:53:36 jdolecek Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.26 2017/03/26 16:53:36 jdolecek Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/atomic.h> 35 #include <sys/bus.h> 36 #include <sys/device.h> 37 #include <sys/kmem.h> 38 #include <sys/module.h> 39 40 #include <dev/pci/pcidevs.h> 41 #include <dev/pci/pcireg.h> 42 #include <dev/pci/pcivar.h> 43 44 #define VIRTIO_PRIVATE 45 46 #include <dev/pci/virtioreg.h> 47 #include <dev/pci/virtiovar.h> 48 49 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 50 51 static int virtio_match(device_t, cfdata_t, void *); 52 static void virtio_attach(device_t, device_t, void *); 53 static int virtio_rescan(device_t, const char *, const int *); 54 static int virtio_detach(device_t, int); 55 static int virtio_intr(void *arg); 56 static int virtio_msix_queue_intr(void *); 57 static int virtio_msix_config_intr(void *); 58 static int virtio_setup_msix_vectors(struct virtio_softc *); 59 static int virtio_setup_msix_interrupts(struct virtio_softc *, 60 struct pci_attach_args *); 61 static int virtio_setup_intx_interrupt(struct virtio_softc *, 62 struct pci_attach_args *); 63 static int virtio_setup_interrupts(struct virtio_softc *); 64 static void virtio_free_interrupts(struct virtio_softc *); 65 static void virtio_soft_intr(void *arg); 66 static void virtio_init_vq(struct virtio_softc *, 67 struct virtqueue *, const bool); 68 69 CFATTACH_DECL3_NEW(virtio, sizeof(struct virtio_softc), 70 virtio_match, virtio_attach, virtio_detach, NULL, virtio_rescan, NULL, 71 DVF_DETACH_SHUTDOWN); 72 73 /* we use the legacy virtio spec, so the pci registers are host native 74 * byte order, not pci (i.e. LE) byte order */ 75 static inline uint16_t 76 nbo_bus_space_read_2(bus_space_tag_t space, bus_space_handle_t handle, 77 bus_size_t offset) 78 { 79 return le16toh(bus_space_read_2(space, handle, offset)); 80 } 81 82 static inline uint32_t 83 nbo_bus_space_read_4(bus_space_tag_t space, bus_space_handle_t handle, 84 bus_size_t offset) 85 { 86 return le32toh(bus_space_read_4(space, handle, offset)); 87 } 88 89 static void 90 nbo_bus_space_write_2(bus_space_tag_t space, bus_space_handle_t handle, 91 bus_size_t offset, uint16_t value) 92 { 93 bus_space_write_2(space, handle, offset, htole16(value)); 94 } 95 96 static void 97 nbo_bus_space_write_4(bus_space_tag_t space, bus_space_handle_t handle, 98 bus_size_t offset, uint32_t value) 99 { 100 bus_space_write_4(space, handle, offset, htole32(value)); 101 } 102 103 /* some functions access registers at 4 byte offset for little/high halves */ 104 #if BYTE_ORDER == BIG_ENDIAN 105 #define REG_HI_OFF 0 106 #define REG_LO_OFF 4 107 #else 108 #define REG_HI_OFF 4 109 #define REG_LO_OFF 0 110 #endif 111 112 static void 113 virtio_set_status(struct virtio_softc *sc, int status) 114 { 115 int old = 0; 116 117 if (status != 0) 118 old = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 119 VIRTIO_CONFIG_DEVICE_STATUS); 120 bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS, 121 status|old); 122 } 123 124 #define virtio_device_reset(sc) virtio_set_status((sc), 0) 125 126 static int 127 virtio_match(device_t parent, cfdata_t match, void *aux) 128 { 129 struct pci_attach_args *pa; 130 131 pa = (struct pci_attach_args *)aux; 132 switch (PCI_VENDOR(pa->pa_id)) { 133 case PCI_VENDOR_QUMRANET: 134 if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 135 PCI_PRODUCT(pa->pa_id)) && 136 (PCI_PRODUCT(pa->pa_id) <= 137 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) 138 return 1; 139 break; 140 } 141 142 return 0; 143 } 144 145 static const char *virtio_device_name[] = { 146 "Unknown (0)", /* 0 */ 147 "Network", /* 1 */ 148 "Block", /* 2 */ 149 "Console", /* 3 */ 150 "Entropy", /* 4 */ 151 "Memory Balloon", /* 5 */ 152 "I/O Memory", /* 6 */ 153 "Remote Processor Messaging", /* 7 */ 154 "SCSI", /* 8 */ 155 "9P Transport", /* 9 */ 156 "mac80211 wlan", /* 10 */ 157 }; 158 #define NDEVNAMES __arraycount(virtio_device_name) 159 160 #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0 161 #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1 162 163 static int 164 virtio_setup_msix_vectors(struct virtio_softc *sc) 165 { 166 int offset, vector, ret, qid; 167 168 offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR; 169 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 170 171 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); 172 ret = nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); 173 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 174 vector, ret); 175 if (ret != vector) 176 return -1; 177 178 for (qid = 0; qid < sc->sc_nvqs; qid++) { 179 offset = VIRTIO_CONFIG_QUEUE_SELECT; 180 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, qid); 181 182 offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR; 183 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 184 185 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); 186 ret = nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); 187 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 188 vector, ret); 189 if (ret != vector) 190 return -1; 191 } 192 193 return 0; 194 } 195 196 static int 197 virtio_setup_msix_interrupts(struct virtio_softc *sc, 198 struct pci_attach_args *pa) 199 { 200 device_t self = sc->sc_dev; 201 pci_chipset_tag_t pc = pa->pa_pc; 202 char intrbuf[PCI_INTRSTR_LEN]; 203 char const *intrstr; 204 int idx; 205 206 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 207 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 208 pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 209 210 sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], 211 sc->sc_ipl, virtio_msix_config_intr, sc, device_xname(sc->sc_dev)); 212 if (sc->sc_ihs[idx] == NULL) { 213 aprint_error_dev(self, "couldn't establish MSI-X for config\n"); 214 goto error; 215 } 216 217 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 218 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 219 pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 220 221 sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], 222 sc->sc_ipl, virtio_msix_queue_intr, sc, device_xname(sc->sc_dev)); 223 if (sc->sc_ihs[idx] == NULL) { 224 aprint_error_dev(self, "couldn't establish MSI-X for queues\n"); 225 goto error; 226 } 227 228 if (virtio_setup_msix_vectors(sc) != 0) { 229 aprint_error_dev(self, "couldn't setup MSI-X vectors\n"); 230 goto error; 231 } 232 233 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 234 intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 235 aprint_normal_dev(self, "config interrupting at %s\n", intrstr); 236 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 237 intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 238 aprint_normal_dev(self, "queues interrupting at %s\n", intrstr); 239 240 return 0; 241 242 error: 243 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 244 if (sc->sc_ihs[idx] != NULL) 245 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); 246 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 247 if (sc->sc_ihs[idx] != NULL) 248 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); 249 250 return -1; 251 } 252 253 static int 254 virtio_setup_intx_interrupt(struct virtio_softc *sc, 255 struct pci_attach_args *pa) 256 { 257 device_t self = sc->sc_dev; 258 pci_chipset_tag_t pc = pa->pa_pc; 259 char intrbuf[PCI_INTRSTR_LEN]; 260 char const *intrstr; 261 262 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 263 pci_intr_setattr(pc, &sc->sc_ihp[0], PCI_INTR_MPSAFE, true); 264 265 sc->sc_ihs[0] = pci_intr_establish_xname(pc, sc->sc_ihp[0], 266 sc->sc_ipl, virtio_intr, sc, device_xname(sc->sc_dev)); 267 if (sc->sc_ihs[0] == NULL) { 268 aprint_error_dev(self, "couldn't establish INTx\n"); 269 return -1; 270 } 271 272 intrstr = pci_intr_string(pc, sc->sc_ihp[0], intrbuf, sizeof(intrbuf)); 273 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 274 275 return 0; 276 } 277 278 static int 279 virtio_setup_interrupts(struct virtio_softc *sc) 280 { 281 device_t self = sc->sc_dev; 282 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 283 int error; 284 int nmsix; 285 int counts[PCI_INTR_TYPE_SIZE]; 286 pci_intr_type_t max_type; 287 288 nmsix = pci_msix_count(sc->sc_pa.pa_pc, sc->sc_pa.pa_tag); 289 aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix); 290 291 /* We need at least two: one for config and the other for queues */ 292 if ((sc->sc_flags & VIRTIO_F_PCI_INTR_MSIX) == 0 || nmsix < 2) { 293 /* Try INTx only */ 294 max_type = PCI_INTR_TYPE_INTX; 295 counts[PCI_INTR_TYPE_INTX] = 1; 296 } else { 297 /* Try MSI-X first and INTx second */ 298 max_type = PCI_INTR_TYPE_MSIX; 299 counts[PCI_INTR_TYPE_MSIX] = 2; 300 counts[PCI_INTR_TYPE_MSI] = 0; 301 counts[PCI_INTR_TYPE_INTX] = 1; 302 } 303 304 retry: 305 error = pci_intr_alloc(&sc->sc_pa, &sc->sc_ihp, counts, max_type); 306 if (error != 0) { 307 aprint_error_dev(self, "couldn't map interrupt\n"); 308 return -1; 309 } 310 311 if (pci_intr_type(pc, sc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) { 312 sc->sc_ihs = kmem_alloc(sizeof(*sc->sc_ihs) * 2, 313 KM_SLEEP); 314 if (sc->sc_ihs == NULL) { 315 pci_intr_release(pc, sc->sc_ihp, 2); 316 317 /* Retry INTx */ 318 max_type = PCI_INTR_TYPE_INTX; 319 counts[PCI_INTR_TYPE_INTX] = 1; 320 goto retry; 321 } 322 323 error = virtio_setup_msix_interrupts(sc, &sc->sc_pa); 324 if (error != 0) { 325 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 2); 326 pci_intr_release(pc, sc->sc_ihp, 2); 327 328 /* Retry INTx */ 329 max_type = PCI_INTR_TYPE_INTX; 330 counts[PCI_INTR_TYPE_INTX] = 1; 331 goto retry; 332 } 333 334 sc->sc_ihs_num = 2; 335 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI; 336 } else if (pci_intr_type(pc, sc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) { 337 sc->sc_ihs = kmem_alloc(sizeof(*sc->sc_ihs) * 1, 338 KM_SLEEP); 339 if (sc->sc_ihs == NULL) { 340 pci_intr_release(pc, sc->sc_ihp, 1); 341 return -1; 342 } 343 344 error = virtio_setup_intx_interrupt(sc, &sc->sc_pa); 345 if (error != 0) { 346 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 1); 347 pci_intr_release(pc, sc->sc_ihp, 1); 348 return -1; 349 } 350 351 sc->sc_ihs_num = 1; 352 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 353 } 354 355 KASSERT(sc->sc_soft_ih == NULL); 356 if (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) { 357 u_int flags = SOFTINT_NET; 358 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 359 flags |= SOFTINT_MPSAFE; 360 361 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, sc); 362 if (sc->sc_soft_ih == NULL) { 363 virtio_free_interrupts(sc); 364 aprint_error_dev(sc->sc_dev, 365 "failed to establish soft interrupt\n"); 366 return -1; 367 } 368 } 369 370 return 0; 371 } 372 373 static void 374 virtio_free_interrupts(struct virtio_softc *sc) 375 { 376 for (int i = 0; i < sc->sc_ihs_num; i++) { 377 if (sc->sc_ihs[i] == NULL) 378 continue; 379 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]); 380 sc->sc_ihs[i] = NULL; 381 } 382 383 if (sc->sc_ihs_num > 0) 384 pci_intr_release(sc->sc_pc, sc->sc_ihp, sc->sc_ihs_num); 385 386 if (sc->sc_soft_ih) { 387 softint_disestablish(sc->sc_soft_ih); 388 sc->sc_soft_ih = NULL; 389 } 390 391 if (sc->sc_ihs != NULL) { 392 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * sc->sc_ihs_num); 393 sc->sc_ihs = NULL; 394 } 395 sc->sc_ihs_num = 0; 396 } 397 398 static void 399 virtio_attach(device_t parent, device_t self, void *aux) 400 { 401 struct virtio_softc *sc = device_private(self); 402 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 403 pci_chipset_tag_t pc = pa->pa_pc; 404 pcitag_t tag = pa->pa_tag; 405 int revision; 406 pcireg_t id; 407 408 revision = PCI_REVISION(pa->pa_class); 409 if (revision != 0) { 410 aprint_normal(": unknown revision 0x%02x; giving up\n", 411 revision); 412 return; 413 } 414 aprint_normal("\n"); 415 aprint_naive("\n"); 416 417 /* subsystem ID shows what I am */ 418 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 419 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 420 (PCI_SUBSYS_ID(id) < NDEVNAMES? 421 virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"), 422 revision); 423 424 sc->sc_dev = self; 425 sc->sc_pc = pc; 426 sc->sc_tag = tag; 427 sc->sc_iot = pa->pa_iot; 428 if (pci_dma64_available(pa)) 429 sc->sc_dmat = pa->pa_dmat64; 430 else 431 sc->sc_dmat = pa->pa_dmat; 432 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 433 434 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 435 &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) { 436 aprint_error_dev(self, "can't map i/o space\n"); 437 return; 438 } 439 440 virtio_device_reset(sc); 441 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 442 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 443 444 sc->sc_childdevid = PCI_SUBSYS_ID(id); 445 sc->sc_child = NULL; 446 sc->sc_pa = *pa; 447 virtio_rescan(self, "virtio", 0); 448 return; 449 } 450 451 /* ARGSUSED */ 452 static int 453 virtio_rescan(device_t self, const char *attr, const int *scan_flags) 454 { 455 struct virtio_softc *sc; 456 struct virtio_attach_args va; 457 458 sc = device_private(self); 459 if (sc->sc_child) /* Child already attached? */ 460 return 0; 461 462 memset(&va, 0, sizeof(va)); 463 va.sc_childdevid = sc->sc_childdevid; 464 465 config_found_ia(self, attr, &va, NULL); 466 467 if (sc->sc_child == NULL) { 468 aprint_error_dev(self, 469 "no matching child driver; not configured\n"); 470 return 0; 471 } 472 473 if (sc->sc_child == VIRTIO_CHILD_FAILED) { 474 aprint_error_dev(self, 475 "virtio configuration failed\n"); 476 return 0; 477 } 478 479 /* 480 * Make sure child drivers initialize interrupts via call 481 * to virtio_child_attach_finish(). 482 */ 483 KASSERT(sc->sc_ihs_num != 0); 484 485 return 0; 486 } 487 488 static int 489 virtio_detach(device_t self, int flags) 490 { 491 struct virtio_softc *sc = device_private(self); 492 int r; 493 494 if (sc->sc_child != NULL) { 495 r = config_detach(sc->sc_child, flags); 496 if (r) 497 return r; 498 } 499 500 /* Check that child detached properly */ 501 KASSERT(sc->sc_child == NULL); 502 KASSERT(sc->sc_vqs == NULL); 503 KASSERT(sc->sc_ihs_num == 0); 504 505 if (sc->sc_iosize) 506 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 507 sc->sc_iosize = 0; 508 509 return 0; 510 } 511 512 /* 513 * Reset the device. 514 */ 515 /* 516 * To reset the device to a known state, do following: 517 * virtio_reset(sc); // this will stop the device activity 518 * <dequeue finished requests>; // virtio_dequeue() still can be called 519 * <revoke pending requests in the vqs if any>; 520 * virtio_reinit_begin(sc); // dequeue prohibitted 521 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 522 * <some other initialization>; 523 * virtio_reinit_end(sc); // device activated; enqueue allowed 524 * Once attached, feature negotiation can only be allowed after virtio_reset. 525 */ 526 void 527 virtio_reset(struct virtio_softc *sc) 528 { 529 virtio_device_reset(sc); 530 } 531 532 void 533 virtio_reinit_start(struct virtio_softc *sc) 534 { 535 int i; 536 537 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 538 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 539 for (i = 0; i < sc->sc_nvqs; i++) { 540 int n; 541 struct virtqueue *vq = &sc->sc_vqs[i]; 542 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 543 VIRTIO_CONFIG_QUEUE_SELECT, 544 vq->vq_index); 545 n = nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, 546 VIRTIO_CONFIG_QUEUE_SIZE); 547 if (n == 0) /* vq disappeared */ 548 continue; 549 if (n != vq->vq_num) { 550 panic("%s: virtqueue size changed, vq index %d\n", 551 device_xname(sc->sc_dev), 552 vq->vq_index); 553 } 554 virtio_init_vq(sc, vq, true); 555 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 556 VIRTIO_CONFIG_QUEUE_ADDRESS, 557 (vq->vq_dmamap->dm_segs[0].ds_addr 558 / VIRTIO_PAGE_SIZE)); 559 } 560 561 /* MSI-X should have more than one handles where INTx has just one */ 562 if (sc->sc_ihs_num > 1) { 563 if (virtio_setup_msix_vectors(sc) != 0) { 564 aprint_error_dev(sc->sc_dev, 565 "couldn't setup MSI-X vectors\n"); 566 return; 567 } 568 } 569 } 570 571 void 572 virtio_reinit_end(struct virtio_softc *sc) 573 { 574 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 575 } 576 577 /* 578 * Feature negotiation. 579 */ 580 uint32_t 581 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 582 { 583 uint32_t r; 584 585 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 586 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 587 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 588 r = nbo_bus_space_read_4(sc->sc_iot, sc->sc_ioh, 589 VIRTIO_CONFIG_DEVICE_FEATURES); 590 r &= guest_features; 591 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 592 VIRTIO_CONFIG_GUEST_FEATURES, r); 593 sc->sc_features = r; 594 if (r & VIRTIO_F_RING_INDIRECT_DESC) 595 sc->sc_indirect = true; 596 else 597 sc->sc_indirect = false; 598 599 return r; 600 } 601 602 /* 603 * Device configuration registers. 604 */ 605 uint8_t 606 virtio_read_device_config_1(struct virtio_softc *sc, int index) 607 { 608 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 609 sc->sc_config_offset + index); 610 } 611 612 uint16_t 613 virtio_read_device_config_2(struct virtio_softc *sc, int index) 614 { 615 return nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, 616 sc->sc_config_offset + index); 617 } 618 619 uint32_t 620 virtio_read_device_config_4(struct virtio_softc *sc, int index) 621 { 622 return nbo_bus_space_read_4(sc->sc_iot, sc->sc_ioh, 623 sc->sc_config_offset + index); 624 } 625 626 uint64_t 627 virtio_read_device_config_8(struct virtio_softc *sc, int index) 628 { 629 uint64_t r; 630 631 r = nbo_bus_space_read_4(sc->sc_iot, sc->sc_ioh, 632 sc->sc_config_offset + index + REG_HI_OFF); 633 r <<= 32; 634 r |= nbo_bus_space_read_4(sc->sc_iot, sc->sc_ioh, 635 sc->sc_config_offset + index + REG_LO_OFF); 636 637 return r; 638 } 639 640 void 641 virtio_write_device_config_1(struct virtio_softc *sc, 642 int index, uint8_t value) 643 { 644 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 645 sc->sc_config_offset + index, value); 646 } 647 648 void 649 virtio_write_device_config_2(struct virtio_softc *sc, 650 int index, uint16_t value) 651 { 652 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 653 sc->sc_config_offset + index, value); 654 } 655 656 void 657 virtio_write_device_config_4(struct virtio_softc *sc, 658 int index, uint32_t value) 659 { 660 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 661 sc->sc_config_offset + index, value); 662 } 663 664 void 665 virtio_write_device_config_8(struct virtio_softc *sc, 666 int index, uint64_t value) 667 { 668 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 669 sc->sc_config_offset + index + REG_LO_OFF, 670 value & 0xffffffff); 671 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 672 sc->sc_config_offset + index + REG_HI_OFF, 673 value >> 32); 674 } 675 676 /* 677 * Interrupt handler. 678 */ 679 static int 680 virtio_intr(void *arg) 681 { 682 struct virtio_softc *sc = arg; 683 int isr, r = 0; 684 685 /* check and ack the interrupt */ 686 isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 687 VIRTIO_CONFIG_ISR_STATUS); 688 if (isr == 0) 689 return 0; 690 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 691 (sc->sc_config_change != NULL)) 692 r = (sc->sc_config_change)(sc); 693 if (sc->sc_intrhand != NULL) { 694 if (sc->sc_soft_ih != NULL) 695 softint_schedule(sc->sc_soft_ih); 696 else 697 r |= (sc->sc_intrhand)(sc); 698 } 699 700 return r; 701 } 702 703 static int 704 virtio_msix_queue_intr(void *arg) 705 { 706 struct virtio_softc *sc = arg; 707 int r = 0; 708 709 if (sc->sc_intrhand != NULL) { 710 if (sc->sc_soft_ih != NULL) 711 softint_schedule(sc->sc_soft_ih); 712 else 713 r |= (sc->sc_intrhand)(sc); 714 } 715 716 return r; 717 } 718 719 static int 720 virtio_msix_config_intr(void *arg) 721 { 722 struct virtio_softc *sc = arg; 723 724 /* TODO: handle events */ 725 aprint_debug_dev(sc->sc_dev, "%s\n", __func__); 726 return 1; 727 } 728 729 static void 730 virtio_soft_intr(void *arg) 731 { 732 struct virtio_softc *sc = arg; 733 734 KASSERT(sc->sc_intrhand != NULL); 735 736 (sc->sc_intrhand)(sc); 737 } 738 739 /* 740 * dmamap sync operations for a virtqueue. 741 */ 742 static inline void 743 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 744 { 745 /* availoffset == sizeof(vring_desc)*vq_num */ 746 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 747 ops); 748 } 749 750 static inline void 751 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 752 { 753 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 754 vq->vq_availoffset, 755 offsetof(struct vring_avail, ring) 756 + vq->vq_num * sizeof(uint16_t), 757 ops); 758 } 759 760 static inline void 761 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 762 { 763 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 764 vq->vq_usedoffset, 765 offsetof(struct vring_used, ring) 766 + vq->vq_num * sizeof(struct vring_used_elem), 767 ops); 768 } 769 770 static inline void 771 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 772 int ops) 773 { 774 int offset = vq->vq_indirectoffset 775 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 776 777 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 778 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 779 ops); 780 } 781 782 /* 783 * Can be used as sc_intrhand. 784 */ 785 /* 786 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 787 * and calls (*vq_done)() if some entries are consumed. 788 */ 789 int 790 virtio_vq_intr(struct virtio_softc *sc) 791 { 792 struct virtqueue *vq; 793 int i, r = 0; 794 795 for (i = 0; i < sc->sc_nvqs; i++) { 796 vq = &sc->sc_vqs[i]; 797 if (vq->vq_queued) { 798 vq->vq_queued = 0; 799 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 800 } 801 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 802 membar_consumer(); 803 if (vq->vq_used_idx != vq->vq_used->idx) { 804 if (vq->vq_done) 805 r |= (vq->vq_done)(vq); 806 } 807 } 808 809 return r; 810 } 811 812 /* 813 * Start/stop vq interrupt. No guarantee. 814 */ 815 void 816 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 817 { 818 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 819 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 820 vq->vq_queued++; 821 } 822 823 void 824 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 825 { 826 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 827 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 828 vq->vq_queued++; 829 } 830 831 /* 832 * Initialize vq structure. 833 */ 834 static void 835 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, 836 const bool reinit) 837 { 838 int i, j; 839 int vq_size = vq->vq_num; 840 841 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 842 843 /* build the indirect descriptor chain */ 844 if (vq->vq_indirect != NULL) { 845 struct vring_desc *vd; 846 847 for (i = 0; i < vq_size; i++) { 848 vd = vq->vq_indirect; 849 vd += vq->vq_maxnsegs * i; 850 for (j = 0; j < vq->vq_maxnsegs-1; j++) { 851 vd[j].next = j + 1; 852 } 853 } 854 } 855 856 /* free slot management */ 857 SIMPLEQ_INIT(&vq->vq_freelist); 858 for (i = 0; i < vq_size; i++) { 859 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 860 &vq->vq_entries[i], qe_list); 861 vq->vq_entries[i].qe_index = i; 862 } 863 if (!reinit) 864 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 865 866 /* enqueue/dequeue status */ 867 vq->vq_avail_idx = 0; 868 vq->vq_used_idx = 0; 869 vq->vq_queued = 0; 870 if (!reinit) { 871 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 872 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 873 } 874 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 875 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 876 vq->vq_queued++; 877 } 878 879 /* 880 * Allocate/free a vq. 881 */ 882 int 883 virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, 884 int maxsegsize, int maxnsegs, const char *name) 885 { 886 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 887 int rsegs, r; 888 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 889 ~(VIRTIO_PAGE_SIZE-1)) 890 891 /* Make sure callers allocate vqs in order */ 892 KASSERT(sc->sc_nvqs == index); 893 894 memset(vq, 0, sizeof(*vq)); 895 896 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 897 VIRTIO_CONFIG_QUEUE_SELECT, index); 898 vq_size = nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, 899 VIRTIO_CONFIG_QUEUE_SIZE); 900 if (vq_size == 0) { 901 aprint_error_dev(sc->sc_dev, 902 "virtqueue not exist, index %d for %s\n", 903 index, name); 904 goto err; 905 } 906 /* allocsize1: descriptor table + avail ring + pad */ 907 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 908 + sizeof(uint16_t)*(2+vq_size)); 909 /* allocsize2: used ring + pad */ 910 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 911 + sizeof(struct vring_used_elem)*vq_size); 912 /* allocsize3: indirect table */ 913 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 914 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 915 else 916 allocsize3 = 0; 917 allocsize = allocsize1 + allocsize2 + allocsize3; 918 919 /* alloc and map the memory */ 920 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 921 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 922 if (r != 0) { 923 aprint_error_dev(sc->sc_dev, 924 "virtqueue %d for %s allocation failed, " 925 "error code %d\n", index, name, r); 926 goto err; 927 } 928 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 929 &vq->vq_vaddr, BUS_DMA_NOWAIT); 930 if (r != 0) { 931 aprint_error_dev(sc->sc_dev, 932 "virtqueue %d for %s map failed, " 933 "error code %d\n", index, name, r); 934 goto err; 935 } 936 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 937 BUS_DMA_NOWAIT, &vq->vq_dmamap); 938 if (r != 0) { 939 aprint_error_dev(sc->sc_dev, 940 "virtqueue %d for %s dmamap creation failed, " 941 "error code %d\n", index, name, r); 942 goto err; 943 } 944 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 945 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 946 if (r != 0) { 947 aprint_error_dev(sc->sc_dev, 948 "virtqueue %d for %s dmamap load failed, " 949 "error code %d\n", index, name, r); 950 goto err; 951 } 952 953 /* set the vq address */ 954 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 955 VIRTIO_CONFIG_QUEUE_ADDRESS, 956 (vq->vq_dmamap->dm_segs[0].ds_addr 957 / VIRTIO_PAGE_SIZE)); 958 959 /* remember addresses and offsets for later use */ 960 vq->vq_owner = sc; 961 vq->vq_num = vq_size; 962 vq->vq_index = index; 963 vq->vq_desc = vq->vq_vaddr; 964 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 965 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 966 vq->vq_usedoffset = allocsize1; 967 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 968 if (allocsize3 > 0) { 969 vq->vq_indirectoffset = allocsize1 + allocsize2; 970 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 971 + vq->vq_indirectoffset); 972 } 973 vq->vq_bytesize = allocsize; 974 vq->vq_maxsegsize = maxsegsize; 975 vq->vq_maxnsegs = maxnsegs; 976 977 /* free slot management */ 978 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 979 KM_NOSLEEP); 980 if (vq->vq_entries == NULL) { 981 r = ENOMEM; 982 goto err; 983 } 984 985 virtio_init_vq(sc, vq, false); 986 987 aprint_verbose_dev(sc->sc_dev, 988 "allocated %u byte for virtqueue %d for %s, " 989 "size %d\n", allocsize, index, name, vq_size); 990 if (allocsize3 > 0) 991 aprint_verbose_dev(sc->sc_dev, 992 "using %d byte (%d entries) " 993 "indirect descriptors\n", 994 allocsize3, maxnsegs * vq_size); 995 996 sc->sc_nvqs++; 997 998 return 0; 999 1000 err: 1001 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 1002 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 1003 if (vq->vq_dmamap) 1004 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 1005 if (vq->vq_vaddr) 1006 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 1007 if (vq->vq_segs[0].ds_addr) 1008 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 1009 memset(vq, 0, sizeof(*vq)); 1010 1011 return -1; 1012 } 1013 1014 int 1015 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 1016 { 1017 struct vq_entry *qe; 1018 int i = 0; 1019 1020 /* device must be already deactivated */ 1021 /* confirm the vq is empty */ 1022 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 1023 i++; 1024 } 1025 if (i != vq->vq_num) { 1026 printf("%s: freeing non-empty vq, index %d\n", 1027 device_xname(sc->sc_dev), vq->vq_index); 1028 return EBUSY; 1029 } 1030 1031 /* tell device that there's no virtqueue any longer */ 1032 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 1033 VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index); 1034 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 1035 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 1036 1037 kmem_free(vq->vq_entries, sizeof(*vq->vq_entries) * vq->vq_num); 1038 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 1039 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 1040 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 1041 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 1042 mutex_destroy(&vq->vq_freelist_lock); 1043 mutex_destroy(&vq->vq_uring_lock); 1044 mutex_destroy(&vq->vq_aring_lock); 1045 memset(vq, 0, sizeof(*vq)); 1046 1047 sc->sc_nvqs--; 1048 1049 return 0; 1050 } 1051 1052 /* 1053 * Free descriptor management. 1054 */ 1055 static struct vq_entry * 1056 vq_alloc_entry(struct virtqueue *vq) 1057 { 1058 struct vq_entry *qe; 1059 1060 mutex_enter(&vq->vq_freelist_lock); 1061 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 1062 mutex_exit(&vq->vq_freelist_lock); 1063 return NULL; 1064 } 1065 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 1066 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 1067 mutex_exit(&vq->vq_freelist_lock); 1068 1069 return qe; 1070 } 1071 1072 static void 1073 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 1074 { 1075 mutex_enter(&vq->vq_freelist_lock); 1076 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 1077 mutex_exit(&vq->vq_freelist_lock); 1078 1079 return; 1080 } 1081 1082 /* 1083 * Enqueue several dmamaps as a single request. 1084 */ 1085 /* 1086 * Typical usage: 1087 * <queue size> number of followings are stored in arrays 1088 * - command blocks (in dmamem) should be pre-allocated and mapped 1089 * - dmamaps for command blocks should be pre-allocated and loaded 1090 * - dmamaps for payload should be pre-allocated 1091 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 1092 * if (r) // currently 0 or EAGAIN 1093 * return r; 1094 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 1095 * if (r) { 1096 * virtio_enqueue_abort(sc, vq, slot); 1097 * return r; 1098 * } 1099 * r = virtio_enqueue_reserve(sc, vq, slot, 1100 * dmamap_payload[slot]->dm_nsegs+1); 1101 * // ^ +1 for command 1102 * if (r) { // currently 0 or EAGAIN 1103 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 1104 * return r; // do not call abort() 1105 * } 1106 * <setup and prepare commands> 1107 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 1108 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 1109 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 1110 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 1111 * virtio_enqueue_commit(sc, vq, slot, true); 1112 */ 1113 1114 /* 1115 * enqueue_prep: allocate a slot number 1116 */ 1117 int 1118 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 1119 { 1120 struct vq_entry *qe1; 1121 1122 KASSERT(slotp != NULL); 1123 1124 qe1 = vq_alloc_entry(vq); 1125 if (qe1 == NULL) 1126 return EAGAIN; 1127 /* next slot is not allocated yet */ 1128 qe1->qe_next = -1; 1129 *slotp = qe1->qe_index; 1130 1131 return 0; 1132 } 1133 1134 /* 1135 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 1136 */ 1137 int 1138 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 1139 int slot, int nsegs) 1140 { 1141 int indirect; 1142 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1143 1144 KASSERT(qe1->qe_next == -1); 1145 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 1146 1147 if ((vq->vq_indirect != NULL) && 1148 (nsegs >= MINSEG_INDIRECT) && 1149 (nsegs <= vq->vq_maxnsegs)) 1150 indirect = 1; 1151 else 1152 indirect = 0; 1153 qe1->qe_indirect = indirect; 1154 1155 if (indirect) { 1156 struct vring_desc *vd; 1157 int i; 1158 1159 vd = &vq->vq_desc[qe1->qe_index]; 1160 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 1161 + vq->vq_indirectoffset; 1162 vd->addr += sizeof(struct vring_desc) 1163 * vq->vq_maxnsegs * qe1->qe_index; 1164 vd->len = sizeof(struct vring_desc) * nsegs; 1165 vd->flags = VRING_DESC_F_INDIRECT; 1166 1167 vd = vq->vq_indirect; 1168 vd += vq->vq_maxnsegs * qe1->qe_index; 1169 qe1->qe_desc_base = vd; 1170 1171 for (i = 0; i < nsegs-1; i++) { 1172 vd[i].flags = VRING_DESC_F_NEXT; 1173 } 1174 vd[i].flags = 0; 1175 qe1->qe_next = 0; 1176 1177 return 0; 1178 } else { 1179 struct vring_desc *vd; 1180 struct vq_entry *qe; 1181 int i, s; 1182 1183 vd = &vq->vq_desc[0]; 1184 qe1->qe_desc_base = vd; 1185 qe1->qe_next = qe1->qe_index; 1186 s = slot; 1187 for (i = 0; i < nsegs - 1; i++) { 1188 qe = vq_alloc_entry(vq); 1189 if (qe == NULL) { 1190 vd[s].flags = 0; 1191 virtio_enqueue_abort(sc, vq, slot); 1192 return EAGAIN; 1193 } 1194 vd[s].flags = VRING_DESC_F_NEXT; 1195 vd[s].next = qe->qe_index; 1196 s = qe->qe_index; 1197 } 1198 vd[s].flags = 0; 1199 1200 return 0; 1201 } 1202 } 1203 1204 /* 1205 * enqueue: enqueue a single dmamap. 1206 */ 1207 int 1208 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1209 bus_dmamap_t dmamap, bool write) 1210 { 1211 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1212 struct vring_desc *vd = qe1->qe_desc_base; 1213 int i; 1214 int s = qe1->qe_next; 1215 1216 KASSERT(s >= 0); 1217 KASSERT(dmamap->dm_nsegs > 0); 1218 1219 for (i = 0; i < dmamap->dm_nsegs; i++) { 1220 vd[s].addr = dmamap->dm_segs[i].ds_addr; 1221 vd[s].len = dmamap->dm_segs[i].ds_len; 1222 if (!write) 1223 vd[s].flags |= VRING_DESC_F_WRITE; 1224 s = vd[s].next; 1225 } 1226 qe1->qe_next = s; 1227 1228 return 0; 1229 } 1230 1231 int 1232 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1233 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 1234 bool write) 1235 { 1236 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1237 struct vring_desc *vd = qe1->qe_desc_base; 1238 int s = qe1->qe_next; 1239 1240 KASSERT(s >= 0); 1241 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 1242 KASSERT((dmamap->dm_segs[0].ds_len > start) && 1243 (dmamap->dm_segs[0].ds_len >= start + len)); 1244 1245 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 1246 vd[s].len = len; 1247 if (!write) 1248 vd[s].flags |= VRING_DESC_F_WRITE; 1249 qe1->qe_next = vd[s].next; 1250 1251 return 0; 1252 } 1253 1254 /* 1255 * enqueue_commit: add it to the aring. 1256 */ 1257 int 1258 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1259 bool notifynow) 1260 { 1261 struct vq_entry *qe1; 1262 1263 if (slot < 0) { 1264 mutex_enter(&vq->vq_aring_lock); 1265 goto notify; 1266 } 1267 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 1268 qe1 = &vq->vq_entries[slot]; 1269 if (qe1->qe_indirect) 1270 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 1271 mutex_enter(&vq->vq_aring_lock); 1272 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 1273 1274 notify: 1275 if (notifynow) { 1276 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 1277 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 1278 membar_producer(); 1279 vq->vq_avail->idx = vq->vq_avail_idx; 1280 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 1281 membar_producer(); 1282 vq->vq_queued++; 1283 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 1284 membar_consumer(); 1285 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 1286 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 1287 VIRTIO_CONFIG_QUEUE_NOTIFY, 1288 vq->vq_index); 1289 } 1290 mutex_exit(&vq->vq_aring_lock); 1291 1292 return 0; 1293 } 1294 1295 /* 1296 * enqueue_abort: rollback. 1297 */ 1298 int 1299 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1300 { 1301 struct vq_entry *qe = &vq->vq_entries[slot]; 1302 struct vring_desc *vd; 1303 int s; 1304 1305 if (qe->qe_next < 0) { 1306 vq_free_entry(vq, qe); 1307 return 0; 1308 } 1309 1310 s = slot; 1311 vd = &vq->vq_desc[0]; 1312 while (vd[s].flags & VRING_DESC_F_NEXT) { 1313 s = vd[s].next; 1314 vq_free_entry(vq, qe); 1315 qe = &vq->vq_entries[s]; 1316 } 1317 vq_free_entry(vq, qe); 1318 return 0; 1319 } 1320 1321 /* 1322 * Dequeue a request. 1323 */ 1324 /* 1325 * dequeue: dequeue a request from uring; dmamap_sync for uring is 1326 * already done in the interrupt handler. 1327 */ 1328 int 1329 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 1330 int *slotp, int *lenp) 1331 { 1332 uint16_t slot, usedidx; 1333 struct vq_entry *qe; 1334 1335 if (vq->vq_used_idx == vq->vq_used->idx) 1336 return ENOENT; 1337 mutex_enter(&vq->vq_uring_lock); 1338 usedidx = vq->vq_used_idx++; 1339 mutex_exit(&vq->vq_uring_lock); 1340 usedidx %= vq->vq_num; 1341 slot = vq->vq_used->ring[usedidx].id; 1342 qe = &vq->vq_entries[slot]; 1343 1344 if (qe->qe_indirect) 1345 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 1346 1347 if (slotp) 1348 *slotp = slot; 1349 if (lenp) 1350 *lenp = vq->vq_used->ring[usedidx].len; 1351 1352 return 0; 1353 } 1354 1355 /* 1356 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1357 * if you forget to call this the slot will be leaked. 1358 */ 1359 int 1360 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1361 { 1362 struct vq_entry *qe = &vq->vq_entries[slot]; 1363 struct vring_desc *vd = &vq->vq_desc[0]; 1364 int s = slot; 1365 1366 while (vd[s].flags & VRING_DESC_F_NEXT) { 1367 s = vd[s].next; 1368 vq_free_entry(vq, qe); 1369 qe = &vq->vq_entries[s]; 1370 } 1371 vq_free_entry(vq, qe); 1372 1373 return 0; 1374 } 1375 1376 /* 1377 * Attach a child, fill all the members. 1378 */ 1379 void 1380 virtio_child_attach_start(struct virtio_softc *sc, device_t child, int ipl, 1381 struct virtqueue *vqs, 1382 virtio_callback config_change, 1383 virtio_callback intr_hand, 1384 int req_flags, int req_features, const char *feat_bits) 1385 { 1386 char buf[256]; 1387 int features; 1388 1389 sc->sc_child = child; 1390 sc->sc_ipl = ipl; 1391 sc->sc_vqs = vqs; 1392 sc->sc_config_change = config_change; 1393 sc->sc_intrhand = intr_hand; 1394 sc->sc_flags = req_flags; 1395 1396 features = virtio_negotiate_features(sc, req_features); 1397 snprintb(buf, sizeof(buf), feat_bits, features); 1398 aprint_normal(": Features: %s\n", buf); 1399 aprint_naive("\n"); 1400 } 1401 1402 int 1403 virtio_child_attach_finish(struct virtio_softc *sc) 1404 { 1405 int r; 1406 1407 r = virtio_setup_interrupts(sc); 1408 if (r != 0) { 1409 aprint_error_dev(sc->sc_dev, "failed to setup interrupts\n"); 1410 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 1411 return 1; 1412 } 1413 1414 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 1415 1416 return 0; 1417 } 1418 1419 void 1420 virtio_child_detach(struct virtio_softc *sc) 1421 { 1422 sc->sc_child = NULL; 1423 sc->sc_vqs = NULL; 1424 1425 virtio_device_reset(sc); 1426 1427 virtio_free_interrupts(sc); 1428 } 1429 1430 void 1431 virtio_child_attach_failed(struct virtio_softc *sc) 1432 { 1433 virtio_child_detach(sc); 1434 1435 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 1436 1437 sc->sc_child = VIRTIO_CHILD_FAILED; 1438 } 1439 1440 bus_dma_tag_t 1441 virtio_dmat(struct virtio_softc *sc) 1442 { 1443 return sc->sc_dmat; 1444 } 1445 1446 device_t 1447 virtio_child(struct virtio_softc *sc) 1448 { 1449 return sc->sc_child; 1450 } 1451 1452 int 1453 virtio_intrhand(struct virtio_softc *sc) 1454 { 1455 return (sc->sc_intrhand)(sc); 1456 } 1457 1458 uint32_t 1459 virtio_features(struct virtio_softc *sc) 1460 { 1461 return sc->sc_features; 1462 } 1463 1464 MODULE(MODULE_CLASS_DRIVER, virtio, "pci"); 1465 1466 #ifdef _MODULE 1467 #include "ioconf.c" 1468 #endif 1469 1470 static int 1471 virtio_modcmd(modcmd_t cmd, void *opaque) 1472 { 1473 int error = 0; 1474 1475 #ifdef _MODULE 1476 switch (cmd) { 1477 case MODULE_CMD_INIT: 1478 error = config_init_component(cfdriver_ioconf_virtio, 1479 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1480 break; 1481 case MODULE_CMD_FINI: 1482 error = config_fini_component(cfdriver_ioconf_virtio, 1483 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1484 break; 1485 default: 1486 error = ENOTTY; 1487 break; 1488 } 1489 #endif 1490 1491 return error; 1492 } 1493
Indexes created Thu Oct 02 14:10:14 GMT 2025