virtio.c revision 1.16.2.2
1 /* $NetBSD: virtio.c,v 1.16.2.2 2017/01/07 08:56:40 pgoyette Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.16.2.2 2017/01/07 08:56:40 pgoyette Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/atomic.h> 35 #include <sys/bus.h> 36 #include <sys/device.h> 37 #include <sys/kmem.h> 38 #include <sys/module.h> 39 40 #include <dev/pci/pcidevs.h> 41 #include <dev/pci/pcireg.h> 42 #include <dev/pci/pcivar.h> 43 44 #include <dev/pci/virtioreg.h> 45 #include <dev/pci/virtiovar.h> 46 47 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 48 49 static int virtio_match(device_t, cfdata_t, void *); 50 static void virtio_attach(device_t, device_t, void *); 51 static int virtio_rescan(device_t, const char *, const int *); 52 static int virtio_detach(device_t, int); 53 static int virtio_intr(void *arg); 54 static int virtio_msix_queue_intr(void *); 55 static int virtio_msix_config_intr(void *); 56 static int virtio_setup_msix_vectors(struct virtio_softc *); 57 static int virtio_setup_msix_interrupts(struct virtio_softc *, 58 struct pci_attach_args *); 59 static int virtio_setup_intx_interrupt(struct virtio_softc *, 60 struct pci_attach_args *); 61 static int virtio_setup_interrupts(struct virtio_softc *); 62 static void virtio_soft_intr(void *arg); 63 static void virtio_init_vq(struct virtio_softc *, 64 struct virtqueue *, const bool); 65 66 CFATTACH_DECL3_NEW(virtio, sizeof(struct virtio_softc), 67 virtio_match, virtio_attach, virtio_detach, NULL, virtio_rescan, NULL, 68 DVF_DETACH_SHUTDOWN); 69 70 static void 71 virtio_set_status(struct virtio_softc *sc, int status) 72 { 73 int old = 0; 74 75 if (status != 0) 76 old = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 77 VIRTIO_CONFIG_DEVICE_STATUS); 78 bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS, 79 status|old); 80 } 81 82 #define virtio_device_reset(sc) virtio_set_status((sc), 0) 83 84 static int 85 virtio_match(device_t parent, cfdata_t match, void *aux) 86 { 87 struct pci_attach_args *pa; 88 89 pa = (struct pci_attach_args *)aux; 90 switch (PCI_VENDOR(pa->pa_id)) { 91 case PCI_VENDOR_QUMRANET: 92 if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 93 PCI_PRODUCT(pa->pa_id)) && 94 (PCI_PRODUCT(pa->pa_id) <= 95 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) 96 return 1; 97 break; 98 } 99 100 return 0; 101 } 102 103 static const char *virtio_device_name[] = { 104 "Unknown (0)", /* 0 */ 105 "Network", /* 1 */ 106 "Block", /* 2 */ 107 "Console", /* 3 */ 108 "Entropy", /* 4 */ 109 "Memory Balloon", /* 5 */ 110 "I/O Memory", /* 6 */ 111 "Remote Processor Messaging", /* 7 */ 112 "SCSI", /* 8 */ 113 "9P Transport", /* 9 */ 114 "mac80211 wlan", /* 10 */ 115 }; 116 #define NDEVNAMES __arraycount(virtio_device_name) 117 118 #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0 119 #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1 120 121 static int 122 virtio_setup_msix_vectors(struct virtio_softc *sc) 123 { 124 int offset, vector, ret, qid; 125 126 offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR; 127 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 128 129 bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); 130 ret = bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); 131 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 132 vector, ret); 133 if (ret != vector) 134 return -1; 135 136 for (qid = 0; qid < sc->sc_nvqs; qid++) { 137 offset = VIRTIO_CONFIG_QUEUE_SELECT; 138 bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, qid); 139 140 offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR; 141 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 142 143 bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); 144 ret = bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); 145 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 146 vector, ret); 147 if (ret != vector) 148 return -1; 149 } 150 151 return 0; 152 } 153 154 static int 155 virtio_setup_msix_interrupts(struct virtio_softc *sc, 156 struct pci_attach_args *pa) 157 { 158 device_t self = sc->sc_dev; 159 pci_chipset_tag_t pc = pa->pa_pc; 160 char intrbuf[PCI_INTRSTR_LEN]; 161 char const *intrstr; 162 int idx; 163 164 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 165 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 166 pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 167 168 sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], IPL_NET, 169 virtio_msix_config_intr, sc, device_xname(sc->sc_dev)); 170 if (sc->sc_ihs[idx] == NULL) { 171 aprint_error_dev(self, "couldn't establish MSI-X for config\n"); 172 goto error; 173 } 174 175 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 176 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 177 pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 178 179 sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], IPL_NET, 180 virtio_msix_queue_intr, sc, device_xname(sc->sc_dev)); 181 if (sc->sc_ihs[idx] == NULL) { 182 aprint_error_dev(self, "couldn't establish MSI-X for queues\n"); 183 goto error; 184 } 185 186 if (virtio_setup_msix_vectors(sc) != 0) { 187 aprint_error_dev(self, "couldn't setup MSI-X vectors\n"); 188 goto error; 189 } 190 191 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 192 intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 193 aprint_normal_dev(self, "config interrupting at %s\n", intrstr); 194 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 195 intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 196 aprint_normal_dev(self, "queues interrupting at %s\n", intrstr); 197 198 return 0; 199 200 error: 201 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 202 if (sc->sc_ihs[idx] != NULL) 203 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); 204 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 205 if (sc->sc_ihs[idx] != NULL) 206 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); 207 208 return -1; 209 } 210 211 static int 212 virtio_setup_intx_interrupt(struct virtio_softc *sc, 213 struct pci_attach_args *pa) 214 { 215 device_t self = sc->sc_dev; 216 pci_chipset_tag_t pc = pa->pa_pc; 217 char intrbuf[PCI_INTRSTR_LEN]; 218 char const *intrstr; 219 220 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 221 pci_intr_setattr(pc, &sc->sc_ihp[0], PCI_INTR_MPSAFE, true); 222 223 sc->sc_ihs[0] = pci_intr_establish_xname(pc, sc->sc_ihp[0], 224 IPL_NET, virtio_intr, sc, device_xname(sc->sc_dev)); 225 if (sc->sc_ihs[0] == NULL) { 226 aprint_error_dev(self, "couldn't establish INTx\n"); 227 return -1; 228 } 229 230 intrstr = pci_intr_string(pc, sc->sc_ihp[0], intrbuf, sizeof(intrbuf)); 231 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 232 233 return 0; 234 } 235 236 static int 237 virtio_setup_interrupts(struct virtio_softc *sc) 238 { 239 device_t self = sc->sc_dev; 240 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 241 int error; 242 int nmsix; 243 int counts[PCI_INTR_TYPE_SIZE]; 244 pci_intr_type_t max_type; 245 246 nmsix = pci_msix_count(sc->sc_pa.pa_pc, sc->sc_pa.pa_tag); 247 aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix); 248 249 /* We need at least two: one for config and the other for queues */ 250 if ((sc->sc_flags & VIRTIO_F_PCI_INTR_MSIX) == 0 || nmsix < 2) { 251 /* Try INTx only */ 252 max_type = PCI_INTR_TYPE_INTX; 253 counts[PCI_INTR_TYPE_INTX] = 1; 254 } else { 255 /* Try MSI-X first and INTx second */ 256 max_type = PCI_INTR_TYPE_MSIX; 257 counts[PCI_INTR_TYPE_MSIX] = 2; 258 counts[PCI_INTR_TYPE_MSI] = 0; 259 counts[PCI_INTR_TYPE_INTX] = 1; 260 } 261 262 retry: 263 error = pci_intr_alloc(&sc->sc_pa, &sc->sc_ihp, counts, max_type); 264 if (error != 0) { 265 aprint_error_dev(self, "couldn't map interrupt\n"); 266 return -1; 267 } 268 269 if (pci_intr_type(pc, sc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) { 270 sc->sc_ihs = kmem_alloc(sizeof(*sc->sc_ihs) * 2, 271 KM_SLEEP); 272 if (sc->sc_ihs == NULL) { 273 pci_intr_release(pc, sc->sc_ihp, 2); 274 275 /* Retry INTx */ 276 max_type = PCI_INTR_TYPE_INTX; 277 counts[PCI_INTR_TYPE_INTX] = 1; 278 goto retry; 279 } 280 281 error = virtio_setup_msix_interrupts(sc, &sc->sc_pa); 282 if (error != 0) { 283 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 2); 284 pci_intr_release(pc, sc->sc_ihp, 2); 285 286 /* Retry INTx */ 287 max_type = PCI_INTR_TYPE_INTX; 288 counts[PCI_INTR_TYPE_INTX] = 1; 289 goto retry; 290 } 291 292 sc->sc_ihs_num = 2; 293 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI; 294 } else if (pci_intr_type(pc, sc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) { 295 sc->sc_ihs = kmem_alloc(sizeof(*sc->sc_ihs) * 1, 296 KM_SLEEP); 297 if (sc->sc_ihs == NULL) { 298 pci_intr_release(pc, sc->sc_ihp, 1); 299 return -1; 300 } 301 302 error = virtio_setup_intx_interrupt(sc, &sc->sc_pa); 303 if (error != 0) { 304 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 1); 305 pci_intr_release(pc, sc->sc_ihp, 1); 306 return -1; 307 } 308 309 sc->sc_ihs_num = 1; 310 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 311 } 312 313 return 0; 314 } 315 316 static void 317 virtio_attach(device_t parent, device_t self, void *aux) 318 { 319 struct virtio_softc *sc = device_private(self); 320 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 321 pci_chipset_tag_t pc = pa->pa_pc; 322 pcitag_t tag = pa->pa_tag; 323 int revision; 324 pcireg_t id; 325 326 revision = PCI_REVISION(pa->pa_class); 327 if (revision != 0) { 328 aprint_normal(": unknown revision 0x%02x; giving up\n", 329 revision); 330 return; 331 } 332 aprint_normal("\n"); 333 aprint_naive("\n"); 334 335 /* subsystem ID shows what I am */ 336 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 337 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 338 (PCI_SUBSYS_ID(id) < NDEVNAMES? 339 virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"), 340 revision); 341 342 sc->sc_dev = self; 343 sc->sc_pc = pc; 344 sc->sc_tag = tag; 345 sc->sc_iot = pa->pa_iot; 346 if (pci_dma64_available(pa)) 347 sc->sc_dmat = pa->pa_dmat64; 348 else 349 sc->sc_dmat = pa->pa_dmat; 350 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 351 352 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 353 &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) { 354 aprint_error_dev(self, "can't map i/o space\n"); 355 return; 356 } 357 358 virtio_device_reset(sc); 359 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 360 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 361 362 /* XXX: use softc as aux... */ 363 sc->sc_childdevid = PCI_SUBSYS_ID(id); 364 sc->sc_child = NULL; 365 sc->sc_pa = *pa; 366 virtio_rescan(self, "virtio", 0); 367 return; 368 } 369 370 /* ARGSUSED */ 371 static int 372 virtio_rescan(device_t self, const char *attr, const int *scan_flags) 373 { 374 struct virtio_softc *sc; 375 int r; 376 377 sc = device_private(self); 378 if (sc->sc_child) /* Child already attached? */ 379 return 0; 380 config_found_ia(self, attr, sc, NULL); 381 if (sc->sc_child == NULL) { 382 aprint_error_dev(self, 383 "no matching child driver; not configured\n"); 384 return 0; 385 } 386 if (sc->sc_child == (void*)1) { /* this shows error */ 387 aprint_error_dev(self, 388 "virtio configuration failed\n"); 389 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 390 return 0; 391 } 392 393 r = virtio_setup_interrupts(sc); 394 if (r != 0) { 395 aprint_error_dev(self, "failed to setup interrupts\n"); 396 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 397 return 0; 398 } 399 400 sc->sc_soft_ih = NULL; 401 if (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) { 402 u_int flags = SOFTINT_NET; 403 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 404 flags |= SOFTINT_MPSAFE; 405 406 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, sc); 407 if (sc->sc_soft_ih == NULL) 408 aprint_error(": failed to establish soft interrupt\n"); 409 } 410 411 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 412 413 return 0; 414 } 415 416 static int 417 virtio_detach(device_t self, int flags) 418 { 419 struct virtio_softc *sc = device_private(self); 420 int r; 421 int i; 422 423 if (sc->sc_child != 0 && sc->sc_child != (void*)1) { 424 r = config_detach(sc->sc_child, flags); 425 if (r) 426 return r; 427 } 428 KASSERT(sc->sc_child == 0 || sc->sc_child == (void*)1); 429 KASSERT(sc->sc_vqs == 0); 430 for (i = 0; i < sc->sc_ihs_num; i++) { 431 if (sc->sc_ihs[i] == NULL) 432 continue; 433 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]); 434 } 435 pci_intr_release(sc->sc_pc, sc->sc_ihp, sc->sc_ihs_num); 436 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * sc->sc_ihs_num); 437 sc->sc_ihs_num = 0; 438 if (sc->sc_iosize) 439 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 440 sc->sc_iosize = 0; 441 442 return 0; 443 } 444 445 /* 446 * Reset the device. 447 */ 448 /* 449 * To reset the device to a known state, do following: 450 * virtio_reset(sc); // this will stop the device activity 451 * <dequeue finished requests>; // virtio_dequeue() still can be called 452 * <revoke pending requests in the vqs if any>; 453 * virtio_reinit_begin(sc); // dequeue prohibitted 454 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 455 * <some other initialization>; 456 * virtio_reinit_end(sc); // device activated; enqueue allowed 457 * Once attached, feature negotiation can only be allowed after virtio_reset. 458 */ 459 void 460 virtio_reset(struct virtio_softc *sc) 461 { 462 virtio_device_reset(sc); 463 } 464 465 void 466 virtio_reinit_start(struct virtio_softc *sc) 467 { 468 int i; 469 470 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 471 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 472 for (i = 0; i < sc->sc_nvqs; i++) { 473 int n; 474 struct virtqueue *vq = &sc->sc_vqs[i]; 475 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 476 VIRTIO_CONFIG_QUEUE_SELECT, 477 vq->vq_index); 478 n = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 479 VIRTIO_CONFIG_QUEUE_SIZE); 480 if (n == 0) /* vq disappeared */ 481 continue; 482 if (n != vq->vq_num) { 483 panic("%s: virtqueue size changed, vq index %d\n", 484 device_xname(sc->sc_dev), 485 vq->vq_index); 486 } 487 virtio_init_vq(sc, vq, true); 488 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 489 VIRTIO_CONFIG_QUEUE_ADDRESS, 490 (vq->vq_dmamap->dm_segs[0].ds_addr 491 / VIRTIO_PAGE_SIZE)); 492 } 493 494 /* MSI-X should have more than one handles where INTx has just one */ 495 if (sc->sc_ihs_num > 1) { 496 if (virtio_setup_msix_vectors(sc) != 0) { 497 aprint_error_dev(sc->sc_dev, 498 "couldn't setup MSI-X vectors\n"); 499 return; 500 } 501 } 502 } 503 504 void 505 virtio_reinit_end(struct virtio_softc *sc) 506 { 507 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 508 } 509 510 /* 511 * Feature negotiation. 512 */ 513 uint32_t 514 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 515 { 516 uint32_t r; 517 518 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 519 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 520 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 521 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 522 VIRTIO_CONFIG_DEVICE_FEATURES); 523 r &= guest_features; 524 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 525 VIRTIO_CONFIG_GUEST_FEATURES, r); 526 sc->sc_features = r; 527 if (r & VIRTIO_F_RING_INDIRECT_DESC) 528 sc->sc_indirect = true; 529 else 530 sc->sc_indirect = false; 531 532 return r; 533 } 534 535 /* 536 * Device configuration registers. 537 */ 538 uint8_t 539 virtio_read_device_config_1(struct virtio_softc *sc, int index) 540 { 541 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 542 sc->sc_config_offset + index); 543 } 544 545 uint16_t 546 virtio_read_device_config_2(struct virtio_softc *sc, int index) 547 { 548 return bus_space_read_2(sc->sc_iot, sc->sc_ioh, 549 sc->sc_config_offset + index); 550 } 551 552 uint32_t 553 virtio_read_device_config_4(struct virtio_softc *sc, int index) 554 { 555 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 556 sc->sc_config_offset + index); 557 } 558 559 uint64_t 560 virtio_read_device_config_8(struct virtio_softc *sc, int index) 561 { 562 uint64_t r; 563 564 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 565 sc->sc_config_offset + index + sizeof(uint32_t)); 566 r <<= 32; 567 r += bus_space_read_4(sc->sc_iot, sc->sc_ioh, 568 sc->sc_config_offset + index); 569 return r; 570 } 571 572 void 573 virtio_write_device_config_1(struct virtio_softc *sc, 574 int index, uint8_t value) 575 { 576 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 577 sc->sc_config_offset + index, value); 578 } 579 580 void 581 virtio_write_device_config_2(struct virtio_softc *sc, 582 int index, uint16_t value) 583 { 584 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 585 sc->sc_config_offset + index, value); 586 } 587 588 void 589 virtio_write_device_config_4(struct virtio_softc *sc, 590 int index, uint32_t value) 591 { 592 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 593 sc->sc_config_offset + index, value); 594 } 595 596 void 597 virtio_write_device_config_8(struct virtio_softc *sc, 598 int index, uint64_t value) 599 { 600 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 601 sc->sc_config_offset + index, 602 value & 0xffffffff); 603 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 604 sc->sc_config_offset + index + sizeof(uint32_t), 605 value >> 32); 606 } 607 608 /* 609 * Interrupt handler. 610 */ 611 static int 612 virtio_intr(void *arg) 613 { 614 struct virtio_softc *sc = arg; 615 int isr, r = 0; 616 617 /* check and ack the interrupt */ 618 isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 619 VIRTIO_CONFIG_ISR_STATUS); 620 if (isr == 0) 621 return 0; 622 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 623 (sc->sc_config_change != NULL)) 624 r = (sc->sc_config_change)(sc); 625 if (sc->sc_intrhand != NULL) { 626 if (sc->sc_soft_ih != NULL) 627 softint_schedule(sc->sc_soft_ih); 628 else 629 r |= (sc->sc_intrhand)(sc); 630 } 631 632 return r; 633 } 634 635 static int 636 virtio_msix_queue_intr(void *arg) 637 { 638 struct virtio_softc *sc = arg; 639 int r = 0; 640 641 if (sc->sc_intrhand != NULL) { 642 if (sc->sc_soft_ih != NULL) 643 softint_schedule(sc->sc_soft_ih); 644 else 645 r |= (sc->sc_intrhand)(sc); 646 } 647 648 return r; 649 } 650 651 static int 652 virtio_msix_config_intr(void *arg) 653 { 654 struct virtio_softc *sc = arg; 655 656 /* TODO: handle events */ 657 aprint_debug_dev(sc->sc_dev, "%s\n", __func__); 658 return 1; 659 } 660 661 static void 662 virtio_soft_intr(void *arg) 663 { 664 struct virtio_softc *sc = arg; 665 666 KASSERT(sc->sc_intrhand != NULL); 667 668 (sc->sc_intrhand)(sc); 669 } 670 671 /* 672 * dmamap sync operations for a virtqueue. 673 */ 674 static inline void 675 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 676 { 677 /* availoffset == sizeof(vring_desc)*vq_num */ 678 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 679 ops); 680 } 681 682 static inline void 683 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 684 { 685 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 686 vq->vq_availoffset, 687 offsetof(struct vring_avail, ring) 688 + vq->vq_num * sizeof(uint16_t), 689 ops); 690 } 691 692 static inline void 693 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 694 { 695 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 696 vq->vq_usedoffset, 697 offsetof(struct vring_used, ring) 698 + vq->vq_num * sizeof(struct vring_used_elem), 699 ops); 700 } 701 702 static inline void 703 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 704 int ops) 705 { 706 int offset = vq->vq_indirectoffset 707 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 708 709 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 710 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 711 ops); 712 } 713 714 /* 715 * Can be used as sc_intrhand. 716 */ 717 /* 718 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 719 * and calls (*vq_done)() if some entries are consumed. 720 */ 721 int 722 virtio_vq_intr(struct virtio_softc *sc) 723 { 724 struct virtqueue *vq; 725 int i, r = 0; 726 727 for (i = 0; i < sc->sc_nvqs; i++) { 728 vq = &sc->sc_vqs[i]; 729 if (vq->vq_queued) { 730 vq->vq_queued = 0; 731 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 732 } 733 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 734 membar_consumer(); 735 if (vq->vq_used_idx != vq->vq_used->idx) { 736 if (vq->vq_done) 737 r |= (vq->vq_done)(vq); 738 } 739 } 740 741 return r; 742 } 743 744 /* 745 * Start/stop vq interrupt. No guarantee. 746 */ 747 void 748 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 749 { 750 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 751 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 752 vq->vq_queued++; 753 } 754 755 void 756 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 757 { 758 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 759 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 760 vq->vq_queued++; 761 } 762 763 /* 764 * Initialize vq structure. 765 */ 766 static void 767 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, 768 const bool reinit) 769 { 770 int i, j; 771 int vq_size = vq->vq_num; 772 773 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 774 775 /* build the indirect descriptor chain */ 776 if (vq->vq_indirect != NULL) { 777 struct vring_desc *vd; 778 779 for (i = 0; i < vq_size; i++) { 780 vd = vq->vq_indirect; 781 vd += vq->vq_maxnsegs * i; 782 for (j = 0; j < vq->vq_maxnsegs-1; j++) 783 vd[j].next = j + 1; 784 } 785 } 786 787 /* free slot management */ 788 SIMPLEQ_INIT(&vq->vq_freelist); 789 for (i = 0; i < vq_size; i++) { 790 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 791 &vq->vq_entries[i], qe_list); 792 vq->vq_entries[i].qe_index = i; 793 } 794 if (!reinit) 795 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 796 797 /* enqueue/dequeue status */ 798 vq->vq_avail_idx = 0; 799 vq->vq_used_idx = 0; 800 vq->vq_queued = 0; 801 if (!reinit) { 802 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 803 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 804 } 805 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 806 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 807 vq->vq_queued++; 808 } 809 810 /* 811 * Allocate/free a vq. 812 */ 813 int 814 virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, 815 int maxsegsize, int maxnsegs, const char *name) 816 { 817 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 818 int rsegs, r; 819 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 820 ~(VIRTIO_PAGE_SIZE-1)) 821 822 memset(vq, 0, sizeof(*vq)); 823 824 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 825 VIRTIO_CONFIG_QUEUE_SELECT, index); 826 vq_size = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 827 VIRTIO_CONFIG_QUEUE_SIZE); 828 if (vq_size == 0) { 829 aprint_error_dev(sc->sc_dev, 830 "virtqueue not exist, index %d for %s\n", 831 index, name); 832 goto err; 833 } 834 /* allocsize1: descriptor table + avail ring + pad */ 835 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 836 + sizeof(uint16_t)*(2+vq_size)); 837 /* allocsize2: used ring + pad */ 838 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 839 + sizeof(struct vring_used_elem)*vq_size); 840 /* allocsize3: indirect table */ 841 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 842 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 843 else 844 allocsize3 = 0; 845 allocsize = allocsize1 + allocsize2 + allocsize3; 846 847 /* alloc and map the memory */ 848 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 849 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 850 if (r != 0) { 851 aprint_error_dev(sc->sc_dev, 852 "virtqueue %d for %s allocation failed, " 853 "error code %d\n", index, name, r); 854 goto err; 855 } 856 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 857 &vq->vq_vaddr, BUS_DMA_NOWAIT); 858 if (r != 0) { 859 aprint_error_dev(sc->sc_dev, 860 "virtqueue %d for %s map failed, " 861 "error code %d\n", index, name, r); 862 goto err; 863 } 864 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 865 BUS_DMA_NOWAIT, &vq->vq_dmamap); 866 if (r != 0) { 867 aprint_error_dev(sc->sc_dev, 868 "virtqueue %d for %s dmamap creation failed, " 869 "error code %d\n", index, name, r); 870 goto err; 871 } 872 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 873 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 874 if (r != 0) { 875 aprint_error_dev(sc->sc_dev, 876 "virtqueue %d for %s dmamap load failed, " 877 "error code %d\n", index, name, r); 878 goto err; 879 } 880 881 /* set the vq address */ 882 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 883 VIRTIO_CONFIG_QUEUE_ADDRESS, 884 (vq->vq_dmamap->dm_segs[0].ds_addr 885 / VIRTIO_PAGE_SIZE)); 886 887 /* remember addresses and offsets for later use */ 888 vq->vq_owner = sc; 889 vq->vq_num = vq_size; 890 vq->vq_index = index; 891 vq->vq_desc = vq->vq_vaddr; 892 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 893 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 894 vq->vq_usedoffset = allocsize1; 895 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 896 if (allocsize3 > 0) { 897 vq->vq_indirectoffset = allocsize1 + allocsize2; 898 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 899 + vq->vq_indirectoffset); 900 } 901 vq->vq_bytesize = allocsize; 902 vq->vq_maxsegsize = maxsegsize; 903 vq->vq_maxnsegs = maxnsegs; 904 905 /* free slot management */ 906 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 907 KM_NOSLEEP); 908 if (vq->vq_entries == NULL) { 909 r = ENOMEM; 910 goto err; 911 } 912 913 virtio_init_vq(sc, vq, false); 914 915 aprint_verbose_dev(sc->sc_dev, 916 "allocated %u byte for virtqueue %d for %s, " 917 "size %d\n", allocsize, index, name, vq_size); 918 if (allocsize3 > 0) 919 aprint_verbose_dev(sc->sc_dev, 920 "using %d byte (%d entries) " 921 "indirect descriptors\n", 922 allocsize3, maxnsegs * vq_size); 923 return 0; 924 925 err: 926 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 927 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 928 if (vq->vq_dmamap) 929 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 930 if (vq->vq_vaddr) 931 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 932 if (vq->vq_segs[0].ds_addr) 933 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 934 memset(vq, 0, sizeof(*vq)); 935 936 return -1; 937 } 938 939 int 940 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 941 { 942 struct vq_entry *qe; 943 int i = 0; 944 945 /* device must be already deactivated */ 946 /* confirm the vq is empty */ 947 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 948 i++; 949 } 950 if (i != vq->vq_num) { 951 printf("%s: freeing non-empty vq, index %d\n", 952 device_xname(sc->sc_dev), vq->vq_index); 953 return EBUSY; 954 } 955 956 /* tell device that there's no virtqueue any longer */ 957 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 958 VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index); 959 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 960 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 961 962 kmem_free(vq->vq_entries, sizeof(*vq->vq_entries) * vq->vq_num); 963 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 964 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 965 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 966 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 967 mutex_destroy(&vq->vq_freelist_lock); 968 mutex_destroy(&vq->vq_uring_lock); 969 mutex_destroy(&vq->vq_aring_lock); 970 memset(vq, 0, sizeof(*vq)); 971 972 return 0; 973 } 974 975 /* 976 * Free descriptor management. 977 */ 978 static struct vq_entry * 979 vq_alloc_entry(struct virtqueue *vq) 980 { 981 struct vq_entry *qe; 982 983 mutex_enter(&vq->vq_freelist_lock); 984 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 985 mutex_exit(&vq->vq_freelist_lock); 986 return NULL; 987 } 988 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 989 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 990 mutex_exit(&vq->vq_freelist_lock); 991 992 return qe; 993 } 994 995 static void 996 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 997 { 998 mutex_enter(&vq->vq_freelist_lock); 999 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 1000 mutex_exit(&vq->vq_freelist_lock); 1001 1002 return; 1003 } 1004 1005 /* 1006 * Enqueue several dmamaps as a single request. 1007 */ 1008 /* 1009 * Typical usage: 1010 * <queue size> number of followings are stored in arrays 1011 * - command blocks (in dmamem) should be pre-allocated and mapped 1012 * - dmamaps for command blocks should be pre-allocated and loaded 1013 * - dmamaps for payload should be pre-allocated 1014 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 1015 * if (r) // currently 0 or EAGAIN 1016 * return r; 1017 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 1018 * if (r) { 1019 * virtio_enqueue_abort(sc, vq, slot); 1020 * return r; 1021 * } 1022 * r = virtio_enqueue_reserve(sc, vq, slot, 1023 * dmamap_payload[slot]->dm_nsegs+1); 1024 * // ^ +1 for command 1025 * if (r) { // currently 0 or EAGAIN 1026 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 1027 * return r; // do not call abort() 1028 * } 1029 * <setup and prepare commands> 1030 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 1031 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 1032 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 1033 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 1034 * virtio_enqueue_commit(sc, vq, slot, true); 1035 */ 1036 1037 /* 1038 * enqueue_prep: allocate a slot number 1039 */ 1040 int 1041 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 1042 { 1043 struct vq_entry *qe1; 1044 1045 KASSERT(slotp != NULL); 1046 1047 qe1 = vq_alloc_entry(vq); 1048 if (qe1 == NULL) 1049 return EAGAIN; 1050 /* next slot is not allocated yet */ 1051 qe1->qe_next = -1; 1052 *slotp = qe1->qe_index; 1053 1054 return 0; 1055 } 1056 1057 /* 1058 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 1059 */ 1060 int 1061 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 1062 int slot, int nsegs) 1063 { 1064 int indirect; 1065 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1066 1067 KASSERT(qe1->qe_next == -1); 1068 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 1069 1070 if ((vq->vq_indirect != NULL) && 1071 (nsegs >= MINSEG_INDIRECT) && 1072 (nsegs <= vq->vq_maxnsegs)) 1073 indirect = 1; 1074 else 1075 indirect = 0; 1076 qe1->qe_indirect = indirect; 1077 1078 if (indirect) { 1079 struct vring_desc *vd; 1080 int i; 1081 1082 vd = &vq->vq_desc[qe1->qe_index]; 1083 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 1084 + vq->vq_indirectoffset; 1085 vd->addr += sizeof(struct vring_desc) 1086 * vq->vq_maxnsegs * qe1->qe_index; 1087 vd->len = sizeof(struct vring_desc) * nsegs; 1088 vd->flags = VRING_DESC_F_INDIRECT; 1089 1090 vd = vq->vq_indirect; 1091 vd += vq->vq_maxnsegs * qe1->qe_index; 1092 qe1->qe_desc_base = vd; 1093 1094 for (i = 0; i < nsegs-1; i++) { 1095 vd[i].flags = VRING_DESC_F_NEXT; 1096 } 1097 vd[i].flags = 0; 1098 qe1->qe_next = 0; 1099 1100 return 0; 1101 } else { 1102 struct vring_desc *vd; 1103 struct vq_entry *qe; 1104 int i, s; 1105 1106 vd = &vq->vq_desc[0]; 1107 qe1->qe_desc_base = vd; 1108 qe1->qe_next = qe1->qe_index; 1109 s = slot; 1110 for (i = 0; i < nsegs - 1; i++) { 1111 qe = vq_alloc_entry(vq); 1112 if (qe == NULL) { 1113 vd[s].flags = 0; 1114 virtio_enqueue_abort(sc, vq, slot); 1115 return EAGAIN; 1116 } 1117 vd[s].flags = VRING_DESC_F_NEXT; 1118 vd[s].next = qe->qe_index; 1119 s = qe->qe_index; 1120 } 1121 vd[s].flags = 0; 1122 1123 return 0; 1124 } 1125 } 1126 1127 /* 1128 * enqueue: enqueue a single dmamap. 1129 */ 1130 int 1131 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1132 bus_dmamap_t dmamap, bool write) 1133 { 1134 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1135 struct vring_desc *vd = qe1->qe_desc_base; 1136 int i; 1137 int s = qe1->qe_next; 1138 1139 KASSERT(s >= 0); 1140 KASSERT(dmamap->dm_nsegs > 0); 1141 1142 for (i = 0; i < dmamap->dm_nsegs; i++) { 1143 vd[s].addr = dmamap->dm_segs[i].ds_addr; 1144 vd[s].len = dmamap->dm_segs[i].ds_len; 1145 if (!write) 1146 vd[s].flags |= VRING_DESC_F_WRITE; 1147 s = vd[s].next; 1148 } 1149 qe1->qe_next = s; 1150 1151 return 0; 1152 } 1153 1154 int 1155 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1156 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 1157 bool write) 1158 { 1159 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1160 struct vring_desc *vd = qe1->qe_desc_base; 1161 int s = qe1->qe_next; 1162 1163 KASSERT(s >= 0); 1164 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 1165 KASSERT((dmamap->dm_segs[0].ds_len > start) && 1166 (dmamap->dm_segs[0].ds_len >= start + len)); 1167 1168 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 1169 vd[s].len = len; 1170 if (!write) 1171 vd[s].flags |= VRING_DESC_F_WRITE; 1172 qe1->qe_next = vd[s].next; 1173 1174 return 0; 1175 } 1176 1177 /* 1178 * enqueue_commit: add it to the aring. 1179 */ 1180 int 1181 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1182 bool notifynow) 1183 { 1184 struct vq_entry *qe1; 1185 1186 if (slot < 0) { 1187 mutex_enter(&vq->vq_aring_lock); 1188 goto notify; 1189 } 1190 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 1191 qe1 = &vq->vq_entries[slot]; 1192 if (qe1->qe_indirect) 1193 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 1194 mutex_enter(&vq->vq_aring_lock); 1195 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 1196 1197 notify: 1198 if (notifynow) { 1199 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 1200 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 1201 membar_producer(); 1202 vq->vq_avail->idx = vq->vq_avail_idx; 1203 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 1204 membar_producer(); 1205 vq->vq_queued++; 1206 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 1207 membar_consumer(); 1208 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 1209 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 1210 VIRTIO_CONFIG_QUEUE_NOTIFY, 1211 vq->vq_index); 1212 } 1213 mutex_exit(&vq->vq_aring_lock); 1214 1215 return 0; 1216 } 1217 1218 /* 1219 * enqueue_abort: rollback. 1220 */ 1221 int 1222 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1223 { 1224 struct vq_entry *qe = &vq->vq_entries[slot]; 1225 struct vring_desc *vd; 1226 int s; 1227 1228 if (qe->qe_next < 0) { 1229 vq_free_entry(vq, qe); 1230 return 0; 1231 } 1232 1233 s = slot; 1234 vd = &vq->vq_desc[0]; 1235 while (vd[s].flags & VRING_DESC_F_NEXT) { 1236 s = vd[s].next; 1237 vq_free_entry(vq, qe); 1238 qe = &vq->vq_entries[s]; 1239 } 1240 vq_free_entry(vq, qe); 1241 return 0; 1242 } 1243 1244 /* 1245 * Dequeue a request. 1246 */ 1247 /* 1248 * dequeue: dequeue a request from uring; dmamap_sync for uring is 1249 * already done in the interrupt handler. 1250 */ 1251 int 1252 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 1253 int *slotp, int *lenp) 1254 { 1255 uint16_t slot, usedidx; 1256 struct vq_entry *qe; 1257 1258 if (vq->vq_used_idx == vq->vq_used->idx) 1259 return ENOENT; 1260 mutex_enter(&vq->vq_uring_lock); 1261 usedidx = vq->vq_used_idx++; 1262 mutex_exit(&vq->vq_uring_lock); 1263 usedidx %= vq->vq_num; 1264 slot = vq->vq_used->ring[usedidx].id; 1265 qe = &vq->vq_entries[slot]; 1266 1267 if (qe->qe_indirect) 1268 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 1269 1270 if (slotp) 1271 *slotp = slot; 1272 if (lenp) 1273 *lenp = vq->vq_used->ring[usedidx].len; 1274 1275 return 0; 1276 } 1277 1278 /* 1279 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1280 * if you forget to call this the slot will be leaked. 1281 */ 1282 int 1283 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1284 { 1285 struct vq_entry *qe = &vq->vq_entries[slot]; 1286 struct vring_desc *vd = &vq->vq_desc[0]; 1287 int s = slot; 1288 1289 while (vd[s].flags & VRING_DESC_F_NEXT) { 1290 s = vd[s].next; 1291 vq_free_entry(vq, qe); 1292 qe = &vq->vq_entries[s]; 1293 } 1294 vq_free_entry(vq, qe); 1295 1296 return 0; 1297 } 1298 1299 MODULE(MODULE_CLASS_DRIVER, virtio, "pci"); 1300 1301 #ifdef _MODULE 1302 #include "ioconf.c" 1303 #endif 1304 1305 static int 1306 virtio_modcmd(modcmd_t cmd, void *opaque) 1307 { 1308 int error = 0; 1309 1310 #ifdef _MODULE 1311 switch (cmd) { 1312 case MODULE_CMD_INIT: 1313 error = config_init_component(cfdriver_ioconf_virtio, 1314 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1315 break; 1316 case MODULE_CMD_FINI: 1317 error = config_fini_component(cfdriver_ioconf_virtio, 1318 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1319 break; 1320 default: 1321 error = ENOTTY; 1322 break; 1323 } 1324 #endif 1325 1326 return error; 1327 } 1328
Indexes created Thu Oct 02 14:10:14 GMT 2025