virtio.c revision 1.5
1 /* $NetBSD: virtio.c,v 1.5 2014/03/29 19:28:25 christos Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.5 2014/03/29 19:28:25 christos Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/atomic.h> 35 #include <sys/bus.h> 36 #include <sys/device.h> 37 #include <sys/kmem.h> 38 39 #include <dev/pci/pcidevs.h> 40 #include <dev/pci/pcireg.h> 41 #include <dev/pci/pcivar.h> 42 43 #include <dev/pci/virtioreg.h> 44 #include <dev/pci/virtiovar.h> 45 46 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 47 48 static int virtio_match(device_t, cfdata_t, void *); 49 static void virtio_attach(device_t, device_t, void *); 50 static int virtio_detach(device_t, int); 51 static int virtio_intr(void *arg); 52 static void virtio_init_vq(struct virtio_softc *, 53 struct virtqueue *, const bool); 54 55 CFATTACH_DECL3_NEW(virtio, sizeof(struct virtio_softc), 56 virtio_match, virtio_attach, virtio_detach, NULL, NULL, NULL, 57 DVF_DETACH_SHUTDOWN); 58 59 static void 60 virtio_set_status(struct virtio_softc *sc, int status) 61 { 62 int old = 0; 63 64 if (status != 0) 65 old = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 66 VIRTIO_CONFIG_DEVICE_STATUS); 67 bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS, 68 status|old); 69 } 70 71 #define virtio_device_reset(sc) virtio_set_status((sc), 0) 72 73 static int 74 virtio_match(device_t parent, cfdata_t match, void *aux) 75 { 76 struct pci_attach_args *pa; 77 78 pa = (struct pci_attach_args *)aux; 79 switch (PCI_VENDOR(pa->pa_id)) { 80 case PCI_VENDOR_QUMRANET: 81 if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 82 PCI_PRODUCT(pa->pa_id)) && 83 (PCI_PRODUCT(pa->pa_id) <= 84 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) 85 return 1; 86 break; 87 } 88 89 return 0; 90 } 91 92 static const char *virtio_device_name[] = { 93 "Unknown (0)", /* 0 */ 94 "Network", /* 1 */ 95 "Block", /* 2 */ 96 "Console", /* 3 */ 97 "Entropy", /* 4 */ 98 "Memory Balloon", /* 5 */ 99 "Unknown (6)", /* 6 */ 100 "Unknown (7)", /* 7 */ 101 "Unknown (8)", /* 8 */ 102 "9P Transport" /* 9 */ 103 }; 104 #define NDEVNAMES (sizeof(virtio_device_name)/sizeof(char*)) 105 106 static void 107 virtio_attach(device_t parent, device_t self, void *aux) 108 { 109 struct virtio_softc *sc = device_private(self); 110 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 111 pci_chipset_tag_t pc = pa->pa_pc; 112 pcitag_t tag = pa->pa_tag; 113 int revision; 114 pcireg_t id; 115 char const *intrstr; 116 pci_intr_handle_t ih; 117 char intrbuf[PCI_INTRSTR_LEN]; 118 119 revision = PCI_REVISION(pa->pa_class); 120 if (revision != 0) { 121 aprint_normal(": unknown revision 0x%02x; giving up\n", 122 revision); 123 return; 124 } 125 aprint_normal("\n"); 126 aprint_naive("\n"); 127 128 /* subsystem ID shows what I am */ 129 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 130 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 131 (PCI_PRODUCT(id) < NDEVNAMES? 132 virtio_device_name[PCI_PRODUCT(id)] : "Unknown"), 133 revision); 134 135 sc->sc_dev = self; 136 sc->sc_pc = pc; 137 sc->sc_tag = tag; 138 sc->sc_iot = pa->pa_iot; 139 sc->sc_dmat = pa->pa_dmat; 140 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 141 142 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 143 &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) { 144 aprint_error_dev(self, "can't map i/o space\n"); 145 return; 146 } 147 148 virtio_device_reset(sc); 149 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 150 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 151 152 /* XXX: use softc as aux... */ 153 sc->sc_childdevid = PCI_PRODUCT(id); 154 sc->sc_child = NULL; 155 config_found(self, sc, NULL); 156 if (sc->sc_child == NULL) { 157 aprint_error_dev(self, 158 "no matching child driver; not configured\n"); 159 return; 160 } 161 if (sc->sc_child == (void*)1) { /* this shows error */ 162 aprint_error_dev(self, 163 "virtio configuration failed\n"); 164 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 165 return; 166 } 167 168 if (pci_intr_map(pa, &ih)) { 169 aprint_error_dev(self, "couldn't map interrupt\n"); 170 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 171 return; 172 } 173 intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf)); 174 sc->sc_ih = pci_intr_establish(pc, ih, sc->sc_ipl, virtio_intr, sc); 175 if (sc->sc_ih == NULL) { 176 aprint_error_dev(self, "couldn't establish interrupt"); 177 if (intrstr != NULL) 178 aprint_error(" at %s", intrstr); 179 aprint_error("\n"); 180 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 181 return; 182 } 183 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 184 185 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 186 187 return; 188 } 189 190 static int 191 virtio_detach(device_t self, int flags) 192 { 193 struct virtio_softc *sc = device_private(self); 194 int r; 195 196 if (sc->sc_child != 0 && sc->sc_child != (void*)1) { 197 r = config_detach(sc->sc_child, flags); 198 if (r) 199 return r; 200 } 201 KASSERT(sc->sc_child == 0 || sc->sc_child == (void*)1); 202 KASSERT(sc->sc_vqs == 0); 203 if (sc->sc_ih != NULL) { 204 pci_intr_disestablish(sc->sc_pc, sc->sc_ih); 205 sc->sc_ih = NULL; 206 } 207 if (sc->sc_iosize) 208 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 209 sc->sc_iosize = 0; 210 211 return 0; 212 } 213 214 /* 215 * Reset the device. 216 */ 217 /* 218 * To reset the device to a known state, do following: 219 * virtio_reset(sc); // this will stop the device activity 220 * <dequeue finished requests>; // virtio_dequeue() still can be called 221 * <revoke pending requests in the vqs if any>; 222 * virtio_reinit_begin(sc); // dequeue prohibitted 223 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 224 * <some other initialization>; 225 * virtio_reinit_end(sc); // device activated; enqueue allowed 226 * Once attached, feature negotiation can only be allowed after virtio_reset. 227 */ 228 void 229 virtio_reset(struct virtio_softc *sc) 230 { 231 virtio_device_reset(sc); 232 } 233 234 void 235 virtio_reinit_start(struct virtio_softc *sc) 236 { 237 int i; 238 239 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 240 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 241 for (i = 0; i < sc->sc_nvqs; i++) { 242 int n; 243 struct virtqueue *vq = &sc->sc_vqs[i]; 244 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 245 VIRTIO_CONFIG_QUEUE_SELECT, 246 vq->vq_index); 247 n = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 248 VIRTIO_CONFIG_QUEUE_SIZE); 249 if (n == 0) /* vq disappeared */ 250 continue; 251 if (n != vq->vq_num) { 252 panic("%s: virtqueue size changed, vq index %d\n", 253 device_xname(sc->sc_dev), 254 vq->vq_index); 255 } 256 virtio_init_vq(sc, vq, true); 257 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 258 VIRTIO_CONFIG_QUEUE_ADDRESS, 259 (vq->vq_dmamap->dm_segs[0].ds_addr 260 / VIRTIO_PAGE_SIZE)); 261 } 262 } 263 264 void 265 virtio_reinit_end(struct virtio_softc *sc) 266 { 267 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 268 } 269 270 /* 271 * Feature negotiation. 272 */ 273 uint32_t 274 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 275 { 276 uint32_t r; 277 278 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 279 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 280 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 281 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 282 VIRTIO_CONFIG_DEVICE_FEATURES); 283 r &= guest_features; 284 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 285 VIRTIO_CONFIG_GUEST_FEATURES, r); 286 sc->sc_features = r; 287 if (r & VIRTIO_F_RING_INDIRECT_DESC) 288 sc->sc_indirect = true; 289 else 290 sc->sc_indirect = false; 291 292 return r; 293 } 294 295 /* 296 * Device configuration registers. 297 */ 298 uint8_t 299 virtio_read_device_config_1(struct virtio_softc *sc, int index) 300 { 301 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 302 sc->sc_config_offset + index); 303 } 304 305 uint16_t 306 virtio_read_device_config_2(struct virtio_softc *sc, int index) 307 { 308 return bus_space_read_2(sc->sc_iot, sc->sc_ioh, 309 sc->sc_config_offset + index); 310 } 311 312 uint32_t 313 virtio_read_device_config_4(struct virtio_softc *sc, int index) 314 { 315 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 316 sc->sc_config_offset + index); 317 } 318 319 uint64_t 320 virtio_read_device_config_8(struct virtio_softc *sc, int index) 321 { 322 uint64_t r; 323 324 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 325 sc->sc_config_offset + index + sizeof(uint32_t)); 326 r <<= 32; 327 r += bus_space_read_4(sc->sc_iot, sc->sc_ioh, 328 sc->sc_config_offset + index); 329 return r; 330 } 331 332 void 333 virtio_write_device_config_1(struct virtio_softc *sc, 334 int index, uint8_t value) 335 { 336 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 337 sc->sc_config_offset + index, value); 338 } 339 340 void 341 virtio_write_device_config_2(struct virtio_softc *sc, 342 int index, uint16_t value) 343 { 344 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 345 sc->sc_config_offset + index, value); 346 } 347 348 void 349 virtio_write_device_config_4(struct virtio_softc *sc, 350 int index, uint32_t value) 351 { 352 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 353 sc->sc_config_offset + index, value); 354 } 355 356 void 357 virtio_write_device_config_8(struct virtio_softc *sc, 358 int index, uint64_t value) 359 { 360 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 361 sc->sc_config_offset + index, 362 value & 0xffffffff); 363 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 364 sc->sc_config_offset + index + sizeof(uint32_t), 365 value >> 32); 366 } 367 368 /* 369 * Interrupt handler. 370 */ 371 static int 372 virtio_intr(void *arg) 373 { 374 struct virtio_softc *sc = arg; 375 int isr, r = 0; 376 377 /* check and ack the interrupt */ 378 isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 379 VIRTIO_CONFIG_ISR_STATUS); 380 if (isr == 0) 381 return 0; 382 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 383 (sc->sc_config_change != NULL)) 384 r = (sc->sc_config_change)(sc); 385 if (sc->sc_intrhand != NULL) 386 r |= (sc->sc_intrhand)(sc); 387 388 return r; 389 } 390 391 /* 392 * dmamap sync operations for a virtqueue. 393 */ 394 static inline void 395 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 396 { 397 /* availoffset == sizeof(vring_desc)*vq_num */ 398 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 399 ops); 400 } 401 402 static inline void 403 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 404 { 405 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 406 vq->vq_availoffset, 407 offsetof(struct vring_avail, ring) 408 + vq->vq_num * sizeof(uint16_t), 409 ops); 410 } 411 412 static inline void 413 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 414 { 415 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 416 vq->vq_usedoffset, 417 offsetof(struct vring_used, ring) 418 + vq->vq_num * sizeof(struct vring_used_elem), 419 ops); 420 } 421 422 static inline void 423 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 424 int ops) 425 { 426 int offset = vq->vq_indirectoffset 427 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 428 429 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 430 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 431 ops); 432 } 433 434 /* 435 * Can be used as sc_intrhand. 436 */ 437 /* 438 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 439 * and calls (*vq_done)() if some entries are consumed. 440 */ 441 int 442 virtio_vq_intr(struct virtio_softc *sc) 443 { 444 struct virtqueue *vq; 445 int i, r = 0; 446 447 for (i = 0; i < sc->sc_nvqs; i++) { 448 vq = &sc->sc_vqs[i]; 449 if (vq->vq_queued) { 450 vq->vq_queued = 0; 451 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 452 } 453 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 454 membar_consumer(); 455 if (vq->vq_used_idx != vq->vq_used->idx) { 456 if (vq->vq_done) 457 r |= (vq->vq_done)(vq); 458 } 459 } 460 461 462 return r; 463 } 464 465 /* 466 * Start/stop vq interrupt. No guarantee. 467 */ 468 void 469 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 470 { 471 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 472 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 473 vq->vq_queued++; 474 } 475 476 void 477 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 478 { 479 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 480 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 481 vq->vq_queued++; 482 } 483 484 /* 485 * Initialize vq structure. 486 */ 487 static void 488 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, const bool reinit) 489 { 490 int i, j; 491 int vq_size = vq->vq_num; 492 493 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 494 495 /* build the indirect descriptor chain */ 496 if (vq->vq_indirect != NULL) { 497 struct vring_desc *vd; 498 499 for (i = 0; i < vq_size; i++) { 500 vd = vq->vq_indirect; 501 vd += vq->vq_maxnsegs * i; 502 for (j = 0; j < vq->vq_maxnsegs-1; j++) 503 vd[j].next = j + 1; 504 } 505 } 506 507 /* free slot management */ 508 SIMPLEQ_INIT(&vq->vq_freelist); 509 for (i = 0; i < vq_size; i++) { 510 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 511 &vq->vq_entries[i], qe_list); 512 vq->vq_entries[i].qe_index = i; 513 } 514 if (!reinit) 515 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 516 517 /* enqueue/dequeue status */ 518 vq->vq_avail_idx = 0; 519 vq->vq_used_idx = 0; 520 vq->vq_queued = 0; 521 if (!reinit) { 522 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 523 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 524 } 525 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 526 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 527 vq->vq_queued++; 528 } 529 530 /* 531 * Allocate/free a vq. 532 */ 533 int 534 virtio_alloc_vq(struct virtio_softc *sc, 535 struct virtqueue *vq, int index, int maxsegsize, int maxnsegs, 536 const char *name) 537 { 538 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 539 int rsegs, r; 540 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 541 ~(VIRTIO_PAGE_SIZE-1)) 542 543 memset(vq, 0, sizeof(*vq)); 544 545 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 546 VIRTIO_CONFIG_QUEUE_SELECT, index); 547 vq_size = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 548 VIRTIO_CONFIG_QUEUE_SIZE); 549 if (vq_size == 0) { 550 aprint_error_dev(sc->sc_dev, 551 "virtqueue not exist, index %d for %s\n", 552 index, name); 553 goto err; 554 } 555 /* allocsize1: descriptor table + avail ring + pad */ 556 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 557 + sizeof(uint16_t)*(2+vq_size)); 558 /* allocsize2: used ring + pad */ 559 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 560 + sizeof(struct vring_used_elem)*vq_size); 561 /* allocsize3: indirect table */ 562 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 563 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 564 else 565 allocsize3 = 0; 566 allocsize = allocsize1 + allocsize2 + allocsize3; 567 568 /* alloc and map the memory */ 569 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 570 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 571 if (r != 0) { 572 aprint_error_dev(sc->sc_dev, 573 "virtqueue %d for %s allocation failed, " 574 "error code %d\n", index, name, r); 575 goto err; 576 } 577 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 578 &vq->vq_vaddr, BUS_DMA_NOWAIT); 579 if (r != 0) { 580 aprint_error_dev(sc->sc_dev, 581 "virtqueue %d for %s map failed, " 582 "error code %d\n", index, name, r); 583 goto err; 584 } 585 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 586 BUS_DMA_NOWAIT, &vq->vq_dmamap); 587 if (r != 0) { 588 aprint_error_dev(sc->sc_dev, 589 "virtqueue %d for %s dmamap creation failed, " 590 "error code %d\n", index, name, r); 591 goto err; 592 } 593 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 594 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 595 if (r != 0) { 596 aprint_error_dev(sc->sc_dev, 597 "virtqueue %d for %s dmamap load failed, " 598 "error code %d\n", index, name, r); 599 goto err; 600 } 601 602 /* set the vq address */ 603 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 604 VIRTIO_CONFIG_QUEUE_ADDRESS, 605 (vq->vq_dmamap->dm_segs[0].ds_addr 606 / VIRTIO_PAGE_SIZE)); 607 608 /* remember addresses and offsets for later use */ 609 vq->vq_owner = sc; 610 vq->vq_num = vq_size; 611 vq->vq_index = index; 612 vq->vq_desc = vq->vq_vaddr; 613 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 614 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 615 vq->vq_usedoffset = allocsize1; 616 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 617 if (allocsize3 > 0) { 618 vq->vq_indirectoffset = allocsize1 + allocsize2; 619 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 620 + vq->vq_indirectoffset); 621 } 622 vq->vq_bytesize = allocsize; 623 vq->vq_maxsegsize = maxsegsize; 624 vq->vq_maxnsegs = maxnsegs; 625 626 /* free slot management */ 627 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 628 KM_NOSLEEP); 629 if (vq->vq_entries == NULL) { 630 r = ENOMEM; 631 goto err; 632 } 633 634 virtio_init_vq(sc, vq, false); 635 636 aprint_verbose_dev(sc->sc_dev, 637 "allocated %u byte for virtqueue %d for %s, " 638 "size %d\n", allocsize, index, name, vq_size); 639 if (allocsize3 > 0) 640 aprint_verbose_dev(sc->sc_dev, 641 "using %d byte (%d entries) " 642 "indirect descriptors\n", 643 allocsize3, maxnsegs * vq_size); 644 return 0; 645 646 err: 647 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 648 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 649 if (vq->vq_dmamap) 650 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 651 if (vq->vq_vaddr) 652 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 653 if (vq->vq_segs[0].ds_addr) 654 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 655 memset(vq, 0, sizeof(*vq)); 656 657 return -1; 658 } 659 660 int 661 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 662 { 663 struct vq_entry *qe; 664 int i = 0; 665 666 /* device must be already deactivated */ 667 /* confirm the vq is empty */ 668 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 669 i++; 670 } 671 if (i != vq->vq_num) { 672 printf("%s: freeing non-empty vq, index %d\n", 673 device_xname(sc->sc_dev), vq->vq_index); 674 return EBUSY; 675 } 676 677 /* tell device that there's no virtqueue any longer */ 678 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 679 VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index); 680 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 681 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 682 683 kmem_free(vq->vq_entries, vq->vq_bytesize); 684 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 685 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 686 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 687 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 688 mutex_destroy(&vq->vq_freelist_lock); 689 mutex_destroy(&vq->vq_uring_lock); 690 mutex_destroy(&vq->vq_aring_lock); 691 memset(vq, 0, sizeof(*vq)); 692 693 return 0; 694 } 695 696 /* 697 * Free descriptor management. 698 */ 699 static struct vq_entry * 700 vq_alloc_entry(struct virtqueue *vq) 701 { 702 struct vq_entry *qe; 703 704 mutex_enter(&vq->vq_freelist_lock); 705 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 706 mutex_exit(&vq->vq_freelist_lock); 707 return NULL; 708 } 709 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 710 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 711 mutex_exit(&vq->vq_freelist_lock); 712 713 return qe; 714 } 715 716 static void 717 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 718 { 719 mutex_enter(&vq->vq_freelist_lock); 720 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 721 mutex_exit(&vq->vq_freelist_lock); 722 723 return; 724 } 725 726 /* 727 * Enqueue several dmamaps as a single request. 728 */ 729 /* 730 * Typical usage: 731 * <queue size> number of followings are stored in arrays 732 * - command blocks (in dmamem) should be pre-allocated and mapped 733 * - dmamaps for command blocks should be pre-allocated and loaded 734 * - dmamaps for payload should be pre-allocated 735 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 736 * if (r) // currently 0 or EAGAIN 737 * return r; 738 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 739 * if (r) { 740 * virtio_enqueue_abort(sc, vq, slot); 741 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 742 * return r; 743 * } 744 * r = virtio_enqueue_reserve(sc, vq, slot, 745 * dmamap_payload[slot]->dm_nsegs+1); 746 * // ^ +1 for command 747 * if (r) { // currently 0 or EAGAIN 748 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 749 * return r; // do not call abort() 750 * } 751 * <setup and prepare commands> 752 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 753 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 754 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 755 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 756 * virtio_enqueue_commit(sc, vq, slot, true); 757 */ 758 759 /* 760 * enqueue_prep: allocate a slot number 761 */ 762 int 763 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 764 { 765 struct vq_entry *qe1; 766 767 KASSERT(slotp != NULL); 768 769 qe1 = vq_alloc_entry(vq); 770 if (qe1 == NULL) 771 return EAGAIN; 772 /* next slot is not allocated yet */ 773 qe1->qe_next = -1; 774 *slotp = qe1->qe_index; 775 776 return 0; 777 } 778 779 /* 780 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 781 */ 782 int 783 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 784 int slot, int nsegs) 785 { 786 int indirect; 787 struct vq_entry *qe1 = &vq->vq_entries[slot]; 788 789 KASSERT(qe1->qe_next == -1); 790 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 791 792 if ((vq->vq_indirect != NULL) && 793 (nsegs >= MINSEG_INDIRECT) && 794 (nsegs <= vq->vq_maxnsegs)) 795 indirect = 1; 796 else 797 indirect = 0; 798 qe1->qe_indirect = indirect; 799 800 if (indirect) { 801 struct vring_desc *vd; 802 int i; 803 804 vd = &vq->vq_desc[qe1->qe_index]; 805 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 806 + vq->vq_indirectoffset; 807 vd->addr += sizeof(struct vring_desc) 808 * vq->vq_maxnsegs * qe1->qe_index; 809 vd->len = sizeof(struct vring_desc) * nsegs; 810 vd->flags = VRING_DESC_F_INDIRECT; 811 812 vd = vq->vq_indirect; 813 vd += vq->vq_maxnsegs * qe1->qe_index; 814 qe1->qe_desc_base = vd; 815 816 for (i = 0; i < nsegs-1; i++) { 817 vd[i].flags = VRING_DESC_F_NEXT; 818 } 819 vd[i].flags = 0; 820 qe1->qe_next = 0; 821 822 return 0; 823 } else { 824 struct vring_desc *vd; 825 struct vq_entry *qe; 826 int i, s; 827 828 vd = &vq->vq_desc[0]; 829 qe1->qe_desc_base = vd; 830 qe1->qe_next = qe1->qe_index; 831 s = slot; 832 for (i = 0; i < nsegs - 1; i++) { 833 qe = vq_alloc_entry(vq); 834 if (qe == NULL) { 835 vd[s].flags = 0; 836 virtio_enqueue_abort(sc, vq, slot); 837 return EAGAIN; 838 } 839 vd[s].flags = VRING_DESC_F_NEXT; 840 vd[s].next = qe->qe_index; 841 s = qe->qe_index; 842 } 843 vd[s].flags = 0; 844 845 return 0; 846 } 847 } 848 849 /* 850 * enqueue: enqueue a single dmamap. 851 */ 852 int 853 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 854 bus_dmamap_t dmamap, bool write) 855 { 856 struct vq_entry *qe1 = &vq->vq_entries[slot]; 857 struct vring_desc *vd = qe1->qe_desc_base; 858 int i; 859 int s = qe1->qe_next; 860 861 KASSERT(s >= 0); 862 KASSERT(dmamap->dm_nsegs > 0); 863 864 for (i = 0; i < dmamap->dm_nsegs; i++) { 865 vd[s].addr = dmamap->dm_segs[i].ds_addr; 866 vd[s].len = dmamap->dm_segs[i].ds_len; 867 if (!write) 868 vd[s].flags |= VRING_DESC_F_WRITE; 869 s = vd[s].next; 870 } 871 qe1->qe_next = s; 872 873 return 0; 874 } 875 876 int 877 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 878 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 879 bool write) 880 { 881 struct vq_entry *qe1 = &vq->vq_entries[slot]; 882 struct vring_desc *vd = qe1->qe_desc_base; 883 int s = qe1->qe_next; 884 885 KASSERT(s >= 0); 886 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 887 KASSERT((dmamap->dm_segs[0].ds_len > start) && 888 (dmamap->dm_segs[0].ds_len >= start + len)); 889 890 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 891 vd[s].len = len; 892 if (!write) 893 vd[s].flags |= VRING_DESC_F_WRITE; 894 qe1->qe_next = vd[s].next; 895 896 return 0; 897 } 898 899 /* 900 * enqueue_commit: add it to the aring. 901 */ 902 int 903 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 904 bool notifynow) 905 { 906 struct vq_entry *qe1; 907 908 if (slot < 0) { 909 mutex_enter(&vq->vq_aring_lock); 910 goto notify; 911 } 912 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 913 qe1 = &vq->vq_entries[slot]; 914 if (qe1->qe_indirect) 915 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 916 mutex_enter(&vq->vq_aring_lock); 917 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 918 919 notify: 920 if (notifynow) { 921 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 922 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 923 membar_producer(); 924 vq->vq_avail->idx = vq->vq_avail_idx; 925 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 926 membar_producer(); 927 vq->vq_queued++; 928 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 929 membar_consumer(); 930 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 931 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 932 VIRTIO_CONFIG_QUEUE_NOTIFY, 933 vq->vq_index); 934 } 935 mutex_exit(&vq->vq_aring_lock); 936 937 return 0; 938 } 939 940 /* 941 * enqueue_abort: rollback. 942 */ 943 int 944 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 945 { 946 struct vq_entry *qe = &vq->vq_entries[slot]; 947 struct vring_desc *vd; 948 int s; 949 950 if (qe->qe_next < 0) { 951 vq_free_entry(vq, qe); 952 return 0; 953 } 954 955 s = slot; 956 vd = &vq->vq_desc[0]; 957 while (vd[s].flags & VRING_DESC_F_NEXT) { 958 s = vd[s].next; 959 vq_free_entry(vq, qe); 960 qe = &vq->vq_entries[s]; 961 } 962 vq_free_entry(vq, qe); 963 return 0; 964 } 965 966 /* 967 * Dequeue a request. 968 */ 969 /* 970 * dequeue: dequeue a request from uring; dmamap_sync for uring is 971 * already done in the interrupt handler. 972 */ 973 int 974 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 975 int *slotp, int *lenp) 976 { 977 uint16_t slot, usedidx; 978 struct vq_entry *qe; 979 980 if (vq->vq_used_idx == vq->vq_used->idx) 981 return ENOENT; 982 mutex_enter(&vq->vq_uring_lock); 983 usedidx = vq->vq_used_idx++; 984 mutex_exit(&vq->vq_uring_lock); 985 usedidx %= vq->vq_num; 986 slot = vq->vq_used->ring[usedidx].id; 987 qe = &vq->vq_entries[slot]; 988 989 if (qe->qe_indirect) 990 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 991 992 if (slotp) 993 *slotp = slot; 994 if (lenp) 995 *lenp = vq->vq_used->ring[usedidx].len; 996 997 return 0; 998 } 999 1000 /* 1001 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1002 * if you forget to call this the slot will be leaked. 1003 */ 1004 int 1005 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1006 { 1007 struct vq_entry *qe = &vq->vq_entries[slot]; 1008 struct vring_desc *vd = &vq->vq_desc[0]; 1009 int s = slot; 1010 1011 while (vd[s].flags & VRING_DESC_F_NEXT) { 1012 s = vd[s].next; 1013 vq_free_entry(vq, qe); 1014 qe = &vq->vq_entries[s]; 1015 } 1016 vq_free_entry(vq, qe); 1017 1018 return 0; 1019 } 1020
Indexes created Thu Oct 02 14:10:14 GMT 2025