virtio.c revision 1.3.8.1
1 /* $NetBSD: virtio.c,v 1.3.8.1 2013/09/07 16:01:03 bouyer Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.3.8.1 2013/09/07 16:01:03 bouyer Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/atomic.h> 35 #include <sys/bus.h> 36 #include <sys/device.h> 37 #include <sys/kmem.h> 38 39 #include <dev/pci/pcidevs.h> 40 #include <dev/pci/pcireg.h> 41 #include <dev/pci/pcivar.h> 42 43 #include <dev/pci/virtioreg.h> 44 #include <dev/pci/virtiovar.h> 45 46 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 47 48 static int virtio_match(device_t, cfdata_t, void *); 49 static void virtio_attach(device_t, device_t, void *); 50 static int virtio_detach(device_t, int); 51 static int virtio_intr(void *arg); 52 static void virtio_init_vq(struct virtio_softc *, 53 struct virtqueue *, const bool); 54 55 CFATTACH_DECL3_NEW(virtio, sizeof(struct virtio_softc), 56 virtio_match, virtio_attach, virtio_detach, NULL, NULL, NULL, 57 DVF_DETACH_SHUTDOWN); 58 59 static void 60 virtio_set_status(struct virtio_softc *sc, int status) 61 { 62 int old = 0; 63 64 if (status != 0) 65 old = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 66 VIRTIO_CONFIG_DEVICE_STATUS); 67 bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS, 68 status|old); 69 } 70 71 #define virtio_device_reset(sc) virtio_set_status((sc), 0) 72 73 static int 74 virtio_match(device_t parent, cfdata_t match, void *aux) 75 { 76 struct pci_attach_args *pa; 77 78 pa = (struct pci_attach_args *)aux; 79 switch (PCI_VENDOR(pa->pa_id)) { 80 case PCI_VENDOR_QUMRANET: 81 if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 82 PCI_PRODUCT(pa->pa_id)) && 83 (PCI_PRODUCT(pa->pa_id) <= 84 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) 85 return 1; 86 break; 87 } 88 89 return 0; 90 } 91 92 static const char *virtio_device_name[] = { 93 "Unknown (0)", /* 0 */ 94 "Network", /* 1 */ 95 "Block", /* 2 */ 96 "Console", /* 3 */ 97 "Entropy", /* 4 */ 98 "Memory Balloon", /* 5 */ 99 "Unknown (6)", /* 6 */ 100 "Unknown (7)", /* 7 */ 101 "Unknown (8)", /* 8 */ 102 "9P Transport" /* 9 */ 103 }; 104 #define NDEVNAMES (sizeof(virtio_device_name)/sizeof(char*)) 105 106 static void 107 virtio_attach(device_t parent, device_t self, void *aux) 108 { 109 struct virtio_softc *sc = device_private(self); 110 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 111 pci_chipset_tag_t pc = pa->pa_pc; 112 pcitag_t tag = pa->pa_tag; 113 int revision; 114 pcireg_t id; 115 char const *intrstr; 116 pci_intr_handle_t ih; 117 118 revision = PCI_REVISION(pa->pa_class); 119 if (revision != 0) { 120 aprint_normal(": unknown revision 0x%02x; giving up\n", 121 revision); 122 return; 123 } 124 aprint_normal("\n"); 125 aprint_naive("\n"); 126 127 /* subsystem ID shows what I am */ 128 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 129 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 130 (PCI_PRODUCT(id) < NDEVNAMES? 131 virtio_device_name[PCI_PRODUCT(id)] : "Unknown"), 132 revision); 133 134 sc->sc_dev = self; 135 sc->sc_pc = pc; 136 sc->sc_tag = tag; 137 sc->sc_iot = pa->pa_iot; 138 sc->sc_dmat = pa->pa_dmat; 139 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 140 141 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 142 &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) { 143 aprint_error_dev(self, "can't map i/o space\n"); 144 return; 145 } 146 147 virtio_device_reset(sc); 148 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 149 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 150 151 /* XXX: use softc as aux... */ 152 sc->sc_childdevid = PCI_PRODUCT(id); 153 sc->sc_child = NULL; 154 config_found(self, sc, NULL); 155 if (sc->sc_child == NULL) { 156 aprint_error_dev(self, 157 "no matching child driver; not configured\n"); 158 return; 159 } 160 if (sc->sc_child == (void*)1) { /* this shows error */ 161 aprint_error_dev(self, 162 "virtio configuration failed\n"); 163 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 164 return; 165 } 166 167 if (pci_intr_map(pa, &ih)) { 168 aprint_error_dev(self, "couldn't map interrupt\n"); 169 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 170 return; 171 } 172 intrstr = pci_intr_string(pc, ih); 173 sc->sc_ih = pci_intr_establish(pc, ih, sc->sc_ipl, virtio_intr, sc); 174 if (sc->sc_ih == NULL) { 175 aprint_error_dev(self, "couldn't establish interrupt"); 176 if (intrstr != NULL) 177 aprint_error(" at %s", intrstr); 178 aprint_error("\n"); 179 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 180 return; 181 } 182 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 183 184 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 185 186 return; 187 } 188 189 static int 190 virtio_detach(device_t self, int flags) 191 { 192 struct virtio_softc *sc = device_private(self); 193 int r; 194 195 if (sc->sc_child != 0 && sc->sc_child != (void*)1) { 196 r = config_detach(sc->sc_child, flags); 197 if (r) 198 return r; 199 } 200 KASSERT(sc->sc_child == 0 || sc->sc_child == (void*)1); 201 KASSERT(sc->sc_vqs == 0); 202 if (sc->sc_ih != NULL) { 203 pci_intr_disestablish(sc->sc_pc, sc->sc_ih); 204 sc->sc_ih = NULL; 205 } 206 if (sc->sc_iosize) 207 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 208 sc->sc_iosize = 0; 209 210 return 0; 211 } 212 213 /* 214 * Reset the device. 215 */ 216 /* 217 * To reset the device to a known state, do following: 218 * virtio_reset(sc); // this will stop the device activity 219 * <dequeue finished requests>; // virtio_dequeue() still can be called 220 * <revoke pending requests in the vqs if any>; 221 * virtio_reinit_begin(sc); // dequeue prohibitted 222 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 223 * <some other initialization>; 224 * virtio_reinit_end(sc); // device activated; enqueue allowed 225 * Once attached, feature negotiation can only be allowed after virtio_reset. 226 */ 227 void 228 virtio_reset(struct virtio_softc *sc) 229 { 230 virtio_device_reset(sc); 231 } 232 233 void 234 virtio_reinit_start(struct virtio_softc *sc) 235 { 236 int i; 237 238 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 239 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 240 for (i = 0; i < sc->sc_nvqs; i++) { 241 int n; 242 struct virtqueue *vq = &sc->sc_vqs[i]; 243 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 244 VIRTIO_CONFIG_QUEUE_SELECT, 245 vq->vq_index); 246 n = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 247 VIRTIO_CONFIG_QUEUE_SIZE); 248 if (n == 0) /* vq disappeared */ 249 continue; 250 if (n != vq->vq_num) { 251 panic("%s: virtqueue size changed, vq index %d\n", 252 device_xname(sc->sc_dev), 253 vq->vq_index); 254 } 255 virtio_init_vq(sc, vq, true); 256 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 257 VIRTIO_CONFIG_QUEUE_ADDRESS, 258 (vq->vq_dmamap->dm_segs[0].ds_addr 259 / VIRTIO_PAGE_SIZE)); 260 } 261 } 262 263 void 264 virtio_reinit_end(struct virtio_softc *sc) 265 { 266 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 267 } 268 269 /* 270 * Feature negotiation. 271 */ 272 uint32_t 273 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 274 { 275 uint32_t r; 276 277 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 278 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 279 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 280 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 281 VIRTIO_CONFIG_DEVICE_FEATURES); 282 r &= guest_features; 283 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 284 VIRTIO_CONFIG_GUEST_FEATURES, r); 285 sc->sc_features = r; 286 if (r & VIRTIO_F_RING_INDIRECT_DESC) 287 sc->sc_indirect = true; 288 else 289 sc->sc_indirect = false; 290 291 return r; 292 } 293 294 /* 295 * Device configuration registers. 296 */ 297 uint8_t 298 virtio_read_device_config_1(struct virtio_softc *sc, int index) 299 { 300 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 301 sc->sc_config_offset + index); 302 } 303 304 uint16_t 305 virtio_read_device_config_2(struct virtio_softc *sc, int index) 306 { 307 return bus_space_read_2(sc->sc_iot, sc->sc_ioh, 308 sc->sc_config_offset + index); 309 } 310 311 uint32_t 312 virtio_read_device_config_4(struct virtio_softc *sc, int index) 313 { 314 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 315 sc->sc_config_offset + index); 316 } 317 318 uint64_t 319 virtio_read_device_config_8(struct virtio_softc *sc, int index) 320 { 321 uint64_t r; 322 323 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 324 sc->sc_config_offset + index + sizeof(uint32_t)); 325 r <<= 32; 326 r += bus_space_read_4(sc->sc_iot, sc->sc_ioh, 327 sc->sc_config_offset + index); 328 return r; 329 } 330 331 void 332 virtio_write_device_config_1(struct virtio_softc *sc, 333 int index, uint8_t value) 334 { 335 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 336 sc->sc_config_offset + index, value); 337 } 338 339 void 340 virtio_write_device_config_2(struct virtio_softc *sc, 341 int index, uint16_t value) 342 { 343 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 344 sc->sc_config_offset + index, value); 345 } 346 347 void 348 virtio_write_device_config_4(struct virtio_softc *sc, 349 int index, uint32_t value) 350 { 351 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 352 sc->sc_config_offset + index, value); 353 } 354 355 void 356 virtio_write_device_config_8(struct virtio_softc *sc, 357 int index, uint64_t value) 358 { 359 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 360 sc->sc_config_offset + index, 361 value & 0xffffffff); 362 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 363 sc->sc_config_offset + index + sizeof(uint32_t), 364 value >> 32); 365 } 366 367 /* 368 * Interrupt handler. 369 */ 370 static int 371 virtio_intr(void *arg) 372 { 373 struct virtio_softc *sc = arg; 374 int isr, r = 0; 375 376 /* check and ack the interrupt */ 377 isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 378 VIRTIO_CONFIG_ISR_STATUS); 379 if (isr == 0) 380 return 0; 381 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 382 (sc->sc_config_change != NULL)) 383 r = (sc->sc_config_change)(sc); 384 if (sc->sc_intrhand != NULL) 385 r |= (sc->sc_intrhand)(sc); 386 387 return r; 388 } 389 390 /* 391 * dmamap sync operations for a virtqueue. 392 */ 393 static inline void 394 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 395 { 396 /* availoffset == sizeof(vring_desc)*vq_num */ 397 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 398 ops); 399 } 400 401 static inline void 402 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 403 { 404 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 405 vq->vq_availoffset, 406 offsetof(struct vring_avail, ring) 407 + vq->vq_num * sizeof(uint16_t), 408 ops); 409 } 410 411 static inline void 412 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 413 { 414 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 415 vq->vq_usedoffset, 416 offsetof(struct vring_used, ring) 417 + vq->vq_num * sizeof(struct vring_used_elem), 418 ops); 419 } 420 421 static inline void 422 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 423 int ops) 424 { 425 int offset = vq->vq_indirectoffset 426 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 427 428 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 429 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 430 ops); 431 } 432 433 /* 434 * Can be used as sc_intrhand. 435 */ 436 /* 437 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 438 * and calls (*vq_done)() if some entries are consumed. 439 */ 440 int 441 virtio_vq_intr(struct virtio_softc *sc) 442 { 443 struct virtqueue *vq; 444 int i, r = 0; 445 446 for (i = 0; i < sc->sc_nvqs; i++) { 447 vq = &sc->sc_vqs[i]; 448 if (vq->vq_queued) { 449 vq->vq_queued = 0; 450 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 451 } 452 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 453 membar_consumer(); 454 if (vq->vq_used_idx != vq->vq_used->idx) { 455 if (vq->vq_done) 456 r |= (vq->vq_done)(vq); 457 } 458 } 459 460 461 return r; 462 } 463 464 /* 465 * Start/stop vq interrupt. No guarantee. 466 */ 467 void 468 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 469 { 470 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 471 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 472 vq->vq_queued++; 473 } 474 475 void 476 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 477 { 478 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 479 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 480 vq->vq_queued++; 481 } 482 483 /* 484 * Initialize vq structure. 485 */ 486 static void 487 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, const bool reinit) 488 { 489 int i, j; 490 int vq_size = vq->vq_num; 491 492 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 493 494 /* build the indirect descriptor chain */ 495 if (vq->vq_indirect != NULL) { 496 struct vring_desc *vd; 497 498 for (i = 0; i < vq_size; i++) { 499 vd = vq->vq_indirect; 500 vd += vq->vq_maxnsegs * i; 501 for (j = 0; j < vq->vq_maxnsegs-1; j++) 502 vd[j].next = j + 1; 503 } 504 } 505 506 /* free slot management */ 507 SIMPLEQ_INIT(&vq->vq_freelist); 508 for (i = 0; i < vq_size; i++) { 509 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 510 &vq->vq_entries[i], qe_list); 511 vq->vq_entries[i].qe_index = i; 512 } 513 if (!reinit) 514 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 515 516 /* enqueue/dequeue status */ 517 vq->vq_avail_idx = 0; 518 vq->vq_used_idx = 0; 519 vq->vq_queued = 0; 520 if (!reinit) { 521 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 522 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 523 } 524 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 525 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 526 vq->vq_queued++; 527 } 528 529 /* 530 * Allocate/free a vq. 531 */ 532 int 533 virtio_alloc_vq(struct virtio_softc *sc, 534 struct virtqueue *vq, int index, int maxsegsize, int maxnsegs, 535 const char *name) 536 { 537 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 538 int rsegs, r; 539 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 540 ~(VIRTIO_PAGE_SIZE-1)) 541 542 memset(vq, 0, sizeof(*vq)); 543 544 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 545 VIRTIO_CONFIG_QUEUE_SELECT, index); 546 vq_size = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 547 VIRTIO_CONFIG_QUEUE_SIZE); 548 if (vq_size == 0) { 549 aprint_error_dev(sc->sc_dev, 550 "virtqueue not exist, index %d for %s\n", 551 index, name); 552 goto err; 553 } 554 /* allocsize1: descriptor table + avail ring + pad */ 555 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 556 + sizeof(uint16_t)*(2+vq_size)); 557 /* allocsize2: used ring + pad */ 558 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 559 + sizeof(struct vring_used_elem)*vq_size); 560 /* allocsize3: indirect table */ 561 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 562 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 563 else 564 allocsize3 = 0; 565 allocsize = allocsize1 + allocsize2 + allocsize3; 566 567 /* alloc and map the memory */ 568 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 569 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 570 if (r != 0) { 571 aprint_error_dev(sc->sc_dev, 572 "virtqueue %d for %s allocation failed, " 573 "error code %d\n", index, name, r); 574 goto err; 575 } 576 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 577 &vq->vq_vaddr, BUS_DMA_NOWAIT); 578 if (r != 0) { 579 aprint_error_dev(sc->sc_dev, 580 "virtqueue %d for %s map failed, " 581 "error code %d\n", index, name, r); 582 goto err; 583 } 584 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 585 BUS_DMA_NOWAIT, &vq->vq_dmamap); 586 if (r != 0) { 587 aprint_error_dev(sc->sc_dev, 588 "virtqueue %d for %s dmamap creation failed, " 589 "error code %d\n", index, name, r); 590 goto err; 591 } 592 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 593 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 594 if (r != 0) { 595 aprint_error_dev(sc->sc_dev, 596 "virtqueue %d for %s dmamap load failed, " 597 "error code %d\n", index, name, r); 598 goto err; 599 } 600 601 /* set the vq address */ 602 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 603 VIRTIO_CONFIG_QUEUE_ADDRESS, 604 (vq->vq_dmamap->dm_segs[0].ds_addr 605 / VIRTIO_PAGE_SIZE)); 606 607 /* remember addresses and offsets for later use */ 608 vq->vq_owner = sc; 609 vq->vq_num = vq_size; 610 vq->vq_index = index; 611 vq->vq_desc = vq->vq_vaddr; 612 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 613 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 614 vq->vq_usedoffset = allocsize1; 615 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 616 if (allocsize3 > 0) { 617 vq->vq_indirectoffset = allocsize1 + allocsize2; 618 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 619 + vq->vq_indirectoffset); 620 } 621 vq->vq_bytesize = allocsize; 622 vq->vq_maxsegsize = maxsegsize; 623 vq->vq_maxnsegs = maxnsegs; 624 625 /* free slot management */ 626 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 627 KM_NOSLEEP); 628 if (vq->vq_entries == NULL) { 629 r = ENOMEM; 630 goto err; 631 } 632 633 virtio_init_vq(sc, vq, false); 634 635 aprint_verbose_dev(sc->sc_dev, 636 "allocated %u byte for virtqueue %d for %s, " 637 "size %d\n", allocsize, index, name, vq_size); 638 if (allocsize3 > 0) 639 aprint_verbose_dev(sc->sc_dev, 640 "using %d byte (%d entries) " 641 "indirect descriptors\n", 642 allocsize3, maxnsegs * vq_size); 643 return 0; 644 645 err: 646 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 647 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 648 if (vq->vq_dmamap) 649 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 650 if (vq->vq_vaddr) 651 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 652 if (vq->vq_segs[0].ds_addr) 653 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 654 memset(vq, 0, sizeof(*vq)); 655 656 return -1; 657 } 658 659 int 660 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 661 { 662 struct vq_entry *qe; 663 int i = 0; 664 665 /* device must be already deactivated */ 666 /* confirm the vq is empty */ 667 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 668 i++; 669 } 670 if (i != vq->vq_num) { 671 printf("%s: freeing non-empty vq, index %d\n", 672 device_xname(sc->sc_dev), vq->vq_index); 673 return EBUSY; 674 } 675 676 /* tell device that there's no virtqueue any longer */ 677 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 678 VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index); 679 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 680 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 681 682 kmem_free(vq->vq_entries, vq->vq_bytesize); 683 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 684 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 685 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 686 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 687 mutex_destroy(&vq->vq_freelist_lock); 688 mutex_destroy(&vq->vq_uring_lock); 689 mutex_destroy(&vq->vq_aring_lock); 690 memset(vq, 0, sizeof(*vq)); 691 692 return 0; 693 } 694 695 /* 696 * Free descriptor management. 697 */ 698 static struct vq_entry * 699 vq_alloc_entry(struct virtqueue *vq) 700 { 701 struct vq_entry *qe; 702 703 mutex_enter(&vq->vq_freelist_lock); 704 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 705 mutex_exit(&vq->vq_freelist_lock); 706 return NULL; 707 } 708 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 709 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 710 mutex_exit(&vq->vq_freelist_lock); 711 712 return qe; 713 } 714 715 static void 716 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 717 { 718 mutex_enter(&vq->vq_freelist_lock); 719 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 720 mutex_exit(&vq->vq_freelist_lock); 721 722 return; 723 } 724 725 /* 726 * Enqueue several dmamaps as a single request. 727 */ 728 /* 729 * Typical usage: 730 * <queue size> number of followings are stored in arrays 731 * - command blocks (in dmamem) should be pre-allocated and mapped 732 * - dmamaps for command blocks should be pre-allocated and loaded 733 * - dmamaps for payload should be pre-allocated 734 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 735 * if (r) // currently 0 or EAGAIN 736 * return r; 737 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 738 * if (r) { 739 * virtio_enqueue_abort(sc, vq, slot); 740 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 741 * return r; 742 * } 743 * r = virtio_enqueue_reserve(sc, vq, slot, 744 * dmamap_payload[slot]->dm_nsegs+1); 745 * // ^ +1 for command 746 * if (r) { // currently 0 or EAGAIN 747 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 748 * return r; // do not call abort() 749 * } 750 * <setup and prepare commands> 751 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 752 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 753 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 754 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 755 * virtio_enqueue_commit(sc, vq, slot, true); 756 */ 757 758 /* 759 * enqueue_prep: allocate a slot number 760 */ 761 int 762 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 763 { 764 struct vq_entry *qe1; 765 766 KASSERT(slotp != NULL); 767 768 qe1 = vq_alloc_entry(vq); 769 if (qe1 == NULL) 770 return EAGAIN; 771 /* next slot is not allocated yet */ 772 qe1->qe_next = -1; 773 *slotp = qe1->qe_index; 774 775 return 0; 776 } 777 778 /* 779 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 780 */ 781 int 782 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 783 int slot, int nsegs) 784 { 785 int indirect; 786 struct vq_entry *qe1 = &vq->vq_entries[slot]; 787 788 KASSERT(qe1->qe_next == -1); 789 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 790 791 if ((vq->vq_indirect != NULL) && 792 (nsegs >= MINSEG_INDIRECT) && 793 (nsegs <= vq->vq_maxnsegs)) 794 indirect = 1; 795 else 796 indirect = 0; 797 qe1->qe_indirect = indirect; 798 799 if (indirect) { 800 struct vring_desc *vd; 801 int i; 802 803 vd = &vq->vq_desc[qe1->qe_index]; 804 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 805 + vq->vq_indirectoffset; 806 vd->addr += sizeof(struct vring_desc) 807 * vq->vq_maxnsegs * qe1->qe_index; 808 vd->len = sizeof(struct vring_desc) * nsegs; 809 vd->flags = VRING_DESC_F_INDIRECT; 810 811 vd = vq->vq_indirect; 812 vd += vq->vq_maxnsegs * qe1->qe_index; 813 qe1->qe_desc_base = vd; 814 815 for (i = 0; i < nsegs-1; i++) { 816 vd[i].flags = VRING_DESC_F_NEXT; 817 } 818 vd[i].flags = 0; 819 qe1->qe_next = 0; 820 821 return 0; 822 } else { 823 struct vring_desc *vd; 824 struct vq_entry *qe; 825 int i, s; 826 827 vd = &vq->vq_desc[0]; 828 qe1->qe_desc_base = vd; 829 qe1->qe_next = qe1->qe_index; 830 s = slot; 831 for (i = 0; i < nsegs - 1; i++) { 832 qe = vq_alloc_entry(vq); 833 if (qe == NULL) { 834 vd[s].flags = 0; 835 virtio_enqueue_abort(sc, vq, slot); 836 return EAGAIN; 837 } 838 vd[s].flags = VRING_DESC_F_NEXT; 839 vd[s].next = qe->qe_index; 840 s = qe->qe_index; 841 } 842 vd[s].flags = 0; 843 844 return 0; 845 } 846 } 847 848 /* 849 * enqueue: enqueue a single dmamap. 850 */ 851 int 852 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 853 bus_dmamap_t dmamap, bool write) 854 { 855 struct vq_entry *qe1 = &vq->vq_entries[slot]; 856 struct vring_desc *vd = qe1->qe_desc_base; 857 int i; 858 int s = qe1->qe_next; 859 860 KASSERT(s >= 0); 861 KASSERT(dmamap->dm_nsegs > 0); 862 863 for (i = 0; i < dmamap->dm_nsegs; i++) { 864 vd[s].addr = dmamap->dm_segs[i].ds_addr; 865 vd[s].len = dmamap->dm_segs[i].ds_len; 866 if (!write) 867 vd[s].flags |= VRING_DESC_F_WRITE; 868 s = vd[s].next; 869 } 870 qe1->qe_next = s; 871 872 return 0; 873 } 874 875 int 876 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 877 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 878 bool write) 879 { 880 struct vq_entry *qe1 = &vq->vq_entries[slot]; 881 struct vring_desc *vd = qe1->qe_desc_base; 882 int s = qe1->qe_next; 883 884 KASSERT(s >= 0); 885 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 886 KASSERT((dmamap->dm_segs[0].ds_len > start) && 887 (dmamap->dm_segs[0].ds_len >= start + len)); 888 889 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 890 vd[s].len = len; 891 if (!write) 892 vd[s].flags |= VRING_DESC_F_WRITE; 893 qe1->qe_next = vd[s].next; 894 895 return 0; 896 } 897 898 /* 899 * enqueue_commit: add it to the aring. 900 */ 901 int 902 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 903 bool notifynow) 904 { 905 struct vq_entry *qe1; 906 907 if (slot < 0) { 908 mutex_enter(&vq->vq_aring_lock); 909 goto notify; 910 } 911 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 912 qe1 = &vq->vq_entries[slot]; 913 if (qe1->qe_indirect) 914 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 915 mutex_enter(&vq->vq_aring_lock); 916 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 917 918 notify: 919 if (notifynow) { 920 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 921 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 922 membar_producer(); 923 vq->vq_avail->idx = vq->vq_avail_idx; 924 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 925 membar_producer(); 926 vq->vq_queued++; 927 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 928 membar_consumer(); 929 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 930 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 931 VIRTIO_CONFIG_QUEUE_NOTIFY, 932 vq->vq_index); 933 } 934 mutex_exit(&vq->vq_aring_lock); 935 936 return 0; 937 } 938 939 /* 940 * enqueue_abort: rollback. 941 */ 942 int 943 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 944 { 945 struct vq_entry *qe = &vq->vq_entries[slot]; 946 struct vring_desc *vd; 947 int s; 948 949 if (qe->qe_next < 0) { 950 vq_free_entry(vq, qe); 951 return 0; 952 } 953 954 s = slot; 955 vd = &vq->vq_desc[0]; 956 while (vd[s].flags & VRING_DESC_F_NEXT) { 957 s = vd[s].next; 958 vq_free_entry(vq, qe); 959 qe = &vq->vq_entries[s]; 960 } 961 vq_free_entry(vq, qe); 962 return 0; 963 } 964 965 /* 966 * Dequeue a request. 967 */ 968 /* 969 * dequeue: dequeue a request from uring; dmamap_sync for uring is 970 * already done in the interrupt handler. 971 */ 972 int 973 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 974 int *slotp, int *lenp) 975 { 976 uint16_t slot, usedidx; 977 struct vq_entry *qe; 978 979 if (vq->vq_used_idx == vq->vq_used->idx) 980 return ENOENT; 981 mutex_enter(&vq->vq_uring_lock); 982 usedidx = vq->vq_used_idx++; 983 mutex_exit(&vq->vq_uring_lock); 984 usedidx %= vq->vq_num; 985 slot = vq->vq_used->ring[usedidx].id; 986 qe = &vq->vq_entries[slot]; 987 988 if (qe->qe_indirect) 989 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 990 991 if (slotp) 992 *slotp = slot; 993 if (lenp) 994 *lenp = vq->vq_used->ring[usedidx].len; 995 996 return 0; 997 } 998 999 /* 1000 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1001 * if you forget to call this the slot will be leaked. 1002 */ 1003 int 1004 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1005 { 1006 struct vq_entry *qe = &vq->vq_entries[slot]; 1007 struct vring_desc *vd = &vq->vq_desc[0]; 1008 int s = slot; 1009 1010 while (vd[s].flags & VRING_DESC_F_NEXT) { 1011 s = vd[s].next; 1012 vq_free_entry(vq, qe); 1013 qe = &vq->vq_entries[s]; 1014 } 1015 vq_free_entry(vq, qe); 1016 1017 return 0; 1018 } 1019
Indexes created Thu Oct 02 14:10:14 GMT 2025