virtio.c revision 1.3.6.2
1 /* $NetBSD: virtio.c,v 1.3.6.2 2012/01/25 21:18:15 riz Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.3.6.2 2012/01/25 21:18:15 riz Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/atomic.h> 35 #include <sys/bus.h> 36 #include <sys/device.h> 37 #include <sys/kmem.h> 38 39 #include <dev/pci/pcidevs.h> 40 #include <dev/pci/pcireg.h> 41 #include <dev/pci/pcivar.h> 42 43 #include <dev/pci/virtioreg.h> 44 #include <dev/pci/virtiovar.h> 45 46 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 47 48 static int virtio_match(device_t, cfdata_t, void *); 49 static void virtio_attach(device_t, device_t, void *); 50 static int virtio_detach(device_t, int); 51 static int virtio_intr(void *arg); 52 static void virtio_init_vq(struct virtio_softc *, 53 struct virtqueue *, const bool); 54 55 CFATTACH_DECL_NEW(virtio, sizeof(struct virtio_softc), 56 virtio_match, virtio_attach, virtio_detach, NULL); 57 58 static void 59 virtio_set_status(struct virtio_softc *sc, int status) 60 { 61 int old = 0; 62 63 if (status != 0) 64 old = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 65 VIRTIO_CONFIG_DEVICE_STATUS); 66 bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS, 67 status|old); 68 } 69 70 #define virtio_device_reset(sc) virtio_set_status((sc), 0) 71 72 static int 73 virtio_match(device_t parent, cfdata_t match, void *aux) 74 { 75 struct pci_attach_args *pa; 76 77 pa = (struct pci_attach_args *)aux; 78 switch (PCI_VENDOR(pa->pa_id)) { 79 case 0x1af4 /*PCI_VENDOR_QUMRANET*/: 80 if ((0x1000 /*PCI_PRODUCT_QUMRANET_VIRTIO_1000*/ <= 81 PCI_PRODUCT(pa->pa_id)) && 82 (PCI_PRODUCT(pa->pa_id) <= 83 0x103f /*PCI_PRODUCT_QUMRANET_VIRTIO_103F*/)) 84 return 1; 85 break; 86 } 87 88 return 0; 89 } 90 91 static const char *virtio_device_name[] = { 92 "Unknown (0)", /* 0 */ 93 "Network", /* 1 */ 94 "Block", /* 2 */ 95 "Console", /* 3 */ 96 "Entropy", /* 4 */ 97 "Memory Balloon", /* 5 */ 98 "Unknown (6)", /* 6 */ 99 "Unknown (7)", /* 7 */ 100 "Unknown (8)", /* 8 */ 101 "9P Transport" /* 9 */ 102 }; 103 #define NDEVNAMES (sizeof(virtio_device_name)/sizeof(char*)) 104 105 static void 106 virtio_attach(device_t parent, device_t self, void *aux) 107 { 108 struct virtio_softc *sc = device_private(self); 109 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 110 pci_chipset_tag_t pc = pa->pa_pc; 111 pcitag_t tag = pa->pa_tag; 112 int revision; 113 pcireg_t id; 114 char const *intrstr; 115 pci_intr_handle_t ih; 116 117 revision = PCI_REVISION(pa->pa_class); 118 if (revision != 0) { 119 aprint_normal(": unknown revision 0x%02x; giving up\n", 120 revision); 121 return; 122 } 123 aprint_normal("\n"); 124 aprint_naive("\n"); 125 126 /* subsystem ID shows what I am */ 127 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 128 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 129 (PCI_PRODUCT(id) < NDEVNAMES? 130 virtio_device_name[PCI_PRODUCT(id)] : "Unknown"), 131 revision); 132 133 sc->sc_dev = self; 134 sc->sc_pc = pc; 135 sc->sc_tag = tag; 136 sc->sc_iot = pa->pa_iot; 137 sc->sc_dmat = pa->pa_dmat; 138 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 139 140 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 141 &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) { 142 aprint_error_dev(self, "can't map i/o space\n"); 143 return; 144 } 145 146 virtio_device_reset(sc); 147 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 148 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 149 150 /* XXX: use softc as aux... */ 151 sc->sc_childdevid = PCI_PRODUCT(id); 152 sc->sc_child = NULL; 153 config_found(self, sc, NULL); 154 if (sc->sc_child == NULL) { 155 aprint_error_dev(self, 156 "no matching child driver; not configured\n"); 157 return; 158 } 159 if (sc->sc_child == (void*)1) { /* this shows error */ 160 aprint_error_dev(self, 161 "virtio configuration failed\n"); 162 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 163 return; 164 } 165 166 if (pci_intr_map(pa, &ih)) { 167 aprint_error_dev(self, "couldn't map interrupt\n"); 168 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 169 return; 170 } 171 intrstr = pci_intr_string(pc, ih); 172 sc->sc_ih = pci_intr_establish(pc, ih, sc->sc_ipl, virtio_intr, sc); 173 if (sc->sc_ih == NULL) { 174 aprint_error_dev(self, "couldn't establish interrupt"); 175 if (intrstr != NULL) 176 aprint_error(" at %s", intrstr); 177 aprint_error("\n"); 178 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 179 return; 180 } 181 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 182 183 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 184 185 return; 186 } 187 188 static int 189 virtio_detach(device_t self, int flags) 190 { 191 struct virtio_softc *sc = device_private(self); 192 int r; 193 194 if (sc->sc_child != 0 && sc->sc_child != (void*)1) { 195 r = config_detach(sc->sc_child, flags); 196 if (r) 197 return r; 198 } 199 KASSERT(sc->sc_child == 0 || sc->sc_child == (void*)1); 200 KASSERT(sc->sc_vqs == 0); 201 pci_intr_disestablish(sc->sc_pc, sc->sc_ih); 202 sc->sc_ih = 0; 203 if (sc->sc_iosize) 204 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 205 sc->sc_iosize = 0; 206 207 return 0; 208 } 209 210 /* 211 * Reset the device. 212 */ 213 /* 214 * To reset the device to a known state, do following: 215 * virtio_reset(sc); // this will stop the device activity 216 * <dequeue finished requests>; // virtio_dequeue() still can be called 217 * <revoke pending requests in the vqs if any>; 218 * virtio_reinit_begin(sc); // dequeue prohibitted 219 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 220 * <some other initialization>; 221 * virtio_reinit_end(sc); // device activated; enqueue allowed 222 * Once attached, feature negotiation can only be allowed after virtio_reset. 223 */ 224 void 225 virtio_reset(struct virtio_softc *sc) 226 { 227 virtio_device_reset(sc); 228 } 229 230 void 231 virtio_reinit_start(struct virtio_softc *sc) 232 { 233 int i; 234 235 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 236 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 237 for (i = 0; i < sc->sc_nvqs; i++) { 238 int n; 239 struct virtqueue *vq = &sc->sc_vqs[i]; 240 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 241 VIRTIO_CONFIG_QUEUE_SELECT, 242 vq->vq_index); 243 n = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 244 VIRTIO_CONFIG_QUEUE_SIZE); 245 if (n == 0) /* vq disappeared */ 246 continue; 247 if (n != vq->vq_num) { 248 panic("%s: virtqueue size changed, vq index %d\n", 249 device_xname(sc->sc_dev), 250 vq->vq_index); 251 } 252 virtio_init_vq(sc, vq, true); 253 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 254 VIRTIO_CONFIG_QUEUE_ADDRESS, 255 (vq->vq_dmamap->dm_segs[0].ds_addr 256 / VIRTIO_PAGE_SIZE)); 257 } 258 } 259 260 void 261 virtio_reinit_end(struct virtio_softc *sc) 262 { 263 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 264 } 265 266 /* 267 * Feature negotiation. 268 */ 269 uint32_t 270 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 271 { 272 uint32_t r; 273 274 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 275 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 276 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 277 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 278 VIRTIO_CONFIG_DEVICE_FEATURES); 279 r &= guest_features; 280 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 281 VIRTIO_CONFIG_GUEST_FEATURES, r); 282 sc->sc_features = r; 283 if (r & VIRTIO_F_RING_INDIRECT_DESC) 284 sc->sc_indirect = true; 285 else 286 sc->sc_indirect = false; 287 288 return r; 289 } 290 291 /* 292 * Device configuration registers. 293 */ 294 uint8_t 295 virtio_read_device_config_1(struct virtio_softc *sc, int index) 296 { 297 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 298 sc->sc_config_offset + index); 299 } 300 301 uint16_t 302 virtio_read_device_config_2(struct virtio_softc *sc, int index) 303 { 304 return bus_space_read_2(sc->sc_iot, sc->sc_ioh, 305 sc->sc_config_offset + index); 306 } 307 308 uint32_t 309 virtio_read_device_config_4(struct virtio_softc *sc, int index) 310 { 311 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 312 sc->sc_config_offset + index); 313 } 314 315 uint64_t 316 virtio_read_device_config_8(struct virtio_softc *sc, int index) 317 { 318 uint64_t r; 319 320 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 321 sc->sc_config_offset + index + sizeof(uint32_t)); 322 r <<= 32; 323 r += bus_space_read_4(sc->sc_iot, sc->sc_ioh, 324 sc->sc_config_offset + index); 325 return r; 326 } 327 328 void 329 virtio_write_device_config_1(struct virtio_softc *sc, 330 int index, uint8_t value) 331 { 332 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 333 sc->sc_config_offset + index, value); 334 } 335 336 void 337 virtio_write_device_config_2(struct virtio_softc *sc, 338 int index, uint16_t value) 339 { 340 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 341 sc->sc_config_offset + index, value); 342 } 343 344 void 345 virtio_write_device_config_4(struct virtio_softc *sc, 346 int index, uint32_t value) 347 { 348 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 349 sc->sc_config_offset + index, value); 350 } 351 352 void 353 virtio_write_device_config_8(struct virtio_softc *sc, 354 int index, uint64_t value) 355 { 356 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 357 sc->sc_config_offset + index, 358 value & 0xffffffff); 359 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 360 sc->sc_config_offset + index + sizeof(uint32_t), 361 value >> 32); 362 } 363 364 /* 365 * Interrupt handler. 366 */ 367 static int 368 virtio_intr(void *arg) 369 { 370 struct virtio_softc *sc = arg; 371 int isr, r = 0; 372 373 /* check and ack the interrupt */ 374 isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 375 VIRTIO_CONFIG_ISR_STATUS); 376 if (isr == 0) 377 return 0; 378 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 379 (sc->sc_config_change != NULL)) 380 r = (sc->sc_config_change)(sc); 381 if (sc->sc_intrhand != NULL) 382 r |= (sc->sc_intrhand)(sc); 383 384 return r; 385 } 386 387 /* 388 * dmamap sync operations for a virtqueue. 389 */ 390 static inline void 391 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 392 { 393 /* availoffset == sizeof(vring_desc)*vq_num */ 394 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 395 ops); 396 } 397 398 static inline void 399 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 400 { 401 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 402 vq->vq_availoffset, 403 offsetof(struct vring_avail, ring) 404 + vq->vq_num * sizeof(uint16_t), 405 ops); 406 } 407 408 static inline void 409 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 410 { 411 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 412 vq->vq_usedoffset, 413 offsetof(struct vring_used, ring) 414 + vq->vq_num * sizeof(struct vring_used_elem), 415 ops); 416 } 417 418 static inline void 419 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 420 int ops) 421 { 422 int offset = vq->vq_indirectoffset 423 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 424 425 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 426 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 427 ops); 428 } 429 430 /* 431 * Can be used as sc_intrhand. 432 */ 433 /* 434 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 435 * and calls (*vq_done)() if some entries are consumed. 436 */ 437 int 438 virtio_vq_intr(struct virtio_softc *sc) 439 { 440 struct virtqueue *vq; 441 int i, r = 0; 442 443 for (i = 0; i < sc->sc_nvqs; i++) { 444 vq = &sc->sc_vqs[i]; 445 if (vq->vq_queued) { 446 vq->vq_queued = 0; 447 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 448 } 449 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 450 membar_consumer(); 451 if (vq->vq_used_idx != vq->vq_used->idx) { 452 if (vq->vq_done) 453 r |= (vq->vq_done)(vq); 454 } 455 } 456 457 458 return r; 459 } 460 461 /* 462 * Start/stop vq interrupt. No guarantee. 463 */ 464 void 465 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 466 { 467 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 468 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 469 vq->vq_queued++; 470 } 471 472 void 473 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 474 { 475 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 476 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 477 vq->vq_queued++; 478 } 479 480 /* 481 * Initialize vq structure. 482 */ 483 static void 484 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, const bool reinit) 485 { 486 int i, j; 487 int vq_size = vq->vq_num; 488 489 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 490 491 /* build the indirect descriptor chain */ 492 if (vq->vq_indirect != NULL) { 493 struct vring_desc *vd; 494 495 for (i = 0; i < vq_size; i++) { 496 vd = vq->vq_indirect; 497 vd += vq->vq_maxnsegs * i; 498 for (j = 0; j < vq->vq_maxnsegs-1; j++) 499 vd[j].next = j + 1; 500 } 501 } 502 503 /* free slot management */ 504 SIMPLEQ_INIT(&vq->vq_freelist); 505 for (i = 0; i < vq_size; i++) { 506 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 507 &vq->vq_entries[i], qe_list); 508 vq->vq_entries[i].qe_index = i; 509 } 510 if (!reinit) 511 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 512 513 /* enqueue/dequeue status */ 514 vq->vq_avail_idx = 0; 515 vq->vq_used_idx = 0; 516 vq->vq_queued = 0; 517 if (!reinit) { 518 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 519 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 520 } 521 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 522 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 523 vq->vq_queued++; 524 } 525 526 /* 527 * Allocate/free a vq. 528 */ 529 int 530 virtio_alloc_vq(struct virtio_softc *sc, 531 struct virtqueue *vq, int index, int maxsegsize, int maxnsegs, 532 const char *name) 533 { 534 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 535 int rsegs, r; 536 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 537 ~(VIRTIO_PAGE_SIZE-1)) 538 539 memset(vq, 0, sizeof(*vq)); 540 541 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 542 VIRTIO_CONFIG_QUEUE_SELECT, index); 543 vq_size = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 544 VIRTIO_CONFIG_QUEUE_SIZE); 545 if (vq_size == 0) { 546 aprint_error_dev(sc->sc_dev, 547 "virtqueue not exist, index %d for %s\n", 548 index, name); 549 goto err; 550 } 551 /* allocsize1: descriptor table + avail ring + pad */ 552 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 553 + sizeof(uint16_t)*(2+vq_size)); 554 /* allocsize2: used ring + pad */ 555 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 556 + sizeof(struct vring_used_elem)*vq_size); 557 /* allocsize3: indirect table */ 558 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 559 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 560 else 561 allocsize3 = 0; 562 allocsize = allocsize1 + allocsize2 + allocsize3; 563 564 /* alloc and map the memory */ 565 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 566 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 567 if (r != 0) { 568 aprint_error_dev(sc->sc_dev, 569 "virtqueue %d for %s allocation failed, " 570 "error code %d\n", index, name, r); 571 goto err; 572 } 573 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 574 &vq->vq_vaddr, BUS_DMA_NOWAIT); 575 if (r != 0) { 576 aprint_error_dev(sc->sc_dev, 577 "virtqueue %d for %s map failed, " 578 "error code %d\n", index, name, r); 579 goto err; 580 } 581 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 582 BUS_DMA_NOWAIT, &vq->vq_dmamap); 583 if (r != 0) { 584 aprint_error_dev(sc->sc_dev, 585 "virtqueue %d for %s dmamap creation failed, " 586 "error code %d\n", index, name, r); 587 goto err; 588 } 589 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 590 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 591 if (r != 0) { 592 aprint_error_dev(sc->sc_dev, 593 "virtqueue %d for %s dmamap load failed, " 594 "error code %d\n", index, name, r); 595 goto err; 596 } 597 598 /* set the vq address */ 599 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 600 VIRTIO_CONFIG_QUEUE_ADDRESS, 601 (vq->vq_dmamap->dm_segs[0].ds_addr 602 / VIRTIO_PAGE_SIZE)); 603 604 /* remember addresses and offsets for later use */ 605 vq->vq_owner = sc; 606 vq->vq_num = vq_size; 607 vq->vq_index = index; 608 vq->vq_desc = vq->vq_vaddr; 609 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 610 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 611 vq->vq_usedoffset = allocsize1; 612 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 613 if (allocsize3 > 0) { 614 vq->vq_indirectoffset = allocsize1 + allocsize2; 615 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 616 + vq->vq_indirectoffset); 617 } 618 vq->vq_bytesize = allocsize; 619 vq->vq_maxsegsize = maxsegsize; 620 vq->vq_maxnsegs = maxnsegs; 621 622 /* free slot management */ 623 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 624 KM_NOSLEEP); 625 if (vq->vq_entries == NULL) { 626 r = ENOMEM; 627 goto err; 628 } 629 630 virtio_init_vq(sc, vq, false); 631 632 aprint_verbose_dev(sc->sc_dev, 633 "allocated %u byte for virtqueue %d for %s, " 634 "size %d\n", allocsize, index, name, vq_size); 635 if (allocsize3 > 0) 636 aprint_verbose_dev(sc->sc_dev, 637 "using %d byte (%d entries) " 638 "indirect descriptors\n", 639 allocsize3, maxnsegs * vq_size); 640 return 0; 641 642 err: 643 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 644 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 645 if (vq->vq_dmamap) 646 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 647 if (vq->vq_vaddr) 648 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 649 if (vq->vq_segs[0].ds_addr) 650 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 651 memset(vq, 0, sizeof(*vq)); 652 653 return -1; 654 } 655 656 int 657 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 658 { 659 struct vq_entry *qe; 660 int i = 0; 661 662 /* device must be already deactivated */ 663 /* confirm the vq is empty */ 664 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 665 i++; 666 } 667 if (i != vq->vq_num) { 668 printf("%s: freeing non-empty vq, index %d\n", 669 device_xname(sc->sc_dev), vq->vq_index); 670 return EBUSY; 671 } 672 673 /* tell device that there's no virtqueue any longer */ 674 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 675 VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index); 676 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 677 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 678 679 kmem_free(vq->vq_entries, vq->vq_bytesize); 680 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 681 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 682 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 683 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 684 mutex_destroy(&vq->vq_freelist_lock); 685 mutex_destroy(&vq->vq_uring_lock); 686 mutex_destroy(&vq->vq_aring_lock); 687 memset(vq, 0, sizeof(*vq)); 688 689 return 0; 690 } 691 692 /* 693 * Free descriptor management. 694 */ 695 static struct vq_entry * 696 vq_alloc_entry(struct virtqueue *vq) 697 { 698 struct vq_entry *qe; 699 700 mutex_enter(&vq->vq_freelist_lock); 701 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 702 mutex_exit(&vq->vq_freelist_lock); 703 return NULL; 704 } 705 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 706 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 707 mutex_exit(&vq->vq_freelist_lock); 708 709 return qe; 710 } 711 712 static void 713 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 714 { 715 mutex_enter(&vq->vq_freelist_lock); 716 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 717 mutex_exit(&vq->vq_freelist_lock); 718 719 return; 720 } 721 722 /* 723 * Enqueue several dmamaps as a single request. 724 */ 725 /* 726 * Typical usage: 727 * <queue size> number of followings are stored in arrays 728 * - command blocks (in dmamem) should be pre-allocated and mapped 729 * - dmamaps for command blocks should be pre-allocated and loaded 730 * - dmamaps for payload should be pre-allocated 731 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 732 * if (r) // currently 0 or EAGAIN 733 * return r; 734 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 735 * if (r) { 736 * virtio_enqueue_abort(sc, vq, slot); 737 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 738 * return r; 739 * } 740 * r = virtio_enqueue_reserve(sc, vq, slot, 741 * dmamap_payload[slot]->dm_nsegs+1); 742 * // ^ +1 for command 743 * if (r) { // currently 0 or EAGAIN 744 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 745 * return r; // do not call abort() 746 * } 747 * <setup and prepare commands> 748 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 749 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 750 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 751 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 752 * virtio_enqueue_commit(sc, vq, slot, true); 753 */ 754 755 /* 756 * enqueue_prep: allocate a slot number 757 */ 758 int 759 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 760 { 761 struct vq_entry *qe1; 762 763 KASSERT(slotp != NULL); 764 765 qe1 = vq_alloc_entry(vq); 766 if (qe1 == NULL) 767 return EAGAIN; 768 /* next slot is not allocated yet */ 769 qe1->qe_next = -1; 770 *slotp = qe1->qe_index; 771 772 return 0; 773 } 774 775 /* 776 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 777 */ 778 int 779 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 780 int slot, int nsegs) 781 { 782 int indirect; 783 struct vq_entry *qe1 = &vq->vq_entries[slot]; 784 785 KASSERT(qe1->qe_next == -1); 786 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 787 788 if ((vq->vq_indirect != NULL) && 789 (nsegs >= MINSEG_INDIRECT) && 790 (nsegs <= vq->vq_maxnsegs)) 791 indirect = 1; 792 else 793 indirect = 0; 794 qe1->qe_indirect = indirect; 795 796 if (indirect) { 797 struct vring_desc *vd; 798 int i; 799 800 vd = &vq->vq_desc[qe1->qe_index]; 801 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 802 + vq->vq_indirectoffset; 803 vd->addr += sizeof(struct vring_desc) 804 * vq->vq_maxnsegs * qe1->qe_index; 805 vd->len = sizeof(struct vring_desc) * nsegs; 806 vd->flags = VRING_DESC_F_INDIRECT; 807 808 vd = vq->vq_indirect; 809 vd += vq->vq_maxnsegs * qe1->qe_index; 810 qe1->qe_desc_base = vd; 811 812 for (i = 0; i < nsegs-1; i++) { 813 vd[i].flags = VRING_DESC_F_NEXT; 814 } 815 vd[i].flags = 0; 816 qe1->qe_next = 0; 817 818 return 0; 819 } else { 820 struct vring_desc *vd; 821 struct vq_entry *qe; 822 int i, s; 823 824 vd = &vq->vq_desc[0]; 825 qe1->qe_desc_base = vd; 826 qe1->qe_next = qe1->qe_index; 827 s = slot; 828 for (i = 0; i < nsegs - 1; i++) { 829 qe = vq_alloc_entry(vq); 830 if (qe == NULL) { 831 vd[s].flags = 0; 832 virtio_enqueue_abort(sc, vq, slot); 833 return EAGAIN; 834 } 835 vd[s].flags = VRING_DESC_F_NEXT; 836 vd[s].next = qe->qe_index; 837 s = qe->qe_index; 838 } 839 vd[s].flags = 0; 840 841 return 0; 842 } 843 } 844 845 /* 846 * enqueue: enqueue a single dmamap. 847 */ 848 int 849 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 850 bus_dmamap_t dmamap, bool write) 851 { 852 struct vq_entry *qe1 = &vq->vq_entries[slot]; 853 struct vring_desc *vd = qe1->qe_desc_base; 854 int i; 855 int s = qe1->qe_next; 856 857 KASSERT(s >= 0); 858 KASSERT(dmamap->dm_nsegs > 0); 859 860 for (i = 0; i < dmamap->dm_nsegs; i++) { 861 vd[s].addr = dmamap->dm_segs[i].ds_addr; 862 vd[s].len = dmamap->dm_segs[i].ds_len; 863 if (!write) 864 vd[s].flags |= VRING_DESC_F_WRITE; 865 s = vd[s].next; 866 } 867 qe1->qe_next = s; 868 869 return 0; 870 } 871 872 int 873 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 874 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 875 bool write) 876 { 877 struct vq_entry *qe1 = &vq->vq_entries[slot]; 878 struct vring_desc *vd = qe1->qe_desc_base; 879 int s = qe1->qe_next; 880 881 KASSERT(s >= 0); 882 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 883 KASSERT((dmamap->dm_segs[0].ds_len > start) && 884 (dmamap->dm_segs[0].ds_len >= start + len)); 885 886 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 887 vd[s].len = len; 888 if (!write) 889 vd[s].flags |= VRING_DESC_F_WRITE; 890 qe1->qe_next = vd[s].next; 891 892 return 0; 893 } 894 895 /* 896 * enqueue_commit: add it to the aring. 897 */ 898 int 899 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 900 bool notifynow) 901 { 902 struct vq_entry *qe1; 903 904 if (slot < 0) { 905 mutex_enter(&vq->vq_aring_lock); 906 goto notify; 907 } 908 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 909 qe1 = &vq->vq_entries[slot]; 910 if (qe1->qe_indirect) 911 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 912 mutex_enter(&vq->vq_aring_lock); 913 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 914 915 notify: 916 if (notifynow) { 917 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 918 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 919 membar_producer(); 920 vq->vq_avail->idx = vq->vq_avail_idx; 921 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 922 membar_producer(); 923 vq->vq_queued++; 924 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 925 membar_consumer(); 926 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 927 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 928 VIRTIO_CONFIG_QUEUE_NOTIFY, 929 vq->vq_index); 930 } 931 mutex_exit(&vq->vq_aring_lock); 932 933 return 0; 934 } 935 936 /* 937 * enqueue_abort: rollback. 938 */ 939 int 940 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 941 { 942 struct vq_entry *qe = &vq->vq_entries[slot]; 943 struct vring_desc *vd; 944 int s; 945 946 if (qe->qe_next < 0) { 947 vq_free_entry(vq, qe); 948 return 0; 949 } 950 951 s = slot; 952 vd = &vq->vq_desc[0]; 953 while (vd[s].flags & VRING_DESC_F_NEXT) { 954 s = vd[s].next; 955 vq_free_entry(vq, qe); 956 qe = &vq->vq_entries[s]; 957 } 958 vq_free_entry(vq, qe); 959 return 0; 960 } 961 962 /* 963 * Dequeue a request. 964 */ 965 /* 966 * dequeue: dequeue a request from uring; dmamap_sync for uring is 967 * already done in the interrupt handler. 968 */ 969 int 970 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 971 int *slotp, int *lenp) 972 { 973 uint16_t slot, usedidx; 974 struct vq_entry *qe; 975 976 if (vq->vq_used_idx == vq->vq_used->idx) 977 return ENOENT; 978 mutex_enter(&vq->vq_uring_lock); 979 usedidx = vq->vq_used_idx++; 980 mutex_exit(&vq->vq_uring_lock); 981 usedidx %= vq->vq_num; 982 slot = vq->vq_used->ring[usedidx].id; 983 qe = &vq->vq_entries[slot]; 984 985 if (qe->qe_indirect) 986 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 987 988 if (slotp) 989 *slotp = slot; 990 if (lenp) 991 *lenp = vq->vq_used->ring[usedidx].len; 992 993 return 0; 994 } 995 996 /* 997 * dequeue_commit: complete dequeue; the slot is recycled for future use. 998 * if you forget to call this the slot will be leaked. 999 */ 1000 int 1001 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1002 { 1003 struct vq_entry *qe = &vq->vq_entries[slot]; 1004 struct vring_desc *vd = &vq->vq_desc[0]; 1005 int s = slot; 1006 1007 while (vd[s].flags & VRING_DESC_F_NEXT) { 1008 s = vd[s].next; 1009 vq_free_entry(vq, qe); 1010 qe = &vq->vq_entries[s]; 1011 } 1012 vq_free_entry(vq, qe); 1013 1014 return 0; 1015 } 1016
Indexes created Thu Oct 02 14:10:14 GMT 2025