virtio.c revision 1.31
1 /* $NetBSD: virtio.c,v 1.31 2018/06/02 22:43:15 jakllsch Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.31 2018/06/02 22:43:15 jakllsch Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/atomic.h> 35 #include <sys/bus.h> 36 #include <sys/device.h> 37 #include <sys/kmem.h> 38 #include <sys/module.h> 39 40 #include <dev/pci/pcidevs.h> 41 #include <dev/pci/pcireg.h> 42 #include <dev/pci/pcivar.h> 43 44 #define VIRTIO_PRIVATE 45 46 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */ 47 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */ 48 49 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 50 51 static void virtio_init_vq(struct virtio_softc *, 52 struct virtqueue *, const bool); 53 54 void 55 virtio_set_status(struct virtio_softc *sc, int status) 56 { 57 sc->sc_ops->set_status(sc, status); 58 } 59 60 /* 61 * Reset the device. 62 */ 63 /* 64 * To reset the device to a known state, do following: 65 * virtio_reset(sc); // this will stop the device activity 66 * <dequeue finished requests>; // virtio_dequeue() still can be called 67 * <revoke pending requests in the vqs if any>; 68 * virtio_reinit_begin(sc); // dequeue prohibitted 69 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 70 * <some other initialization>; 71 * virtio_reinit_end(sc); // device activated; enqueue allowed 72 * Once attached, feature negotiation can only be allowed after virtio_reset. 73 */ 74 void 75 virtio_reset(struct virtio_softc *sc) 76 { 77 virtio_device_reset(sc); 78 } 79 80 void 81 virtio_reinit_start(struct virtio_softc *sc) 82 { 83 int i; 84 85 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 86 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 87 for (i = 0; i < sc->sc_nvqs; i++) { 88 int n; 89 struct virtqueue *vq = &sc->sc_vqs[i]; 90 n = sc->sc_ops->read_queue_size(sc, vq->vq_index); 91 if (n == 0) /* vq disappeared */ 92 continue; 93 if (n != vq->vq_num) { 94 panic("%s: virtqueue size changed, vq index %d\n", 95 device_xname(sc->sc_dev), 96 vq->vq_index); 97 } 98 virtio_init_vq(sc, vq, true); 99 sc->sc_ops->setup_queue(sc, vq->vq_index, 100 vq->vq_dmamap->dm_segs[0].ds_addr / VIRTIO_PAGE_SIZE); 101 } 102 } 103 104 void 105 virtio_reinit_end(struct virtio_softc *sc) 106 { 107 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 108 } 109 110 /* 111 * Feature negotiation. 112 */ 113 uint32_t 114 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 115 { 116 uint32_t r; 117 118 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 119 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 120 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 121 r = sc->sc_ops->neg_features(sc, guest_features); 122 sc->sc_features = r; 123 if (r & VIRTIO_F_RING_INDIRECT_DESC) 124 sc->sc_indirect = true; 125 else 126 sc->sc_indirect = false; 127 128 return r; 129 } 130 131 /* 132 * Device configuration registers. 133 */ 134 uint8_t 135 virtio_read_device_config_1(struct virtio_softc *sc, int index) 136 { 137 return sc->sc_ops->read_dev_cfg_1(sc, index); 138 } 139 140 uint16_t 141 virtio_read_device_config_2(struct virtio_softc *sc, int index) 142 { 143 return sc->sc_ops->read_dev_cfg_2(sc, index); 144 } 145 146 uint32_t 147 virtio_read_device_config_4(struct virtio_softc *sc, int index) 148 { 149 return sc->sc_ops->read_dev_cfg_4(sc, index); 150 } 151 152 uint64_t 153 virtio_read_device_config_8(struct virtio_softc *sc, int index) 154 { 155 return sc->sc_ops->read_dev_cfg_8(sc, index); 156 } 157 158 void 159 virtio_write_device_config_1(struct virtio_softc *sc, 160 int index, uint8_t value) 161 { 162 return sc->sc_ops->write_dev_cfg_1(sc, index, value); 163 } 164 165 void 166 virtio_write_device_config_2(struct virtio_softc *sc, 167 int index, uint16_t value) 168 { 169 return sc->sc_ops->write_dev_cfg_2(sc, index, value); 170 } 171 172 void 173 virtio_write_device_config_4(struct virtio_softc *sc, 174 int index, uint32_t value) 175 { 176 return sc->sc_ops->write_dev_cfg_4(sc, index, value); 177 } 178 179 void 180 virtio_write_device_config_8(struct virtio_softc *sc, 181 int index, uint64_t value) 182 { 183 return sc->sc_ops->write_dev_cfg_8(sc, index, value); 184 } 185 186 /* 187 * Interrupt handler. 188 */ 189 static void 190 virtio_soft_intr(void *arg) 191 { 192 struct virtio_softc *sc = arg; 193 194 KASSERT(sc->sc_intrhand != NULL); 195 196 (sc->sc_intrhand)(sc); 197 } 198 199 /* 200 * dmamap sync operations for a virtqueue. 201 */ 202 static inline void 203 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 204 { 205 /* availoffset == sizeof(vring_desc)*vq_num */ 206 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 207 ops); 208 } 209 210 static inline void 211 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 212 { 213 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 214 vq->vq_availoffset, 215 offsetof(struct vring_avail, ring) 216 + vq->vq_num * sizeof(uint16_t), 217 ops); 218 } 219 220 static inline void 221 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 222 { 223 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 224 vq->vq_usedoffset, 225 offsetof(struct vring_used, ring) 226 + vq->vq_num * sizeof(struct vring_used_elem), 227 ops); 228 } 229 230 static inline void 231 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 232 int ops) 233 { 234 int offset = vq->vq_indirectoffset 235 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 236 237 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 238 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 239 ops); 240 } 241 242 /* 243 * Can be used as sc_intrhand. 244 */ 245 /* 246 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 247 * and calls (*vq_done)() if some entries are consumed. 248 */ 249 int 250 virtio_vq_intr(struct virtio_softc *sc) 251 { 252 struct virtqueue *vq; 253 int i, r = 0; 254 255 for (i = 0; i < sc->sc_nvqs; i++) { 256 vq = &sc->sc_vqs[i]; 257 if (vq->vq_queued) { 258 vq->vq_queued = 0; 259 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 260 } 261 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 262 membar_consumer(); 263 if (vq->vq_used_idx != vq->vq_used->idx) { 264 if (vq->vq_done) 265 r |= (vq->vq_done)(vq); 266 } 267 } 268 269 return r; 270 } 271 272 /* 273 * Start/stop vq interrupt. No guarantee. 274 */ 275 void 276 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 277 { 278 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 279 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 280 vq->vq_queued++; 281 } 282 283 void 284 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 285 { 286 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 287 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 288 vq->vq_queued++; 289 } 290 291 /* 292 * Initialize vq structure. 293 */ 294 static void 295 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, 296 const bool reinit) 297 { 298 int i, j; 299 int vq_size = vq->vq_num; 300 301 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 302 303 /* build the indirect descriptor chain */ 304 if (vq->vq_indirect != NULL) { 305 struct vring_desc *vd; 306 307 for (i = 0; i < vq_size; i++) { 308 vd = vq->vq_indirect; 309 vd += vq->vq_maxnsegs * i; 310 for (j = 0; j < vq->vq_maxnsegs-1; j++) { 311 vd[j].next = j + 1; 312 } 313 } 314 } 315 316 /* free slot management */ 317 SIMPLEQ_INIT(&vq->vq_freelist); 318 for (i = 0; i < vq_size; i++) { 319 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 320 &vq->vq_entries[i], qe_list); 321 vq->vq_entries[i].qe_index = i; 322 } 323 if (!reinit) 324 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 325 326 /* enqueue/dequeue status */ 327 vq->vq_avail_idx = 0; 328 vq->vq_used_idx = 0; 329 vq->vq_queued = 0; 330 if (!reinit) { 331 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 332 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 333 } 334 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 335 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 336 vq->vq_queued++; 337 } 338 339 /* 340 * Allocate/free a vq. 341 */ 342 int 343 virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, 344 int maxsegsize, int maxnsegs, const char *name) 345 { 346 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 347 int rsegs, r; 348 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 349 ~(VIRTIO_PAGE_SIZE-1)) 350 351 /* Make sure callers allocate vqs in order */ 352 KASSERT(sc->sc_nvqs == index); 353 354 memset(vq, 0, sizeof(*vq)); 355 356 vq_size = sc->sc_ops->read_queue_size(sc, index); 357 if (vq_size == 0) { 358 aprint_error_dev(sc->sc_dev, 359 "virtqueue not exist, index %d for %s\n", 360 index, name); 361 goto err; 362 } 363 /* allocsize1: descriptor table + avail ring + pad */ 364 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 365 + sizeof(uint16_t)*(2+vq_size)); 366 /* allocsize2: used ring + pad */ 367 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 368 + sizeof(struct vring_used_elem)*vq_size); 369 /* allocsize3: indirect table */ 370 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 371 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 372 else 373 allocsize3 = 0; 374 allocsize = allocsize1 + allocsize2 + allocsize3; 375 376 /* alloc and map the memory */ 377 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 378 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 379 if (r != 0) { 380 aprint_error_dev(sc->sc_dev, 381 "virtqueue %d for %s allocation failed, " 382 "error code %d\n", index, name, r); 383 goto err; 384 } 385 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 386 &vq->vq_vaddr, BUS_DMA_NOWAIT); 387 if (r != 0) { 388 aprint_error_dev(sc->sc_dev, 389 "virtqueue %d for %s map failed, " 390 "error code %d\n", index, name, r); 391 goto err; 392 } 393 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 394 BUS_DMA_NOWAIT, &vq->vq_dmamap); 395 if (r != 0) { 396 aprint_error_dev(sc->sc_dev, 397 "virtqueue %d for %s dmamap creation failed, " 398 "error code %d\n", index, name, r); 399 goto err; 400 } 401 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 402 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 403 if (r != 0) { 404 aprint_error_dev(sc->sc_dev, 405 "virtqueue %d for %s dmamap load failed, " 406 "error code %d\n", index, name, r); 407 goto err; 408 } 409 410 /* set the vq address */ 411 sc->sc_ops->setup_queue(sc, vq->vq_index, 412 vq->vq_dmamap->dm_segs[0].ds_addr / VIRTIO_PAGE_SIZE); 413 414 /* remember addresses and offsets for later use */ 415 vq->vq_owner = sc; 416 vq->vq_num = vq_size; 417 vq->vq_index = index; 418 vq->vq_desc = vq->vq_vaddr; 419 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 420 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 421 vq->vq_usedoffset = allocsize1; 422 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 423 if (allocsize3 > 0) { 424 vq->vq_indirectoffset = allocsize1 + allocsize2; 425 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 426 + vq->vq_indirectoffset); 427 } 428 vq->vq_bytesize = allocsize; 429 vq->vq_maxsegsize = maxsegsize; 430 vq->vq_maxnsegs = maxnsegs; 431 432 /* free slot management */ 433 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 434 KM_NOSLEEP); 435 if (vq->vq_entries == NULL) { 436 r = ENOMEM; 437 goto err; 438 } 439 440 virtio_init_vq(sc, vq, false); 441 442 aprint_verbose_dev(sc->sc_dev, 443 "allocated %u byte for virtqueue %d for %s, " 444 "size %d\n", allocsize, index, name, vq_size); 445 if (allocsize3 > 0) 446 aprint_verbose_dev(sc->sc_dev, 447 "using %d byte (%d entries) " 448 "indirect descriptors\n", 449 allocsize3, maxnsegs * vq_size); 450 451 sc->sc_nvqs++; 452 453 return 0; 454 455 err: 456 sc->sc_ops->setup_queue(sc, vq->vq_index, 0); 457 if (vq->vq_dmamap) 458 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 459 if (vq->vq_vaddr) 460 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 461 if (vq->vq_segs[0].ds_addr) 462 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 463 memset(vq, 0, sizeof(*vq)); 464 465 return -1; 466 } 467 468 int 469 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 470 { 471 struct vq_entry *qe; 472 int i = 0; 473 474 /* device must be already deactivated */ 475 /* confirm the vq is empty */ 476 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 477 i++; 478 } 479 if (i != vq->vq_num) { 480 printf("%s: freeing non-empty vq, index %d\n", 481 device_xname(sc->sc_dev), vq->vq_index); 482 return EBUSY; 483 } 484 485 /* tell device that there's no virtqueue any longer */ 486 sc->sc_ops->setup_queue(sc, vq->vq_index, 0); 487 488 kmem_free(vq->vq_entries, sizeof(*vq->vq_entries) * vq->vq_num); 489 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 490 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 491 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 492 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 493 mutex_destroy(&vq->vq_freelist_lock); 494 mutex_destroy(&vq->vq_uring_lock); 495 mutex_destroy(&vq->vq_aring_lock); 496 memset(vq, 0, sizeof(*vq)); 497 498 sc->sc_nvqs--; 499 500 return 0; 501 } 502 503 /* 504 * Free descriptor management. 505 */ 506 static struct vq_entry * 507 vq_alloc_entry(struct virtqueue *vq) 508 { 509 struct vq_entry *qe; 510 511 mutex_enter(&vq->vq_freelist_lock); 512 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 513 mutex_exit(&vq->vq_freelist_lock); 514 return NULL; 515 } 516 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 517 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 518 mutex_exit(&vq->vq_freelist_lock); 519 520 return qe; 521 } 522 523 static void 524 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 525 { 526 mutex_enter(&vq->vq_freelist_lock); 527 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 528 mutex_exit(&vq->vq_freelist_lock); 529 530 return; 531 } 532 533 /* 534 * Enqueue several dmamaps as a single request. 535 */ 536 /* 537 * Typical usage: 538 * <queue size> number of followings are stored in arrays 539 * - command blocks (in dmamem) should be pre-allocated and mapped 540 * - dmamaps for command blocks should be pre-allocated and loaded 541 * - dmamaps for payload should be pre-allocated 542 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 543 * if (r) // currently 0 or EAGAIN 544 * return r; 545 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 546 * if (r) { 547 * virtio_enqueue_abort(sc, vq, slot); 548 * return r; 549 * } 550 * r = virtio_enqueue_reserve(sc, vq, slot, 551 * dmamap_payload[slot]->dm_nsegs+1); 552 * // ^ +1 for command 553 * if (r) { // currently 0 or EAGAIN 554 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 555 * return r; // do not call abort() 556 * } 557 * <setup and prepare commands> 558 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 559 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 560 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 561 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 562 * virtio_enqueue_commit(sc, vq, slot, true); 563 */ 564 565 /* 566 * enqueue_prep: allocate a slot number 567 */ 568 int 569 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 570 { 571 struct vq_entry *qe1; 572 573 KASSERT(slotp != NULL); 574 575 qe1 = vq_alloc_entry(vq); 576 if (qe1 == NULL) 577 return EAGAIN; 578 /* next slot is not allocated yet */ 579 qe1->qe_next = -1; 580 *slotp = qe1->qe_index; 581 582 return 0; 583 } 584 585 /* 586 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 587 */ 588 int 589 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 590 int slot, int nsegs) 591 { 592 int indirect; 593 struct vq_entry *qe1 = &vq->vq_entries[slot]; 594 595 KASSERT(qe1->qe_next == -1); 596 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 597 598 if ((vq->vq_indirect != NULL) && 599 (nsegs >= MINSEG_INDIRECT) && 600 (nsegs <= vq->vq_maxnsegs)) 601 indirect = 1; 602 else 603 indirect = 0; 604 qe1->qe_indirect = indirect; 605 606 if (indirect) { 607 struct vring_desc *vd; 608 int i; 609 610 vd = &vq->vq_desc[qe1->qe_index]; 611 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 612 + vq->vq_indirectoffset; 613 vd->addr += sizeof(struct vring_desc) 614 * vq->vq_maxnsegs * qe1->qe_index; 615 vd->len = sizeof(struct vring_desc) * nsegs; 616 vd->flags = VRING_DESC_F_INDIRECT; 617 618 vd = vq->vq_indirect; 619 vd += vq->vq_maxnsegs * qe1->qe_index; 620 qe1->qe_desc_base = vd; 621 622 for (i = 0; i < nsegs-1; i++) { 623 vd[i].flags = VRING_DESC_F_NEXT; 624 } 625 vd[i].flags = 0; 626 qe1->qe_next = 0; 627 628 return 0; 629 } else { 630 struct vring_desc *vd; 631 struct vq_entry *qe; 632 int i, s; 633 634 vd = &vq->vq_desc[0]; 635 qe1->qe_desc_base = vd; 636 qe1->qe_next = qe1->qe_index; 637 s = slot; 638 for (i = 0; i < nsegs - 1; i++) { 639 qe = vq_alloc_entry(vq); 640 if (qe == NULL) { 641 vd[s].flags = 0; 642 virtio_enqueue_abort(sc, vq, slot); 643 return EAGAIN; 644 } 645 vd[s].flags = VRING_DESC_F_NEXT; 646 vd[s].next = qe->qe_index; 647 s = qe->qe_index; 648 } 649 vd[s].flags = 0; 650 651 return 0; 652 } 653 } 654 655 /* 656 * enqueue: enqueue a single dmamap. 657 */ 658 int 659 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 660 bus_dmamap_t dmamap, bool write) 661 { 662 struct vq_entry *qe1 = &vq->vq_entries[slot]; 663 struct vring_desc *vd = qe1->qe_desc_base; 664 int i; 665 int s = qe1->qe_next; 666 667 KASSERT(s >= 0); 668 KASSERT(dmamap->dm_nsegs > 0); 669 670 for (i = 0; i < dmamap->dm_nsegs; i++) { 671 vd[s].addr = dmamap->dm_segs[i].ds_addr; 672 vd[s].len = dmamap->dm_segs[i].ds_len; 673 if (!write) 674 vd[s].flags |= VRING_DESC_F_WRITE; 675 s = vd[s].next; 676 } 677 qe1->qe_next = s; 678 679 return 0; 680 } 681 682 int 683 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 684 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 685 bool write) 686 { 687 struct vq_entry *qe1 = &vq->vq_entries[slot]; 688 struct vring_desc *vd = qe1->qe_desc_base; 689 int s = qe1->qe_next; 690 691 KASSERT(s >= 0); 692 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 693 KASSERT((dmamap->dm_segs[0].ds_len > start) && 694 (dmamap->dm_segs[0].ds_len >= start + len)); 695 696 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 697 vd[s].len = len; 698 if (!write) 699 vd[s].flags |= VRING_DESC_F_WRITE; 700 qe1->qe_next = vd[s].next; 701 702 return 0; 703 } 704 705 /* 706 * enqueue_commit: add it to the aring. 707 */ 708 int 709 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 710 bool notifynow) 711 { 712 struct vq_entry *qe1; 713 714 if (slot < 0) { 715 mutex_enter(&vq->vq_aring_lock); 716 goto notify; 717 } 718 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 719 qe1 = &vq->vq_entries[slot]; 720 if (qe1->qe_indirect) 721 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 722 mutex_enter(&vq->vq_aring_lock); 723 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 724 725 notify: 726 if (notifynow) { 727 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 728 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 729 membar_producer(); 730 vq->vq_avail->idx = vq->vq_avail_idx; 731 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 732 membar_producer(); 733 vq->vq_queued++; 734 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 735 membar_consumer(); 736 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 737 sc->sc_ops->kick(sc, vq->vq_index); 738 } 739 mutex_exit(&vq->vq_aring_lock); 740 741 return 0; 742 } 743 744 /* 745 * enqueue_abort: rollback. 746 */ 747 int 748 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 749 { 750 struct vq_entry *qe = &vq->vq_entries[slot]; 751 struct vring_desc *vd; 752 int s; 753 754 if (qe->qe_next < 0) { 755 vq_free_entry(vq, qe); 756 return 0; 757 } 758 759 s = slot; 760 vd = &vq->vq_desc[0]; 761 while (vd[s].flags & VRING_DESC_F_NEXT) { 762 s = vd[s].next; 763 vq_free_entry(vq, qe); 764 qe = &vq->vq_entries[s]; 765 } 766 vq_free_entry(vq, qe); 767 return 0; 768 } 769 770 /* 771 * Dequeue a request. 772 */ 773 /* 774 * dequeue: dequeue a request from uring; dmamap_sync for uring is 775 * already done in the interrupt handler. 776 */ 777 int 778 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 779 int *slotp, int *lenp) 780 { 781 uint16_t slot, usedidx; 782 struct vq_entry *qe; 783 784 if (vq->vq_used_idx == vq->vq_used->idx) 785 return ENOENT; 786 mutex_enter(&vq->vq_uring_lock); 787 usedidx = vq->vq_used_idx++; 788 mutex_exit(&vq->vq_uring_lock); 789 usedidx %= vq->vq_num; 790 slot = vq->vq_used->ring[usedidx].id; 791 qe = &vq->vq_entries[slot]; 792 793 if (qe->qe_indirect) 794 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 795 796 if (slotp) 797 *slotp = slot; 798 if (lenp) 799 *lenp = vq->vq_used->ring[usedidx].len; 800 801 return 0; 802 } 803 804 /* 805 * dequeue_commit: complete dequeue; the slot is recycled for future use. 806 * if you forget to call this the slot will be leaked. 807 */ 808 int 809 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 810 { 811 struct vq_entry *qe = &vq->vq_entries[slot]; 812 struct vring_desc *vd = &vq->vq_desc[0]; 813 int s = slot; 814 815 while (vd[s].flags & VRING_DESC_F_NEXT) { 816 s = vd[s].next; 817 vq_free_entry(vq, qe); 818 qe = &vq->vq_entries[s]; 819 } 820 vq_free_entry(vq, qe); 821 822 return 0; 823 } 824 825 /* 826 * Attach a child, fill all the members. 827 */ 828 void 829 virtio_child_attach_start(struct virtio_softc *sc, device_t child, int ipl, 830 struct virtqueue *vqs, 831 virtio_callback config_change, 832 virtio_callback intr_hand, 833 int req_flags, int req_features, const char *feat_bits) 834 { 835 char buf[256]; 836 int features; 837 838 sc->sc_child = child; 839 sc->sc_ipl = ipl; 840 sc->sc_vqs = vqs; 841 sc->sc_config_change = config_change; 842 sc->sc_intrhand = intr_hand; 843 sc->sc_flags = req_flags; 844 845 features = virtio_negotiate_features(sc, req_features); 846 snprintb(buf, sizeof(buf), feat_bits, features); 847 aprint_normal(": Features: %s\n", buf); 848 aprint_naive("\n"); 849 } 850 851 int 852 virtio_child_attach_finish(struct virtio_softc *sc) 853 { 854 int r; 855 856 r = sc->sc_ops->setup_interrupts(sc); 857 if (r != 0) { 858 aprint_error_dev(sc->sc_dev, "failed to setup interrupts\n"); 859 goto fail; 860 } 861 862 KASSERT(sc->sc_soft_ih == NULL); 863 if (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) { 864 u_int flags = SOFTINT_NET; 865 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 866 flags |= SOFTINT_MPSAFE; 867 868 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, sc); 869 if (sc->sc_soft_ih == NULL) { 870 sc->sc_ops->free_interrupts(sc); 871 aprint_error_dev(sc->sc_dev, 872 "failed to establish soft interrupt\n"); 873 goto fail; 874 } 875 } 876 877 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 878 return 0; 879 880 fail: 881 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 882 return 1; 883 } 884 885 void 886 virtio_child_detach(struct virtio_softc *sc) 887 { 888 sc->sc_child = NULL; 889 sc->sc_vqs = NULL; 890 891 virtio_device_reset(sc); 892 893 sc->sc_ops->free_interrupts(sc); 894 895 if (sc->sc_soft_ih) { 896 softint_disestablish(sc->sc_soft_ih); 897 sc->sc_soft_ih = NULL; 898 } 899 } 900 901 void 902 virtio_child_attach_failed(struct virtio_softc *sc) 903 { 904 virtio_child_detach(sc); 905 906 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 907 908 sc->sc_child = VIRTIO_CHILD_FAILED; 909 } 910 911 bus_dma_tag_t 912 virtio_dmat(struct virtio_softc *sc) 913 { 914 return sc->sc_dmat; 915 } 916 917 device_t 918 virtio_child(struct virtio_softc *sc) 919 { 920 return sc->sc_child; 921 } 922 923 int 924 virtio_intrhand(struct virtio_softc *sc) 925 { 926 return (sc->sc_intrhand)(sc); 927 } 928 929 uint32_t 930 virtio_features(struct virtio_softc *sc) 931 { 932 return sc->sc_features; 933 } 934 935 MODULE(MODULE_CLASS_DRIVER, virtio, "pci"); 936 937 #ifdef _MODULE 938 #include "ioconf.c" 939 #endif 940 941 static int 942 virtio_modcmd(modcmd_t cmd, void *opaque) 943 { 944 int error = 0; 945 946 #ifdef _MODULE 947 switch (cmd) { 948 case MODULE_CMD_INIT: 949 error = config_init_component(cfdriver_ioconf_virtio, 950 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 951 break; 952 case MODULE_CMD_FINI: 953 error = config_fini_component(cfdriver_ioconf_virtio, 954 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 955 break; 956 default: 957 error = ENOTTY; 958 break; 959 } 960 #endif 961 962 return error; 963 } 964
Indexes created Sun Sep 21 20:09:37 GMT 2025