virtio.c revision 1.63.2.4
1 /* $NetBSD: virtio.c,v 1.63.2.4 2023/05/13 10:56:10 martin Exp $ */ 2 3 /* 4 * Copyright (c) 2020 The NetBSD Foundation, Inc. 5 * Copyright (c) 2012 Stefan Fritsch, Alexander Fiveg. 6 * Copyright (c) 2010 Minoura Makoto. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.63.2.4 2023/05/13 10:56:10 martin Exp $"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kernel.h> 36 #include <sys/atomic.h> 37 #include <sys/bus.h> 38 #include <sys/device.h> 39 #include <sys/kmem.h> 40 #include <sys/module.h> 41 42 #define VIRTIO_PRIVATE 43 44 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */ 45 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */ 46 47 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 48 49 /* 50 * The maximum descriptor size is 2^15. Use that value as the end of 51 * descriptor chain terminator since it will never be a valid index 52 * in the descriptor table. 53 */ 54 #define VRING_DESC_CHAIN_END 32768 55 56 /* incomplete list */ 57 static const char *virtio_device_name[] = { 58 "unknown (0)", /* 0 */ 59 "network", /* 1 */ 60 "block", /* 2 */ 61 "console", /* 3 */ 62 "entropy", /* 4 */ 63 "memory balloon", /* 5 */ 64 "I/O memory", /* 6 */ 65 "remote processor messaging", /* 7 */ 66 "SCSI", /* 8 */ 67 "9P transport", /* 9 */ 68 }; 69 #define NDEVNAMES __arraycount(virtio_device_name) 70 71 static void virtio_reset_vq(struct virtio_softc *, 72 struct virtqueue *); 73 74 void 75 virtio_set_status(struct virtio_softc *sc, int status) 76 { 77 sc->sc_ops->set_status(sc, status); 78 } 79 80 /* 81 * Reset the device. 82 */ 83 /* 84 * To reset the device to a known state, do following: 85 * virtio_reset(sc); // this will stop the device activity 86 * <dequeue finished requests>; // virtio_dequeue() still can be called 87 * <revoke pending requests in the vqs if any>; 88 * virtio_reinit_start(sc); // dequeue prohibitted 89 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 90 * <some other initialization>; 91 * virtio_reinit_end(sc); // device activated; enqueue allowed 92 * Once attached, feature negotiation can only be allowed after virtio_reset. 93 */ 94 void 95 virtio_reset(struct virtio_softc *sc) 96 { 97 virtio_device_reset(sc); 98 } 99 100 int 101 virtio_reinit_start(struct virtio_softc *sc) 102 { 103 int i, r; 104 105 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 106 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 107 for (i = 0; i < sc->sc_nvqs; i++) { 108 int n; 109 struct virtqueue *vq = &sc->sc_vqs[i]; 110 n = sc->sc_ops->read_queue_size(sc, vq->vq_index); 111 if (n == 0) /* vq disappeared */ 112 continue; 113 if (n != vq->vq_num) { 114 panic("%s: virtqueue size changed, vq index %d\n", 115 device_xname(sc->sc_dev), 116 vq->vq_index); 117 } 118 virtio_reset_vq(sc, vq); 119 sc->sc_ops->setup_queue(sc, vq->vq_index, 120 vq->vq_dmamap->dm_segs[0].ds_addr); 121 } 122 123 r = sc->sc_ops->setup_interrupts(sc, 1); 124 if (r != 0) 125 goto fail; 126 127 return 0; 128 129 fail: 130 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 131 132 return 1; 133 } 134 135 void 136 virtio_reinit_end(struct virtio_softc *sc) 137 { 138 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 139 } 140 141 /* 142 * Feature negotiation. 143 */ 144 void 145 virtio_negotiate_features(struct virtio_softc *sc, uint64_t guest_features) 146 { 147 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 148 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 149 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 150 sc->sc_ops->neg_features(sc, guest_features); 151 if (sc->sc_active_features & VIRTIO_F_RING_INDIRECT_DESC) 152 sc->sc_indirect = true; 153 else 154 sc->sc_indirect = false; 155 } 156 157 158 /* 159 * Device configuration registers readers/writers 160 */ 161 #if 0 162 #define DPRINTFR(n, fmt, val, index, num) \ 163 printf("\n%s (", n); \ 164 for (int i = 0; i < num; i++) \ 165 printf("%02x ", bus_space_read_1(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index+i)); \ 166 printf(") -> "); printf(fmt, val); printf("\n"); 167 #define DPRINTFR2(n, fmt, val_s, val_n) \ 168 printf("%s ", n); \ 169 printf("\n stream "); printf(fmt, val_s); printf(" norm "); printf(fmt, val_n); printf("\n"); 170 #else 171 #define DPRINTFR(n, fmt, val, index, num) 172 #define DPRINTFR2(n, fmt, val_s, val_n) 173 #endif 174 175 176 uint8_t 177 virtio_read_device_config_1(struct virtio_softc *sc, int index) 178 { 179 bus_space_tag_t iot = sc->sc_devcfg_iot; 180 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 181 uint8_t val; 182 183 val = bus_space_read_1(iot, ioh, index); 184 185 DPRINTFR("read_1", "%02x", val, index, 1); 186 return val; 187 } 188 189 uint16_t 190 virtio_read_device_config_2(struct virtio_softc *sc, int index) 191 { 192 bus_space_tag_t iot = sc->sc_devcfg_iot; 193 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 194 uint16_t val; 195 196 val = bus_space_read_2(iot, ioh, index); 197 if (BYTE_ORDER != sc->sc_bus_endian) 198 val = bswap16(val); 199 200 DPRINTFR("read_2", "%04x", val, index, 2); 201 DPRINTFR2("read_2", "%04x", 202 bus_space_read_stream_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 203 index), 204 bus_space_read_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index)); 205 return val; 206 } 207 208 uint32_t 209 virtio_read_device_config_4(struct virtio_softc *sc, int index) 210 { 211 bus_space_tag_t iot = sc->sc_devcfg_iot; 212 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 213 uint32_t val; 214 215 val = bus_space_read_4(iot, ioh, index); 216 if (BYTE_ORDER != sc->sc_bus_endian) 217 val = bswap32(val); 218 219 DPRINTFR("read_4", "%08x", val, index, 4); 220 DPRINTFR2("read_4", "%08x", 221 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 222 index), 223 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index)); 224 return val; 225 } 226 227 /* 228 * The Virtio spec explicitly tells that reading and writing 8 bytes are not 229 * considered atomic and no triggers may be connected to reading or writing 230 * it. We access it using two 32 reads. See virtio spec 4.1.3.1. 231 */ 232 uint64_t 233 virtio_read_device_config_8(struct virtio_softc *sc, int index) 234 { 235 bus_space_tag_t iot = sc->sc_devcfg_iot; 236 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 237 union { 238 uint64_t u64; 239 uint32_t l[2]; 240 } v; 241 uint64_t val; 242 243 v.l[0] = bus_space_read_4(iot, ioh, index); 244 v.l[1] = bus_space_read_4(iot, ioh, index + 4); 245 if (sc->sc_bus_endian != sc->sc_struct_endian) { 246 v.l[0] = bswap32(v.l[0]); 247 v.l[1] = bswap32(v.l[1]); 248 } 249 val = v.u64; 250 251 if (BYTE_ORDER != sc->sc_struct_endian) 252 val = bswap64(val); 253 254 DPRINTFR("read_8", "%08"PRIx64, val, index, 8); 255 DPRINTFR2("read_8 low ", "%08x", 256 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 257 index), 258 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index)); 259 DPRINTFR2("read_8 high ", "%08x", 260 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 261 index + 4), 262 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index + 4)); 263 return val; 264 } 265 266 /* 267 * In the older virtio spec, device config registers are host endian. On newer 268 * they are little endian. Some newer devices however explicitly specify their 269 * register to always be little endian. These functions cater for these. 270 */ 271 uint16_t 272 virtio_read_device_config_le_2(struct virtio_softc *sc, int index) 273 { 274 bus_space_tag_t iot = sc->sc_devcfg_iot; 275 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 276 uint16_t val; 277 278 val = bus_space_read_2(iot, ioh, index); 279 if (sc->sc_bus_endian != LITTLE_ENDIAN) 280 val = bswap16(val); 281 282 DPRINTFR("read_le_2", "%04x", val, index, 2); 283 DPRINTFR2("read_le_2", "%04x", 284 bus_space_read_stream_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0), 285 bus_space_read_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0)); 286 return val; 287 } 288 289 uint32_t 290 virtio_read_device_config_le_4(struct virtio_softc *sc, int index) 291 { 292 bus_space_tag_t iot = sc->sc_devcfg_iot; 293 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 294 uint32_t val; 295 296 val = bus_space_read_4(iot, ioh, index); 297 if (sc->sc_bus_endian != LITTLE_ENDIAN) 298 val = bswap32(val); 299 300 DPRINTFR("read_le_4", "%08x", val, index, 4); 301 DPRINTFR2("read_le_4", "%08x", 302 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0), 303 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0)); 304 return val; 305 } 306 307 void 308 virtio_write_device_config_1(struct virtio_softc *sc, int index, uint8_t value) 309 { 310 bus_space_tag_t iot = sc->sc_devcfg_iot; 311 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 312 313 bus_space_write_1(iot, ioh, index, value); 314 } 315 316 void 317 virtio_write_device_config_2(struct virtio_softc *sc, int index, 318 uint16_t value) 319 { 320 bus_space_tag_t iot = sc->sc_devcfg_iot; 321 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 322 323 if (BYTE_ORDER != sc->sc_bus_endian) 324 value = bswap16(value); 325 bus_space_write_2(iot, ioh, index, value); 326 } 327 328 void 329 virtio_write_device_config_4(struct virtio_softc *sc, int index, 330 uint32_t value) 331 { 332 bus_space_tag_t iot = sc->sc_devcfg_iot; 333 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 334 335 if (BYTE_ORDER != sc->sc_bus_endian) 336 value = bswap32(value); 337 bus_space_write_4(iot, ioh, index, value); 338 } 339 340 /* 341 * The Virtio spec explicitly tells that reading and writing 8 bytes are not 342 * considered atomic and no triggers may be connected to reading or writing 343 * it. We access it using two 32 bit writes. For good measure it is stated to 344 * always write lsb first just in case of a hypervisor bug. See See virtio 345 * spec 4.1.3.1. 346 */ 347 void 348 virtio_write_device_config_8(struct virtio_softc *sc, int index, 349 uint64_t value) 350 { 351 bus_space_tag_t iot = sc->sc_devcfg_iot; 352 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 353 union { 354 uint64_t u64; 355 uint32_t l[2]; 356 } v; 357 358 if (BYTE_ORDER != sc->sc_struct_endian) 359 value = bswap64(value); 360 361 v.u64 = value; 362 if (sc->sc_bus_endian != sc->sc_struct_endian) { 363 v.l[0] = bswap32(v.l[0]); 364 v.l[1] = bswap32(v.l[1]); 365 } 366 367 if (sc->sc_struct_endian == LITTLE_ENDIAN) { 368 bus_space_write_4(iot, ioh, index, v.l[0]); 369 bus_space_write_4(iot, ioh, index + 4, v.l[1]); 370 } else { 371 bus_space_write_4(iot, ioh, index + 4, v.l[1]); 372 bus_space_write_4(iot, ioh, index, v.l[0]); 373 } 374 } 375 376 /* 377 * In the older virtio spec, device config registers are host endian. On newer 378 * they are little endian. Some newer devices however explicitly specify their 379 * register to always be little endian. These functions cater for these. 380 */ 381 void 382 virtio_write_device_config_le_2(struct virtio_softc *sc, int index, 383 uint16_t value) 384 { 385 bus_space_tag_t iot = sc->sc_devcfg_iot; 386 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 387 388 if (sc->sc_bus_endian != LITTLE_ENDIAN) 389 value = bswap16(value); 390 bus_space_write_2(iot, ioh, index, value); 391 } 392 393 void 394 virtio_write_device_config_le_4(struct virtio_softc *sc, int index, 395 uint32_t value) 396 { 397 bus_space_tag_t iot = sc->sc_devcfg_iot; 398 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 399 400 if (sc->sc_bus_endian != LITTLE_ENDIAN) 401 value = bswap32(value); 402 bus_space_write_4(iot, ioh, index, value); 403 } 404 405 406 /* 407 * data structures endian helpers 408 */ 409 uint16_t 410 virtio_rw16(struct virtio_softc *sc, uint16_t val) 411 { 412 KASSERT(sc); 413 return BYTE_ORDER != sc->sc_struct_endian ? bswap16(val) : val; 414 } 415 416 uint32_t 417 virtio_rw32(struct virtio_softc *sc, uint32_t val) 418 { 419 KASSERT(sc); 420 return BYTE_ORDER != sc->sc_struct_endian ? bswap32(val) : val; 421 } 422 423 uint64_t 424 virtio_rw64(struct virtio_softc *sc, uint64_t val) 425 { 426 KASSERT(sc); 427 return BYTE_ORDER != sc->sc_struct_endian ? bswap64(val) : val; 428 } 429 430 431 /* 432 * Interrupt handler. 433 */ 434 static void 435 virtio_soft_intr(void *arg) 436 { 437 struct virtio_softc *sc = arg; 438 439 KASSERT(sc->sc_intrhand != NULL); 440 441 (*sc->sc_intrhand)(sc); 442 } 443 444 /* set to vq->vq_intrhand in virtio_init_vq_vqdone() */ 445 static int 446 virtio_vq_done(void *xvq) 447 { 448 struct virtqueue *vq = xvq; 449 450 return vq->vq_done(vq); 451 } 452 453 static int 454 virtio_vq_intr(struct virtio_softc *sc) 455 { 456 struct virtqueue *vq; 457 int i, r = 0; 458 459 for (i = 0; i < sc->sc_nvqs; i++) { 460 vq = &sc->sc_vqs[i]; 461 if (virtio_vq_is_enqueued(sc, vq) == 1) { 462 r |= (*vq->vq_intrhand)(vq->vq_intrhand_arg); 463 } 464 } 465 466 return r; 467 } 468 469 /* 470 * dmamap sync operations for a virtqueue. 471 */ 472 static inline void 473 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 474 { 475 476 /* availoffset == sizeof(vring_desc) * vq_num */ 477 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 478 ops); 479 } 480 481 static inline void 482 vq_sync_aring_all(struct virtio_softc *sc, struct virtqueue *vq, int ops) 483 { 484 uint16_t hdrlen = offsetof(struct vring_avail, ring); 485 size_t payloadlen = vq->vq_num * sizeof(uint16_t); 486 size_t usedlen = 0; 487 488 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) 489 usedlen = sizeof(uint16_t); 490 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 491 vq->vq_availoffset, hdrlen + payloadlen + usedlen, ops); 492 } 493 494 static inline void 495 vq_sync_aring_header(struct virtio_softc *sc, struct virtqueue *vq, int ops) 496 { 497 uint16_t hdrlen = offsetof(struct vring_avail, ring); 498 499 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 500 vq->vq_availoffset, hdrlen, ops); 501 } 502 503 static inline void 504 vq_sync_aring_payload(struct virtio_softc *sc, struct virtqueue *vq, int ops) 505 { 506 uint16_t hdrlen = offsetof(struct vring_avail, ring); 507 size_t payloadlen = vq->vq_num * sizeof(uint16_t); 508 509 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 510 vq->vq_availoffset + hdrlen, payloadlen, ops); 511 } 512 513 static inline void 514 vq_sync_aring_used(struct virtio_softc *sc, struct virtqueue *vq, int ops) 515 { 516 uint16_t hdrlen = offsetof(struct vring_avail, ring); 517 size_t payloadlen = vq->vq_num * sizeof(uint16_t); 518 size_t usedlen = sizeof(uint16_t); 519 520 if ((sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) == 0) 521 return; 522 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 523 vq->vq_availoffset + hdrlen + payloadlen, usedlen, ops); 524 } 525 526 static inline void 527 vq_sync_uring_all(struct virtio_softc *sc, struct virtqueue *vq, int ops) 528 { 529 uint16_t hdrlen = offsetof(struct vring_used, ring); 530 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem); 531 size_t availlen = 0; 532 533 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) 534 availlen = sizeof(uint16_t); 535 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 536 vq->vq_usedoffset, hdrlen + payloadlen + availlen, ops); 537 } 538 539 static inline void 540 vq_sync_uring_header(struct virtio_softc *sc, struct virtqueue *vq, int ops) 541 { 542 uint16_t hdrlen = offsetof(struct vring_used, ring); 543 544 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 545 vq->vq_usedoffset, hdrlen, ops); 546 } 547 548 static inline void 549 vq_sync_uring_payload(struct virtio_softc *sc, struct virtqueue *vq, int ops) 550 { 551 uint16_t hdrlen = offsetof(struct vring_used, ring); 552 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem); 553 554 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 555 vq->vq_usedoffset + hdrlen, payloadlen, ops); 556 } 557 558 static inline void 559 vq_sync_uring_avail(struct virtio_softc *sc, struct virtqueue *vq, int ops) 560 { 561 uint16_t hdrlen = offsetof(struct vring_used, ring); 562 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem); 563 size_t availlen = sizeof(uint16_t); 564 565 if ((sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) == 0) 566 return; 567 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 568 vq->vq_usedoffset + hdrlen + payloadlen, availlen, ops); 569 } 570 571 static inline void 572 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 573 int ops) 574 { 575 int offset = vq->vq_indirectoffset + 576 sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 577 578 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 579 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, ops); 580 } 581 582 bool 583 virtio_vq_is_enqueued(struct virtio_softc *sc, struct virtqueue *vq) 584 { 585 586 if (vq->vq_queued) { 587 vq->vq_queued = 0; 588 vq_sync_aring_all(sc, vq, BUS_DMASYNC_POSTWRITE); 589 } 590 591 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD); 592 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx)) 593 return 0; 594 vq_sync_uring_payload(sc, vq, BUS_DMASYNC_POSTREAD); 595 return 1; 596 } 597 598 /* 599 * Increase the event index in order to delay interrupts. 600 */ 601 int 602 virtio_postpone_intr(struct virtio_softc *sc, struct virtqueue *vq, 603 uint16_t nslots) 604 { 605 uint16_t idx, nused; 606 607 idx = vq->vq_used_idx + nslots; 608 609 /* set the new event index: avail_ring->used_event = idx */ 610 *vq->vq_used_event = virtio_rw16(sc, idx); 611 vq_sync_aring_used(vq->vq_owner, vq, BUS_DMASYNC_PREWRITE); 612 vq->vq_queued++; 613 614 nused = (uint16_t) 615 (virtio_rw16(sc, vq->vq_used->idx) - vq->vq_used_idx); 616 KASSERT(nused <= vq->vq_num); 617 618 return nslots < nused; 619 } 620 621 /* 622 * Postpone interrupt until 3/4 of the available descriptors have been 623 * consumed. 624 */ 625 int 626 virtio_postpone_intr_smart(struct virtio_softc *sc, struct virtqueue *vq) 627 { 628 uint16_t nslots; 629 630 nslots = (uint16_t) 631 (virtio_rw16(sc, vq->vq_avail->idx) - vq->vq_used_idx) * 3 / 4; 632 633 return virtio_postpone_intr(sc, vq, nslots); 634 } 635 636 /* 637 * Postpone interrupt until all of the available descriptors have been 638 * consumed. 639 */ 640 int 641 virtio_postpone_intr_far(struct virtio_softc *sc, struct virtqueue *vq) 642 { 643 uint16_t nslots; 644 645 nslots = (uint16_t) 646 (virtio_rw16(sc, vq->vq_avail->idx) - vq->vq_used_idx); 647 648 return virtio_postpone_intr(sc, vq, nslots); 649 } 650 651 /* 652 * Start/stop vq interrupt. No guarantee. 653 */ 654 void 655 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 656 { 657 658 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) { 659 /* 660 * No way to disable the interrupt completely with 661 * RingEventIdx. Instead advance used_event by half the 662 * possible value. This won't happen soon and is far enough in 663 * the past to not trigger a spurios interrupt. 664 */ 665 *vq->vq_used_event = virtio_rw16(sc, vq->vq_used_idx + 0x8000); 666 vq_sync_aring_used(sc, vq, BUS_DMASYNC_PREWRITE); 667 } else { 668 vq->vq_avail->flags |= 669 virtio_rw16(sc, VRING_AVAIL_F_NO_INTERRUPT); 670 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE); 671 } 672 vq->vq_queued++; 673 } 674 675 int 676 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 677 { 678 679 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) { 680 /* 681 * If event index feature is negotiated, enabling interrupts 682 * is done through setting the latest consumed index in the 683 * used_event field 684 */ 685 *vq->vq_used_event = virtio_rw16(sc, vq->vq_used_idx); 686 vq_sync_aring_used(sc, vq, BUS_DMASYNC_PREWRITE); 687 } else { 688 vq->vq_avail->flags &= 689 ~virtio_rw16(sc, VRING_AVAIL_F_NO_INTERRUPT); 690 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE); 691 } 692 vq->vq_queued++; 693 694 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD); 695 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx)) 696 return 0; 697 vq_sync_uring_payload(sc, vq, BUS_DMASYNC_POSTREAD); 698 return 1; 699 } 700 701 /* 702 * Initialize vq structure. 703 */ 704 /* 705 * Reset virtqueue parameters 706 */ 707 static void 708 virtio_reset_vq(struct virtio_softc *sc, struct virtqueue *vq) 709 { 710 struct vring_desc *vds; 711 int i, j; 712 int vq_size = vq->vq_num; 713 714 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 715 716 /* build the descriptor chain for free slot management */ 717 vds = vq->vq_desc; 718 for (i = 0; i < vq_size - 1; i++) { 719 vds[i].next = virtio_rw16(sc, i + 1); 720 } 721 vds[i].next = virtio_rw16(sc, VRING_DESC_CHAIN_END); 722 vq->vq_free_idx = 0; 723 724 /* build the indirect descriptor chain */ 725 if (vq->vq_indirect != NULL) { 726 struct vring_desc *vd; 727 728 for (i = 0; i < vq_size; i++) { 729 vd = vq->vq_indirect; 730 vd += vq->vq_maxnsegs * i; 731 for (j = 0; j < vq->vq_maxnsegs - 1; j++) { 732 vd[j].next = virtio_rw16(sc, j + 1); 733 } 734 } 735 } 736 737 /* enqueue/dequeue status */ 738 vq->vq_avail_idx = 0; 739 vq->vq_used_idx = 0; 740 vq->vq_queued = 0; 741 vq_sync_uring_all(sc, vq, BUS_DMASYNC_PREREAD); 742 vq->vq_queued++; 743 } 744 745 /* Initialize vq */ 746 void 747 virtio_init_vq_vqdone(struct virtio_softc *sc, struct virtqueue *vq, 748 int index, int (*vq_done)(struct virtqueue *)) 749 { 750 751 virtio_init_vq(sc, vq, index, virtio_vq_done, vq); 752 vq->vq_done = vq_done; 753 } 754 755 void 756 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, 757 int (*func)(void *), void *arg) 758 { 759 760 memset(vq, 0, sizeof(*vq)); 761 762 vq->vq_owner = sc; 763 vq->vq_num = sc->sc_ops->read_queue_size(sc, index); 764 vq->vq_index = index; 765 vq->vq_intrhand = func; 766 vq->vq_intrhand_arg = arg; 767 } 768 769 /* 770 * Allocate/free a vq. 771 */ 772 int 773 virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, 774 int maxsegsize, int maxnsegs, const char *name) 775 { 776 bus_size_t size_desc, size_avail, size_used, size_indirect; 777 bus_size_t allocsize = 0, size_desc_avail; 778 int rsegs, r, hdrlen; 779 unsigned int vq_num; 780 #define VIRTQUEUE_ALIGN(n) roundup(n, VIRTIO_PAGE_SIZE) 781 782 vq_num = vq->vq_num; 783 784 if (vq_num == 0) { 785 aprint_error_dev(sc->sc_dev, 786 "virtqueue not exist, index %d for %s\n", 787 vq->vq_index, name); 788 goto err; 789 } 790 791 hdrlen = sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX ? 3 : 2; 792 793 size_desc = sizeof(vq->vq_desc[0]) * vq_num; 794 size_avail = sizeof(uint16_t) * hdrlen 795 + sizeof(vq->vq_avail[0].ring[0]) * vq_num; 796 size_used = sizeof(uint16_t) *hdrlen 797 + sizeof(vq->vq_used[0].ring[0]) * vq_num; 798 size_indirect = (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) ? 799 sizeof(struct vring_desc) * maxnsegs * vq_num : 0; 800 801 size_desc_avail = VIRTQUEUE_ALIGN(size_desc + size_avail); 802 size_used = VIRTQUEUE_ALIGN(size_used); 803 804 allocsize = size_desc_avail + size_used + size_indirect; 805 806 /* alloc and map the memory */ 807 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 808 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_WAITOK); 809 if (r != 0) { 810 aprint_error_dev(sc->sc_dev, 811 "virtqueue %d for %s allocation failed, " 812 "error code %d\n", vq->vq_index, name, r); 813 goto err; 814 } 815 816 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], rsegs, allocsize, 817 &vq->vq_vaddr, BUS_DMA_WAITOK); 818 if (r != 0) { 819 aprint_error_dev(sc->sc_dev, 820 "virtqueue %d for %s map failed, " 821 "error code %d\n", vq->vq_index, name, r); 822 goto err; 823 } 824 825 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 826 BUS_DMA_WAITOK, &vq->vq_dmamap); 827 if (r != 0) { 828 aprint_error_dev(sc->sc_dev, 829 "virtqueue %d for %s dmamap creation failed, " 830 "error code %d\n", vq->vq_index, name, r); 831 goto err; 832 } 833 834 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 835 vq->vq_vaddr, allocsize, NULL, BUS_DMA_WAITOK); 836 if (r != 0) { 837 aprint_error_dev(sc->sc_dev, 838 "virtqueue %d for %s dmamap load failed, " 839 "error code %d\n", vq->vq_index, name, r); 840 goto err; 841 } 842 843 vq->vq_bytesize = allocsize; 844 vq->vq_maxsegsize = maxsegsize; 845 vq->vq_maxnsegs = maxnsegs; 846 847 #define VIRTIO_PTR(base, offset) (void *)((intptr_t)(base) + (offset)) 848 /* initialize vring pointers */ 849 vq->vq_desc = VIRTIO_PTR(vq->vq_vaddr, 0); 850 vq->vq_availoffset = size_desc; 851 vq->vq_avail = VIRTIO_PTR(vq->vq_vaddr, vq->vq_availoffset); 852 vq->vq_used_event = VIRTIO_PTR(vq->vq_avail, 853 offsetof(struct vring_avail, ring[vq_num])); 854 vq->vq_usedoffset = size_desc_avail; 855 vq->vq_used = VIRTIO_PTR(vq->vq_vaddr, vq->vq_usedoffset); 856 vq->vq_avail_event = VIRTIO_PTR(vq->vq_used, 857 offsetof(struct vring_used, ring[vq_num])); 858 859 if (size_indirect > 0) { 860 vq->vq_indirectoffset = size_desc_avail + size_used; 861 vq->vq_indirect = VIRTIO_PTR(vq->vq_vaddr, 862 vq->vq_indirectoffset); 863 } 864 #undef VIRTIO_PTR 865 866 vq->vq_descx = kmem_zalloc(sizeof(vq->vq_descx[0]) * vq_num, 867 KM_SLEEP); 868 869 mutex_init(&vq->vq_freedesc_lock, MUTEX_SPIN, sc->sc_ipl); 870 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 871 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 872 873 virtio_reset_vq(sc, vq); 874 875 aprint_verbose_dev(sc->sc_dev, 876 "allocated %" PRIuBUSSIZE " byte for virtqueue %d for %s, " 877 "size %d\n", allocsize, vq->vq_index, name, vq_num); 878 if (size_indirect > 0) 879 aprint_verbose_dev(sc->sc_dev, 880 "using %" PRIuBUSSIZE " byte (%d entries) indirect " 881 "descriptors\n", size_indirect, maxnsegs * vq_num); 882 883 return 0; 884 885 err: 886 sc->sc_ops->setup_queue(sc, vq->vq_index, 0); 887 if (vq->vq_dmamap) 888 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 889 if (vq->vq_vaddr) 890 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 891 if (vq->vq_segs[0].ds_addr) 892 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 893 memset(vq, 0, sizeof(*vq)); 894 895 return -1; 896 } 897 898 int 899 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 900 { 901 uint16_t s; 902 size_t i; 903 904 if (vq->vq_vaddr == NULL) 905 return 0; 906 907 /* device must be already deactivated */ 908 /* confirm the vq is empty */ 909 s = vq->vq_free_idx; 910 i = 0; 911 while (s != virtio_rw16(sc, VRING_DESC_CHAIN_END)) { 912 s = vq->vq_desc[s].next; 913 i++; 914 } 915 if (i != vq->vq_num) { 916 printf("%s: freeing non-empty vq, index %d\n", 917 device_xname(sc->sc_dev), vq->vq_index); 918 return EBUSY; 919 } 920 921 /* tell device that there's no virtqueue any longer */ 922 sc->sc_ops->setup_queue(sc, vq->vq_index, 0); 923 924 vq_sync_aring_all(sc, vq, BUS_DMASYNC_POSTWRITE); 925 926 kmem_free(vq->vq_descx, sizeof(vq->vq_descx[0]) * vq->vq_num); 927 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 928 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 929 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 930 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 931 mutex_destroy(&vq->vq_freedesc_lock); 932 mutex_destroy(&vq->vq_uring_lock); 933 mutex_destroy(&vq->vq_aring_lock); 934 memset(vq, 0, sizeof(*vq)); 935 936 return 0; 937 } 938 939 /* 940 * Free descriptor management. 941 */ 942 static int 943 vq_alloc_slot_locked(struct virtio_softc *sc, struct virtqueue *vq, 944 size_t nslots) 945 { 946 struct vring_desc *vd; 947 uint16_t rv, tail; 948 size_t i; 949 950 KASSERT(mutex_owned(&vq->vq_freedesc_lock)); 951 952 tail = virtio_rw16(sc, vq->vq_free_idx); 953 for (i = 0; i < nslots - 1; i++) { 954 if (tail == VRING_DESC_CHAIN_END) 955 return VRING_DESC_CHAIN_END; 956 957 vd = &vq->vq_desc[tail]; 958 vd->flags = virtio_rw16(sc, VRING_DESC_F_NEXT); 959 tail = virtio_rw16(sc, vd->next); 960 } 961 962 if (tail == VRING_DESC_CHAIN_END) 963 return VRING_DESC_CHAIN_END; 964 965 rv = virtio_rw16(sc, vq->vq_free_idx); 966 967 vd = &vq->vq_desc[tail]; 968 vd->flags = virtio_rw16(sc, 0); 969 vq->vq_free_idx = vd->next; 970 971 return rv; 972 } 973 static uint16_t 974 vq_alloc_slot(struct virtio_softc *sc, struct virtqueue *vq, size_t nslots) 975 { 976 uint16_t rv; 977 978 mutex_enter(&vq->vq_freedesc_lock); 979 rv = vq_alloc_slot_locked(sc, vq, nslots); 980 mutex_exit(&vq->vq_freedesc_lock); 981 982 return rv; 983 } 984 985 static void 986 vq_free_slot(struct virtio_softc *sc, struct virtqueue *vq, uint16_t slot) 987 { 988 struct vring_desc *vd; 989 uint16_t s; 990 991 mutex_enter(&vq->vq_freedesc_lock); 992 vd = &vq->vq_desc[slot]; 993 while ((vd->flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) != 0) { 994 s = virtio_rw16(sc, vd->next); 995 vd = &vq->vq_desc[s]; 996 } 997 vd->next = vq->vq_free_idx; 998 vq->vq_free_idx = virtio_rw16(sc, slot); 999 mutex_exit(&vq->vq_freedesc_lock); 1000 } 1001 1002 /* 1003 * Enqueue several dmamaps as a single request. 1004 */ 1005 /* 1006 * Typical usage: 1007 * <queue size> number of followings are stored in arrays 1008 * - command blocks (in dmamem) should be pre-allocated and mapped 1009 * - dmamaps for command blocks should be pre-allocated and loaded 1010 * - dmamaps for payload should be pre-allocated 1011 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 1012 * if (r) // currently 0 or EAGAIN 1013 * return r; 1014 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 1015 * if (r) { 1016 * virtio_enqueue_abort(sc, vq, slot); 1017 * return r; 1018 * } 1019 * r = virtio_enqueue_reserve(sc, vq, slot, 1020 * dmamap_payload[slot]->dm_nsegs + 1); 1021 * // ^ +1 for command 1022 * if (r) { // currently 0 or EAGAIN 1023 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 1024 * return r; // do not call abort() 1025 * } 1026 * <setup and prepare commands> 1027 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 1028 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 1029 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 1030 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 1031 * virtio_enqueue_commit(sc, vq, slot, true); 1032 */ 1033 1034 /* 1035 * enqueue_prep: allocate a slot number 1036 */ 1037 int 1038 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 1039 { 1040 uint16_t slot; 1041 1042 KASSERT(slotp != NULL); 1043 1044 slot = vq_alloc_slot(sc, vq, 1); 1045 if (slot == VRING_DESC_CHAIN_END) 1046 return EAGAIN; 1047 1048 *slotp = slot; 1049 1050 return 0; 1051 } 1052 1053 /* 1054 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 1055 */ 1056 int 1057 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 1058 int slot, int nsegs) 1059 { 1060 struct vring_desc *vd; 1061 struct vring_desc_extra *vdx; 1062 int i; 1063 1064 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 1065 1066 vdx = &vq->vq_descx[slot]; 1067 vd = &vq->vq_desc[slot]; 1068 1069 KASSERT((vd->flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0); 1070 1071 if ((vq->vq_indirect != NULL) && 1072 (nsegs >= MINSEG_INDIRECT) && 1073 (nsegs <= vq->vq_maxnsegs)) 1074 vdx->use_indirect = true; 1075 else 1076 vdx->use_indirect = false; 1077 1078 if (vdx->use_indirect) { 1079 uint64_t addr; 1080 1081 addr = vq->vq_dmamap->dm_segs[0].ds_addr 1082 + vq->vq_indirectoffset; 1083 addr += sizeof(struct vring_desc) 1084 * vq->vq_maxnsegs * slot; 1085 1086 vd->addr = virtio_rw64(sc, addr); 1087 vd->len = virtio_rw32(sc, sizeof(struct vring_desc) * nsegs); 1088 vd->flags = virtio_rw16(sc, VRING_DESC_F_INDIRECT); 1089 1090 vd = &vq->vq_indirect[vq->vq_maxnsegs * slot]; 1091 vdx->desc_base = vd; 1092 vdx->desc_free_idx = 0; 1093 1094 for (i = 0; i < nsegs - 1; i++) { 1095 vd[i].flags = virtio_rw16(sc, VRING_DESC_F_NEXT); 1096 } 1097 vd[i].flags = virtio_rw16(sc, 0); 1098 } else { 1099 uint16_t s; 1100 1101 s = vq_alloc_slot(sc, vq, nsegs - 1); 1102 if (s == VRING_DESC_CHAIN_END) { 1103 vq_free_slot(sc, vq, slot); 1104 return EAGAIN; 1105 } 1106 1107 vd->next = virtio_rw16(sc, s); 1108 vd->flags = virtio_rw16(sc, VRING_DESC_F_NEXT); 1109 1110 vdx->desc_base = &vq->vq_desc[0]; 1111 vdx->desc_free_idx = slot; 1112 } 1113 1114 return 0; 1115 } 1116 1117 /* 1118 * enqueue: enqueue a single dmamap. 1119 */ 1120 int 1121 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1122 bus_dmamap_t dmamap, bool write) 1123 { 1124 struct vring_desc *vds; 1125 struct vring_desc_extra *vdx; 1126 uint16_t s; 1127 int i; 1128 1129 KASSERT(dmamap->dm_nsegs > 0); 1130 1131 vdx = &vq->vq_descx[slot]; 1132 vds = vdx->desc_base; 1133 s = vdx->desc_free_idx; 1134 1135 KASSERT(vds != NULL); 1136 1137 for (i = 0; i < dmamap->dm_nsegs; i++) { 1138 KASSERT(s != VRING_DESC_CHAIN_END); 1139 1140 vds[s].addr = virtio_rw64(sc, dmamap->dm_segs[i].ds_addr); 1141 vds[s].len = virtio_rw32(sc, dmamap->dm_segs[i].ds_len); 1142 if (!write) 1143 vds[s].flags |= virtio_rw16(sc, VRING_DESC_F_WRITE); 1144 1145 if ((vds[s].flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0) { 1146 s = VRING_DESC_CHAIN_END; 1147 } else { 1148 s = virtio_rw16(sc, vds[s].next); 1149 } 1150 } 1151 1152 vdx->desc_free_idx = s; 1153 1154 return 0; 1155 } 1156 1157 int 1158 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1159 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 1160 bool write) 1161 { 1162 struct vring_desc_extra *vdx; 1163 struct vring_desc *vds; 1164 uint16_t s; 1165 1166 vdx = &vq->vq_descx[slot]; 1167 vds = vdx->desc_base; 1168 s = vdx->desc_free_idx; 1169 1170 KASSERT(s != VRING_DESC_CHAIN_END); 1171 KASSERT(vds != NULL); 1172 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 1173 KASSERT(dmamap->dm_segs[0].ds_len > start); 1174 KASSERT(dmamap->dm_segs[0].ds_len >= start + len); 1175 1176 vds[s].addr = virtio_rw64(sc, dmamap->dm_segs[0].ds_addr + start); 1177 vds[s].len = virtio_rw32(sc, len); 1178 if (!write) 1179 vds[s].flags |= virtio_rw16(sc, VRING_DESC_F_WRITE); 1180 1181 if ((vds[s].flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0) { 1182 s = VRING_DESC_CHAIN_END; 1183 } else { 1184 s = virtio_rw16(sc, vds[s].next); 1185 } 1186 1187 vdx->desc_free_idx = s; 1188 1189 return 0; 1190 } 1191 1192 /* 1193 * enqueue_commit: add it to the aring. 1194 */ 1195 int 1196 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1197 bool notifynow) 1198 { 1199 1200 if (slot < 0) { 1201 mutex_enter(&vq->vq_aring_lock); 1202 goto notify; 1203 } 1204 1205 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 1206 if (vq->vq_descx[slot].use_indirect) 1207 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 1208 1209 mutex_enter(&vq->vq_aring_lock); 1210 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = 1211 virtio_rw16(sc, slot); 1212 1213 notify: 1214 if (notifynow) { 1215 uint16_t o, n, t; 1216 uint16_t flags; 1217 1218 o = virtio_rw16(sc, vq->vq_avail->idx) - 1; 1219 n = vq->vq_avail_idx; 1220 1221 /* 1222 * Prepare for `device->CPU' (host->guest) transfer 1223 * into the buffer. This must happen before we commit 1224 * the vq->vq_avail->idx update to ensure we're not 1225 * still using the buffer in case program-prior loads 1226 * or stores in it get delayed past the store to 1227 * vq->vq_avail->idx. 1228 */ 1229 vq_sync_uring_all(sc, vq, BUS_DMASYNC_PREREAD); 1230 1231 /* ensure payload is published, then avail idx */ 1232 vq_sync_aring_payload(sc, vq, BUS_DMASYNC_PREWRITE); 1233 vq->vq_avail->idx = virtio_rw16(sc, vq->vq_avail_idx); 1234 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE); 1235 vq->vq_queued++; 1236 1237 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) { 1238 vq_sync_uring_avail(sc, vq, BUS_DMASYNC_POSTREAD); 1239 t = virtio_rw16(sc, *vq->vq_avail_event) + 1; 1240 if ((uint16_t) (n - t) < (uint16_t) (n - o)) 1241 sc->sc_ops->kick(sc, vq->vq_index); 1242 } else { 1243 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD); 1244 flags = virtio_rw16(sc, vq->vq_used->flags); 1245 if (!(flags & VRING_USED_F_NO_NOTIFY)) 1246 sc->sc_ops->kick(sc, vq->vq_index); 1247 } 1248 } 1249 mutex_exit(&vq->vq_aring_lock); 1250 1251 return 0; 1252 } 1253 1254 /* 1255 * enqueue_abort: rollback. 1256 */ 1257 int 1258 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1259 { 1260 struct vring_desc_extra *vdx; 1261 1262 vq_free_slot(sc, vq, slot); 1263 1264 vdx = &vq->vq_descx[slot]; 1265 vdx->desc_free_idx = VRING_DESC_CHAIN_END; 1266 vdx->desc_base = NULL; 1267 1268 return 0; 1269 } 1270 1271 /* 1272 * Dequeue a request. 1273 */ 1274 /* 1275 * dequeue: dequeue a request from uring; dmamap_sync for uring is 1276 * already done in the interrupt handler. 1277 */ 1278 int 1279 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 1280 int *slotp, int *lenp) 1281 { 1282 uint16_t slot, usedidx; 1283 1284 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx)) 1285 return ENOENT; 1286 mutex_enter(&vq->vq_uring_lock); 1287 usedidx = vq->vq_used_idx++; 1288 mutex_exit(&vq->vq_uring_lock); 1289 usedidx %= vq->vq_num; 1290 slot = virtio_rw32(sc, vq->vq_used->ring[usedidx].id); 1291 1292 if (vq->vq_descx[slot].use_indirect) 1293 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 1294 1295 if (slotp) 1296 *slotp = slot; 1297 if (lenp) 1298 *lenp = virtio_rw32(sc, vq->vq_used->ring[usedidx].len); 1299 1300 return 0; 1301 } 1302 1303 /* 1304 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1305 * if you forget to call this the slot will be leaked. 1306 */ 1307 int 1308 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1309 { 1310 struct vring_desc_extra *vdx; 1311 1312 vq_free_slot(sc, vq, slot); 1313 1314 vdx = &vq->vq_descx[slot]; 1315 vdx->desc_base = NULL; 1316 vdx->desc_free_idx = VRING_DESC_CHAIN_END; 1317 1318 return 0; 1319 } 1320 1321 /* 1322 * Attach a child, fill all the members. 1323 */ 1324 void 1325 virtio_child_attach_start(struct virtio_softc *sc, device_t child, int ipl, 1326 uint64_t req_features, const char *feat_bits) 1327 { 1328 char buf[1024]; 1329 1330 KASSERT(sc->sc_child == NULL); 1331 KASSERT(!ISSET(sc->sc_child_flags, VIRTIO_CHILD_DETACHED)); 1332 1333 sc->sc_child = child; 1334 sc->sc_ipl = ipl; 1335 1336 virtio_negotiate_features(sc, req_features); 1337 snprintb(buf, sizeof(buf), feat_bits, sc->sc_active_features); 1338 aprint_normal(": features: %s\n", buf); 1339 aprint_naive("\n"); 1340 } 1341 1342 int 1343 virtio_child_attach_finish(struct virtio_softc *sc, 1344 struct virtqueue *vqs, size_t nvqs, 1345 virtio_callback config_change, 1346 int req_flags) 1347 { 1348 size_t i; 1349 int r; 1350 1351 #ifdef DIAGNOSTIC 1352 KASSERT(nvqs > 0); 1353 #define VIRTIO_ASSERT_FLAGS (VIRTIO_F_INTR_SOFTINT | VIRTIO_F_INTR_PERVQ) 1354 KASSERT((req_flags & VIRTIO_ASSERT_FLAGS) != VIRTIO_ASSERT_FLAGS); 1355 #undef VIRTIO_ASSERT_FLAGS 1356 1357 for (i = 0; i < nvqs; i++){ 1358 KASSERT(vqs[i].vq_index == i); 1359 KASSERT(vqs[i].vq_intrhand != NULL); 1360 KASSERT(vqs[i].vq_done == NULL || 1361 vqs[i].vq_intrhand == virtio_vq_done); 1362 } 1363 #endif 1364 1365 1366 sc->sc_vqs = vqs; 1367 sc->sc_nvqs = nvqs; 1368 sc->sc_config_change = config_change; 1369 sc->sc_intrhand = virtio_vq_intr; 1370 sc->sc_flags = req_flags; 1371 1372 /* set the vq address */ 1373 for (i = 0; i < nvqs; i++) { 1374 sc->sc_ops->setup_queue(sc, vqs[i].vq_index, 1375 vqs[i].vq_dmamap->dm_segs[0].ds_addr); 1376 } 1377 1378 r = sc->sc_ops->alloc_interrupts(sc); 1379 if (r != 0) { 1380 aprint_error_dev(sc->sc_dev, 1381 "failed to allocate interrupts\n"); 1382 goto fail; 1383 } 1384 1385 r = sc->sc_ops->setup_interrupts(sc, 0); 1386 if (r != 0) { 1387 aprint_error_dev(sc->sc_dev, "failed to setup interrupts\n"); 1388 goto fail; 1389 } 1390 1391 KASSERT(sc->sc_soft_ih == NULL); 1392 if (sc->sc_flags & VIRTIO_F_INTR_SOFTINT) { 1393 u_int flags = SOFTINT_NET; 1394 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) 1395 flags |= SOFTINT_MPSAFE; 1396 1397 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, 1398 sc); 1399 if (sc->sc_soft_ih == NULL) { 1400 sc->sc_ops->free_interrupts(sc); 1401 aprint_error_dev(sc->sc_dev, 1402 "failed to establish soft interrupt\n"); 1403 goto fail; 1404 } 1405 } 1406 1407 SET(sc->sc_child_flags, VIRTIO_CHILD_ATTACH_FINISHED); 1408 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 1409 return 0; 1410 1411 fail: 1412 if (sc->sc_soft_ih) { 1413 softint_disestablish(sc->sc_soft_ih); 1414 sc->sc_soft_ih = NULL; 1415 } 1416 1417 sc->sc_ops->free_interrupts(sc); 1418 1419 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 1420 return 1; 1421 } 1422 1423 void 1424 virtio_child_detach(struct virtio_softc *sc) 1425 { 1426 1427 /* already detached */ 1428 if (ISSET(sc->sc_child_flags, VIRTIO_CHILD_DETACHED)) 1429 return; 1430 1431 sc->sc_vqs = NULL; 1432 1433 virtio_device_reset(sc); 1434 1435 sc->sc_ops->free_interrupts(sc); 1436 1437 if (sc->sc_soft_ih) { 1438 softint_disestablish(sc->sc_soft_ih); 1439 sc->sc_soft_ih = NULL; 1440 } 1441 1442 SET(sc->sc_child_flags, VIRTIO_CHILD_DETACHED); 1443 } 1444 1445 void 1446 virtio_child_attach_failed(struct virtio_softc *sc) 1447 { 1448 virtio_child_detach(sc); 1449 1450 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 1451 1452 SET(sc->sc_child_flags, VIRTIO_CHILD_ATTACH_FAILED); 1453 } 1454 1455 bus_dma_tag_t 1456 virtio_dmat(struct virtio_softc *sc) 1457 { 1458 return sc->sc_dmat; 1459 } 1460 1461 device_t 1462 virtio_child(struct virtio_softc *sc) 1463 { 1464 return sc->sc_child; 1465 } 1466 1467 int 1468 virtio_intrhand(struct virtio_softc *sc) 1469 { 1470 return (*sc->sc_intrhand)(sc); 1471 } 1472 1473 uint64_t 1474 virtio_features(struct virtio_softc *sc) 1475 { 1476 return sc->sc_active_features; 1477 } 1478 1479 int 1480 virtio_attach_failed(struct virtio_softc *sc) 1481 { 1482 device_t self = sc->sc_dev; 1483 1484 /* no error if its not connected, but its failed */ 1485 if (sc->sc_childdevid == 0) 1486 return 1; 1487 1488 if (ISSET(sc->sc_child_flags, VIRTIO_CHILD_ATTACH_FAILED)) { 1489 aprint_error_dev(self, "virtio configuration failed\n"); 1490 return 1; 1491 } 1492 1493 if (sc->sc_child == NULL) { 1494 aprint_error_dev(self, 1495 "no matching child driver; not configured\n"); 1496 return 1; 1497 } 1498 1499 /* sanity check */ 1500 if (!ISSET(sc->sc_child_flags, VIRTIO_CHILD_ATTACH_FINISHED)) { 1501 aprint_error_dev(self, "virtio internal error, child driver " 1502 "signaled OK but didn't initialize interrupts\n"); 1503 return 1; 1504 } 1505 1506 return 0; 1507 } 1508 1509 void 1510 virtio_print_device_type(device_t self, int id, int revision) 1511 { 1512 aprint_normal_dev(self, "%s device (id %d, rev. 0x%02x)\n", 1513 (id < NDEVNAMES ? virtio_device_name[id] : "Unknown"), 1514 id, 1515 revision); 1516 } 1517 1518 1519 MODULE(MODULE_CLASS_DRIVER, virtio, NULL); 1520 1521 #ifdef _MODULE 1522 #include "ioconf.c" 1523 #endif 1524 1525 static int 1526 virtio_modcmd(modcmd_t cmd, void *opaque) 1527 { 1528 int error = 0; 1529 1530 #ifdef _MODULE 1531 switch (cmd) { 1532 case MODULE_CMD_INIT: 1533 error = config_init_component(cfdriver_ioconf_virtio, 1534 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1535 break; 1536 case MODULE_CMD_FINI: 1537 error = config_fini_component(cfdriver_ioconf_virtio, 1538 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1539 break; 1540 default: 1541 error = ENOTTY; 1542 break; 1543 } 1544 #endif 1545 1546 return error; 1547 } 1548
Indexes created Thu Oct 02 14:10:14 GMT 2025