1 /* $NetBSD: virtio.c,v 1.84 2025/09/06 02:56:18 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2020 The NetBSD Foundation, Inc. 5 * Copyright (c) 2012 Stefan Fritsch, Alexander Fiveg. 6 * Copyright (c) 2010 Minoura Makoto. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.84 2025/09/06 02:56:18 riastradh Exp $"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kernel.h> 36 #include <sys/atomic.h> 37 #include <sys/bus.h> 38 #include <sys/device.h> 39 #include <sys/kmem.h> 40 #include <sys/module.h> 41 #include <sys/paravirt_membar.h> 42 43 #define VIRTIO_PRIVATE 44 45 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */ 46 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */ 47 48 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 49 50 /* 51 * The maximum descriptor size is 2^15. Use that value as the end of 52 * descriptor chain terminator since it will never be a valid index 53 * in the descriptor table. 54 */ 55 #define VRING_DESC_CHAIN_END 32768 56 57 /* incomplete list */ 58 static const char *virtio_device_name[] = { 59 "unknown (0)", /* 0 */ 60 "network", /* 1 */ 61 "block", /* 2 */ 62 "console", /* 3 */ 63 "entropy", /* 4 */ 64 "memory balloon", /* 5 */ 65 "I/O memory", /* 6 */ 66 "remote processor messaging", /* 7 */ 67 "SCSI", /* 8 */ 68 "9P transport", /* 9 */ 69 NULL, /* 10 */ 70 NULL, /* 11 */ 71 NULL, /* 12 */ 72 NULL, /* 13 */ 73 NULL, /* 14 */ 74 NULL, /* 15 */ 75 "GPU", /* 16 */ 76 }; 77 #define NDEVNAMES __arraycount(virtio_device_name) 78 79 static void virtio_reset_vq(struct virtio_softc *, 80 struct virtqueue *); 81 82 void 83 virtio_set_status(struct virtio_softc *sc, int status) 84 { 85 sc->sc_ops->set_status(sc, status); 86 } 87 88 /* 89 * Reset the device. 90 */ 91 /* 92 * To reset the device to a known state, do following: 93 * virtio_reset(sc); // this will stop the device activity 94 * <dequeue finished requests>; // virtio_dequeue() still can be called 95 * <revoke pending requests in the vqs if any>; 96 * virtio_reinit_start(sc); // dequeue prohibited 97 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 98 * <some other initialization>; 99 * virtio_reinit_end(sc); // device activated; enqueue allowed 100 * Once attached, feature negotiation can only be allowed after virtio_reset. 101 */ 102 void 103 virtio_reset(struct virtio_softc *sc) 104 { 105 virtio_device_reset(sc); 106 } 107 108 int 109 virtio_reinit_start(struct virtio_softc *sc) 110 { 111 int i, r; 112 113 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 114 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 115 for (i = 0; i < sc->sc_nvqs; i++) { 116 int n; 117 struct virtqueue *vq = &sc->sc_vqs[i]; 118 n = sc->sc_ops->read_queue_size(sc, vq->vq_index); 119 if (n == 0) /* vq disappeared */ 120 continue; 121 if (n != vq->vq_num) { 122 panic("%s: virtqueue size changed, vq index %d\n", 123 device_xname(sc->sc_dev), 124 vq->vq_index); 125 } 126 virtio_reset_vq(sc, vq); 127 sc->sc_ops->setup_queue(sc, vq->vq_index, 128 vq->vq_dmamap->dm_segs[0].ds_addr); 129 } 130 131 r = sc->sc_ops->setup_interrupts(sc, 1); 132 if (r != 0) 133 goto fail; 134 135 return 0; 136 137 fail: 138 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 139 140 return 1; 141 } 142 143 void 144 virtio_reinit_end(struct virtio_softc *sc) 145 { 146 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 147 } 148 149 /* 150 * Feature negotiation. 151 */ 152 void 153 virtio_negotiate_features(struct virtio_softc *sc, uint64_t guest_features) 154 { 155 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 156 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 157 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 158 sc->sc_ops->neg_features(sc, guest_features); 159 if (sc->sc_active_features & VIRTIO_F_RING_INDIRECT_DESC) 160 sc->sc_indirect = true; 161 else 162 sc->sc_indirect = false; 163 } 164 165 166 /* 167 * Device configuration registers readers/writers 168 */ 169 #if 0 170 #define DPRINTFR(n, fmt, val, index, num) \ 171 printf("\n%s (", n); \ 172 for (int i = 0; i < num; i++) \ 173 printf("%02x ", bus_space_read_1(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index+i)); \ 174 printf(") -> "); printf(fmt, val); printf("\n"); 175 #define DPRINTFR2(n, fmt, val_s, val_n) \ 176 printf("%s ", n); \ 177 printf("\n stream "); printf(fmt, val_s); printf(" norm "); printf(fmt, val_n); printf("\n"); 178 #else 179 #define DPRINTFR(n, fmt, val, index, num) 180 #define DPRINTFR2(n, fmt, val_s, val_n) 181 #endif 182 183 184 uint8_t 185 virtio_read_device_config_1(struct virtio_softc *sc, int index) 186 { 187 bus_space_tag_t iot = sc->sc_devcfg_iot; 188 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 189 uint8_t val; 190 191 val = bus_space_read_1(iot, ioh, index); 192 193 DPRINTFR("read_1", "%02x", val, index, 1); 194 return val; 195 } 196 197 uint16_t 198 virtio_read_device_config_2(struct virtio_softc *sc, int index) 199 { 200 bus_space_tag_t iot = sc->sc_devcfg_iot; 201 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 202 uint16_t val; 203 204 val = bus_space_read_2(iot, ioh, index); 205 if (BYTE_ORDER != sc->sc_bus_endian) 206 val = bswap16(val); 207 208 DPRINTFR("read_2", "%04x", val, index, 2); 209 DPRINTFR2("read_2", "%04x", 210 bus_space_read_stream_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 211 index), 212 bus_space_read_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index)); 213 return val; 214 } 215 216 uint32_t 217 virtio_read_device_config_4(struct virtio_softc *sc, int index) 218 { 219 bus_space_tag_t iot = sc->sc_devcfg_iot; 220 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 221 uint32_t val; 222 223 val = bus_space_read_4(iot, ioh, index); 224 if (BYTE_ORDER != sc->sc_bus_endian) 225 val = bswap32(val); 226 227 DPRINTFR("read_4", "%08x", val, index, 4); 228 DPRINTFR2("read_4", "%08x", 229 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 230 index), 231 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index)); 232 return val; 233 } 234 235 /* 236 * The Virtio spec explicitly tells that reading and writing 8 bytes are not 237 * considered atomic and no triggers may be connected to reading or writing 238 * it. We access it using two 32 reads. See virtio spec 4.1.3.1. 239 */ 240 uint64_t 241 virtio_read_device_config_8(struct virtio_softc *sc, int index) 242 { 243 bus_space_tag_t iot = sc->sc_devcfg_iot; 244 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 245 union { 246 uint64_t u64; 247 uint32_t l[2]; 248 } v; 249 uint64_t val; 250 251 v.l[0] = bus_space_read_4(iot, ioh, index); 252 v.l[1] = bus_space_read_4(iot, ioh, index + 4); 253 if (sc->sc_bus_endian != sc->sc_struct_endian) { 254 v.l[0] = bswap32(v.l[0]); 255 v.l[1] = bswap32(v.l[1]); 256 } 257 val = v.u64; 258 259 if (BYTE_ORDER != sc->sc_struct_endian) 260 val = bswap64(val); 261 262 DPRINTFR("read_8", "%08"PRIx64, val, index, 8); 263 DPRINTFR2("read_8 low ", "%08x", 264 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 265 index), 266 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index)); 267 DPRINTFR2("read_8 high ", "%08x", 268 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 269 index + 4), 270 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index + 4)); 271 return val; 272 } 273 274 /* 275 * In the older virtio spec, device config registers are host endian. On newer 276 * they are little endian. Some newer devices however explicitly specify their 277 * register to always be little endian. These functions cater for these. 278 */ 279 uint16_t 280 virtio_read_device_config_le_2(struct virtio_softc *sc, int index) 281 { 282 bus_space_tag_t iot = sc->sc_devcfg_iot; 283 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 284 uint16_t val; 285 286 val = bus_space_read_2(iot, ioh, index); 287 #if !defined(__aarch64__) && !defined(__arm__) 288 /* 289 * For big-endian aarch64/armv7, bus endian is always LSB, but 290 * byte-order is automatically swapped by bus_space(9) (see also 291 * comments in virtio_pci.c). Therefore, no need to swap here. 292 */ 293 if (sc->sc_bus_endian != LITTLE_ENDIAN) 294 val = bswap16(val); 295 #endif 296 297 DPRINTFR("read_le_2", "%04x", val, index, 2); 298 DPRINTFR2("read_le_2", "%04x", 299 bus_space_read_stream_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0), 300 bus_space_read_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0)); 301 return val; 302 } 303 304 uint32_t 305 virtio_read_device_config_le_4(struct virtio_softc *sc, int index) 306 { 307 bus_space_tag_t iot = sc->sc_devcfg_iot; 308 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 309 uint32_t val; 310 311 val = bus_space_read_4(iot, ioh, index); 312 #if !defined(__aarch64__) && !defined(__arm__) 313 /* See virtio_read_device_config_le_2() above. */ 314 if (sc->sc_bus_endian != LITTLE_ENDIAN) 315 val = bswap32(val); 316 #endif 317 318 DPRINTFR("read_le_4", "%08x", val, index, 4); 319 DPRINTFR2("read_le_4", "%08x", 320 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0), 321 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0)); 322 return val; 323 } 324 325 void 326 virtio_write_device_config_1(struct virtio_softc *sc, int index, uint8_t value) 327 { 328 bus_space_tag_t iot = sc->sc_devcfg_iot; 329 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 330 331 bus_space_write_1(iot, ioh, index, value); 332 } 333 334 void 335 virtio_write_device_config_2(struct virtio_softc *sc, int index, 336 uint16_t value) 337 { 338 bus_space_tag_t iot = sc->sc_devcfg_iot; 339 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 340 341 if (BYTE_ORDER != sc->sc_bus_endian) 342 value = bswap16(value); 343 bus_space_write_2(iot, ioh, index, value); 344 } 345 346 void 347 virtio_write_device_config_4(struct virtio_softc *sc, int index, 348 uint32_t value) 349 { 350 bus_space_tag_t iot = sc->sc_devcfg_iot; 351 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 352 353 if (BYTE_ORDER != sc->sc_bus_endian) 354 value = bswap32(value); 355 bus_space_write_4(iot, ioh, index, value); 356 } 357 358 /* 359 * The Virtio spec explicitly tells that reading and writing 8 bytes are not 360 * considered atomic and no triggers may be connected to reading or writing 361 * it. We access it using two 32 bit writes. For good measure it is stated to 362 * always write lsb first just in case of a hypervisor bug. See See virtio 363 * spec 4.1.3.1. 364 */ 365 void 366 virtio_write_device_config_8(struct virtio_softc *sc, int index, 367 uint64_t value) 368 { 369 bus_space_tag_t iot = sc->sc_devcfg_iot; 370 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 371 union { 372 uint64_t u64; 373 uint32_t l[2]; 374 } v; 375 376 if (BYTE_ORDER != sc->sc_struct_endian) 377 value = bswap64(value); 378 379 v.u64 = value; 380 if (sc->sc_bus_endian != sc->sc_struct_endian) { 381 v.l[0] = bswap32(v.l[0]); 382 v.l[1] = bswap32(v.l[1]); 383 } 384 385 if (sc->sc_struct_endian == LITTLE_ENDIAN) { 386 bus_space_write_4(iot, ioh, index, v.l[0]); 387 bus_space_write_4(iot, ioh, index + 4, v.l[1]); 388 } else { 389 bus_space_write_4(iot, ioh, index + 4, v.l[1]); 390 bus_space_write_4(iot, ioh, index, v.l[0]); 391 } 392 } 393 394 /* 395 * In the older virtio spec, device config registers are host endian. On newer 396 * they are little endian. Some newer devices however explicitly specify their 397 * register to always be little endian. These functions cater for these. 398 */ 399 void 400 virtio_write_device_config_le_2(struct virtio_softc *sc, int index, 401 uint16_t value) 402 { 403 bus_space_tag_t iot = sc->sc_devcfg_iot; 404 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 405 406 if (sc->sc_bus_endian != LITTLE_ENDIAN) 407 value = bswap16(value); 408 bus_space_write_2(iot, ioh, index, value); 409 } 410 411 void 412 virtio_write_device_config_le_4(struct virtio_softc *sc, int index, 413 uint32_t value) 414 { 415 bus_space_tag_t iot = sc->sc_devcfg_iot; 416 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 417 418 if (sc->sc_bus_endian != LITTLE_ENDIAN) 419 value = bswap32(value); 420 bus_space_write_4(iot, ioh, index, value); 421 } 422 423 424 /* 425 * data structures endian helpers 426 */ 427 uint16_t 428 virtio_rw16(struct virtio_softc *sc, uint16_t val) 429 { 430 KASSERT(sc); 431 return BYTE_ORDER != sc->sc_struct_endian ? bswap16(val) : val; 432 } 433 434 uint32_t 435 virtio_rw32(struct virtio_softc *sc, uint32_t val) 436 { 437 KASSERT(sc); 438 return BYTE_ORDER != sc->sc_struct_endian ? bswap32(val) : val; 439 } 440 441 uint64_t 442 virtio_rw64(struct virtio_softc *sc, uint64_t val) 443 { 444 KASSERT(sc); 445 return BYTE_ORDER != sc->sc_struct_endian ? bswap64(val) : val; 446 } 447 448 449 /* 450 * Interrupt handler. 451 */ 452 static void 453 virtio_soft_intr(void *arg) 454 { 455 struct virtio_softc *sc = arg; 456 457 KASSERT(sc->sc_intrhand != NULL); 458 459 (*sc->sc_intrhand)(sc); 460 } 461 462 /* set to vq->vq_intrhand in virtio_init_vq_vqdone() */ 463 static int 464 virtio_vq_done(void *xvq) 465 { 466 struct virtqueue *vq = xvq; 467 468 return vq->vq_done(vq); 469 } 470 471 static int 472 virtio_vq_intr(struct virtio_softc *sc) 473 { 474 struct virtqueue *vq; 475 int i, r = 0; 476 477 for (i = 0; i < sc->sc_nvqs; i++) { 478 vq = &sc->sc_vqs[i]; 479 if (virtio_vq_is_enqueued(sc, vq) == 1) { 480 r |= (*vq->vq_intrhand)(vq->vq_intrhand_arg); 481 } 482 } 483 484 return r; 485 } 486 487 /* 488 * dmamap sync operations for a virtqueue. 489 */ 490 static inline void 491 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 492 { 493 494 /* availoffset == sizeof(vring_desc) * vq_num */ 495 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 496 ops); 497 } 498 499 static inline void 500 vq_sync_aring_all(struct virtio_softc *sc, struct virtqueue *vq, int ops) 501 { 502 uint16_t hdrlen = offsetof(struct vring_avail, ring); 503 size_t payloadlen = vq->vq_num * sizeof(uint16_t); 504 size_t usedlen = 0; 505 506 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) 507 usedlen = sizeof(uint16_t); 508 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 509 vq->vq_availoffset, hdrlen + payloadlen + usedlen, ops); 510 } 511 512 static inline void 513 vq_sync_aring_header(struct virtio_softc *sc, struct virtqueue *vq, int ops) 514 { 515 uint16_t hdrlen = offsetof(struct vring_avail, ring); 516 517 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 518 vq->vq_availoffset, hdrlen, ops); 519 } 520 521 static inline void 522 vq_sync_aring_payload(struct virtio_softc *sc, struct virtqueue *vq, int ops) 523 { 524 uint16_t hdrlen = offsetof(struct vring_avail, ring); 525 size_t payloadlen = vq->vq_num * sizeof(uint16_t); 526 527 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 528 vq->vq_availoffset + hdrlen, payloadlen, ops); 529 } 530 531 static inline void 532 vq_sync_aring_used(struct virtio_softc *sc, struct virtqueue *vq, int ops) 533 { 534 uint16_t hdrlen = offsetof(struct vring_avail, ring); 535 size_t payloadlen = vq->vq_num * sizeof(uint16_t); 536 size_t usedlen = sizeof(uint16_t); 537 538 if ((sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) == 0) 539 return; 540 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 541 vq->vq_availoffset + hdrlen + payloadlen, usedlen, ops); 542 } 543 544 static inline void 545 vq_sync_uring_all(struct virtio_softc *sc, struct virtqueue *vq, int ops) 546 { 547 uint16_t hdrlen = offsetof(struct vring_used, ring); 548 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem); 549 size_t availlen = 0; 550 551 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) 552 availlen = sizeof(uint16_t); 553 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 554 vq->vq_usedoffset, hdrlen + payloadlen + availlen, ops); 555 } 556 557 static inline void 558 vq_sync_uring_header(struct virtio_softc *sc, struct virtqueue *vq, int ops) 559 { 560 uint16_t hdrlen = offsetof(struct vring_used, ring); 561 562 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 563 vq->vq_usedoffset, hdrlen, ops); 564 } 565 566 static inline void 567 vq_sync_uring_payload(struct virtio_softc *sc, struct virtqueue *vq, int ops) 568 { 569 uint16_t hdrlen = offsetof(struct vring_used, ring); 570 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem); 571 572 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 573 vq->vq_usedoffset + hdrlen, payloadlen, ops); 574 } 575 576 static inline void 577 vq_sync_uring_avail(struct virtio_softc *sc, struct virtqueue *vq, int ops) 578 { 579 uint16_t hdrlen = offsetof(struct vring_used, ring); 580 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem); 581 size_t availlen = sizeof(uint16_t); 582 583 if ((sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) == 0) 584 return; 585 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 586 vq->vq_usedoffset + hdrlen + payloadlen, availlen, ops); 587 } 588 589 static inline void 590 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 591 int ops) 592 { 593 int offset = vq->vq_indirectoffset + 594 sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 595 596 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 597 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, ops); 598 } 599 600 bool 601 virtio_vq_is_enqueued(struct virtio_softc *sc, struct virtqueue *vq) 602 { 603 604 if (vq->vq_queued) { 605 vq->vq_queued = 0; 606 vq_sync_aring_all(sc, vq, BUS_DMASYNC_POSTWRITE); 607 } 608 609 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD); 610 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx)) 611 return 0; 612 vq_sync_uring_payload(sc, vq, BUS_DMASYNC_POSTREAD); 613 return 1; 614 } 615 616 /* 617 * Increase the event index in order to delay interrupts. 618 */ 619 int 620 virtio_postpone_intr(struct virtio_softc *sc, struct virtqueue *vq, 621 uint16_t nslots) 622 { 623 uint16_t idx, nused; 624 625 idx = vq->vq_used_idx + nslots; 626 627 /* set the new event index: avail_ring->used_event = idx */ 628 *vq->vq_used_event = virtio_rw16(sc, idx); 629 vq_sync_aring_used(vq->vq_owner, vq, BUS_DMASYNC_PREWRITE); 630 vq->vq_queued++; 631 632 nused = (uint16_t) 633 (virtio_rw16(sc, vq->vq_used->idx) - vq->vq_used_idx); 634 KASSERT(nused <= vq->vq_num); 635 636 return nslots < nused; 637 } 638 639 /* 640 * Postpone interrupt until 3/4 of the available descriptors have been 641 * consumed. 642 */ 643 int 644 virtio_postpone_intr_smart(struct virtio_softc *sc, struct virtqueue *vq) 645 { 646 uint16_t nslots; 647 648 nslots = (uint16_t) 649 (virtio_rw16(sc, vq->vq_avail->idx) - vq->vq_used_idx) * 3 / 4; 650 651 return virtio_postpone_intr(sc, vq, nslots); 652 } 653 654 /* 655 * Postpone interrupt until all of the available descriptors have been 656 * consumed. 657 */ 658 int 659 virtio_postpone_intr_far(struct virtio_softc *sc, struct virtqueue *vq) 660 { 661 uint16_t nslots; 662 663 nslots = (uint16_t) 664 (virtio_rw16(sc, vq->vq_avail->idx) - vq->vq_used_idx); 665 666 return virtio_postpone_intr(sc, vq, nslots); 667 } 668 669 /* 670 * Start/stop vq interrupt. No guarantee. 671 */ 672 void 673 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 674 { 675 676 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) { 677 /* 678 * No way to disable the interrupt completely with 679 * RingEventIdx. Instead advance used_event by half the 680 * possible value. This won't happen soon and is far enough in 681 * the past to not trigger a spurious interrupt. 682 */ 683 *vq->vq_used_event = virtio_rw16(sc, vq->vq_used_idx + 0x8000); 684 vq_sync_aring_used(sc, vq, BUS_DMASYNC_PREWRITE); 685 } else { 686 vq->vq_avail->flags |= 687 virtio_rw16(sc, VRING_AVAIL_F_NO_INTERRUPT); 688 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE); 689 } 690 vq->vq_queued++; 691 } 692 693 int 694 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 695 { 696 697 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) { 698 /* 699 * If event index feature is negotiated, enabling interrupts 700 * is done through setting the latest consumed index in the 701 * used_event field 702 */ 703 *vq->vq_used_event = virtio_rw16(sc, vq->vq_used_idx); 704 vq_sync_aring_used(sc, vq, BUS_DMASYNC_PREWRITE); 705 } else { 706 vq->vq_avail->flags &= 707 ~virtio_rw16(sc, VRING_AVAIL_F_NO_INTERRUPT); 708 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE); 709 } 710 vq->vq_queued++; 711 712 /* 713 * Ensure we announce to the host side that we are accepting 714 * interrupts _before_ we check whether any pending events had 715 * come over the queue while we weren't accepting interrupts. 716 */ 717 paravirt_membar_sync(); 718 719 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD); 720 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx)) 721 return 0; 722 vq_sync_uring_payload(sc, vq, BUS_DMASYNC_POSTREAD); 723 return 1; 724 } 725 726 /* 727 * Initialize vq structure. 728 */ 729 /* 730 * Reset virtqueue parameters 731 */ 732 static void 733 virtio_reset_vq(struct virtio_softc *sc, struct virtqueue *vq) 734 { 735 struct vring_desc *vds; 736 int i, j; 737 int vq_size = vq->vq_num; 738 739 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 740 741 /* build the descriptor chain for free slot management */ 742 vds = vq->vq_desc; 743 for (i = 0; i < vq_size - 1; i++) { 744 vds[i].next = virtio_rw16(sc, i + 1); 745 } 746 vds[i].next = virtio_rw16(sc, VRING_DESC_CHAIN_END); 747 vq->vq_free_idx = 0; 748 749 /* build the indirect descriptor chain */ 750 if (vq->vq_indirect != NULL) { 751 struct vring_desc *vd; 752 753 for (i = 0; i < vq_size; i++) { 754 vd = vq->vq_indirect; 755 vd += vq->vq_maxnsegs * i; 756 for (j = 0; j < vq->vq_maxnsegs - 1; j++) { 757 vd[j].next = virtio_rw16(sc, j + 1); 758 } 759 } 760 } 761 762 /* enqueue/dequeue status */ 763 vq->vq_avail_idx = 0; 764 vq->vq_used_idx = 0; 765 vq->vq_queued = 0; 766 vq_sync_uring_all(sc, vq, BUS_DMASYNC_PREREAD); 767 vq->vq_queued++; 768 } 769 770 /* Initialize vq */ 771 void 772 virtio_init_vq_vqdone(struct virtio_softc *sc, struct virtqueue *vq, 773 int index, int (*vq_done)(struct virtqueue *)) 774 { 775 776 virtio_init_vq(sc, vq, index, virtio_vq_done, vq); 777 vq->vq_done = vq_done; 778 } 779 780 void 781 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, 782 int (*func)(void *), void *arg) 783 { 784 785 memset(vq, 0, sizeof(*vq)); 786 787 vq->vq_owner = sc; 788 vq->vq_num = sc->sc_ops->read_queue_size(sc, index); 789 vq->vq_index = index; 790 vq->vq_intrhand = func; 791 vq->vq_intrhand_arg = arg; 792 } 793 794 /* 795 * Allocate/free a vq. 796 */ 797 int 798 virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, 799 int maxsegsize, int maxnsegs, const char *name) 800 { 801 bus_size_t size_desc, size_avail, size_used, size_indirect; 802 bus_size_t allocsize = 0, size_desc_avail; 803 int rsegs, r, hdrlen; 804 unsigned int vq_num; 805 #define VIRTQUEUE_ALIGN(n) roundup(n, VIRTIO_PAGE_SIZE) 806 807 vq_num = vq->vq_num; 808 809 if (vq_num == 0) { 810 aprint_error_dev(sc->sc_dev, 811 "virtqueue not exist, index %d for %s\n", 812 vq->vq_index, name); 813 goto err; 814 } 815 816 hdrlen = sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX ? 3 : 2; 817 818 size_desc = sizeof(vq->vq_desc[0]) * vq_num; 819 size_avail = sizeof(uint16_t) * hdrlen 820 + sizeof(vq->vq_avail[0].ring[0]) * vq_num; 821 size_used = sizeof(uint16_t) *hdrlen 822 + sizeof(vq->vq_used[0].ring[0]) * vq_num; 823 size_indirect = (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) ? 824 sizeof(struct vring_desc) * maxnsegs * vq_num : 0; 825 826 size_desc_avail = VIRTQUEUE_ALIGN(size_desc + size_avail); 827 size_used = VIRTQUEUE_ALIGN(size_used); 828 829 allocsize = size_desc_avail + size_used + size_indirect; 830 831 /* alloc and map the memory */ 832 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 833 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_WAITOK); 834 if (r != 0) { 835 aprint_error_dev(sc->sc_dev, 836 "virtqueue %d for %s allocation failed, " 837 "error code %d\n", vq->vq_index, name, r); 838 goto err; 839 } 840 841 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], rsegs, allocsize, 842 &vq->vq_vaddr, BUS_DMA_WAITOK); 843 if (r != 0) { 844 aprint_error_dev(sc->sc_dev, 845 "virtqueue %d for %s map failed, " 846 "error code %d\n", vq->vq_index, name, r); 847 goto err; 848 } 849 850 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 851 BUS_DMA_WAITOK, &vq->vq_dmamap); 852 if (r != 0) { 853 aprint_error_dev(sc->sc_dev, 854 "virtqueue %d for %s dmamap creation failed, " 855 "error code %d\n", vq->vq_index, name, r); 856 goto err; 857 } 858 859 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 860 vq->vq_vaddr, allocsize, NULL, BUS_DMA_WAITOK); 861 if (r != 0) { 862 aprint_error_dev(sc->sc_dev, 863 "virtqueue %d for %s dmamap load failed, " 864 "error code %d\n", vq->vq_index, name, r); 865 goto err; 866 } 867 868 vq->vq_bytesize = allocsize; 869 vq->vq_maxsegsize = maxsegsize; 870 vq->vq_maxnsegs = maxnsegs; 871 872 #define VIRTIO_PTR(base, offset) (void *)((intptr_t)(base) + (offset)) 873 /* initialize vring pointers */ 874 vq->vq_desc = VIRTIO_PTR(vq->vq_vaddr, 0); 875 vq->vq_availoffset = size_desc; 876 vq->vq_avail = VIRTIO_PTR(vq->vq_vaddr, vq->vq_availoffset); 877 vq->vq_used_event = VIRTIO_PTR(vq->vq_avail, 878 offsetof(struct vring_avail, ring[vq_num])); 879 vq->vq_usedoffset = size_desc_avail; 880 vq->vq_used = VIRTIO_PTR(vq->vq_vaddr, vq->vq_usedoffset); 881 vq->vq_avail_event = VIRTIO_PTR(vq->vq_used, 882 offsetof(struct vring_used, ring[vq_num])); 883 884 if (size_indirect > 0) { 885 vq->vq_indirectoffset = size_desc_avail + size_used; 886 vq->vq_indirect = VIRTIO_PTR(vq->vq_vaddr, 887 vq->vq_indirectoffset); 888 } 889 #undef VIRTIO_PTR 890 891 vq->vq_descx = kmem_zalloc(sizeof(vq->vq_descx[0]) * vq_num, 892 KM_SLEEP); 893 894 mutex_init(&vq->vq_freedesc_lock, MUTEX_SPIN, sc->sc_ipl); 895 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 896 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 897 898 virtio_reset_vq(sc, vq); 899 900 aprint_verbose_dev(sc->sc_dev, 901 "allocated %" PRIuBUSSIZE " byte for virtqueue %d for %s, " 902 "size %d\n", allocsize, vq->vq_index, name, vq_num); 903 if (size_indirect > 0) 904 aprint_verbose_dev(sc->sc_dev, 905 "using %" PRIuBUSSIZE " byte (%d entries) indirect " 906 "descriptors\n", size_indirect, maxnsegs * vq_num); 907 908 return 0; 909 910 err: 911 sc->sc_ops->setup_queue(sc, vq->vq_index, 0); 912 if (vq->vq_dmamap) 913 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 914 if (vq->vq_vaddr) 915 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 916 if (vq->vq_segs[0].ds_addr) 917 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 918 memset(vq, 0, sizeof(*vq)); 919 920 return -1; 921 } 922 923 int 924 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 925 { 926 uint16_t s; 927 size_t i; 928 929 if (vq->vq_vaddr == NULL) 930 return 0; 931 932 /* device must be already deactivated */ 933 /* confirm the vq is empty */ 934 s = vq->vq_free_idx; 935 i = 0; 936 while (s != virtio_rw16(sc, VRING_DESC_CHAIN_END)) { 937 s = vq->vq_desc[s].next; 938 i++; 939 } 940 if (i != vq->vq_num) { 941 printf("%s: freeing non-empty vq, index %d\n", 942 device_xname(sc->sc_dev), vq->vq_index); 943 return EBUSY; 944 } 945 946 /* tell device that there's no virtqueue any longer */ 947 sc->sc_ops->setup_queue(sc, vq->vq_index, 0); 948 949 vq_sync_aring_all(sc, vq, BUS_DMASYNC_POSTWRITE); 950 951 kmem_free(vq->vq_descx, sizeof(vq->vq_descx[0]) * vq->vq_num); 952 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 953 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 954 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 955 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 956 mutex_destroy(&vq->vq_freedesc_lock); 957 mutex_destroy(&vq->vq_uring_lock); 958 mutex_destroy(&vq->vq_aring_lock); 959 memset(vq, 0, sizeof(*vq)); 960 961 return 0; 962 } 963 964 /* 965 * Free descriptor management. 966 */ 967 static int 968 vq_alloc_slot_locked(struct virtio_softc *sc, struct virtqueue *vq, 969 size_t nslots) 970 { 971 struct vring_desc *vd; 972 uint16_t head, tail; 973 size_t i; 974 975 KASSERT(mutex_owned(&vq->vq_freedesc_lock)); 976 977 head = tail = virtio_rw16(sc, vq->vq_free_idx); 978 for (i = 0; i < nslots - 1; i++) { 979 if (tail == VRING_DESC_CHAIN_END) 980 return VRING_DESC_CHAIN_END; 981 982 vd = &vq->vq_desc[tail]; 983 vd->flags = virtio_rw16(sc, VRING_DESC_F_NEXT); 984 tail = virtio_rw16(sc, vd->next); 985 } 986 987 if (tail == VRING_DESC_CHAIN_END) 988 return VRING_DESC_CHAIN_END; 989 990 vd = &vq->vq_desc[tail]; 991 vd->flags = virtio_rw16(sc, 0); 992 vq->vq_free_idx = vd->next; 993 994 return head; 995 } 996 static uint16_t 997 vq_alloc_slot(struct virtio_softc *sc, struct virtqueue *vq, size_t nslots) 998 { 999 uint16_t rv; 1000 1001 mutex_enter(&vq->vq_freedesc_lock); 1002 rv = vq_alloc_slot_locked(sc, vq, nslots); 1003 mutex_exit(&vq->vq_freedesc_lock); 1004 1005 return rv; 1006 } 1007 1008 static void 1009 vq_free_slot(struct virtio_softc *sc, struct virtqueue *vq, uint16_t slot) 1010 { 1011 struct vring_desc *vd; 1012 uint16_t s; 1013 1014 mutex_enter(&vq->vq_freedesc_lock); 1015 vd = &vq->vq_desc[slot]; 1016 while ((vd->flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) != 0) { 1017 s = virtio_rw16(sc, vd->next); 1018 vd = &vq->vq_desc[s]; 1019 } 1020 vd->next = vq->vq_free_idx; 1021 vq->vq_free_idx = virtio_rw16(sc, slot); 1022 mutex_exit(&vq->vq_freedesc_lock); 1023 } 1024 1025 /* 1026 * Enqueue several dmamaps as a single request. 1027 */ 1028 /* 1029 * Typical usage: 1030 * <queue size> number of followings are stored in arrays 1031 * - command blocks (in dmamem) should be pre-allocated and mapped 1032 * - dmamaps for command blocks should be pre-allocated and loaded 1033 * - dmamaps for payload should be pre-allocated 1034 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 1035 * if (r) // currently 0 or EAGAIN 1036 * return r; 1037 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 1038 * if (r) { 1039 * virtio_enqueue_abort(sc, vq, slot); 1040 * return r; 1041 * } 1042 * r = virtio_enqueue_reserve(sc, vq, slot, 1043 * dmamap_payload[slot]->dm_nsegs + 1); 1044 * // ^ +1 for command 1045 * if (r) { // currently 0 or EAGAIN 1046 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 1047 * return r; // do not call abort() 1048 * } 1049 * <setup and prepare commands> 1050 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 1051 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 1052 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 1053 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 1054 * virtio_enqueue_commit(sc, vq, slot, true); 1055 */ 1056 1057 /* 1058 * enqueue_prep: allocate a slot number 1059 */ 1060 int 1061 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 1062 { 1063 uint16_t slot; 1064 1065 KASSERT(sc->sc_child_state == VIRTIO_CHILD_ATTACH_FINISHED); 1066 KASSERT(slotp != NULL); 1067 1068 slot = vq_alloc_slot(sc, vq, 1); 1069 if (slot == VRING_DESC_CHAIN_END) 1070 return EAGAIN; 1071 1072 *slotp = slot; 1073 1074 return 0; 1075 } 1076 1077 /* 1078 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 1079 */ 1080 int 1081 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 1082 int slot, int nsegs) 1083 { 1084 struct vring_desc *vd; 1085 struct vring_desc_extra *vdx; 1086 int i; 1087 1088 KASSERT(1 <= nsegs); 1089 KASSERT(nsegs <= vq->vq_num); 1090 1091 vdx = &vq->vq_descx[slot]; 1092 vd = &vq->vq_desc[slot]; 1093 1094 KASSERT((vd->flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0); 1095 1096 if ((vq->vq_indirect != NULL) && 1097 (nsegs >= MINSEG_INDIRECT) && 1098 (nsegs <= vq->vq_maxnsegs)) 1099 vdx->use_indirect = true; 1100 else 1101 vdx->use_indirect = false; 1102 1103 if (vdx->use_indirect) { 1104 uint64_t addr; 1105 1106 addr = vq->vq_dmamap->dm_segs[0].ds_addr 1107 + vq->vq_indirectoffset; 1108 addr += sizeof(struct vring_desc) 1109 * vq->vq_maxnsegs * slot; 1110 1111 vd->addr = virtio_rw64(sc, addr); 1112 vd->len = virtio_rw32(sc, sizeof(struct vring_desc) * nsegs); 1113 vd->flags = virtio_rw16(sc, VRING_DESC_F_INDIRECT); 1114 1115 vd = &vq->vq_indirect[vq->vq_maxnsegs * slot]; 1116 vdx->desc_base = vd; 1117 vdx->desc_free_idx = 0; 1118 1119 for (i = 0; i < nsegs - 1; i++) { 1120 vd[i].flags = virtio_rw16(sc, VRING_DESC_F_NEXT); 1121 } 1122 vd[i].flags = virtio_rw16(sc, 0); 1123 } else { 1124 if (nsegs > 1) { 1125 uint16_t s; 1126 1127 s = vq_alloc_slot(sc, vq, nsegs - 1); 1128 if (s == VRING_DESC_CHAIN_END) { 1129 vq_free_slot(sc, vq, slot); 1130 return EAGAIN; 1131 } 1132 vd->next = virtio_rw16(sc, s); 1133 vd->flags = virtio_rw16(sc, VRING_DESC_F_NEXT); 1134 } 1135 1136 vdx->desc_base = &vq->vq_desc[0]; 1137 vdx->desc_free_idx = slot; 1138 } 1139 1140 return 0; 1141 } 1142 1143 /* 1144 * enqueue: enqueue a single dmamap. 1145 */ 1146 int 1147 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1148 bus_dmamap_t dmamap, bool write) 1149 { 1150 struct vring_desc *vds; 1151 struct vring_desc_extra *vdx; 1152 uint16_t s; 1153 int i; 1154 1155 KASSERT(dmamap->dm_nsegs > 0); 1156 1157 vdx = &vq->vq_descx[slot]; 1158 vds = vdx->desc_base; 1159 s = vdx->desc_free_idx; 1160 1161 KASSERT(vds != NULL); 1162 1163 for (i = 0; i < dmamap->dm_nsegs; i++) { 1164 KASSERT(s != VRING_DESC_CHAIN_END); 1165 1166 vds[s].addr = virtio_rw64(sc, dmamap->dm_segs[i].ds_addr); 1167 vds[s].len = virtio_rw32(sc, dmamap->dm_segs[i].ds_len); 1168 if (!write) 1169 vds[s].flags |= virtio_rw16(sc, VRING_DESC_F_WRITE); 1170 1171 if ((vds[s].flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0) { 1172 s = VRING_DESC_CHAIN_END; 1173 } else { 1174 s = virtio_rw16(sc, vds[s].next); 1175 } 1176 } 1177 1178 vdx->desc_free_idx = s; 1179 1180 return 0; 1181 } 1182 1183 int 1184 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1185 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 1186 bool write) 1187 { 1188 struct vring_desc_extra *vdx; 1189 struct vring_desc *vds; 1190 uint16_t s; 1191 1192 vdx = &vq->vq_descx[slot]; 1193 vds = vdx->desc_base; 1194 s = vdx->desc_free_idx; 1195 1196 KASSERT(s != VRING_DESC_CHAIN_END); 1197 KASSERT(vds != NULL); 1198 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 1199 KASSERT(dmamap->dm_segs[0].ds_len > start); 1200 KASSERT(dmamap->dm_segs[0].ds_len >= start + len); 1201 1202 vds[s].addr = virtio_rw64(sc, dmamap->dm_segs[0].ds_addr + start); 1203 vds[s].len = virtio_rw32(sc, len); 1204 if (!write) 1205 vds[s].flags |= virtio_rw16(sc, VRING_DESC_F_WRITE); 1206 1207 if ((vds[s].flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0) { 1208 s = VRING_DESC_CHAIN_END; 1209 } else { 1210 s = virtio_rw16(sc, vds[s].next); 1211 } 1212 1213 vdx->desc_free_idx = s; 1214 1215 return 0; 1216 } 1217 1218 /* 1219 * enqueue_commit: add it to the aring. 1220 */ 1221 int 1222 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1223 bool notifynow) 1224 { 1225 1226 if (slot < 0) { 1227 mutex_enter(&vq->vq_aring_lock); 1228 goto notify; 1229 } 1230 1231 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 1232 if (vq->vq_descx[slot].use_indirect) 1233 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 1234 1235 mutex_enter(&vq->vq_aring_lock); 1236 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = 1237 virtio_rw16(sc, slot); 1238 1239 notify: 1240 if (notifynow) { 1241 uint16_t o, n, t; 1242 uint16_t flags; 1243 1244 o = virtio_rw16(sc, vq->vq_avail->idx) - 1; 1245 n = vq->vq_avail_idx; 1246 1247 /* 1248 * Prepare for `device->CPU' (host->guest) transfer 1249 * into the buffer. This must happen before we commit 1250 * the vq->vq_avail->idx update to ensure we're not 1251 * still using the buffer in case program-prior loads 1252 * or stores in it get delayed past the store to 1253 * vq->vq_avail->idx. 1254 */ 1255 vq_sync_uring_all(sc, vq, BUS_DMASYNC_PREREAD); 1256 1257 /* ensure payload is published, then avail idx */ 1258 vq_sync_aring_payload(sc, vq, BUS_DMASYNC_PREWRITE); 1259 vq->vq_avail->idx = virtio_rw16(sc, vq->vq_avail_idx); 1260 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE); 1261 vq->vq_queued++; 1262 1263 /* 1264 * Ensure we publish the avail idx _before_ we check whether 1265 * the host needs to notified. 1266 */ 1267 paravirt_membar_sync(); 1268 1269 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) { 1270 vq_sync_uring_avail(sc, vq, BUS_DMASYNC_POSTREAD); 1271 t = virtio_rw16(sc, *vq->vq_avail_event) + 1; 1272 if ((uint16_t) (n - t) < (uint16_t) (n - o)) 1273 sc->sc_ops->kick(sc, vq->vq_index); 1274 } else { 1275 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD); 1276 flags = virtio_rw16(sc, vq->vq_used->flags); 1277 if (!(flags & VRING_USED_F_NO_NOTIFY)) 1278 sc->sc_ops->kick(sc, vq->vq_index); 1279 } 1280 } 1281 mutex_exit(&vq->vq_aring_lock); 1282 1283 return 0; 1284 } 1285 1286 /* 1287 * enqueue_abort: rollback. 1288 */ 1289 int 1290 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1291 { 1292 struct vring_desc_extra *vdx; 1293 1294 vdx = &vq->vq_descx[slot]; 1295 vdx->desc_free_idx = VRING_DESC_CHAIN_END; 1296 vdx->desc_base = NULL; 1297 1298 vq_free_slot(sc, vq, slot); 1299 1300 return 0; 1301 } 1302 1303 /* 1304 * Dequeue a request. 1305 */ 1306 /* 1307 * dequeue: dequeue a request from uring; dmamap_sync for uring is 1308 * already done in the interrupt handler. 1309 */ 1310 int 1311 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 1312 int *slotp, int *lenp) 1313 { 1314 uint16_t slot, usedidx; 1315 1316 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx)) 1317 return ENOENT; 1318 mutex_enter(&vq->vq_uring_lock); 1319 usedidx = vq->vq_used_idx++; 1320 mutex_exit(&vq->vq_uring_lock); 1321 usedidx %= vq->vq_num; 1322 slot = virtio_rw32(sc, vq->vq_used->ring[usedidx].id); 1323 1324 if (vq->vq_descx[slot].use_indirect) 1325 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 1326 1327 if (slotp) 1328 *slotp = slot; 1329 if (lenp) 1330 *lenp = virtio_rw32(sc, vq->vq_used->ring[usedidx].len); 1331 1332 return 0; 1333 } 1334 1335 /* 1336 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1337 * if you forget to call this the slot will be leaked. 1338 */ 1339 int 1340 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1341 { 1342 struct vring_desc_extra *vdx; 1343 1344 vdx = &vq->vq_descx[slot]; 1345 vdx->desc_base = NULL; 1346 vdx->desc_free_idx = VRING_DESC_CHAIN_END; 1347 1348 vq_free_slot(sc, vq, slot); 1349 1350 return 0; 1351 } 1352 1353 /* 1354 * Attach a child, fill all the members. 1355 */ 1356 void 1357 virtio_child_attach_start(struct virtio_softc *sc, device_t child, int ipl, 1358 uint64_t req_features, const char *feat_bits) 1359 { 1360 char buf[1024]; 1361 1362 KASSERT(sc->sc_child == NULL); 1363 KASSERT(sc->sc_child_state == VIRTIO_NO_CHILD); 1364 1365 sc->sc_child = child; 1366 sc->sc_ipl = ipl; 1367 1368 virtio_negotiate_features(sc, req_features); 1369 snprintb(buf, sizeof(buf), feat_bits, sc->sc_active_features); 1370 aprint_normal(": features: %s\n", buf); 1371 aprint_naive("\n"); 1372 } 1373 1374 int 1375 virtio_child_attach_finish(struct virtio_softc *sc, 1376 struct virtqueue *vqs, size_t nvqs, 1377 virtio_callback config_change, 1378 int req_flags) 1379 { 1380 size_t i; 1381 int r; 1382 1383 #ifdef DIAGNOSTIC 1384 KASSERT(nvqs > 0); 1385 #define VIRTIO_ASSERT_FLAGS (VIRTIO_F_INTR_SOFTINT | VIRTIO_F_INTR_PERVQ) 1386 KASSERT((req_flags & VIRTIO_ASSERT_FLAGS) != VIRTIO_ASSERT_FLAGS); 1387 #undef VIRTIO_ASSERT_FLAGS 1388 1389 for (i = 0; i < nvqs; i++){ 1390 KASSERT(vqs[i].vq_index == i); 1391 KASSERT(vqs[i].vq_intrhand != NULL); 1392 KASSERT(vqs[i].vq_done == NULL || 1393 vqs[i].vq_intrhand == virtio_vq_done); 1394 } 1395 #endif 1396 1397 1398 sc->sc_vqs = vqs; 1399 sc->sc_nvqs = nvqs; 1400 sc->sc_config_change = config_change; 1401 sc->sc_intrhand = virtio_vq_intr; 1402 sc->sc_flags = req_flags; 1403 1404 /* set the vq address */ 1405 for (i = 0; i < nvqs; i++) { 1406 sc->sc_ops->setup_queue(sc, vqs[i].vq_index, 1407 vqs[i].vq_dmamap->dm_segs[0].ds_addr); 1408 } 1409 1410 r = sc->sc_ops->alloc_interrupts(sc); 1411 if (r != 0) { 1412 aprint_error_dev(sc->sc_dev, 1413 "failed to allocate interrupts\n"); 1414 goto fail; 1415 } 1416 1417 r = sc->sc_ops->setup_interrupts(sc, 0); 1418 if (r != 0) { 1419 aprint_error_dev(sc->sc_dev, "failed to setup interrupts\n"); 1420 goto fail; 1421 } 1422 1423 KASSERT(sc->sc_soft_ih == NULL); 1424 if (sc->sc_flags & VIRTIO_F_INTR_SOFTINT) { 1425 u_int flags = SOFTINT_NET; 1426 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) 1427 flags |= SOFTINT_MPSAFE; 1428 1429 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, 1430 sc); 1431 if (sc->sc_soft_ih == NULL) { 1432 sc->sc_ops->free_interrupts(sc); 1433 aprint_error_dev(sc->sc_dev, 1434 "failed to establish soft interrupt\n"); 1435 goto fail; 1436 } 1437 } 1438 1439 sc->sc_child_state = VIRTIO_CHILD_ATTACH_FINISHED; 1440 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 1441 return 0; 1442 1443 fail: 1444 if (sc->sc_soft_ih) { 1445 softint_disestablish(sc->sc_soft_ih); 1446 sc->sc_soft_ih = NULL; 1447 } 1448 1449 sc->sc_ops->free_interrupts(sc); 1450 1451 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 1452 return 1; 1453 } 1454 1455 void 1456 virtio_child_detach(struct virtio_softc *sc) 1457 { 1458 1459 /* already detached */ 1460 if (sc->sc_child == NULL) 1461 return; 1462 1463 1464 virtio_device_reset(sc); 1465 1466 sc->sc_ops->free_interrupts(sc); 1467 1468 if (sc->sc_soft_ih) { 1469 softint_disestablish(sc->sc_soft_ih); 1470 sc->sc_soft_ih = NULL; 1471 } 1472 1473 sc->sc_vqs = NULL; 1474 sc->sc_child = NULL; 1475 } 1476 1477 void 1478 virtio_child_attach_failed(struct virtio_softc *sc) 1479 { 1480 virtio_child_detach(sc); 1481 1482 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 1483 1484 sc->sc_child_state = VIRTIO_CHILD_ATTACH_FAILED; 1485 } 1486 1487 bus_dma_tag_t 1488 virtio_dmat(struct virtio_softc *sc) 1489 { 1490 return sc->sc_dmat; 1491 } 1492 1493 device_t 1494 virtio_child(struct virtio_softc *sc) 1495 { 1496 return sc->sc_child; 1497 } 1498 1499 int 1500 virtio_intrhand(struct virtio_softc *sc) 1501 { 1502 return (*sc->sc_intrhand)(sc); 1503 } 1504 1505 uint64_t 1506 virtio_features(struct virtio_softc *sc) 1507 { 1508 return sc->sc_active_features; 1509 } 1510 1511 bool 1512 virtio_version_1(struct virtio_softc *sc) 1513 { 1514 return sc->sc_version_1; 1515 } 1516 1517 int 1518 virtio_attach_failed(struct virtio_softc *sc) 1519 { 1520 device_t self = sc->sc_dev; 1521 1522 /* no error if its not connected, but its failed */ 1523 if (sc->sc_childdevid == 0) 1524 return 1; 1525 1526 if (sc->sc_child == NULL) { 1527 switch (sc->sc_child_state) { 1528 case VIRTIO_CHILD_ATTACH_FAILED: 1529 aprint_error_dev(self, 1530 "virtio configuration failed\n"); 1531 break; 1532 case VIRTIO_NO_CHILD: 1533 aprint_error_dev(self, 1534 "no matching child driver; not configured\n"); 1535 break; 1536 default: 1537 /* sanity check */ 1538 aprint_error_dev(self, 1539 "virtio internal error, " 1540 "child driver is not configured\n"); 1541 break; 1542 } 1543 1544 return 1; 1545 } 1546 1547 /* sanity check */ 1548 if (sc->sc_child_state != VIRTIO_CHILD_ATTACH_FINISHED) { 1549 aprint_error_dev(self, "virtio internal error, child driver " 1550 "signaled OK but didn't initialize interrupts\n"); 1551 return 1; 1552 } 1553 1554 return 0; 1555 } 1556 1557 void 1558 virtio_print_device_type(device_t self, int id, int revision) 1559 { 1560 aprint_normal_dev(self, "%s device (id %d, rev. 0x%02x)\n", 1561 (id < NDEVNAMES ? virtio_device_name[id] : "Unknown"), 1562 id, 1563 revision); 1564 } 1565 1566 1567 MODULE(MODULE_CLASS_DRIVER, virtio, NULL); 1568 1569 #ifdef _MODULE 1570 #include "ioconf.c" 1571 #endif 1572 1573 static int 1574 virtio_modcmd(modcmd_t cmd, void *opaque) 1575 { 1576 int error = 0; 1577 1578 #ifdef _MODULE 1579 switch (cmd) { 1580 case MODULE_CMD_INIT: 1581 error = config_init_component(cfdriver_ioconf_virtio, 1582 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1583 break; 1584 case MODULE_CMD_FINI: 1585 error = config_fini_component(cfdriver_ioconf_virtio, 1586 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1587 break; 1588 default: 1589 error = ENOTTY; 1590 break; 1591 } 1592 #endif 1593 1594 return error; 1595 } 1596
Indexes created Sun Sep 21 20:09:37 GMT 2025