virtio.c revision 1.81.2.1
1 /* $NetBSD: virtio.c,v 1.81.2.1 2025/08/02 05:57:00 perseant Exp $ */ 2 3 /* 4 * Copyright (c) 2020 The NetBSD Foundation, Inc. 5 * Copyright (c) 2012 Stefan Fritsch, Alexander Fiveg. 6 * Copyright (c) 2010 Minoura Makoto. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.81.2.1 2025/08/02 05:57:00 perseant Exp $"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kernel.h> 36 #include <sys/atomic.h> 37 #include <sys/bus.h> 38 #include <sys/device.h> 39 #include <sys/kmem.h> 40 #include <sys/module.h> 41 42 #define VIRTIO_PRIVATE 43 44 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */ 45 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */ 46 47 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 48 49 /* 50 * The maximum descriptor size is 2^15. Use that value as the end of 51 * descriptor chain terminator since it will never be a valid index 52 * in the descriptor table. 53 */ 54 #define VRING_DESC_CHAIN_END 32768 55 56 /* incomplete list */ 57 static const char *virtio_device_name[] = { 58 "unknown (0)", /* 0 */ 59 "network", /* 1 */ 60 "block", /* 2 */ 61 "console", /* 3 */ 62 "entropy", /* 4 */ 63 "memory balloon", /* 5 */ 64 "I/O memory", /* 6 */ 65 "remote processor messaging", /* 7 */ 66 "SCSI", /* 8 */ 67 "9P transport", /* 9 */ 68 NULL, /* 10 */ 69 NULL, /* 11 */ 70 NULL, /* 12 */ 71 NULL, /* 13 */ 72 NULL, /* 14 */ 73 NULL, /* 15 */ 74 "GPU", /* 16 */ 75 }; 76 #define NDEVNAMES __arraycount(virtio_device_name) 77 78 static void virtio_reset_vq(struct virtio_softc *, 79 struct virtqueue *); 80 81 void 82 virtio_set_status(struct virtio_softc *sc, int status) 83 { 84 sc->sc_ops->set_status(sc, status); 85 } 86 87 /* 88 * Reset the device. 89 */ 90 /* 91 * To reset the device to a known state, do following: 92 * virtio_reset(sc); // this will stop the device activity 93 * <dequeue finished requests>; // virtio_dequeue() still can be called 94 * <revoke pending requests in the vqs if any>; 95 * virtio_reinit_start(sc); // dequeue prohibited 96 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 97 * <some other initialization>; 98 * virtio_reinit_end(sc); // device activated; enqueue allowed 99 * Once attached, feature negotiation can only be allowed after virtio_reset. 100 */ 101 void 102 virtio_reset(struct virtio_softc *sc) 103 { 104 virtio_device_reset(sc); 105 } 106 107 int 108 virtio_reinit_start(struct virtio_softc *sc) 109 { 110 int i, r; 111 112 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 113 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 114 for (i = 0; i < sc->sc_nvqs; i++) { 115 int n; 116 struct virtqueue *vq = &sc->sc_vqs[i]; 117 n = sc->sc_ops->read_queue_size(sc, vq->vq_index); 118 if (n == 0) /* vq disappeared */ 119 continue; 120 if (n != vq->vq_num) { 121 panic("%s: virtqueue size changed, vq index %d\n", 122 device_xname(sc->sc_dev), 123 vq->vq_index); 124 } 125 virtio_reset_vq(sc, vq); 126 sc->sc_ops->setup_queue(sc, vq->vq_index, 127 vq->vq_dmamap->dm_segs[0].ds_addr); 128 } 129 130 r = sc->sc_ops->setup_interrupts(sc, 1); 131 if (r != 0) 132 goto fail; 133 134 return 0; 135 136 fail: 137 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 138 139 return 1; 140 } 141 142 void 143 virtio_reinit_end(struct virtio_softc *sc) 144 { 145 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 146 } 147 148 /* 149 * Feature negotiation. 150 */ 151 void 152 virtio_negotiate_features(struct virtio_softc *sc, uint64_t guest_features) 153 { 154 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 155 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 156 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 157 sc->sc_ops->neg_features(sc, guest_features); 158 if (sc->sc_active_features & VIRTIO_F_RING_INDIRECT_DESC) 159 sc->sc_indirect = true; 160 else 161 sc->sc_indirect = false; 162 } 163 164 165 /* 166 * Device configuration registers readers/writers 167 */ 168 #if 0 169 #define DPRINTFR(n, fmt, val, index, num) \ 170 printf("\n%s (", n); \ 171 for (int i = 0; i < num; i++) \ 172 printf("%02x ", bus_space_read_1(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index+i)); \ 173 printf(") -> "); printf(fmt, val); printf("\n"); 174 #define DPRINTFR2(n, fmt, val_s, val_n) \ 175 printf("%s ", n); \ 176 printf("\n stream "); printf(fmt, val_s); printf(" norm "); printf(fmt, val_n); printf("\n"); 177 #else 178 #define DPRINTFR(n, fmt, val, index, num) 179 #define DPRINTFR2(n, fmt, val_s, val_n) 180 #endif 181 182 183 uint8_t 184 virtio_read_device_config_1(struct virtio_softc *sc, int index) 185 { 186 bus_space_tag_t iot = sc->sc_devcfg_iot; 187 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 188 uint8_t val; 189 190 val = bus_space_read_1(iot, ioh, index); 191 192 DPRINTFR("read_1", "%02x", val, index, 1); 193 return val; 194 } 195 196 uint16_t 197 virtio_read_device_config_2(struct virtio_softc *sc, int index) 198 { 199 bus_space_tag_t iot = sc->sc_devcfg_iot; 200 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 201 uint16_t val; 202 203 val = bus_space_read_2(iot, ioh, index); 204 if (BYTE_ORDER != sc->sc_bus_endian) 205 val = bswap16(val); 206 207 DPRINTFR("read_2", "%04x", val, index, 2); 208 DPRINTFR2("read_2", "%04x", 209 bus_space_read_stream_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 210 index), 211 bus_space_read_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index)); 212 return val; 213 } 214 215 uint32_t 216 virtio_read_device_config_4(struct virtio_softc *sc, int index) 217 { 218 bus_space_tag_t iot = sc->sc_devcfg_iot; 219 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 220 uint32_t val; 221 222 val = bus_space_read_4(iot, ioh, index); 223 if (BYTE_ORDER != sc->sc_bus_endian) 224 val = bswap32(val); 225 226 DPRINTFR("read_4", "%08x", val, index, 4); 227 DPRINTFR2("read_4", "%08x", 228 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 229 index), 230 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index)); 231 return val; 232 } 233 234 /* 235 * The Virtio spec explicitly tells that reading and writing 8 bytes are not 236 * considered atomic and no triggers may be connected to reading or writing 237 * it. We access it using two 32 reads. See virtio spec 4.1.3.1. 238 */ 239 uint64_t 240 virtio_read_device_config_8(struct virtio_softc *sc, int index) 241 { 242 bus_space_tag_t iot = sc->sc_devcfg_iot; 243 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 244 union { 245 uint64_t u64; 246 uint32_t l[2]; 247 } v; 248 uint64_t val; 249 250 v.l[0] = bus_space_read_4(iot, ioh, index); 251 v.l[1] = bus_space_read_4(iot, ioh, index + 4); 252 if (sc->sc_bus_endian != sc->sc_struct_endian) { 253 v.l[0] = bswap32(v.l[0]); 254 v.l[1] = bswap32(v.l[1]); 255 } 256 val = v.u64; 257 258 if (BYTE_ORDER != sc->sc_struct_endian) 259 val = bswap64(val); 260 261 DPRINTFR("read_8", "%08"PRIx64, val, index, 8); 262 DPRINTFR2("read_8 low ", "%08x", 263 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 264 index), 265 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index)); 266 DPRINTFR2("read_8 high ", "%08x", 267 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 268 index + 4), 269 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index + 4)); 270 return val; 271 } 272 273 /* 274 * In the older virtio spec, device config registers are host endian. On newer 275 * they are little endian. Some newer devices however explicitly specify their 276 * register to always be little endian. These functions cater for these. 277 */ 278 uint16_t 279 virtio_read_device_config_le_2(struct virtio_softc *sc, int index) 280 { 281 bus_space_tag_t iot = sc->sc_devcfg_iot; 282 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 283 uint16_t val; 284 285 val = bus_space_read_2(iot, ioh, index); 286 #if !defined(__aarch64__) && !defined(__arm__) 287 /* 288 * For big-endian aarch64/armv7, bus endian is always LSB, but 289 * byte-order is automatically swapped by bus_space(9) (see also 290 * comments in virtio_pci.c). Therefore, no need to swap here. 291 */ 292 if (sc->sc_bus_endian != LITTLE_ENDIAN) 293 val = bswap16(val); 294 #endif 295 296 DPRINTFR("read_le_2", "%04x", val, index, 2); 297 DPRINTFR2("read_le_2", "%04x", 298 bus_space_read_stream_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0), 299 bus_space_read_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0)); 300 return val; 301 } 302 303 uint32_t 304 virtio_read_device_config_le_4(struct virtio_softc *sc, int index) 305 { 306 bus_space_tag_t iot = sc->sc_devcfg_iot; 307 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 308 uint32_t val; 309 310 val = bus_space_read_4(iot, ioh, index); 311 #if !defined(__aarch64__) && !defined(__arm__) 312 /* See virtio_read_device_config_le_2() above. */ 313 if (sc->sc_bus_endian != LITTLE_ENDIAN) 314 val = bswap32(val); 315 #endif 316 317 DPRINTFR("read_le_4", "%08x", val, index, 4); 318 DPRINTFR2("read_le_4", "%08x", 319 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0), 320 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0)); 321 return val; 322 } 323 324 void 325 virtio_write_device_config_1(struct virtio_softc *sc, int index, uint8_t value) 326 { 327 bus_space_tag_t iot = sc->sc_devcfg_iot; 328 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 329 330 bus_space_write_1(iot, ioh, index, value); 331 } 332 333 void 334 virtio_write_device_config_2(struct virtio_softc *sc, int index, 335 uint16_t value) 336 { 337 bus_space_tag_t iot = sc->sc_devcfg_iot; 338 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 339 340 if (BYTE_ORDER != sc->sc_bus_endian) 341 value = bswap16(value); 342 bus_space_write_2(iot, ioh, index, value); 343 } 344 345 void 346 virtio_write_device_config_4(struct virtio_softc *sc, int index, 347 uint32_t value) 348 { 349 bus_space_tag_t iot = sc->sc_devcfg_iot; 350 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 351 352 if (BYTE_ORDER != sc->sc_bus_endian) 353 value = bswap32(value); 354 bus_space_write_4(iot, ioh, index, value); 355 } 356 357 /* 358 * The Virtio spec explicitly tells that reading and writing 8 bytes are not 359 * considered atomic and no triggers may be connected to reading or writing 360 * it. We access it using two 32 bit writes. For good measure it is stated to 361 * always write lsb first just in case of a hypervisor bug. See See virtio 362 * spec 4.1.3.1. 363 */ 364 void 365 virtio_write_device_config_8(struct virtio_softc *sc, int index, 366 uint64_t value) 367 { 368 bus_space_tag_t iot = sc->sc_devcfg_iot; 369 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 370 union { 371 uint64_t u64; 372 uint32_t l[2]; 373 } v; 374 375 if (BYTE_ORDER != sc->sc_struct_endian) 376 value = bswap64(value); 377 378 v.u64 = value; 379 if (sc->sc_bus_endian != sc->sc_struct_endian) { 380 v.l[0] = bswap32(v.l[0]); 381 v.l[1] = bswap32(v.l[1]); 382 } 383 384 if (sc->sc_struct_endian == LITTLE_ENDIAN) { 385 bus_space_write_4(iot, ioh, index, v.l[0]); 386 bus_space_write_4(iot, ioh, index + 4, v.l[1]); 387 } else { 388 bus_space_write_4(iot, ioh, index + 4, v.l[1]); 389 bus_space_write_4(iot, ioh, index, v.l[0]); 390 } 391 } 392 393 /* 394 * In the older virtio spec, device config registers are host endian. On newer 395 * they are little endian. Some newer devices however explicitly specify their 396 * register to always be little endian. These functions cater for these. 397 */ 398 void 399 virtio_write_device_config_le_2(struct virtio_softc *sc, int index, 400 uint16_t value) 401 { 402 bus_space_tag_t iot = sc->sc_devcfg_iot; 403 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 404 405 if (sc->sc_bus_endian != LITTLE_ENDIAN) 406 value = bswap16(value); 407 bus_space_write_2(iot, ioh, index, value); 408 } 409 410 void 411 virtio_write_device_config_le_4(struct virtio_softc *sc, int index, 412 uint32_t value) 413 { 414 bus_space_tag_t iot = sc->sc_devcfg_iot; 415 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 416 417 if (sc->sc_bus_endian != LITTLE_ENDIAN) 418 value = bswap32(value); 419 bus_space_write_4(iot, ioh, index, value); 420 } 421 422 423 /* 424 * data structures endian helpers 425 */ 426 uint16_t 427 virtio_rw16(struct virtio_softc *sc, uint16_t val) 428 { 429 KASSERT(sc); 430 return BYTE_ORDER != sc->sc_struct_endian ? bswap16(val) : val; 431 } 432 433 uint32_t 434 virtio_rw32(struct virtio_softc *sc, uint32_t val) 435 { 436 KASSERT(sc); 437 return BYTE_ORDER != sc->sc_struct_endian ? bswap32(val) : val; 438 } 439 440 uint64_t 441 virtio_rw64(struct virtio_softc *sc, uint64_t val) 442 { 443 KASSERT(sc); 444 return BYTE_ORDER != sc->sc_struct_endian ? bswap64(val) : val; 445 } 446 447 448 /* 449 * Interrupt handler. 450 */ 451 static void 452 virtio_soft_intr(void *arg) 453 { 454 struct virtio_softc *sc = arg; 455 456 KASSERT(sc->sc_intrhand != NULL); 457 458 (*sc->sc_intrhand)(sc); 459 } 460 461 /* set to vq->vq_intrhand in virtio_init_vq_vqdone() */ 462 static int 463 virtio_vq_done(void *xvq) 464 { 465 struct virtqueue *vq = xvq; 466 467 return vq->vq_done(vq); 468 } 469 470 static int 471 virtio_vq_intr(struct virtio_softc *sc) 472 { 473 struct virtqueue *vq; 474 int i, r = 0; 475 476 for (i = 0; i < sc->sc_nvqs; i++) { 477 vq = &sc->sc_vqs[i]; 478 if (virtio_vq_is_enqueued(sc, vq) == 1) { 479 r |= (*vq->vq_intrhand)(vq->vq_intrhand_arg); 480 } 481 } 482 483 return r; 484 } 485 486 /* 487 * dmamap sync operations for a virtqueue. 488 */ 489 static inline void 490 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 491 { 492 493 /* availoffset == sizeof(vring_desc) * vq_num */ 494 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 495 ops); 496 } 497 498 static inline void 499 vq_sync_aring_all(struct virtio_softc *sc, struct virtqueue *vq, int ops) 500 { 501 uint16_t hdrlen = offsetof(struct vring_avail, ring); 502 size_t payloadlen = vq->vq_num * sizeof(uint16_t); 503 size_t usedlen = 0; 504 505 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) 506 usedlen = sizeof(uint16_t); 507 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 508 vq->vq_availoffset, hdrlen + payloadlen + usedlen, ops); 509 } 510 511 static inline void 512 vq_sync_aring_header(struct virtio_softc *sc, struct virtqueue *vq, int ops) 513 { 514 uint16_t hdrlen = offsetof(struct vring_avail, ring); 515 516 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 517 vq->vq_availoffset, hdrlen, ops); 518 } 519 520 static inline void 521 vq_sync_aring_payload(struct virtio_softc *sc, struct virtqueue *vq, int ops) 522 { 523 uint16_t hdrlen = offsetof(struct vring_avail, ring); 524 size_t payloadlen = vq->vq_num * sizeof(uint16_t); 525 526 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 527 vq->vq_availoffset + hdrlen, payloadlen, ops); 528 } 529 530 static inline void 531 vq_sync_aring_used(struct virtio_softc *sc, struct virtqueue *vq, int ops) 532 { 533 uint16_t hdrlen = offsetof(struct vring_avail, ring); 534 size_t payloadlen = vq->vq_num * sizeof(uint16_t); 535 size_t usedlen = sizeof(uint16_t); 536 537 if ((sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) == 0) 538 return; 539 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 540 vq->vq_availoffset + hdrlen + payloadlen, usedlen, ops); 541 } 542 543 static inline void 544 vq_sync_uring_all(struct virtio_softc *sc, struct virtqueue *vq, int ops) 545 { 546 uint16_t hdrlen = offsetof(struct vring_used, ring); 547 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem); 548 size_t availlen = 0; 549 550 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) 551 availlen = sizeof(uint16_t); 552 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 553 vq->vq_usedoffset, hdrlen + payloadlen + availlen, ops); 554 } 555 556 static inline void 557 vq_sync_uring_header(struct virtio_softc *sc, struct virtqueue *vq, int ops) 558 { 559 uint16_t hdrlen = offsetof(struct vring_used, ring); 560 561 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 562 vq->vq_usedoffset, hdrlen, ops); 563 } 564 565 static inline void 566 vq_sync_uring_payload(struct virtio_softc *sc, struct virtqueue *vq, int ops) 567 { 568 uint16_t hdrlen = offsetof(struct vring_used, ring); 569 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem); 570 571 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 572 vq->vq_usedoffset + hdrlen, payloadlen, ops); 573 } 574 575 static inline void 576 vq_sync_uring_avail(struct virtio_softc *sc, struct virtqueue *vq, int ops) 577 { 578 uint16_t hdrlen = offsetof(struct vring_used, ring); 579 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem); 580 size_t availlen = sizeof(uint16_t); 581 582 if ((sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) == 0) 583 return; 584 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 585 vq->vq_usedoffset + hdrlen + payloadlen, availlen, ops); 586 } 587 588 static inline void 589 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 590 int ops) 591 { 592 int offset = vq->vq_indirectoffset + 593 sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 594 595 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 596 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, ops); 597 } 598 599 bool 600 virtio_vq_is_enqueued(struct virtio_softc *sc, struct virtqueue *vq) 601 { 602 603 if (vq->vq_queued) { 604 vq->vq_queued = 0; 605 vq_sync_aring_all(sc, vq, BUS_DMASYNC_POSTWRITE); 606 } 607 608 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD); 609 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx)) 610 return 0; 611 vq_sync_uring_payload(sc, vq, BUS_DMASYNC_POSTREAD); 612 return 1; 613 } 614 615 /* 616 * Increase the event index in order to delay interrupts. 617 */ 618 int 619 virtio_postpone_intr(struct virtio_softc *sc, struct virtqueue *vq, 620 uint16_t nslots) 621 { 622 uint16_t idx, nused; 623 624 idx = vq->vq_used_idx + nslots; 625 626 /* set the new event index: avail_ring->used_event = idx */ 627 *vq->vq_used_event = virtio_rw16(sc, idx); 628 vq_sync_aring_used(vq->vq_owner, vq, BUS_DMASYNC_PREWRITE); 629 vq->vq_queued++; 630 631 nused = (uint16_t) 632 (virtio_rw16(sc, vq->vq_used->idx) - vq->vq_used_idx); 633 KASSERT(nused <= vq->vq_num); 634 635 return nslots < nused; 636 } 637 638 /* 639 * Postpone interrupt until 3/4 of the available descriptors have been 640 * consumed. 641 */ 642 int 643 virtio_postpone_intr_smart(struct virtio_softc *sc, struct virtqueue *vq) 644 { 645 uint16_t nslots; 646 647 nslots = (uint16_t) 648 (virtio_rw16(sc, vq->vq_avail->idx) - vq->vq_used_idx) * 3 / 4; 649 650 return virtio_postpone_intr(sc, vq, nslots); 651 } 652 653 /* 654 * Postpone interrupt until all of the available descriptors have been 655 * consumed. 656 */ 657 int 658 virtio_postpone_intr_far(struct virtio_softc *sc, struct virtqueue *vq) 659 { 660 uint16_t nslots; 661 662 nslots = (uint16_t) 663 (virtio_rw16(sc, vq->vq_avail->idx) - vq->vq_used_idx); 664 665 return virtio_postpone_intr(sc, vq, nslots); 666 } 667 668 /* 669 * Start/stop vq interrupt. No guarantee. 670 */ 671 void 672 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 673 { 674 675 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) { 676 /* 677 * No way to disable the interrupt completely with 678 * RingEventIdx. Instead advance used_event by half the 679 * possible value. This won't happen soon and is far enough in 680 * the past to not trigger a spurious interrupt. 681 */ 682 *vq->vq_used_event = virtio_rw16(sc, vq->vq_used_idx + 0x8000); 683 vq_sync_aring_used(sc, vq, BUS_DMASYNC_PREWRITE); 684 } else { 685 vq->vq_avail->flags |= 686 virtio_rw16(sc, VRING_AVAIL_F_NO_INTERRUPT); 687 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE); 688 } 689 vq->vq_queued++; 690 } 691 692 int 693 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 694 { 695 696 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) { 697 /* 698 * If event index feature is negotiated, enabling interrupts 699 * is done through setting the latest consumed index in the 700 * used_event field 701 */ 702 *vq->vq_used_event = virtio_rw16(sc, vq->vq_used_idx); 703 vq_sync_aring_used(sc, vq, BUS_DMASYNC_PREWRITE); 704 } else { 705 vq->vq_avail->flags &= 706 ~virtio_rw16(sc, VRING_AVAIL_F_NO_INTERRUPT); 707 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE); 708 } 709 vq->vq_queued++; 710 711 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD); 712 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx)) 713 return 0; 714 vq_sync_uring_payload(sc, vq, BUS_DMASYNC_POSTREAD); 715 return 1; 716 } 717 718 /* 719 * Initialize vq structure. 720 */ 721 /* 722 * Reset virtqueue parameters 723 */ 724 static void 725 virtio_reset_vq(struct virtio_softc *sc, struct virtqueue *vq) 726 { 727 struct vring_desc *vds; 728 int i, j; 729 int vq_size = vq->vq_num; 730 731 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 732 733 /* build the descriptor chain for free slot management */ 734 vds = vq->vq_desc; 735 for (i = 0; i < vq_size - 1; i++) { 736 vds[i].next = virtio_rw16(sc, i + 1); 737 } 738 vds[i].next = virtio_rw16(sc, VRING_DESC_CHAIN_END); 739 vq->vq_free_idx = 0; 740 741 /* build the indirect descriptor chain */ 742 if (vq->vq_indirect != NULL) { 743 struct vring_desc *vd; 744 745 for (i = 0; i < vq_size; i++) { 746 vd = vq->vq_indirect; 747 vd += vq->vq_maxnsegs * i; 748 for (j = 0; j < vq->vq_maxnsegs - 1; j++) { 749 vd[j].next = virtio_rw16(sc, j + 1); 750 } 751 } 752 } 753 754 /* enqueue/dequeue status */ 755 vq->vq_avail_idx = 0; 756 vq->vq_used_idx = 0; 757 vq->vq_queued = 0; 758 vq_sync_uring_all(sc, vq, BUS_DMASYNC_PREREAD); 759 vq->vq_queued++; 760 } 761 762 /* Initialize vq */ 763 void 764 virtio_init_vq_vqdone(struct virtio_softc *sc, struct virtqueue *vq, 765 int index, int (*vq_done)(struct virtqueue *)) 766 { 767 768 virtio_init_vq(sc, vq, index, virtio_vq_done, vq); 769 vq->vq_done = vq_done; 770 } 771 772 void 773 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, 774 int (*func)(void *), void *arg) 775 { 776 777 memset(vq, 0, sizeof(*vq)); 778 779 vq->vq_owner = sc; 780 vq->vq_num = sc->sc_ops->read_queue_size(sc, index); 781 vq->vq_index = index; 782 vq->vq_intrhand = func; 783 vq->vq_intrhand_arg = arg; 784 } 785 786 /* 787 * Allocate/free a vq. 788 */ 789 int 790 virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, 791 int maxsegsize, int maxnsegs, const char *name) 792 { 793 bus_size_t size_desc, size_avail, size_used, size_indirect; 794 bus_size_t allocsize = 0, size_desc_avail; 795 int rsegs, r, hdrlen; 796 unsigned int vq_num; 797 #define VIRTQUEUE_ALIGN(n) roundup(n, VIRTIO_PAGE_SIZE) 798 799 vq_num = vq->vq_num; 800 801 if (vq_num == 0) { 802 aprint_error_dev(sc->sc_dev, 803 "virtqueue not exist, index %d for %s\n", 804 vq->vq_index, name); 805 goto err; 806 } 807 808 hdrlen = sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX ? 3 : 2; 809 810 size_desc = sizeof(vq->vq_desc[0]) * vq_num; 811 size_avail = sizeof(uint16_t) * hdrlen 812 + sizeof(vq->vq_avail[0].ring[0]) * vq_num; 813 size_used = sizeof(uint16_t) *hdrlen 814 + sizeof(vq->vq_used[0].ring[0]) * vq_num; 815 size_indirect = (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) ? 816 sizeof(struct vring_desc) * maxnsegs * vq_num : 0; 817 818 size_desc_avail = VIRTQUEUE_ALIGN(size_desc + size_avail); 819 size_used = VIRTQUEUE_ALIGN(size_used); 820 821 allocsize = size_desc_avail + size_used + size_indirect; 822 823 /* alloc and map the memory */ 824 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 825 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_WAITOK); 826 if (r != 0) { 827 aprint_error_dev(sc->sc_dev, 828 "virtqueue %d for %s allocation failed, " 829 "error code %d\n", vq->vq_index, name, r); 830 goto err; 831 } 832 833 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], rsegs, allocsize, 834 &vq->vq_vaddr, BUS_DMA_WAITOK); 835 if (r != 0) { 836 aprint_error_dev(sc->sc_dev, 837 "virtqueue %d for %s map failed, " 838 "error code %d\n", vq->vq_index, name, r); 839 goto err; 840 } 841 842 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 843 BUS_DMA_WAITOK, &vq->vq_dmamap); 844 if (r != 0) { 845 aprint_error_dev(sc->sc_dev, 846 "virtqueue %d for %s dmamap creation failed, " 847 "error code %d\n", vq->vq_index, name, r); 848 goto err; 849 } 850 851 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 852 vq->vq_vaddr, allocsize, NULL, BUS_DMA_WAITOK); 853 if (r != 0) { 854 aprint_error_dev(sc->sc_dev, 855 "virtqueue %d for %s dmamap load failed, " 856 "error code %d\n", vq->vq_index, name, r); 857 goto err; 858 } 859 860 vq->vq_bytesize = allocsize; 861 vq->vq_maxsegsize = maxsegsize; 862 vq->vq_maxnsegs = maxnsegs; 863 864 #define VIRTIO_PTR(base, offset) (void *)((intptr_t)(base) + (offset)) 865 /* initialize vring pointers */ 866 vq->vq_desc = VIRTIO_PTR(vq->vq_vaddr, 0); 867 vq->vq_availoffset = size_desc; 868 vq->vq_avail = VIRTIO_PTR(vq->vq_vaddr, vq->vq_availoffset); 869 vq->vq_used_event = VIRTIO_PTR(vq->vq_avail, 870 offsetof(struct vring_avail, ring[vq_num])); 871 vq->vq_usedoffset = size_desc_avail; 872 vq->vq_used = VIRTIO_PTR(vq->vq_vaddr, vq->vq_usedoffset); 873 vq->vq_avail_event = VIRTIO_PTR(vq->vq_used, 874 offsetof(struct vring_used, ring[vq_num])); 875 876 if (size_indirect > 0) { 877 vq->vq_indirectoffset = size_desc_avail + size_used; 878 vq->vq_indirect = VIRTIO_PTR(vq->vq_vaddr, 879 vq->vq_indirectoffset); 880 } 881 #undef VIRTIO_PTR 882 883 vq->vq_descx = kmem_zalloc(sizeof(vq->vq_descx[0]) * vq_num, 884 KM_SLEEP); 885 886 mutex_init(&vq->vq_freedesc_lock, MUTEX_SPIN, sc->sc_ipl); 887 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 888 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 889 890 virtio_reset_vq(sc, vq); 891 892 aprint_verbose_dev(sc->sc_dev, 893 "allocated %" PRIuBUSSIZE " byte for virtqueue %d for %s, " 894 "size %d\n", allocsize, vq->vq_index, name, vq_num); 895 if (size_indirect > 0) 896 aprint_verbose_dev(sc->sc_dev, 897 "using %" PRIuBUSSIZE " byte (%d entries) indirect " 898 "descriptors\n", size_indirect, maxnsegs * vq_num); 899 900 return 0; 901 902 err: 903 sc->sc_ops->setup_queue(sc, vq->vq_index, 0); 904 if (vq->vq_dmamap) 905 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 906 if (vq->vq_vaddr) 907 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 908 if (vq->vq_segs[0].ds_addr) 909 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 910 memset(vq, 0, sizeof(*vq)); 911 912 return -1; 913 } 914 915 int 916 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 917 { 918 uint16_t s; 919 size_t i; 920 921 if (vq->vq_vaddr == NULL) 922 return 0; 923 924 /* device must be already deactivated */ 925 /* confirm the vq is empty */ 926 s = vq->vq_free_idx; 927 i = 0; 928 while (s != virtio_rw16(sc, VRING_DESC_CHAIN_END)) { 929 s = vq->vq_desc[s].next; 930 i++; 931 } 932 if (i != vq->vq_num) { 933 printf("%s: freeing non-empty vq, index %d\n", 934 device_xname(sc->sc_dev), vq->vq_index); 935 return EBUSY; 936 } 937 938 /* tell device that there's no virtqueue any longer */ 939 sc->sc_ops->setup_queue(sc, vq->vq_index, 0); 940 941 vq_sync_aring_all(sc, vq, BUS_DMASYNC_POSTWRITE); 942 943 kmem_free(vq->vq_descx, sizeof(vq->vq_descx[0]) * vq->vq_num); 944 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 945 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 946 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 947 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 948 mutex_destroy(&vq->vq_freedesc_lock); 949 mutex_destroy(&vq->vq_uring_lock); 950 mutex_destroy(&vq->vq_aring_lock); 951 memset(vq, 0, sizeof(*vq)); 952 953 return 0; 954 } 955 956 /* 957 * Free descriptor management. 958 */ 959 static int 960 vq_alloc_slot_locked(struct virtio_softc *sc, struct virtqueue *vq, 961 size_t nslots) 962 { 963 struct vring_desc *vd; 964 uint16_t head, tail; 965 size_t i; 966 967 KASSERT(mutex_owned(&vq->vq_freedesc_lock)); 968 969 head = tail = virtio_rw16(sc, vq->vq_free_idx); 970 for (i = 0; i < nslots - 1; i++) { 971 if (tail == VRING_DESC_CHAIN_END) 972 return VRING_DESC_CHAIN_END; 973 974 vd = &vq->vq_desc[tail]; 975 vd->flags = virtio_rw16(sc, VRING_DESC_F_NEXT); 976 tail = virtio_rw16(sc, vd->next); 977 } 978 979 if (tail == VRING_DESC_CHAIN_END) 980 return VRING_DESC_CHAIN_END; 981 982 vd = &vq->vq_desc[tail]; 983 vd->flags = virtio_rw16(sc, 0); 984 vq->vq_free_idx = vd->next; 985 986 return head; 987 } 988 static uint16_t 989 vq_alloc_slot(struct virtio_softc *sc, struct virtqueue *vq, size_t nslots) 990 { 991 uint16_t rv; 992 993 mutex_enter(&vq->vq_freedesc_lock); 994 rv = vq_alloc_slot_locked(sc, vq, nslots); 995 mutex_exit(&vq->vq_freedesc_lock); 996 997 return rv; 998 } 999 1000 static void 1001 vq_free_slot(struct virtio_softc *sc, struct virtqueue *vq, uint16_t slot) 1002 { 1003 struct vring_desc *vd; 1004 uint16_t s; 1005 1006 mutex_enter(&vq->vq_freedesc_lock); 1007 vd = &vq->vq_desc[slot]; 1008 while ((vd->flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) != 0) { 1009 s = virtio_rw16(sc, vd->next); 1010 vd = &vq->vq_desc[s]; 1011 } 1012 vd->next = vq->vq_free_idx; 1013 vq->vq_free_idx = virtio_rw16(sc, slot); 1014 mutex_exit(&vq->vq_freedesc_lock); 1015 } 1016 1017 /* 1018 * Enqueue several dmamaps as a single request. 1019 */ 1020 /* 1021 * Typical usage: 1022 * <queue size> number of followings are stored in arrays 1023 * - command blocks (in dmamem) should be pre-allocated and mapped 1024 * - dmamaps for command blocks should be pre-allocated and loaded 1025 * - dmamaps for payload should be pre-allocated 1026 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 1027 * if (r) // currently 0 or EAGAIN 1028 * return r; 1029 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 1030 * if (r) { 1031 * virtio_enqueue_abort(sc, vq, slot); 1032 * return r; 1033 * } 1034 * r = virtio_enqueue_reserve(sc, vq, slot, 1035 * dmamap_payload[slot]->dm_nsegs + 1); 1036 * // ^ +1 for command 1037 * if (r) { // currently 0 or EAGAIN 1038 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 1039 * return r; // do not call abort() 1040 * } 1041 * <setup and prepare commands> 1042 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 1043 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 1044 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 1045 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 1046 * virtio_enqueue_commit(sc, vq, slot, true); 1047 */ 1048 1049 /* 1050 * enqueue_prep: allocate a slot number 1051 */ 1052 int 1053 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 1054 { 1055 uint16_t slot; 1056 1057 KASSERT(sc->sc_child_state == VIRTIO_CHILD_ATTACH_FINISHED); 1058 KASSERT(slotp != NULL); 1059 1060 slot = vq_alloc_slot(sc, vq, 1); 1061 if (slot == VRING_DESC_CHAIN_END) 1062 return EAGAIN; 1063 1064 *slotp = slot; 1065 1066 return 0; 1067 } 1068 1069 /* 1070 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 1071 */ 1072 int 1073 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 1074 int slot, int nsegs) 1075 { 1076 struct vring_desc *vd; 1077 struct vring_desc_extra *vdx; 1078 int i; 1079 1080 KASSERT(1 <= nsegs); 1081 KASSERT(nsegs <= vq->vq_num); 1082 1083 vdx = &vq->vq_descx[slot]; 1084 vd = &vq->vq_desc[slot]; 1085 1086 KASSERT((vd->flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0); 1087 1088 if ((vq->vq_indirect != NULL) && 1089 (nsegs >= MINSEG_INDIRECT) && 1090 (nsegs <= vq->vq_maxnsegs)) 1091 vdx->use_indirect = true; 1092 else 1093 vdx->use_indirect = false; 1094 1095 if (vdx->use_indirect) { 1096 uint64_t addr; 1097 1098 addr = vq->vq_dmamap->dm_segs[0].ds_addr 1099 + vq->vq_indirectoffset; 1100 addr += sizeof(struct vring_desc) 1101 * vq->vq_maxnsegs * slot; 1102 1103 vd->addr = virtio_rw64(sc, addr); 1104 vd->len = virtio_rw32(sc, sizeof(struct vring_desc) * nsegs); 1105 vd->flags = virtio_rw16(sc, VRING_DESC_F_INDIRECT); 1106 1107 vd = &vq->vq_indirect[vq->vq_maxnsegs * slot]; 1108 vdx->desc_base = vd; 1109 vdx->desc_free_idx = 0; 1110 1111 for (i = 0; i < nsegs - 1; i++) { 1112 vd[i].flags = virtio_rw16(sc, VRING_DESC_F_NEXT); 1113 } 1114 vd[i].flags = virtio_rw16(sc, 0); 1115 } else { 1116 if (nsegs > 1) { 1117 uint16_t s; 1118 1119 s = vq_alloc_slot(sc, vq, nsegs - 1); 1120 if (s == VRING_DESC_CHAIN_END) { 1121 vq_free_slot(sc, vq, slot); 1122 return EAGAIN; 1123 } 1124 vd->next = virtio_rw16(sc, s); 1125 vd->flags = virtio_rw16(sc, VRING_DESC_F_NEXT); 1126 } 1127 1128 vdx->desc_base = &vq->vq_desc[0]; 1129 vdx->desc_free_idx = slot; 1130 } 1131 1132 return 0; 1133 } 1134 1135 /* 1136 * enqueue: enqueue a single dmamap. 1137 */ 1138 int 1139 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1140 bus_dmamap_t dmamap, bool write) 1141 { 1142 struct vring_desc *vds; 1143 struct vring_desc_extra *vdx; 1144 uint16_t s; 1145 int i; 1146 1147 KASSERT(dmamap->dm_nsegs > 0); 1148 1149 vdx = &vq->vq_descx[slot]; 1150 vds = vdx->desc_base; 1151 s = vdx->desc_free_idx; 1152 1153 KASSERT(vds != NULL); 1154 1155 for (i = 0; i < dmamap->dm_nsegs; i++) { 1156 KASSERT(s != VRING_DESC_CHAIN_END); 1157 1158 vds[s].addr = virtio_rw64(sc, dmamap->dm_segs[i].ds_addr); 1159 vds[s].len = virtio_rw32(sc, dmamap->dm_segs[i].ds_len); 1160 if (!write) 1161 vds[s].flags |= virtio_rw16(sc, VRING_DESC_F_WRITE); 1162 1163 if ((vds[s].flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0) { 1164 s = VRING_DESC_CHAIN_END; 1165 } else { 1166 s = virtio_rw16(sc, vds[s].next); 1167 } 1168 } 1169 1170 vdx->desc_free_idx = s; 1171 1172 return 0; 1173 } 1174 1175 int 1176 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1177 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 1178 bool write) 1179 { 1180 struct vring_desc_extra *vdx; 1181 struct vring_desc *vds; 1182 uint16_t s; 1183 1184 vdx = &vq->vq_descx[slot]; 1185 vds = vdx->desc_base; 1186 s = vdx->desc_free_idx; 1187 1188 KASSERT(s != VRING_DESC_CHAIN_END); 1189 KASSERT(vds != NULL); 1190 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 1191 KASSERT(dmamap->dm_segs[0].ds_len > start); 1192 KASSERT(dmamap->dm_segs[0].ds_len >= start + len); 1193 1194 vds[s].addr = virtio_rw64(sc, dmamap->dm_segs[0].ds_addr + start); 1195 vds[s].len = virtio_rw32(sc, len); 1196 if (!write) 1197 vds[s].flags |= virtio_rw16(sc, VRING_DESC_F_WRITE); 1198 1199 if ((vds[s].flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0) { 1200 s = VRING_DESC_CHAIN_END; 1201 } else { 1202 s = virtio_rw16(sc, vds[s].next); 1203 } 1204 1205 vdx->desc_free_idx = s; 1206 1207 return 0; 1208 } 1209 1210 /* 1211 * enqueue_commit: add it to the aring. 1212 */ 1213 int 1214 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1215 bool notifynow) 1216 { 1217 1218 if (slot < 0) { 1219 mutex_enter(&vq->vq_aring_lock); 1220 goto notify; 1221 } 1222 1223 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 1224 if (vq->vq_descx[slot].use_indirect) 1225 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 1226 1227 mutex_enter(&vq->vq_aring_lock); 1228 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = 1229 virtio_rw16(sc, slot); 1230 1231 notify: 1232 if (notifynow) { 1233 uint16_t o, n, t; 1234 uint16_t flags; 1235 1236 o = virtio_rw16(sc, vq->vq_avail->idx) - 1; 1237 n = vq->vq_avail_idx; 1238 1239 /* 1240 * Prepare for `device->CPU' (host->guest) transfer 1241 * into the buffer. This must happen before we commit 1242 * the vq->vq_avail->idx update to ensure we're not 1243 * still using the buffer in case program-prior loads 1244 * or stores in it get delayed past the store to 1245 * vq->vq_avail->idx. 1246 */ 1247 vq_sync_uring_all(sc, vq, BUS_DMASYNC_PREREAD); 1248 1249 /* ensure payload is published, then avail idx */ 1250 vq_sync_aring_payload(sc, vq, BUS_DMASYNC_PREWRITE); 1251 vq->vq_avail->idx = virtio_rw16(sc, vq->vq_avail_idx); 1252 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE); 1253 vq->vq_queued++; 1254 1255 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) { 1256 vq_sync_uring_avail(sc, vq, BUS_DMASYNC_POSTREAD); 1257 t = virtio_rw16(sc, *vq->vq_avail_event) + 1; 1258 if ((uint16_t) (n - t) < (uint16_t) (n - o)) 1259 sc->sc_ops->kick(sc, vq->vq_index); 1260 } else { 1261 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD); 1262 flags = virtio_rw16(sc, vq->vq_used->flags); 1263 if (!(flags & VRING_USED_F_NO_NOTIFY)) 1264 sc->sc_ops->kick(sc, vq->vq_index); 1265 } 1266 } 1267 mutex_exit(&vq->vq_aring_lock); 1268 1269 return 0; 1270 } 1271 1272 /* 1273 * enqueue_abort: rollback. 1274 */ 1275 int 1276 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1277 { 1278 struct vring_desc_extra *vdx; 1279 1280 vdx = &vq->vq_descx[slot]; 1281 vdx->desc_free_idx = VRING_DESC_CHAIN_END; 1282 vdx->desc_base = NULL; 1283 1284 vq_free_slot(sc, vq, slot); 1285 1286 return 0; 1287 } 1288 1289 /* 1290 * Dequeue a request. 1291 */ 1292 /* 1293 * dequeue: dequeue a request from uring; dmamap_sync for uring is 1294 * already done in the interrupt handler. 1295 */ 1296 int 1297 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 1298 int *slotp, int *lenp) 1299 { 1300 uint16_t slot, usedidx; 1301 1302 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx)) 1303 return ENOENT; 1304 mutex_enter(&vq->vq_uring_lock); 1305 usedidx = vq->vq_used_idx++; 1306 mutex_exit(&vq->vq_uring_lock); 1307 usedidx %= vq->vq_num; 1308 slot = virtio_rw32(sc, vq->vq_used->ring[usedidx].id); 1309 1310 if (vq->vq_descx[slot].use_indirect) 1311 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 1312 1313 if (slotp) 1314 *slotp = slot; 1315 if (lenp) 1316 *lenp = virtio_rw32(sc, vq->vq_used->ring[usedidx].len); 1317 1318 return 0; 1319 } 1320 1321 /* 1322 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1323 * if you forget to call this the slot will be leaked. 1324 */ 1325 int 1326 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1327 { 1328 struct vring_desc_extra *vdx; 1329 1330 vdx = &vq->vq_descx[slot]; 1331 vdx->desc_base = NULL; 1332 vdx->desc_free_idx = VRING_DESC_CHAIN_END; 1333 1334 vq_free_slot(sc, vq, slot); 1335 1336 return 0; 1337 } 1338 1339 /* 1340 * Attach a child, fill all the members. 1341 */ 1342 void 1343 virtio_child_attach_start(struct virtio_softc *sc, device_t child, int ipl, 1344 uint64_t req_features, const char *feat_bits) 1345 { 1346 char buf[1024]; 1347 1348 KASSERT(sc->sc_child == NULL); 1349 KASSERT(sc->sc_child_state == VIRTIO_NO_CHILD); 1350 1351 sc->sc_child = child; 1352 sc->sc_ipl = ipl; 1353 1354 virtio_negotiate_features(sc, req_features); 1355 snprintb(buf, sizeof(buf), feat_bits, sc->sc_active_features); 1356 aprint_normal(": features: %s\n", buf); 1357 aprint_naive("\n"); 1358 } 1359 1360 int 1361 virtio_child_attach_finish(struct virtio_softc *sc, 1362 struct virtqueue *vqs, size_t nvqs, 1363 virtio_callback config_change, 1364 int req_flags) 1365 { 1366 size_t i; 1367 int r; 1368 1369 #ifdef DIAGNOSTIC 1370 KASSERT(nvqs > 0); 1371 #define VIRTIO_ASSERT_FLAGS (VIRTIO_F_INTR_SOFTINT | VIRTIO_F_INTR_PERVQ) 1372 KASSERT((req_flags & VIRTIO_ASSERT_FLAGS) != VIRTIO_ASSERT_FLAGS); 1373 #undef VIRTIO_ASSERT_FLAGS 1374 1375 for (i = 0; i < nvqs; i++){ 1376 KASSERT(vqs[i].vq_index == i); 1377 KASSERT(vqs[i].vq_intrhand != NULL); 1378 KASSERT(vqs[i].vq_done == NULL || 1379 vqs[i].vq_intrhand == virtio_vq_done); 1380 } 1381 #endif 1382 1383 1384 sc->sc_vqs = vqs; 1385 sc->sc_nvqs = nvqs; 1386 sc->sc_config_change = config_change; 1387 sc->sc_intrhand = virtio_vq_intr; 1388 sc->sc_flags = req_flags; 1389 1390 /* set the vq address */ 1391 for (i = 0; i < nvqs; i++) { 1392 sc->sc_ops->setup_queue(sc, vqs[i].vq_index, 1393 vqs[i].vq_dmamap->dm_segs[0].ds_addr); 1394 } 1395 1396 r = sc->sc_ops->alloc_interrupts(sc); 1397 if (r != 0) { 1398 aprint_error_dev(sc->sc_dev, 1399 "failed to allocate interrupts\n"); 1400 goto fail; 1401 } 1402 1403 r = sc->sc_ops->setup_interrupts(sc, 0); 1404 if (r != 0) { 1405 aprint_error_dev(sc->sc_dev, "failed to setup interrupts\n"); 1406 goto fail; 1407 } 1408 1409 KASSERT(sc->sc_soft_ih == NULL); 1410 if (sc->sc_flags & VIRTIO_F_INTR_SOFTINT) { 1411 u_int flags = SOFTINT_NET; 1412 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) 1413 flags |= SOFTINT_MPSAFE; 1414 1415 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, 1416 sc); 1417 if (sc->sc_soft_ih == NULL) { 1418 sc->sc_ops->free_interrupts(sc); 1419 aprint_error_dev(sc->sc_dev, 1420 "failed to establish soft interrupt\n"); 1421 goto fail; 1422 } 1423 } 1424 1425 sc->sc_child_state = VIRTIO_CHILD_ATTACH_FINISHED; 1426 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 1427 return 0; 1428 1429 fail: 1430 if (sc->sc_soft_ih) { 1431 softint_disestablish(sc->sc_soft_ih); 1432 sc->sc_soft_ih = NULL; 1433 } 1434 1435 sc->sc_ops->free_interrupts(sc); 1436 1437 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 1438 return 1; 1439 } 1440 1441 void 1442 virtio_child_detach(struct virtio_softc *sc) 1443 { 1444 1445 /* already detached */ 1446 if (sc->sc_child == NULL) 1447 return; 1448 1449 1450 virtio_device_reset(sc); 1451 1452 sc->sc_ops->free_interrupts(sc); 1453 1454 if (sc->sc_soft_ih) { 1455 softint_disestablish(sc->sc_soft_ih); 1456 sc->sc_soft_ih = NULL; 1457 } 1458 1459 sc->sc_vqs = NULL; 1460 sc->sc_child = NULL; 1461 } 1462 1463 void 1464 virtio_child_attach_failed(struct virtio_softc *sc) 1465 { 1466 virtio_child_detach(sc); 1467 1468 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 1469 1470 sc->sc_child_state = VIRTIO_CHILD_ATTACH_FAILED; 1471 } 1472 1473 bus_dma_tag_t 1474 virtio_dmat(struct virtio_softc *sc) 1475 { 1476 return sc->sc_dmat; 1477 } 1478 1479 device_t 1480 virtio_child(struct virtio_softc *sc) 1481 { 1482 return sc->sc_child; 1483 } 1484 1485 int 1486 virtio_intrhand(struct virtio_softc *sc) 1487 { 1488 return (*sc->sc_intrhand)(sc); 1489 } 1490 1491 uint64_t 1492 virtio_features(struct virtio_softc *sc) 1493 { 1494 return sc->sc_active_features; 1495 } 1496 1497 bool 1498 virtio_version_1(struct virtio_softc *sc) 1499 { 1500 return sc->sc_version_1; 1501 } 1502 1503 int 1504 virtio_attach_failed(struct virtio_softc *sc) 1505 { 1506 device_t self = sc->sc_dev; 1507 1508 /* no error if its not connected, but its failed */ 1509 if (sc->sc_childdevid == 0) 1510 return 1; 1511 1512 if (sc->sc_child == NULL) { 1513 switch (sc->sc_child_state) { 1514 case VIRTIO_CHILD_ATTACH_FAILED: 1515 aprint_error_dev(self, 1516 "virtio configuration failed\n"); 1517 break; 1518 case VIRTIO_NO_CHILD: 1519 aprint_error_dev(self, 1520 "no matching child driver; not configured\n"); 1521 break; 1522 default: 1523 /* sanity check */ 1524 aprint_error_dev(self, 1525 "virtio internal error, " 1526 "child driver is not configured\n"); 1527 break; 1528 } 1529 1530 return 1; 1531 } 1532 1533 /* sanity check */ 1534 if (sc->sc_child_state != VIRTIO_CHILD_ATTACH_FINISHED) { 1535 aprint_error_dev(self, "virtio internal error, child driver " 1536 "signaled OK but didn't initialize interrupts\n"); 1537 return 1; 1538 } 1539 1540 return 0; 1541 } 1542 1543 void 1544 virtio_print_device_type(device_t self, int id, int revision) 1545 { 1546 aprint_normal_dev(self, "%s device (id %d, rev. 0x%02x)\n", 1547 (id < NDEVNAMES ? virtio_device_name[id] : "Unknown"), 1548 id, 1549 revision); 1550 } 1551 1552 1553 MODULE(MODULE_CLASS_DRIVER, virtio, NULL); 1554 1555 #ifdef _MODULE 1556 #include "ioconf.c" 1557 #endif 1558 1559 static int 1560 virtio_modcmd(modcmd_t cmd, void *opaque) 1561 { 1562 int error = 0; 1563 1564 #ifdef _MODULE 1565 switch (cmd) { 1566 case MODULE_CMD_INIT: 1567 error = config_init_component(cfdriver_ioconf_virtio, 1568 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1569 break; 1570 case MODULE_CMD_FINI: 1571 error = config_fini_component(cfdriver_ioconf_virtio, 1572 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1573 break; 1574 default: 1575 error = ENOTTY; 1576 break; 1577 } 1578 #endif 1579 1580 return error; 1581 } 1582
Indexes created Thu Oct 02 14:10:14 GMT 2025