ld_virtio.c revision 1.40
1 /* $NetBSD: ld_virtio.c,v 1.40 2025/02/22 16:53:37 mlelstv Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: ld_virtio.c,v 1.40 2025/02/22 16:53:37 mlelstv Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/buf.h> 35 #include <sys/bufq.h> 36 #include <sys/bus.h> 37 #include <sys/device.h> 38 #include <sys/disk.h> 39 #include <sys/mutex.h> 40 #include <sys/module.h> 41 #include <sys/kmem.h> 42 43 #include <dev/ldvar.h> 44 #include <dev/pci/virtioreg.h> 45 #include <dev/pci/virtiovar.h> 46 47 #include "ioconf.h" 48 49 /* 50 * ld_virtioreg: 51 */ 52 /* Configuration registers */ 53 #define VIRTIO_BLK_CONFIG_CAPACITY 0 /* 64bit */ 54 #define VIRTIO_BLK_CONFIG_SIZE_MAX 8 /* 32bit */ 55 #define VIRTIO_BLK_CONFIG_SEG_MAX 12 /* 32bit */ 56 #define VIRTIO_BLK_CONFIG_GEOMETRY_C 16 /* 16bit */ 57 #define VIRTIO_BLK_CONFIG_GEOMETRY_H 18 /* 8bit */ 58 #define VIRTIO_BLK_CONFIG_GEOMETRY_S 19 /* 8bit */ 59 #define VIRTIO_BLK_CONFIG_BLK_SIZE 20 /* 32bit */ 60 #define VIRTIO_BLK_CONFIG_WRITEBACK 32 /* 8bit */ 61 #define VIRTIO_BLK_CONFIG_NUM_QUEUES 34 /* 16bit */ 62 #define VIRTIO_BLK_CONFIG_MAX_DISCARD_SECTORS 36 /* 32bit */ 63 #define VIRTIO_BLK_CONFIG_MAX_DISCARD_SEG 40 /* 32bit */ 64 #define VIRTIO_BLK_CONFIG_DISCARD_SECTOR_ALIGNMENT 44 /* 32bit */ 65 66 /* Feature bits */ 67 #define VIRTIO_BLK_F_BARRIER (1<<0) 68 #define VIRTIO_BLK_F_SIZE_MAX (1<<1) 69 #define VIRTIO_BLK_F_SEG_MAX (1<<2) 70 #define VIRTIO_BLK_F_GEOMETRY (1<<4) 71 #define VIRTIO_BLK_F_RO (1<<5) 72 #define VIRTIO_BLK_F_BLK_SIZE (1<<6) 73 #define VIRTIO_BLK_F_SCSI (1<<7) 74 #define VIRTIO_BLK_F_FLUSH (1<<9) 75 #define VIRTIO_BLK_F_TOPOLOGY (1<<10) 76 #define VIRTIO_BLK_F_CONFIG_WCE (1<<11) 77 #define VIRTIO_BLK_F_MQ (1<<12) 78 #define VIRTIO_BLK_F_DISCARD (1<<13) 79 #define VIRTIO_BLK_F_WRITE_ZEROES (1<<14) 80 #define VIRTIO_BLK_F_LIFETIME (1<<15) 81 #define VIRTIO_BLK_F_SECURE_ERASE (1<<16) 82 83 /* 84 * Each block request uses at least two segments - one for the header 85 * and one for the status. 86 */ 87 #define VIRTIO_BLK_CTRL_SEGMENTS 2 88 89 #define VIRTIO_BLK_FLAG_BITS \ 90 VIRTIO_COMMON_FLAG_BITS \ 91 "b\x10" "SECURE_ERASE\0" \ 92 "b\x0f" "LIFETIME\0" \ 93 "b\x0e" "WRITE_ZEROES\0" \ 94 "b\x0d" "DISCARD\0" \ 95 "b\x0c" "MQ\0" \ 96 "b\x0b" "CONFIG_WCE\0" \ 97 "b\x0a" "TOPOLOGY\0" \ 98 "b\x09" "FLUSH\0" \ 99 "b\x07" "SCSI\0" \ 100 "b\x06" "BLK_SIZE\0" \ 101 "b\x05" "RO\0" \ 102 "b\x04" "GEOMETRY\0" \ 103 "b\x02" "SEG_MAX\0" \ 104 "b\x01" "SIZE_MAX\0" \ 105 "b\x00" "BARRIER\0" 106 107 /* Command */ 108 #define VIRTIO_BLK_T_IN 0 109 #define VIRTIO_BLK_T_OUT 1 110 #define VIRTIO_BLK_T_FLUSH 4 111 #define VIRTIO_BLK_T_GET_ID 8 112 #define VIRTIO_BLK_T_GET_LIFETIME 10 113 #define VIRTIO_BLK_T_DISCARD 11 114 #define VIRTIO_BLK_T_WRITE_ZEROES 13 115 #define VIRTIO_BLK_T_SECURE_ERASE 14 116 #define VIRTIO_BLK_T_BARRIER 0x80000000 117 118 /* Sector */ 119 #define VIRTIO_BLK_BSIZE 512 120 121 /* Status */ 122 #define VIRTIO_BLK_S_OK 0 123 #define VIRTIO_BLK_S_IOERR 1 124 #define VIRTIO_BLK_S_UNSUPP 2 125 126 /* Request header structure */ 127 struct virtio_blk_req_hdr { 128 uint32_t type; /* VIRTIO_BLK_T_* */ 129 uint32_t ioprio; 130 uint64_t sector; 131 } __packed; 132 /* payload and 1 byte status follows */ 133 134 struct virtio_blk_discard_write_zeroes { 135 uint64_t sector; 136 uint32_t num_sectors; 137 union { 138 uint32_t flags; 139 struct { 140 uint32_t unmap:1; 141 uint32_t reserved:31; 142 }; 143 }; 144 } __packed; 145 146 /* 147 * ld_virtiovar: 148 */ 149 struct virtio_blk_req { 150 struct virtio_blk_req_hdr vr_hdr; 151 uint8_t vr_status; 152 struct buf *vr_bp; 153 #define DUMMY_VR_BP ((void *)1) 154 bus_dmamap_t vr_cmdsts; 155 bus_dmamap_t vr_payload; 156 void * vr_datap; 157 size_t vr_datas; 158 }; 159 160 struct ld_virtio_softc { 161 struct ld_softc sc_ld; 162 device_t sc_dev; 163 164 uint32_t sc_seg_max; /* max number of segs in xfer */ 165 uint32_t sc_size_max; /* max size of single seg */ 166 167 struct virtio_softc *sc_virtio; 168 struct virtqueue sc_vq; 169 170 struct virtio_blk_req *sc_reqs; 171 bus_dma_segment_t sc_reqs_seg; 172 173 int sc_readonly; 174 175 enum { 176 SYNC_FREE, SYNC_BUSY, SYNC_DONE 177 } sc_sync_use; 178 kcondvar_t sc_sync_wait; 179 kmutex_t sc_sync_wait_lock; 180 uint8_t sc_sync_status; 181 uint8_t *sc_typename; 182 183 uint32_t sc_max_discard_sectors; 184 uint32_t sc_max_discard_seg; 185 #if 0 186 uint32_t sc_discard_sector_alignment; 187 #endif 188 }; 189 190 static int ld_virtio_match(device_t, cfdata_t, void *); 191 static void ld_virtio_attach(device_t, device_t, void *); 192 static int ld_virtio_detach(device_t, int); 193 194 CFATTACH_DECL_NEW(ld_virtio, sizeof(struct ld_virtio_softc), 195 ld_virtio_match, ld_virtio_attach, ld_virtio_detach, NULL); 196 197 static int 198 ld_virtio_match(device_t parent, cfdata_t match, void *aux) 199 { 200 struct virtio_attach_args *va = aux; 201 202 if (va->sc_childdevid == VIRTIO_DEVICE_ID_BLOCK) 203 return 1; 204 205 return 0; 206 } 207 208 static int ld_virtio_vq_done(struct virtqueue *); 209 static int ld_virtio_dump(struct ld_softc *, void *, int, int); 210 static int ld_virtio_start(struct ld_softc *, struct buf *); 211 static int ld_virtio_ioctl(struct ld_softc *, u_long, void *, int32_t, bool); 212 static int ld_virtio_info(struct ld_softc *, bool); 213 static int ld_virtio_discard(struct ld_softc *, struct buf *); 214 215 static int 216 ld_virtio_alloc_reqs(struct ld_virtio_softc *sc, int qsize) 217 { 218 int allocsize, r, rsegs, i; 219 struct ld_softc *ld = &sc->sc_ld; 220 void *vaddr; 221 222 allocsize = sizeof(struct virtio_blk_req) * qsize; 223 r = bus_dmamem_alloc(virtio_dmat(sc->sc_virtio), allocsize, 0, 0, 224 &sc->sc_reqs_seg, 1, &rsegs, BUS_DMA_WAITOK); 225 if (r != 0) { 226 aprint_error_dev(sc->sc_dev, 227 "DMA memory allocation failed, size %d, " 228 "error code %d\n", allocsize, r); 229 goto err_none; 230 } 231 r = bus_dmamem_map(virtio_dmat(sc->sc_virtio), 232 &sc->sc_reqs_seg, 1, allocsize, 233 &vaddr, BUS_DMA_WAITOK); 234 if (r != 0) { 235 aprint_error_dev(sc->sc_dev, 236 "DMA memory map failed, " 237 "error code %d\n", r); 238 goto err_dmamem_alloc; 239 } 240 sc->sc_reqs = vaddr; 241 memset(vaddr, 0, allocsize); 242 for (i = 0; i < qsize; i++) { 243 struct virtio_blk_req *vr = &sc->sc_reqs[i]; 244 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), 245 offsetof(struct virtio_blk_req, vr_bp), 246 1, 247 offsetof(struct virtio_blk_req, vr_bp), 248 0, 249 BUS_DMA_WAITOK|BUS_DMA_ALLOCNOW, 250 &vr->vr_cmdsts); 251 if (r != 0) { 252 aprint_error_dev(sc->sc_dev, 253 "command dmamap creation failed, " 254 "error code %d\n", r); 255 goto err_reqs; 256 } 257 r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), vr->vr_cmdsts, 258 &vr->vr_hdr, 259 offsetof(struct virtio_blk_req, vr_bp), 260 NULL, BUS_DMA_WAITOK); 261 if (r != 0) { 262 aprint_error_dev(sc->sc_dev, 263 "command dmamap load failed, " 264 "error code %d\n", r); 265 goto err_reqs; 266 } 267 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), 268 /*size*/ld->sc_maxxfer, 269 /*nseg*/sc->sc_seg_max, 270 /*maxsegsz*/sc->sc_size_max, 271 /*boundary*/0, 272 BUS_DMA_WAITOK|BUS_DMA_ALLOCNOW, 273 &vr->vr_payload); 274 if (r != 0) { 275 aprint_error_dev(sc->sc_dev, 276 "payload dmamap creation failed, " 277 "error code %d\n", r); 278 goto err_reqs; 279 } 280 vr->vr_datap = NULL; 281 vr->vr_datas = 0; 282 } 283 return 0; 284 285 err_reqs: 286 for (i = 0; i < qsize; i++) { 287 struct virtio_blk_req *vr = &sc->sc_reqs[i]; 288 if (vr->vr_cmdsts) { 289 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), 290 vr->vr_cmdsts); 291 vr->vr_cmdsts = 0; 292 } 293 if (vr->vr_payload) { 294 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), 295 vr->vr_payload); 296 vr->vr_payload = 0; 297 } 298 } 299 bus_dmamem_unmap(virtio_dmat(sc->sc_virtio), sc->sc_reqs, allocsize); 300 err_dmamem_alloc: 301 bus_dmamem_free(virtio_dmat(sc->sc_virtio), &sc->sc_reqs_seg, 1); 302 err_none: 303 return -1; 304 } 305 306 static void 307 ld_virtio_attach(device_t parent, device_t self, void *aux) 308 { 309 struct ld_virtio_softc *sc = device_private(self); 310 struct ld_softc *ld = &sc->sc_ld; 311 struct virtio_softc *vsc = device_private(parent); 312 uint64_t features; 313 int qsize; 314 315 if (virtio_child(vsc) != NULL) { 316 aprint_normal(": child already attached for %s; " 317 "something wrong...\n", device_xname(parent)); 318 return; 319 } 320 321 sc->sc_dev = self; 322 sc->sc_virtio = vsc; 323 324 virtio_child_attach_start(vsc, self, IPL_BIO, 325 (VIRTIO_BLK_F_SIZE_MAX | VIRTIO_BLK_F_SEG_MAX | 326 VIRTIO_BLK_F_GEOMETRY | VIRTIO_BLK_F_RO | VIRTIO_BLK_F_BLK_SIZE | 327 VIRTIO_BLK_F_FLUSH | VIRTIO_BLK_F_CONFIG_WCE | 328 VIRTIO_BLK_F_DISCARD), 329 VIRTIO_BLK_FLAG_BITS); 330 331 features = virtio_features(vsc); 332 if (features == 0) 333 goto err; 334 335 if (features & VIRTIO_BLK_F_RO) 336 sc->sc_readonly = 1; 337 else 338 sc->sc_readonly = 0; 339 340 if (features & VIRTIO_BLK_F_BLK_SIZE) { 341 ld->sc_secsize = virtio_read_device_config_4(vsc, 342 VIRTIO_BLK_CONFIG_BLK_SIZE); 343 } else 344 ld->sc_secsize = VIRTIO_BLK_BSIZE; 345 346 if (features & VIRTIO_BLK_F_SEG_MAX) { 347 sc->sc_seg_max = virtio_read_device_config_4(vsc, 348 VIRTIO_BLK_CONFIG_SEG_MAX); 349 if (sc->sc_seg_max == 0) { 350 aprint_error_dev(sc->sc_dev, 351 "Invalid SEG_MAX %d\n", sc->sc_seg_max); 352 goto err; 353 } 354 } else { 355 sc->sc_seg_max = 1; 356 aprint_verbose_dev(sc->sc_dev, 357 "Unknown SEG_MAX, assuming %"PRIu32"\n", sc->sc_seg_max); 358 } 359 360 /* At least genfs_io assumes size_max*seg_max >= MAXPHYS. */ 361 if (features & VIRTIO_BLK_F_SIZE_MAX) { 362 sc->sc_size_max = virtio_read_device_config_4(vsc, 363 VIRTIO_BLK_CONFIG_SIZE_MAX); 364 if (sc->sc_size_max < MAXPHYS/sc->sc_seg_max) { 365 aprint_error_dev(sc->sc_dev, 366 "Too small SIZE_MAX %d minimum is %d\n", 367 sc->sc_size_max, MAXPHYS/sc->sc_seg_max); 368 // goto err; 369 sc->sc_size_max = MAXPHYS/sc->sc_seg_max; 370 } else if (sc->sc_size_max > MAXPHYS) { 371 aprint_verbose_dev(sc->sc_dev, 372 "Clip SIZE_MAX from %d to %d\n", 373 sc->sc_size_max, MAXPHYS); 374 sc->sc_size_max = MAXPHYS; 375 } 376 } else { 377 sc->sc_size_max = MAXPHYS; 378 aprint_verbose_dev(sc->sc_dev, 379 "Unknown SIZE_MAX, assuming %"PRIu32"\n", 380 sc->sc_size_max); 381 } 382 383 aprint_normal_dev(sc->sc_dev, "max %"PRIu32" segs" 384 " of max %"PRIu32" bytes\n", 385 sc->sc_seg_max, sc->sc_size_max); 386 387 virtio_init_vq_vqdone(vsc, &sc->sc_vq, 0, 388 ld_virtio_vq_done); 389 390 if (virtio_alloc_vq(vsc, &sc->sc_vq, sc->sc_size_max, 391 sc->sc_seg_max + VIRTIO_BLK_CTRL_SEGMENTS, "I/O request") != 0) 392 goto err; 393 qsize = sc->sc_vq.vq_num; 394 395 if (virtio_child_attach_finish(vsc, &sc->sc_vq, 1, 396 NULL, VIRTIO_F_INTR_MSIX) != 0) 397 goto err; 398 399 ld->sc_dv = self; 400 ld->sc_secperunit = virtio_read_device_config_8(vsc, 401 VIRTIO_BLK_CONFIG_CAPACITY) / (ld->sc_secsize / VIRTIO_BLK_BSIZE); 402 403 /* 404 * Clamp ld->sc_maxxfer to MAXPHYS before ld_virtio_alloc_reqs 405 * allocates DMA maps of at most ld->sc_maxxfer bytes. 406 * ldattach will also clamp to MAXPHYS, but not until after 407 * ld_virtio_alloc_reqs is done, so that doesn't help. 408 */ 409 ld->sc_maxxfer = MIN(MAXPHYS, sc->sc_size_max * sc->sc_seg_max); 410 411 if (features & VIRTIO_BLK_F_GEOMETRY) { 412 ld->sc_ncylinders = virtio_read_device_config_2(vsc, 413 VIRTIO_BLK_CONFIG_GEOMETRY_C); 414 ld->sc_nheads = virtio_read_device_config_1(vsc, 415 VIRTIO_BLK_CONFIG_GEOMETRY_H); 416 ld->sc_nsectors = virtio_read_device_config_1(vsc, 417 VIRTIO_BLK_CONFIG_GEOMETRY_S); 418 } 419 ld->sc_maxqueuecnt = qsize - 1; /* reserve slot for dumps, flushes */ 420 421 if (ld_virtio_alloc_reqs(sc, qsize) < 0) 422 goto err; 423 424 cv_init(&sc->sc_sync_wait, "vblksync"); 425 mutex_init(&sc->sc_sync_wait_lock, MUTEX_DEFAULT, IPL_BIO); 426 sc->sc_sync_use = SYNC_FREE; 427 428 ld->sc_dump = ld_virtio_dump; 429 ld->sc_start = ld_virtio_start; 430 ld->sc_ioctl = ld_virtio_ioctl; 431 432 if (ld_virtio_info(ld, true) == 0) 433 ld->sc_typename = sc->sc_typename; 434 else 435 ld->sc_typename = __UNCONST("Virtio Block Device"); 436 437 if (features & VIRTIO_BLK_F_DISCARD) { 438 ld->sc_discard = ld_virtio_discard; 439 sc->sc_max_discard_sectors = virtio_read_device_config_4(vsc, 440 VIRTIO_BLK_CONFIG_MAX_DISCARD_SECTORS); 441 sc->sc_max_discard_seg = virtio_read_device_config_4(vsc, 442 VIRTIO_BLK_CONFIG_MAX_DISCARD_SEG); 443 #if 0 444 sc->sc_discard_sector_alignment = 445 virtio_read_device_config_4(vsc, 446 VIRTIO_BLK_CONFIG_DISCARD_SECTOR_ALIGNMENT); 447 #endif 448 } 449 450 ld->sc_flags = LDF_ENABLED | LDF_MPSAFE; 451 ldattach(ld, BUFQ_DISK_DEFAULT_STRAT); 452 453 return; 454 455 err: 456 virtio_child_attach_failed(vsc); 457 return; 458 } 459 460 static int __used 461 ld_virtio_info(struct ld_softc *ld, bool poll) 462 { 463 struct ld_virtio_softc *sc = device_private(ld->sc_dv); 464 struct virtio_softc *vsc = sc->sc_virtio; 465 struct virtqueue *vq = &sc->sc_vq; 466 struct virtio_blk_req *vr; 467 int r; 468 int slot; 469 uint8_t id_data[20]; /* virtio v1.2 5.2.6 */ 470 bool unload = false; 471 472 if (sc->sc_typename != NULL) { 473 kmem_strfree(sc->sc_typename); 474 sc->sc_typename = NULL; 475 } 476 477 mutex_enter(&sc->sc_sync_wait_lock); 478 while (sc->sc_sync_use != SYNC_FREE) { 479 if (poll) { 480 mutex_exit(&sc->sc_sync_wait_lock); 481 ld_virtio_vq_done(vq); 482 mutex_enter(&sc->sc_sync_wait_lock); 483 continue; 484 } 485 cv_wait(&sc->sc_sync_wait, &sc->sc_sync_wait_lock); 486 } 487 sc->sc_sync_use = SYNC_BUSY; 488 mutex_exit(&sc->sc_sync_wait_lock); 489 490 r = virtio_enqueue_prep(vsc, vq, &slot); 491 if (r != 0) 492 goto done; 493 494 vr = &sc->sc_reqs[slot]; 495 KASSERT(vr->vr_bp == NULL); 496 497 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload, 498 id_data, sizeof(id_data), NULL, 499 BUS_DMA_READ|BUS_DMA_NOWAIT); 500 if (r != 0) { 501 aprint_error_dev(sc->sc_dev, 502 "payload dmamap failed, error code %d\n", r); 503 virtio_enqueue_abort(vsc, vq, slot); 504 goto done; 505 } 506 unload = true; 507 508 KASSERT(vr->vr_payload->dm_nsegs <= sc->sc_seg_max); 509 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs + 510 VIRTIO_BLK_CTRL_SEGMENTS); 511 if (r != 0) { 512 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 513 goto done; 514 } 515 516 vr->vr_bp = DUMMY_VR_BP; 517 vr->vr_hdr.type = virtio_rw32(vsc, VIRTIO_BLK_T_GET_ID); 518 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0); 519 vr->vr_hdr.sector = virtio_rw64(vsc, 0); 520 521 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 522 0, sizeof(struct virtio_blk_req_hdr), 523 BUS_DMASYNC_PREWRITE); 524 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 525 0, sizeof(id_data), 526 BUS_DMASYNC_PREREAD); 527 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 528 offsetof(struct virtio_blk_req, vr_status), 529 sizeof(uint8_t), 530 BUS_DMASYNC_PREREAD); 531 532 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 533 0, sizeof(struct virtio_blk_req_hdr), 534 true); 535 virtio_enqueue(vsc, vq, slot, vr->vr_payload, false); 536 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 537 offsetof(struct virtio_blk_req, vr_status), 538 sizeof(uint8_t), 539 false); 540 virtio_enqueue_commit(vsc, vq, slot, true); 541 542 done: 543 mutex_enter(&sc->sc_sync_wait_lock); 544 while (sc->sc_sync_use != SYNC_DONE) { 545 if (poll) { 546 mutex_exit(&sc->sc_sync_wait_lock); 547 ld_virtio_vq_done(vq); 548 mutex_enter(&sc->sc_sync_wait_lock); 549 continue; 550 } 551 cv_wait(&sc->sc_sync_wait, &sc->sc_sync_wait_lock); 552 } 553 554 if (sc->sc_sync_status == VIRTIO_BLK_S_OK) 555 r = 0; 556 else 557 r = EIO; 558 559 sc->sc_sync_use = SYNC_FREE; 560 cv_broadcast(&sc->sc_sync_wait); 561 mutex_exit(&sc->sc_sync_wait_lock); 562 563 if (unload) { 564 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 565 0, sizeof(id_data), BUS_DMASYNC_POSTREAD); 566 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 567 } 568 569 if (r == 0) 570 sc->sc_typename = kmem_strndup(id_data, sizeof(id_data), KM_NOSLEEP); 571 572 return r; 573 } 574 575 static int 576 ld_virtio_start(struct ld_softc *ld, struct buf *bp) 577 { 578 /* splbio */ 579 struct ld_virtio_softc *sc = device_private(ld->sc_dv); 580 struct virtio_softc *vsc = sc->sc_virtio; 581 struct virtqueue *vq = &sc->sc_vq; 582 struct virtio_blk_req *vr; 583 int r; 584 int isread = (bp->b_flags & B_READ); 585 int slot; 586 587 if (sc->sc_readonly && !isread) 588 return EIO; 589 590 r = virtio_enqueue_prep(vsc, vq, &slot); 591 if (r != 0) 592 return r; 593 594 vr = &sc->sc_reqs[slot]; 595 KASSERT(vr->vr_bp == NULL); 596 597 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload, 598 bp->b_data, bp->b_bcount, NULL, 599 ((isread?BUS_DMA_READ:BUS_DMA_WRITE) 600 |BUS_DMA_NOWAIT)); 601 if (r != 0) { 602 aprint_error_dev(sc->sc_dev, 603 "payload dmamap failed, error code %d\n", r); 604 virtio_enqueue_abort(vsc, vq, slot); 605 return r; 606 } 607 608 KASSERT(vr->vr_payload->dm_nsegs <= sc->sc_seg_max); 609 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs + 610 VIRTIO_BLK_CTRL_SEGMENTS); 611 if (r != 0) { 612 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 613 return r; 614 } 615 616 vr->vr_bp = bp; 617 vr->vr_hdr.type = virtio_rw32(vsc, 618 isread ? VIRTIO_BLK_T_IN : VIRTIO_BLK_T_OUT); 619 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0); 620 vr->vr_hdr.sector = virtio_rw64(vsc, 621 bp->b_rawblkno * sc->sc_ld.sc_secsize / 622 VIRTIO_BLK_BSIZE); 623 624 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 625 0, sizeof(struct virtio_blk_req_hdr), 626 BUS_DMASYNC_PREWRITE); 627 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 628 0, bp->b_bcount, 629 isread?BUS_DMASYNC_PREREAD:BUS_DMASYNC_PREWRITE); 630 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 631 offsetof(struct virtio_blk_req, vr_status), 632 sizeof(uint8_t), 633 BUS_DMASYNC_PREREAD); 634 635 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 636 0, sizeof(struct virtio_blk_req_hdr), 637 true); 638 virtio_enqueue(vsc, vq, slot, vr->vr_payload, !isread); 639 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 640 offsetof(struct virtio_blk_req, vr_status), 641 sizeof(uint8_t), 642 false); 643 virtio_enqueue_commit(vsc, vq, slot, true); 644 645 return 0; 646 } 647 648 static void 649 ld_virtio_vq_done1(struct ld_virtio_softc *sc, struct virtio_softc *vsc, 650 struct virtqueue *vq, int slot) 651 { 652 struct virtio_blk_req *vr = &sc->sc_reqs[slot]; 653 struct buf *bp = vr->vr_bp; 654 const uint32_t rt = virtio_rw32(vsc, vr->vr_hdr.type); 655 656 vr->vr_bp = NULL; 657 658 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 659 0, sizeof(struct virtio_blk_req_hdr), 660 BUS_DMASYNC_POSTWRITE); 661 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 662 sizeof(struct virtio_blk_req_hdr), sizeof(uint8_t), 663 BUS_DMASYNC_POSTREAD); 664 if (bp == DUMMY_VR_BP) { 665 mutex_enter(&sc->sc_sync_wait_lock); 666 sc->sc_sync_status = vr->vr_status; 667 sc->sc_sync_use = SYNC_DONE; 668 cv_broadcast(&sc->sc_sync_wait); 669 mutex_exit(&sc->sc_sync_wait_lock); 670 virtio_dequeue_commit(vsc, vq, slot); 671 return; 672 } 673 switch (rt) { 674 case VIRTIO_BLK_T_OUT: 675 case VIRTIO_BLK_T_IN: 676 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 677 0, bp->b_bcount, 678 (bp->b_flags & B_READ)?BUS_DMASYNC_POSTREAD 679 :BUS_DMASYNC_POSTWRITE); 680 break; 681 default: 682 if (vr->vr_datap == NULL) 683 break; 684 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 685 0, vr->vr_datas, BUS_DMASYNC_POSTREAD | 686 BUS_DMASYNC_POSTWRITE); 687 break; 688 } 689 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 690 691 if (vr->vr_status != VIRTIO_BLK_S_OK) { 692 bp->b_error = EIO; 693 bp->b_resid = bp->b_bcount; 694 } else { 695 bp->b_error = 0; 696 bp->b_resid = 0; 697 } 698 699 if (vr->vr_datap != NULL) { 700 kmem_free(vr->vr_datap, vr->vr_datas); 701 vr->vr_datap = NULL; 702 vr->vr_datas = 0; 703 } 704 705 virtio_dequeue_commit(vsc, vq, slot); 706 707 switch (rt) { 708 case VIRTIO_BLK_T_OUT: 709 case VIRTIO_BLK_T_IN: 710 lddone(&sc->sc_ld, bp); 711 break; 712 case VIRTIO_BLK_T_DISCARD: 713 lddiscardend(&sc->sc_ld, bp); 714 break; 715 } 716 } 717 718 static int 719 ld_virtio_vq_done(struct virtqueue *vq) 720 { 721 struct virtio_softc *vsc = vq->vq_owner; 722 struct ld_virtio_softc *sc = device_private(virtio_child(vsc)); 723 int r = 0; 724 int slot; 725 726 again: 727 if (virtio_dequeue(vsc, vq, &slot, NULL)) 728 return r; 729 r = 1; 730 731 ld_virtio_vq_done1(sc, vsc, vq, slot); 732 goto again; 733 } 734 735 static int 736 ld_virtio_dump(struct ld_softc *ld, void *data, int blkno, int blkcnt) 737 { 738 struct ld_virtio_softc *sc = device_private(ld->sc_dv); 739 struct virtio_softc *vsc = sc->sc_virtio; 740 struct virtqueue *vq = &sc->sc_vq; 741 struct virtio_blk_req *vr; 742 int slot, r; 743 744 if (sc->sc_readonly) 745 return EIO; 746 747 r = virtio_enqueue_prep(vsc, vq, &slot); 748 if (r != 0) { 749 if (r == EAGAIN) { /* no free slot; dequeue first */ 750 delay(100); 751 ld_virtio_vq_done(vq); 752 r = virtio_enqueue_prep(vsc, vq, &slot); 753 if (r != 0) 754 return r; 755 } 756 return r; 757 } 758 vr = &sc->sc_reqs[slot]; 759 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload, 760 data, blkcnt*ld->sc_secsize, NULL, 761 BUS_DMA_WRITE|BUS_DMA_NOWAIT); 762 if (r != 0) 763 return r; 764 765 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs + 766 VIRTIO_BLK_CTRL_SEGMENTS); 767 if (r != 0) { 768 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 769 return r; 770 } 771 772 vr->vr_bp = (void*)0xdeadbeef; 773 vr->vr_hdr.type = virtio_rw32(vsc, VIRTIO_BLK_T_OUT); 774 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0); 775 vr->vr_hdr.sector = virtio_rw64(vsc, 776 (daddr_t) blkno * ld->sc_secsize / 777 VIRTIO_BLK_BSIZE); 778 779 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 780 0, sizeof(struct virtio_blk_req_hdr), 781 BUS_DMASYNC_PREWRITE); 782 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 783 0, blkcnt*ld->sc_secsize, 784 BUS_DMASYNC_PREWRITE); 785 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 786 offsetof(struct virtio_blk_req, vr_status), 787 sizeof(uint8_t), 788 BUS_DMASYNC_PREREAD); 789 790 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 791 0, sizeof(struct virtio_blk_req_hdr), 792 true); 793 virtio_enqueue(vsc, vq, slot, vr->vr_payload, true); 794 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 795 offsetof(struct virtio_blk_req, vr_status), 796 sizeof(uint8_t), 797 false); 798 virtio_enqueue_commit(vsc, vq, slot, true); 799 800 for ( ; ; ) { 801 int dslot; 802 803 r = virtio_dequeue(vsc, vq, &dslot, NULL); 804 if (r != 0) 805 continue; 806 if (dslot != slot) { 807 ld_virtio_vq_done1(sc, vsc, vq, dslot); 808 continue; 809 } else 810 break; 811 } 812 813 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 814 0, sizeof(struct virtio_blk_req_hdr), 815 BUS_DMASYNC_POSTWRITE); 816 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 817 0, blkcnt*ld->sc_secsize, 818 BUS_DMASYNC_POSTWRITE); 819 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 820 offsetof(struct virtio_blk_req, vr_status), 821 sizeof(uint8_t), 822 BUS_DMASYNC_POSTREAD); 823 if (vr->vr_status == VIRTIO_BLK_S_OK) 824 r = 0; 825 else 826 r = EIO; 827 virtio_dequeue_commit(vsc, vq, slot); 828 829 return r; 830 } 831 832 static int 833 ld_virtio_detach(device_t self, int flags) 834 { 835 struct ld_virtio_softc *sc = device_private(self); 836 struct ld_softc *ld = &sc->sc_ld; 837 bus_dma_tag_t dmat = virtio_dmat(sc->sc_virtio); 838 int r, i, qsize; 839 840 qsize = sc->sc_vq.vq_num; 841 r = ldbegindetach(ld, flags); 842 if (r != 0) 843 return r; 844 virtio_reset(sc->sc_virtio); 845 virtio_free_vq(sc->sc_virtio, &sc->sc_vq); 846 847 for (i = 0; i < qsize; i++) { 848 bus_dmamap_destroy(dmat, 849 sc->sc_reqs[i].vr_cmdsts); 850 bus_dmamap_destroy(dmat, 851 sc->sc_reqs[i].vr_payload); 852 } 853 bus_dmamem_unmap(dmat, sc->sc_reqs, 854 sizeof(struct virtio_blk_req) * qsize); 855 bus_dmamem_free(dmat, &sc->sc_reqs_seg, 1); 856 857 ldenddetach(ld); 858 859 if (sc->sc_typename != NULL) 860 kmem_strfree(sc->sc_typename); 861 862 cv_destroy(&sc->sc_sync_wait); 863 mutex_destroy(&sc->sc_sync_wait_lock); 864 865 virtio_child_detach(sc->sc_virtio); 866 867 return 0; 868 } 869 870 static int 871 ld_virtio_flush(struct ld_softc *ld, bool poll) 872 { 873 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 874 struct virtio_softc * const vsc = sc->sc_virtio; 875 const uint64_t features = virtio_features(vsc); 876 struct virtqueue *vq = &sc->sc_vq; 877 struct virtio_blk_req *vr; 878 int slot; 879 int r; 880 881 if ((features & VIRTIO_BLK_F_FLUSH) == 0) 882 return 0; 883 884 mutex_enter(&sc->sc_sync_wait_lock); 885 while (sc->sc_sync_use != SYNC_FREE) { 886 if (poll) { 887 mutex_exit(&sc->sc_sync_wait_lock); 888 ld_virtio_vq_done(vq); 889 mutex_enter(&sc->sc_sync_wait_lock); 890 continue; 891 } 892 cv_wait(&sc->sc_sync_wait, &sc->sc_sync_wait_lock); 893 } 894 sc->sc_sync_use = SYNC_BUSY; 895 mutex_exit(&sc->sc_sync_wait_lock); 896 897 r = virtio_enqueue_prep(vsc, vq, &slot); 898 if (r != 0) { 899 return r; 900 } 901 902 vr = &sc->sc_reqs[slot]; 903 KASSERT(vr->vr_bp == NULL); 904 905 r = virtio_enqueue_reserve(vsc, vq, slot, VIRTIO_BLK_CTRL_SEGMENTS); 906 if (r != 0) { 907 return r; 908 } 909 910 vr->vr_bp = DUMMY_VR_BP; 911 vr->vr_hdr.type = virtio_rw32(vsc, VIRTIO_BLK_T_FLUSH); 912 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0); 913 vr->vr_hdr.sector = virtio_rw64(vsc, 0); 914 915 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 916 0, sizeof(struct virtio_blk_req_hdr), 917 BUS_DMASYNC_PREWRITE); 918 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 919 offsetof(struct virtio_blk_req, vr_status), 920 sizeof(uint8_t), 921 BUS_DMASYNC_PREREAD); 922 923 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 924 0, sizeof(struct virtio_blk_req_hdr), 925 true); 926 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 927 offsetof(struct virtio_blk_req, vr_status), 928 sizeof(uint8_t), 929 false); 930 virtio_enqueue_commit(vsc, vq, slot, true); 931 932 mutex_enter(&sc->sc_sync_wait_lock); 933 while (sc->sc_sync_use != SYNC_DONE) { 934 if (poll) { 935 mutex_exit(&sc->sc_sync_wait_lock); 936 ld_virtio_vq_done(vq); 937 mutex_enter(&sc->sc_sync_wait_lock); 938 continue; 939 } 940 cv_wait(&sc->sc_sync_wait, &sc->sc_sync_wait_lock); 941 } 942 943 if (sc->sc_sync_status == VIRTIO_BLK_S_OK) 944 r = 0; 945 else 946 r = EIO; 947 948 sc->sc_sync_use = SYNC_FREE; 949 cv_broadcast(&sc->sc_sync_wait); 950 mutex_exit(&sc->sc_sync_wait_lock); 951 952 return r; 953 } 954 955 static int 956 ld_virtio_getcache(struct ld_softc *ld, int *bitsp) 957 { 958 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 959 struct virtio_softc * const vsc = sc->sc_virtio; 960 const uint64_t features = virtio_features(vsc); 961 962 *bitsp = DKCACHE_READ; 963 if ((features & VIRTIO_BLK_F_CONFIG_WCE) != 0) 964 *bitsp |= DKCACHE_WCHANGE; 965 if (virtio_read_device_config_1(vsc, 966 VIRTIO_BLK_CONFIG_WRITEBACK) != 0x00) 967 *bitsp |= DKCACHE_WRITE; 968 969 return 0; 970 } 971 972 static int 973 ld_virtio_setcache(struct ld_softc *ld, int bits) 974 { 975 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 976 struct virtio_softc * const vsc = sc->sc_virtio; 977 const uint8_t wce = (bits & DKCACHE_WRITE) ? 0x01 : 0x00; 978 979 virtio_write_device_config_1(vsc, 980 VIRTIO_BLK_CONFIG_WRITEBACK, wce); 981 if (virtio_read_device_config_1(vsc, 982 VIRTIO_BLK_CONFIG_WRITEBACK) != wce) 983 return EIO; 984 985 return 0; 986 } 987 988 static int 989 ld_virtio_ioctl(struct ld_softc *ld, u_long cmd, void *addr, int32_t flag, bool poll) 990 { 991 int error; 992 993 switch (cmd) { 994 case DIOCCACHESYNC: 995 error = ld_virtio_flush(ld, poll); 996 break; 997 998 case DIOCGCACHE: 999 error = ld_virtio_getcache(ld, (int *)addr); 1000 break; 1001 1002 case DIOCSCACHE: 1003 error = ld_virtio_setcache(ld, *(int *)addr); 1004 break; 1005 1006 default: 1007 error = EPASSTHROUGH; 1008 break; 1009 } 1010 1011 return error; 1012 } 1013 1014 static int 1015 ld_virtio_discard(struct ld_softc *ld, struct buf *bp) 1016 { 1017 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 1018 struct virtio_softc * const vsc = sc->sc_virtio; 1019 struct virtqueue * const vq = &sc->sc_vq; 1020 struct virtio_blk_req *vr; 1021 const uint64_t features = virtio_features(vsc); 1022 int r; 1023 int slot; 1024 uint64_t blkno; 1025 uint32_t nblks; 1026 struct virtio_blk_discard_write_zeroes * dwz; 1027 1028 if ((features & VIRTIO_BLK_F_DISCARD) == 0 || 1029 sc->sc_max_discard_seg < 1) 1030 return EINVAL; 1031 1032 if (sc->sc_readonly) 1033 return EIO; 1034 1035 blkno = bp->b_rawblkno * sc->sc_ld.sc_secsize / VIRTIO_BLK_BSIZE; 1036 nblks = bp->b_bcount / VIRTIO_BLK_BSIZE; 1037 1038 if (nblks > sc->sc_max_discard_sectors) 1039 return ERANGE; 1040 1041 r = virtio_enqueue_prep(vsc, vq, &slot); 1042 if (r != 0) { 1043 return r; 1044 } 1045 1046 vr = &sc->sc_reqs[slot]; 1047 KASSERT(vr->vr_bp == NULL); 1048 1049 dwz = kmem_alloc(sizeof(*dwz), KM_SLEEP); 1050 1051 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload, 1052 dwz, sizeof(*dwz), NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1053 if (r != 0) { 1054 device_printf(sc->sc_dev, 1055 "discard payload dmamap failed, error code %d\n", r); 1056 virtio_enqueue_abort(vsc, vq, slot); 1057 kmem_free(dwz, sizeof(*dwz)); 1058 return r; 1059 } 1060 1061 KASSERT(vr->vr_payload->dm_nsegs <= sc->sc_seg_max); 1062 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs + 1063 VIRTIO_BLK_CTRL_SEGMENTS); 1064 if (r != 0) { 1065 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 1066 kmem_free(dwz, sizeof(*dwz)); 1067 return r; 1068 } 1069 1070 vr->vr_hdr.type = virtio_rw32(vsc, VIRTIO_BLK_T_DISCARD); 1071 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0); 1072 vr->vr_hdr.sector = virtio_rw64(vsc, 0); 1073 vr->vr_bp = bp; 1074 1075 KASSERT(vr->vr_datap == NULL); 1076 vr->vr_datap = dwz; 1077 vr->vr_datas = sizeof(*dwz); 1078 1079 dwz->sector = virtio_rw64(vsc, blkno); 1080 dwz->num_sectors = virtio_rw32(vsc, nblks); 1081 dwz->flags = virtio_rw32(vsc, 0); 1082 1083 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 1084 0, sizeof(struct virtio_blk_req_hdr), 1085 BUS_DMASYNC_PREWRITE); 1086 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 1087 0, vr->vr_datas, BUS_DMASYNC_PREWRITE); 1088 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 1089 offsetof(struct virtio_blk_req, vr_status), 1090 sizeof(uint8_t), 1091 BUS_DMASYNC_PREREAD); 1092 1093 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 1094 0, sizeof(struct virtio_blk_req_hdr), 1095 true); 1096 virtio_enqueue(vsc, vq, slot, vr->vr_payload, true); 1097 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 1098 offsetof(struct virtio_blk_req, vr_status), 1099 sizeof(uint8_t), 1100 false); 1101 virtio_enqueue_commit(vsc, vq, slot, true); 1102 1103 return 0; 1104 } 1105 1106 MODULE(MODULE_CLASS_DRIVER, ld_virtio, "ld,virtio"); 1107 1108 static int 1109 ld_virtio_modcmd(modcmd_t cmd, void *opaque) 1110 { 1111 int error = 0; 1112 1113 switch (cmd) { 1114 case MODULE_CMD_INIT: 1115 #ifdef _MODULE 1116 error = config_init_component(cfdriver_ioconf_ld_virtio, 1117 cfattach_ioconf_ld_virtio, cfdata_ioconf_ld_virtio); 1118 #endif 1119 break; 1120 case MODULE_CMD_FINI: 1121 #ifdef _MODULE 1122 error = config_fini_component(cfdriver_ioconf_ld_virtio, 1123 cfattach_ioconf_ld_virtio, cfdata_ioconf_ld_virtio); 1124 #endif 1125 break; 1126 default: 1127 error = ENOTTY; 1128 break; 1129 } 1130 1131 return error; 1132 } 1133
Indexes created Sun Sep 28 16:09:52 GMT 2025