ld_virtio.c revision 1.39
1 /* $NetBSD: ld_virtio.c,v 1.39 2025/02/22 09:57:09 mlelstv Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: ld_virtio.c,v 1.39 2025/02/22 09:57:09 mlelstv Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/buf.h> 35 #include <sys/bufq.h> 36 #include <sys/bus.h> 37 #include <sys/device.h> 38 #include <sys/disk.h> 39 #include <sys/mutex.h> 40 #include <sys/module.h> 41 #include <sys/kmem.h> 42 43 #include <dev/ldvar.h> 44 #include <dev/pci/virtioreg.h> 45 #include <dev/pci/virtiovar.h> 46 47 #include "ioconf.h" 48 49 /* 50 * ld_virtioreg: 51 */ 52 /* Configuration registers */ 53 #define VIRTIO_BLK_CONFIG_CAPACITY 0 /* 64bit */ 54 #define VIRTIO_BLK_CONFIG_SIZE_MAX 8 /* 32bit */ 55 #define VIRTIO_BLK_CONFIG_SEG_MAX 12 /* 32bit */ 56 #define VIRTIO_BLK_CONFIG_GEOMETRY_C 16 /* 16bit */ 57 #define VIRTIO_BLK_CONFIG_GEOMETRY_H 18 /* 8bit */ 58 #define VIRTIO_BLK_CONFIG_GEOMETRY_S 19 /* 8bit */ 59 #define VIRTIO_BLK_CONFIG_BLK_SIZE 20 /* 32bit */ 60 #define VIRTIO_BLK_CONFIG_WRITEBACK 32 /* 8bit */ 61 #define VIRTIO_BLK_CONFIG_NUM_QUEUES 34 /* 16bit */ 62 #define VIRTIO_BLK_CONFIG_MAX_DISCARD_SECTORS 36 /* 32bit */ 63 #define VIRTIO_BLK_CONFIG_MAX_DISCARD_SEG 40 /* 32bit */ 64 #define VIRTIO_BLK_CONFIG_DISCARD_SECTOR_ALIGNMENT 44 /* 32bit */ 65 66 /* Feature bits */ 67 #define VIRTIO_BLK_F_BARRIER (1<<0) 68 #define VIRTIO_BLK_F_SIZE_MAX (1<<1) 69 #define VIRTIO_BLK_F_SEG_MAX (1<<2) 70 #define VIRTIO_BLK_F_GEOMETRY (1<<4) 71 #define VIRTIO_BLK_F_RO (1<<5) 72 #define VIRTIO_BLK_F_BLK_SIZE (1<<6) 73 #define VIRTIO_BLK_F_SCSI (1<<7) 74 #define VIRTIO_BLK_F_FLUSH (1<<9) 75 #define VIRTIO_BLK_F_TOPOLOGY (1<<10) 76 #define VIRTIO_BLK_F_CONFIG_WCE (1<<11) 77 #define VIRTIO_BLK_F_MQ (1<<12) 78 #define VIRTIO_BLK_F_DISCARD (1<<13) 79 #define VIRTIO_BLK_F_WRITE_ZEROES (1<<14) 80 #define VIRTIO_BLK_F_LIFETIME (1<<15) 81 #define VIRTIO_BLK_F_SECURE_ERASE (1<<16) 82 83 /* 84 * Each block request uses at least two segments - one for the header 85 * and one for the status. 86 */ 87 #define VIRTIO_BLK_CTRL_SEGMENTS 2 88 89 #define VIRTIO_BLK_FLAG_BITS \ 90 VIRTIO_COMMON_FLAG_BITS \ 91 "b\x10" "SECURE_ERASE\0" \ 92 "b\x0f" "LIFETIME\0" \ 93 "b\x0e" "WRITE_ZEROES\0" \ 94 "b\x0d" "DISCARD\0" \ 95 "b\x0c" "MQ\0" \ 96 "b\x0b" "CONFIG_WCE\0" \ 97 "b\x0a" "TOPOLOGY\0" \ 98 "b\x09" "FLUSH\0" \ 99 "b\x07" "SCSI\0" \ 100 "b\x06" "BLK_SIZE\0" \ 101 "b\x05" "RO\0" \ 102 "b\x04" "GEOMETRY\0" \ 103 "b\x02" "SEG_MAX\0" \ 104 "b\x01" "SIZE_MAX\0" \ 105 "b\x00" "BARRIER\0" 106 107 /* Command */ 108 #define VIRTIO_BLK_T_IN 0 109 #define VIRTIO_BLK_T_OUT 1 110 #define VIRTIO_BLK_T_FLUSH 4 111 #define VIRTIO_BLK_T_GET_ID 8 112 #define VIRTIO_BLK_T_GET_LIFETIME 10 113 #define VIRTIO_BLK_T_DISCARD 11 114 #define VIRTIO_BLK_T_WRITE_ZEROES 13 115 #define VIRTIO_BLK_T_SECURE_ERASE 14 116 #define VIRTIO_BLK_T_BARRIER 0x80000000 117 118 /* Sector */ 119 #define VIRTIO_BLK_BSIZE 512 120 121 /* Status */ 122 #define VIRTIO_BLK_S_OK 0 123 #define VIRTIO_BLK_S_IOERR 1 124 #define VIRTIO_BLK_S_UNSUPP 2 125 126 /* Request header structure */ 127 struct virtio_blk_req_hdr { 128 uint32_t type; /* VIRTIO_BLK_T_* */ 129 uint32_t ioprio; 130 uint64_t sector; 131 } __packed; 132 /* payload and 1 byte status follows */ 133 134 struct virtio_blk_discard_write_zeroes { 135 uint64_t sector; 136 uint32_t num_sectors; 137 union { 138 uint32_t flags; 139 struct { 140 uint32_t unmap:1; 141 uint32_t reserved:31; 142 }; 143 }; 144 } __packed; 145 146 /* 147 * ld_virtiovar: 148 */ 149 struct virtio_blk_req { 150 struct virtio_blk_req_hdr vr_hdr; 151 uint8_t vr_status; 152 struct buf *vr_bp; 153 #define DUMMY_VR_BP ((void *)1) 154 bus_dmamap_t vr_cmdsts; 155 bus_dmamap_t vr_payload; 156 void * vr_datap; 157 size_t vr_datas; 158 }; 159 160 struct ld_virtio_softc { 161 struct ld_softc sc_ld; 162 device_t sc_dev; 163 164 uint32_t sc_seg_max; /* max number of segs in xfer */ 165 uint32_t sc_size_max; /* max size of single seg */ 166 167 struct virtio_softc *sc_virtio; 168 struct virtqueue sc_vq; 169 170 struct virtio_blk_req *sc_reqs; 171 bus_dma_segment_t sc_reqs_seg; 172 173 int sc_readonly; 174 175 enum { 176 SYNC_FREE, SYNC_BUSY, SYNC_DONE 177 } sc_sync_use; 178 kcondvar_t sc_sync_wait; 179 kmutex_t sc_sync_wait_lock; 180 uint8_t sc_sync_status; 181 uint8_t *sc_typename; 182 183 uint32_t sc_max_discard_sectors; 184 uint32_t sc_max_discard_seg; 185 #if 0 186 uint32_t sc_discard_sector_alignment; 187 #endif 188 }; 189 190 static int ld_virtio_match(device_t, cfdata_t, void *); 191 static void ld_virtio_attach(device_t, device_t, void *); 192 static int ld_virtio_detach(device_t, int); 193 194 CFATTACH_DECL_NEW(ld_virtio, sizeof(struct ld_virtio_softc), 195 ld_virtio_match, ld_virtio_attach, ld_virtio_detach, NULL); 196 197 static int 198 ld_virtio_match(device_t parent, cfdata_t match, void *aux) 199 { 200 struct virtio_attach_args *va = aux; 201 202 if (va->sc_childdevid == VIRTIO_DEVICE_ID_BLOCK) 203 return 1; 204 205 return 0; 206 } 207 208 static int ld_virtio_vq_done(struct virtqueue *); 209 static int ld_virtio_dump(struct ld_softc *, void *, int, int); 210 static int ld_virtio_start(struct ld_softc *, struct buf *); 211 static int ld_virtio_ioctl(struct ld_softc *, u_long, void *, int32_t, bool); 212 static int ld_virtio_info(struct ld_softc *); 213 static int ld_virtio_discard(struct ld_softc *, struct buf *); 214 215 static int 216 ld_virtio_alloc_reqs(struct ld_virtio_softc *sc, int qsize) 217 { 218 int allocsize, r, rsegs, i; 219 struct ld_softc *ld = &sc->sc_ld; 220 void *vaddr; 221 222 allocsize = sizeof(struct virtio_blk_req) * qsize; 223 r = bus_dmamem_alloc(virtio_dmat(sc->sc_virtio), allocsize, 0, 0, 224 &sc->sc_reqs_seg, 1, &rsegs, BUS_DMA_WAITOK); 225 if (r != 0) { 226 aprint_error_dev(sc->sc_dev, 227 "DMA memory allocation failed, size %d, " 228 "error code %d\n", allocsize, r); 229 goto err_none; 230 } 231 r = bus_dmamem_map(virtio_dmat(sc->sc_virtio), 232 &sc->sc_reqs_seg, 1, allocsize, 233 &vaddr, BUS_DMA_WAITOK); 234 if (r != 0) { 235 aprint_error_dev(sc->sc_dev, 236 "DMA memory map failed, " 237 "error code %d\n", r); 238 goto err_dmamem_alloc; 239 } 240 sc->sc_reqs = vaddr; 241 memset(vaddr, 0, allocsize); 242 for (i = 0; i < qsize; i++) { 243 struct virtio_blk_req *vr = &sc->sc_reqs[i]; 244 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), 245 offsetof(struct virtio_blk_req, vr_bp), 246 1, 247 offsetof(struct virtio_blk_req, vr_bp), 248 0, 249 BUS_DMA_WAITOK|BUS_DMA_ALLOCNOW, 250 &vr->vr_cmdsts); 251 if (r != 0) { 252 aprint_error_dev(sc->sc_dev, 253 "command dmamap creation failed, " 254 "error code %d\n", r); 255 goto err_reqs; 256 } 257 r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), vr->vr_cmdsts, 258 &vr->vr_hdr, 259 offsetof(struct virtio_blk_req, vr_bp), 260 NULL, BUS_DMA_WAITOK); 261 if (r != 0) { 262 aprint_error_dev(sc->sc_dev, 263 "command dmamap load failed, " 264 "error code %d\n", r); 265 goto err_reqs; 266 } 267 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), 268 /*size*/ld->sc_maxxfer, 269 /*nseg*/sc->sc_seg_max, 270 /*maxsegsz*/sc->sc_size_max, 271 /*boundary*/0, 272 BUS_DMA_WAITOK|BUS_DMA_ALLOCNOW, 273 &vr->vr_payload); 274 if (r != 0) { 275 aprint_error_dev(sc->sc_dev, 276 "payload dmamap creation failed, " 277 "error code %d\n", r); 278 goto err_reqs; 279 } 280 vr->vr_datap = NULL; 281 vr->vr_datas = 0; 282 } 283 return 0; 284 285 err_reqs: 286 for (i = 0; i < qsize; i++) { 287 struct virtio_blk_req *vr = &sc->sc_reqs[i]; 288 if (vr->vr_cmdsts) { 289 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), 290 vr->vr_cmdsts); 291 vr->vr_cmdsts = 0; 292 } 293 if (vr->vr_payload) { 294 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), 295 vr->vr_payload); 296 vr->vr_payload = 0; 297 } 298 } 299 bus_dmamem_unmap(virtio_dmat(sc->sc_virtio), sc->sc_reqs, allocsize); 300 err_dmamem_alloc: 301 bus_dmamem_free(virtio_dmat(sc->sc_virtio), &sc->sc_reqs_seg, 1); 302 err_none: 303 return -1; 304 } 305 306 static void 307 ld_virtio_attach(device_t parent, device_t self, void *aux) 308 { 309 struct ld_virtio_softc *sc = device_private(self); 310 struct ld_softc *ld = &sc->sc_ld; 311 struct virtio_softc *vsc = device_private(parent); 312 uint64_t features; 313 int qsize; 314 315 if (virtio_child(vsc) != NULL) { 316 aprint_normal(": child already attached for %s; " 317 "something wrong...\n", device_xname(parent)); 318 return; 319 } 320 321 sc->sc_dev = self; 322 sc->sc_virtio = vsc; 323 324 virtio_child_attach_start(vsc, self, IPL_BIO, 325 (VIRTIO_BLK_F_SIZE_MAX | VIRTIO_BLK_F_SEG_MAX | 326 VIRTIO_BLK_F_GEOMETRY | VIRTIO_BLK_F_RO | VIRTIO_BLK_F_BLK_SIZE | 327 VIRTIO_BLK_F_FLUSH | VIRTIO_BLK_F_CONFIG_WCE | 328 VIRTIO_BLK_F_DISCARD), 329 VIRTIO_BLK_FLAG_BITS); 330 331 features = virtio_features(vsc); 332 if (features == 0) 333 goto err; 334 335 if (features & VIRTIO_BLK_F_RO) 336 sc->sc_readonly = 1; 337 else 338 sc->sc_readonly = 0; 339 340 if (features & VIRTIO_BLK_F_BLK_SIZE) { 341 ld->sc_secsize = virtio_read_device_config_4(vsc, 342 VIRTIO_BLK_CONFIG_BLK_SIZE); 343 } else 344 ld->sc_secsize = VIRTIO_BLK_BSIZE; 345 346 if (features & VIRTIO_BLK_F_SEG_MAX) { 347 sc->sc_seg_max = virtio_read_device_config_4(vsc, 348 VIRTIO_BLK_CONFIG_SEG_MAX); 349 if (sc->sc_seg_max == 0) { 350 aprint_error_dev(sc->sc_dev, 351 "Invalid SEG_MAX %d\n", sc->sc_seg_max); 352 goto err; 353 } 354 } else { 355 sc->sc_seg_max = 1; 356 aprint_verbose_dev(sc->sc_dev, 357 "Unknown SEG_MAX, assuming %"PRIu32"\n", sc->sc_seg_max); 358 } 359 360 /* At least genfs_io assumes size_max*seg_max >= MAXPHYS. */ 361 if (features & VIRTIO_BLK_F_SIZE_MAX) { 362 sc->sc_size_max = virtio_read_device_config_4(vsc, 363 VIRTIO_BLK_CONFIG_SIZE_MAX); 364 if (sc->sc_size_max < MAXPHYS/sc->sc_seg_max) { 365 aprint_error_dev(sc->sc_dev, 366 "Too small SIZE_MAX %d minimum is %d\n", 367 sc->sc_size_max, MAXPHYS/sc->sc_seg_max); 368 // goto err; 369 sc->sc_size_max = MAXPHYS/sc->sc_seg_max; 370 } else if (sc->sc_size_max > MAXPHYS) { 371 aprint_verbose_dev(sc->sc_dev, 372 "Clip SIZE_MAX from %d to %d\n", 373 sc->sc_size_max, MAXPHYS); 374 sc->sc_size_max = MAXPHYS; 375 } 376 } else { 377 sc->sc_size_max = MAXPHYS; 378 aprint_verbose_dev(sc->sc_dev, 379 "Unknown SIZE_MAX, assuming %"PRIu32"\n", 380 sc->sc_size_max); 381 } 382 383 aprint_normal_dev(sc->sc_dev, "max %"PRIu32" segs" 384 " of max %"PRIu32" bytes\n", 385 sc->sc_seg_max, sc->sc_size_max); 386 387 virtio_init_vq_vqdone(vsc, &sc->sc_vq, 0, 388 ld_virtio_vq_done); 389 390 if (virtio_alloc_vq(vsc, &sc->sc_vq, sc->sc_size_max, 391 sc->sc_seg_max + VIRTIO_BLK_CTRL_SEGMENTS, "I/O request") != 0) 392 goto err; 393 qsize = sc->sc_vq.vq_num; 394 395 if (virtio_child_attach_finish(vsc, &sc->sc_vq, 1, 396 NULL, VIRTIO_F_INTR_MSIX) != 0) 397 goto err; 398 399 ld->sc_dv = self; 400 ld->sc_secperunit = virtio_read_device_config_8(vsc, 401 VIRTIO_BLK_CONFIG_CAPACITY) / (ld->sc_secsize / VIRTIO_BLK_BSIZE); 402 403 /* 404 * Clamp ld->sc_maxxfer to MAXPHYS before ld_virtio_alloc_reqs 405 * allocates DMA maps of at most ld->sc_maxxfer bytes. 406 * ldattach will also clamp to MAXPHYS, but not until after 407 * ld_virtio_alloc_reqs is done, so that doesn't help. 408 */ 409 ld->sc_maxxfer = MIN(MAXPHYS, sc->sc_size_max * sc->sc_seg_max); 410 411 if (features & VIRTIO_BLK_F_GEOMETRY) { 412 ld->sc_ncylinders = virtio_read_device_config_2(vsc, 413 VIRTIO_BLK_CONFIG_GEOMETRY_C); 414 ld->sc_nheads = virtio_read_device_config_1(vsc, 415 VIRTIO_BLK_CONFIG_GEOMETRY_H); 416 ld->sc_nsectors = virtio_read_device_config_1(vsc, 417 VIRTIO_BLK_CONFIG_GEOMETRY_S); 418 } 419 ld->sc_maxqueuecnt = qsize - 1; /* reserve slot for dumps, flushes */ 420 421 if (ld_virtio_alloc_reqs(sc, qsize) < 0) 422 goto err; 423 424 cv_init(&sc->sc_sync_wait, "vblksync"); 425 mutex_init(&sc->sc_sync_wait_lock, MUTEX_DEFAULT, IPL_BIO); 426 sc->sc_sync_use = SYNC_FREE; 427 428 ld->sc_dump = ld_virtio_dump; 429 ld->sc_start = ld_virtio_start; 430 ld->sc_ioctl = ld_virtio_ioctl; 431 432 if (ld_virtio_info(ld) == 0) 433 ld->sc_typename = sc->sc_typename; 434 else 435 ld->sc_typename = __UNCONST("Virtio Block Device"); 436 437 if (features & VIRTIO_BLK_F_DISCARD) { 438 ld->sc_discard = ld_virtio_discard; 439 sc->sc_max_discard_sectors = virtio_read_device_config_4(vsc, 440 VIRTIO_BLK_CONFIG_MAX_DISCARD_SECTORS); 441 sc->sc_max_discard_seg = virtio_read_device_config_4(vsc, 442 VIRTIO_BLK_CONFIG_MAX_DISCARD_SEG); 443 #if 0 444 sc->sc_discard_sector_alignment = 445 virtio_read_device_config_4(vsc, 446 VIRTIO_BLK_CONFIG_DISCARD_SECTOR_ALIGNMENT); 447 #endif 448 } 449 450 ld->sc_flags = LDF_ENABLED | LDF_MPSAFE; 451 ldattach(ld, BUFQ_DISK_DEFAULT_STRAT); 452 453 return; 454 455 err: 456 virtio_child_attach_failed(vsc); 457 return; 458 } 459 460 static int 461 ld_virtio_info(struct ld_softc *ld) 462 { 463 struct ld_virtio_softc *sc = device_private(ld->sc_dv); 464 struct virtio_softc *vsc = sc->sc_virtio; 465 struct virtqueue *vq = &sc->sc_vq; 466 struct virtio_blk_req *vr; 467 int r; 468 int slot; 469 uint8_t id_data[20]; /* virtio v1.2 5.2.6 */ 470 471 if (sc->sc_typename != NULL) { 472 kmem_strfree(sc->sc_typename); 473 sc->sc_typename = NULL; 474 } 475 476 r = virtio_enqueue_prep(vsc, vq, &slot); 477 if (r != 0) 478 return r; 479 480 vr = &sc->sc_reqs[slot]; 481 KASSERT(vr->vr_bp == NULL); 482 483 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload, 484 id_data, sizeof(id_data), NULL, 485 BUS_DMA_READ|BUS_DMA_NOWAIT); 486 if (r != 0) { 487 aprint_error_dev(sc->sc_dev, 488 "payload dmamap failed, error code %d\n", r); 489 virtio_enqueue_abort(vsc, vq, slot); 490 return r; 491 } 492 493 KASSERT(vr->vr_payload->dm_nsegs <= sc->sc_seg_max); 494 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs + 495 VIRTIO_BLK_CTRL_SEGMENTS); 496 if (r != 0) { 497 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 498 return r; 499 } 500 501 vr->vr_bp = DUMMY_VR_BP; 502 vr->vr_hdr.type = virtio_rw32(vsc, VIRTIO_BLK_T_GET_ID); 503 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0); 504 vr->vr_hdr.sector = virtio_rw64(vsc, 0); 505 506 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 507 0, sizeof(struct virtio_blk_req_hdr), 508 BUS_DMASYNC_PREWRITE); 509 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 510 0, sizeof(id_data), 511 BUS_DMASYNC_PREREAD); 512 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 513 offsetof(struct virtio_blk_req, vr_status), 514 sizeof(uint8_t), 515 BUS_DMASYNC_PREREAD); 516 517 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 518 0, sizeof(struct virtio_blk_req_hdr), 519 true); 520 virtio_enqueue(vsc, vq, slot, vr->vr_payload, false); 521 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 522 offsetof(struct virtio_blk_req, vr_status), 523 sizeof(uint8_t), 524 false); 525 virtio_enqueue_commit(vsc, vq, slot, true); 526 527 mutex_enter(&sc->sc_sync_wait_lock); 528 while (sc->sc_sync_use != SYNC_DONE) 529 cv_wait(&sc->sc_sync_wait, &sc->sc_sync_wait_lock); 530 531 if (sc->sc_sync_status == VIRTIO_BLK_S_OK) 532 r = 0; 533 else 534 r = EIO; 535 536 sc->sc_sync_use = SYNC_FREE; 537 cv_broadcast(&sc->sc_sync_wait); 538 mutex_exit(&sc->sc_sync_wait_lock); 539 540 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 541 0, sizeof(id_data), BUS_DMASYNC_POSTREAD); 542 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 543 544 if (r != 0) 545 return r; 546 547 sc->sc_typename = kmem_strndup(id_data, sizeof(id_data), KM_NOSLEEP); 548 549 return 0; 550 } 551 552 static int 553 ld_virtio_start(struct ld_softc *ld, struct buf *bp) 554 { 555 /* splbio */ 556 struct ld_virtio_softc *sc = device_private(ld->sc_dv); 557 struct virtio_softc *vsc = sc->sc_virtio; 558 struct virtqueue *vq = &sc->sc_vq; 559 struct virtio_blk_req *vr; 560 int r; 561 int isread = (bp->b_flags & B_READ); 562 int slot; 563 564 if (sc->sc_readonly && !isread) 565 return EIO; 566 567 r = virtio_enqueue_prep(vsc, vq, &slot); 568 if (r != 0) 569 return r; 570 571 vr = &sc->sc_reqs[slot]; 572 KASSERT(vr->vr_bp == NULL); 573 574 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload, 575 bp->b_data, bp->b_bcount, NULL, 576 ((isread?BUS_DMA_READ:BUS_DMA_WRITE) 577 |BUS_DMA_NOWAIT)); 578 if (r != 0) { 579 aprint_error_dev(sc->sc_dev, 580 "payload dmamap failed, error code %d\n", r); 581 virtio_enqueue_abort(vsc, vq, slot); 582 return r; 583 } 584 585 KASSERT(vr->vr_payload->dm_nsegs <= sc->sc_seg_max); 586 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs + 587 VIRTIO_BLK_CTRL_SEGMENTS); 588 if (r != 0) { 589 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 590 return r; 591 } 592 593 vr->vr_bp = bp; 594 vr->vr_hdr.type = virtio_rw32(vsc, 595 isread ? VIRTIO_BLK_T_IN : VIRTIO_BLK_T_OUT); 596 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0); 597 vr->vr_hdr.sector = virtio_rw64(vsc, 598 bp->b_rawblkno * sc->sc_ld.sc_secsize / 599 VIRTIO_BLK_BSIZE); 600 601 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 602 0, sizeof(struct virtio_blk_req_hdr), 603 BUS_DMASYNC_PREWRITE); 604 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 605 0, bp->b_bcount, 606 isread?BUS_DMASYNC_PREREAD:BUS_DMASYNC_PREWRITE); 607 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 608 offsetof(struct virtio_blk_req, vr_status), 609 sizeof(uint8_t), 610 BUS_DMASYNC_PREREAD); 611 612 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 613 0, sizeof(struct virtio_blk_req_hdr), 614 true); 615 virtio_enqueue(vsc, vq, slot, vr->vr_payload, !isread); 616 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 617 offsetof(struct virtio_blk_req, vr_status), 618 sizeof(uint8_t), 619 false); 620 virtio_enqueue_commit(vsc, vq, slot, true); 621 622 return 0; 623 } 624 625 static void 626 ld_virtio_vq_done1(struct ld_virtio_softc *sc, struct virtio_softc *vsc, 627 struct virtqueue *vq, int slot) 628 { 629 struct virtio_blk_req *vr = &sc->sc_reqs[slot]; 630 struct buf *bp = vr->vr_bp; 631 const uint32_t rt = virtio_rw32(vsc, vr->vr_hdr.type); 632 633 vr->vr_bp = NULL; 634 635 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 636 0, sizeof(struct virtio_blk_req_hdr), 637 BUS_DMASYNC_POSTWRITE); 638 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 639 sizeof(struct virtio_blk_req_hdr), sizeof(uint8_t), 640 BUS_DMASYNC_POSTREAD); 641 if (bp == DUMMY_VR_BP) { 642 mutex_enter(&sc->sc_sync_wait_lock); 643 sc->sc_sync_status = vr->vr_status; 644 sc->sc_sync_use = SYNC_DONE; 645 cv_broadcast(&sc->sc_sync_wait); 646 mutex_exit(&sc->sc_sync_wait_lock); 647 virtio_dequeue_commit(vsc, vq, slot); 648 return; 649 } 650 switch (rt) { 651 case VIRTIO_BLK_T_OUT: 652 case VIRTIO_BLK_T_IN: 653 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 654 0, bp->b_bcount, 655 (bp->b_flags & B_READ)?BUS_DMASYNC_POSTREAD 656 :BUS_DMASYNC_POSTWRITE); 657 break; 658 default: 659 if (vr->vr_datap == NULL) 660 break; 661 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 662 0, vr->vr_datas, BUS_DMASYNC_POSTREAD | 663 BUS_DMASYNC_POSTWRITE); 664 break; 665 } 666 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 667 668 if (vr->vr_status != VIRTIO_BLK_S_OK) { 669 bp->b_error = EIO; 670 bp->b_resid = bp->b_bcount; 671 } else { 672 bp->b_error = 0; 673 bp->b_resid = 0; 674 } 675 676 if (vr->vr_datap != NULL) { 677 kmem_free(vr->vr_datap, vr->vr_datas); 678 vr->vr_datap = NULL; 679 vr->vr_datas = 0; 680 } 681 682 virtio_dequeue_commit(vsc, vq, slot); 683 684 switch (rt) { 685 case VIRTIO_BLK_T_OUT: 686 case VIRTIO_BLK_T_IN: 687 lddone(&sc->sc_ld, bp); 688 break; 689 case VIRTIO_BLK_T_DISCARD: 690 lddiscardend(&sc->sc_ld, bp); 691 break; 692 } 693 } 694 695 static int 696 ld_virtio_vq_done(struct virtqueue *vq) 697 { 698 struct virtio_softc *vsc = vq->vq_owner; 699 struct ld_virtio_softc *sc = device_private(virtio_child(vsc)); 700 int r = 0; 701 int slot; 702 703 again: 704 if (virtio_dequeue(vsc, vq, &slot, NULL)) 705 return r; 706 r = 1; 707 708 ld_virtio_vq_done1(sc, vsc, vq, slot); 709 goto again; 710 } 711 712 static int 713 ld_virtio_dump(struct ld_softc *ld, void *data, int blkno, int blkcnt) 714 { 715 struct ld_virtio_softc *sc = device_private(ld->sc_dv); 716 struct virtio_softc *vsc = sc->sc_virtio; 717 struct virtqueue *vq = &sc->sc_vq; 718 struct virtio_blk_req *vr; 719 int slot, r; 720 721 if (sc->sc_readonly) 722 return EIO; 723 724 r = virtio_enqueue_prep(vsc, vq, &slot); 725 if (r != 0) { 726 if (r == EAGAIN) { /* no free slot; dequeue first */ 727 delay(100); 728 ld_virtio_vq_done(vq); 729 r = virtio_enqueue_prep(vsc, vq, &slot); 730 if (r != 0) 731 return r; 732 } 733 return r; 734 } 735 vr = &sc->sc_reqs[slot]; 736 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload, 737 data, blkcnt*ld->sc_secsize, NULL, 738 BUS_DMA_WRITE|BUS_DMA_NOWAIT); 739 if (r != 0) 740 return r; 741 742 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs + 743 VIRTIO_BLK_CTRL_SEGMENTS); 744 if (r != 0) { 745 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 746 return r; 747 } 748 749 vr->vr_bp = (void*)0xdeadbeef; 750 vr->vr_hdr.type = virtio_rw32(vsc, VIRTIO_BLK_T_OUT); 751 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0); 752 vr->vr_hdr.sector = virtio_rw64(vsc, 753 (daddr_t) blkno * ld->sc_secsize / 754 VIRTIO_BLK_BSIZE); 755 756 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 757 0, sizeof(struct virtio_blk_req_hdr), 758 BUS_DMASYNC_PREWRITE); 759 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 760 0, blkcnt*ld->sc_secsize, 761 BUS_DMASYNC_PREWRITE); 762 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 763 offsetof(struct virtio_blk_req, vr_status), 764 sizeof(uint8_t), 765 BUS_DMASYNC_PREREAD); 766 767 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 768 0, sizeof(struct virtio_blk_req_hdr), 769 true); 770 virtio_enqueue(vsc, vq, slot, vr->vr_payload, true); 771 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 772 offsetof(struct virtio_blk_req, vr_status), 773 sizeof(uint8_t), 774 false); 775 virtio_enqueue_commit(vsc, vq, slot, true); 776 777 for ( ; ; ) { 778 int dslot; 779 780 r = virtio_dequeue(vsc, vq, &dslot, NULL); 781 if (r != 0) 782 continue; 783 if (dslot != slot) { 784 ld_virtio_vq_done1(sc, vsc, vq, dslot); 785 continue; 786 } else 787 break; 788 } 789 790 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 791 0, sizeof(struct virtio_blk_req_hdr), 792 BUS_DMASYNC_POSTWRITE); 793 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 794 0, blkcnt*ld->sc_secsize, 795 BUS_DMASYNC_POSTWRITE); 796 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 797 offsetof(struct virtio_blk_req, vr_status), 798 sizeof(uint8_t), 799 BUS_DMASYNC_POSTREAD); 800 if (vr->vr_status == VIRTIO_BLK_S_OK) 801 r = 0; 802 else 803 r = EIO; 804 virtio_dequeue_commit(vsc, vq, slot); 805 806 return r; 807 } 808 809 static int 810 ld_virtio_detach(device_t self, int flags) 811 { 812 struct ld_virtio_softc *sc = device_private(self); 813 struct ld_softc *ld = &sc->sc_ld; 814 bus_dma_tag_t dmat = virtio_dmat(sc->sc_virtio); 815 int r, i, qsize; 816 817 qsize = sc->sc_vq.vq_num; 818 r = ldbegindetach(ld, flags); 819 if (r != 0) 820 return r; 821 virtio_reset(sc->sc_virtio); 822 virtio_free_vq(sc->sc_virtio, &sc->sc_vq); 823 824 for (i = 0; i < qsize; i++) { 825 bus_dmamap_destroy(dmat, 826 sc->sc_reqs[i].vr_cmdsts); 827 bus_dmamap_destroy(dmat, 828 sc->sc_reqs[i].vr_payload); 829 } 830 bus_dmamem_unmap(dmat, sc->sc_reqs, 831 sizeof(struct virtio_blk_req) * qsize); 832 bus_dmamem_free(dmat, &sc->sc_reqs_seg, 1); 833 834 ldenddetach(ld); 835 836 if (sc->sc_typename != NULL) 837 kmem_strfree(sc->sc_typename); 838 839 cv_destroy(&sc->sc_sync_wait); 840 mutex_destroy(&sc->sc_sync_wait_lock); 841 842 virtio_child_detach(sc->sc_virtio); 843 844 return 0; 845 } 846 847 static int 848 ld_virtio_flush(struct ld_softc *ld, bool poll) 849 { 850 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 851 struct virtio_softc * const vsc = sc->sc_virtio; 852 const uint64_t features = virtio_features(vsc); 853 struct virtqueue *vq = &sc->sc_vq; 854 struct virtio_blk_req *vr; 855 int slot; 856 int r; 857 858 if ((features & VIRTIO_BLK_F_FLUSH) == 0) 859 return 0; 860 861 mutex_enter(&sc->sc_sync_wait_lock); 862 while (sc->sc_sync_use != SYNC_FREE) { 863 if (poll) { 864 mutex_exit(&sc->sc_sync_wait_lock); 865 ld_virtio_vq_done(vq); 866 mutex_enter(&sc->sc_sync_wait_lock); 867 continue; 868 } 869 cv_wait(&sc->sc_sync_wait, &sc->sc_sync_wait_lock); 870 } 871 sc->sc_sync_use = SYNC_BUSY; 872 mutex_exit(&sc->sc_sync_wait_lock); 873 874 r = virtio_enqueue_prep(vsc, vq, &slot); 875 if (r != 0) { 876 return r; 877 } 878 879 vr = &sc->sc_reqs[slot]; 880 KASSERT(vr->vr_bp == NULL); 881 882 r = virtio_enqueue_reserve(vsc, vq, slot, VIRTIO_BLK_CTRL_SEGMENTS); 883 if (r != 0) { 884 return r; 885 } 886 887 vr->vr_bp = DUMMY_VR_BP; 888 vr->vr_hdr.type = virtio_rw32(vsc, VIRTIO_BLK_T_FLUSH); 889 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0); 890 vr->vr_hdr.sector = virtio_rw64(vsc, 0); 891 892 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 893 0, sizeof(struct virtio_blk_req_hdr), 894 BUS_DMASYNC_PREWRITE); 895 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 896 offsetof(struct virtio_blk_req, vr_status), 897 sizeof(uint8_t), 898 BUS_DMASYNC_PREREAD); 899 900 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 901 0, sizeof(struct virtio_blk_req_hdr), 902 true); 903 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 904 offsetof(struct virtio_blk_req, vr_status), 905 sizeof(uint8_t), 906 false); 907 virtio_enqueue_commit(vsc, vq, slot, true); 908 909 mutex_enter(&sc->sc_sync_wait_lock); 910 while (sc->sc_sync_use != SYNC_DONE) { 911 if (poll) { 912 mutex_exit(&sc->sc_sync_wait_lock); 913 ld_virtio_vq_done(vq); 914 mutex_enter(&sc->sc_sync_wait_lock); 915 continue; 916 } 917 cv_wait(&sc->sc_sync_wait, &sc->sc_sync_wait_lock); 918 } 919 920 if (sc->sc_sync_status == VIRTIO_BLK_S_OK) 921 r = 0; 922 else 923 r = EIO; 924 925 sc->sc_sync_use = SYNC_FREE; 926 cv_broadcast(&sc->sc_sync_wait); 927 mutex_exit(&sc->sc_sync_wait_lock); 928 929 return r; 930 } 931 932 static int 933 ld_virtio_getcache(struct ld_softc *ld, int *bitsp) 934 { 935 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 936 struct virtio_softc * const vsc = sc->sc_virtio; 937 const uint64_t features = virtio_features(vsc); 938 939 *bitsp = DKCACHE_READ; 940 if ((features & VIRTIO_BLK_F_CONFIG_WCE) != 0) 941 *bitsp |= DKCACHE_WCHANGE; 942 if (virtio_read_device_config_1(vsc, 943 VIRTIO_BLK_CONFIG_WRITEBACK) != 0x00) 944 *bitsp |= DKCACHE_WRITE; 945 946 return 0; 947 } 948 949 static int 950 ld_virtio_setcache(struct ld_softc *ld, int bits) 951 { 952 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 953 struct virtio_softc * const vsc = sc->sc_virtio; 954 const uint8_t wce = (bits & DKCACHE_WRITE) ? 0x01 : 0x00; 955 956 virtio_write_device_config_1(vsc, 957 VIRTIO_BLK_CONFIG_WRITEBACK, wce); 958 if (virtio_read_device_config_1(vsc, 959 VIRTIO_BLK_CONFIG_WRITEBACK) != wce) 960 return EIO; 961 962 return 0; 963 } 964 965 static int 966 ld_virtio_ioctl(struct ld_softc *ld, u_long cmd, void *addr, int32_t flag, bool poll) 967 { 968 int error; 969 970 switch (cmd) { 971 case DIOCCACHESYNC: 972 error = ld_virtio_flush(ld, poll); 973 break; 974 975 case DIOCGCACHE: 976 error = ld_virtio_getcache(ld, (int *)addr); 977 break; 978 979 case DIOCSCACHE: 980 error = ld_virtio_setcache(ld, *(int *)addr); 981 break; 982 983 default: 984 error = EPASSTHROUGH; 985 break; 986 } 987 988 return error; 989 } 990 991 static int 992 ld_virtio_discard(struct ld_softc *ld, struct buf *bp) 993 { 994 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 995 struct virtio_softc * const vsc = sc->sc_virtio; 996 struct virtqueue * const vq = &sc->sc_vq; 997 struct virtio_blk_req *vr; 998 const uint64_t features = virtio_features(vsc); 999 int r; 1000 int slot; 1001 uint64_t blkno; 1002 uint32_t nblks; 1003 struct virtio_blk_discard_write_zeroes * dwz; 1004 1005 if ((features & VIRTIO_BLK_F_DISCARD) == 0 || 1006 sc->sc_max_discard_seg < 1) 1007 return EINVAL; 1008 1009 if (sc->sc_readonly) 1010 return EIO; 1011 1012 blkno = bp->b_rawblkno * sc->sc_ld.sc_secsize / VIRTIO_BLK_BSIZE; 1013 nblks = bp->b_bcount / VIRTIO_BLK_BSIZE; 1014 1015 if (nblks > sc->sc_max_discard_sectors) 1016 return ERANGE; 1017 1018 r = virtio_enqueue_prep(vsc, vq, &slot); 1019 if (r != 0) { 1020 return r; 1021 } 1022 1023 vr = &sc->sc_reqs[slot]; 1024 KASSERT(vr->vr_bp == NULL); 1025 1026 dwz = kmem_alloc(sizeof(*dwz), KM_SLEEP); 1027 1028 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload, 1029 dwz, sizeof(*dwz), NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1030 if (r != 0) { 1031 device_printf(sc->sc_dev, 1032 "discard payload dmamap failed, error code %d\n", r); 1033 virtio_enqueue_abort(vsc, vq, slot); 1034 kmem_free(dwz, sizeof(*dwz)); 1035 return r; 1036 } 1037 1038 KASSERT(vr->vr_payload->dm_nsegs <= sc->sc_seg_max); 1039 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs + 1040 VIRTIO_BLK_CTRL_SEGMENTS); 1041 if (r != 0) { 1042 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 1043 kmem_free(dwz, sizeof(*dwz)); 1044 return r; 1045 } 1046 1047 vr->vr_hdr.type = virtio_rw32(vsc, VIRTIO_BLK_T_DISCARD); 1048 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0); 1049 vr->vr_hdr.sector = virtio_rw64(vsc, 0); 1050 vr->vr_bp = bp; 1051 1052 KASSERT(vr->vr_datap == NULL); 1053 vr->vr_datap = dwz; 1054 vr->vr_datas = sizeof(*dwz); 1055 1056 dwz->sector = virtio_rw64(vsc, blkno); 1057 dwz->num_sectors = virtio_rw32(vsc, nblks); 1058 dwz->flags = virtio_rw32(vsc, 0); 1059 1060 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 1061 0, sizeof(struct virtio_blk_req_hdr), 1062 BUS_DMASYNC_PREWRITE); 1063 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 1064 0, vr->vr_datas, BUS_DMASYNC_PREWRITE); 1065 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 1066 offsetof(struct virtio_blk_req, vr_status), 1067 sizeof(uint8_t), 1068 BUS_DMASYNC_PREREAD); 1069 1070 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 1071 0, sizeof(struct virtio_blk_req_hdr), 1072 true); 1073 virtio_enqueue(vsc, vq, slot, vr->vr_payload, true); 1074 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 1075 offsetof(struct virtio_blk_req, vr_status), 1076 sizeof(uint8_t), 1077 false); 1078 virtio_enqueue_commit(vsc, vq, slot, true); 1079 1080 return 0; 1081 } 1082 1083 MODULE(MODULE_CLASS_DRIVER, ld_virtio, "ld,virtio"); 1084 1085 static int 1086 ld_virtio_modcmd(modcmd_t cmd, void *opaque) 1087 { 1088 int error = 0; 1089 1090 switch (cmd) { 1091 case MODULE_CMD_INIT: 1092 #ifdef _MODULE 1093 error = config_init_component(cfdriver_ioconf_ld_virtio, 1094 cfattach_ioconf_ld_virtio, cfdata_ioconf_ld_virtio); 1095 #endif 1096 break; 1097 case MODULE_CMD_FINI: 1098 #ifdef _MODULE 1099 error = config_fini_component(cfdriver_ioconf_ld_virtio, 1100 cfattach_ioconf_ld_virtio, cfdata_ioconf_ld_virtio); 1101 #endif 1102 break; 1103 default: 1104 error = ENOTTY; 1105 break; 1106 } 1107 1108 return error; 1109 } 1110
Indexes created Sun Sep 28 16:09:52 GMT 2025