Home | History | Annotate | Line # | Download | only in pci
ld_virtio.c revision 1.10
      1 /*	$NetBSD: ld_virtio.c,v 1.10 2016/07/07 06:55:41 msaitoh Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 2010 Minoura Makoto.
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26  */
     27 
     28 #include <sys/cdefs.h>
     29 __KERNEL_RCSID(0, "$NetBSD: ld_virtio.c,v 1.10 2016/07/07 06:55:41 msaitoh Exp $");
     30 
     31 #include <sys/param.h>
     32 #include <sys/systm.h>
     33 #include <sys/kernel.h>
     34 #include <sys/buf.h>
     35 #include <sys/bus.h>
     36 #include <sys/device.h>
     37 #include <sys/disk.h>
     38 #include <sys/mutex.h>
     39 
     40 #include <dev/pci/pcidevs.h>
     41 #include <dev/pci/pcireg.h>
     42 #include <dev/pci/pcivar.h>
     43 
     44 #include <dev/ldvar.h>
     45 #include <dev/pci/virtioreg.h>
     46 #include <dev/pci/virtiovar.h>
     47 
     48 /*
     49  * ld_virtioreg:
     50  */
     51 /* Configuration registers */
     52 #define VIRTIO_BLK_CONFIG_CAPACITY	0 /* 64bit */
     53 #define VIRTIO_BLK_CONFIG_SIZE_MAX	8 /* 32bit */
     54 #define VIRTIO_BLK_CONFIG_SEG_MAX	12 /* 32bit */
     55 #define VIRTIO_BLK_CONFIG_GEOMETRY_C	16 /* 16bit */
     56 #define VIRTIO_BLK_CONFIG_GEOMETRY_H	18 /* 8bit */
     57 #define VIRTIO_BLK_CONFIG_GEOMETRY_S	19 /* 8bit */
     58 #define VIRTIO_BLK_CONFIG_BLK_SIZE	20 /* 32bit */
     59 
     60 /* Feature bits */
     61 #define VIRTIO_BLK_F_BARRIER	(1<<0)
     62 #define VIRTIO_BLK_F_SIZE_MAX	(1<<1)
     63 #define VIRTIO_BLK_F_SEG_MAX	(1<<2)
     64 #define VIRTIO_BLK_F_GEOMETRY	(1<<4)
     65 #define VIRTIO_BLK_F_RO		(1<<5)
     66 #define VIRTIO_BLK_F_BLK_SIZE	(1<<6)
     67 #define VIRTIO_BLK_F_SCSI	(1<<7)
     68 #define VIRTIO_BLK_F_FLUSH	(1<<9)
     69 
     70 /*
     71  * Each block request uses at least two segments - one for the header
     72  * and one for the status.
     73 */
     74 #define	VIRTIO_BLK_MIN_SEGMENTS	2
     75 
     76 #define VIRTIO_BLK_FLAG_BITS \
     77 	VIRTIO_COMMON_FLAG_BITS \
     78 	"\x0a""FLUSH" \
     79 	"\x08""SCSI" \
     80 	"\x07""BLK_SIZE" \
     81 	"\x06""RO" \
     82 	"\x05""GEOMETRY" \
     83 	"\x03""SEG_MAX" \
     84 	"\x02""SIZE_MAX" \
     85 	"\x01""BARRIER"
     86 
     87 /* Command */
     88 #define VIRTIO_BLK_T_IN		0
     89 #define VIRTIO_BLK_T_OUT	1
     90 #define VIRTIO_BLK_T_BARRIER	0x80000000
     91 
     92 /* Status */
     93 #define VIRTIO_BLK_S_OK		0
     94 #define VIRTIO_BLK_S_IOERR	1
     95 
     96 /* Request header structure */
     97 struct virtio_blk_req_hdr {
     98 	uint32_t	type;	/* VIRTIO_BLK_T_* */
     99 	uint32_t	ioprio;
    100 	uint64_t	sector;
    101 } __packed;
    102 /* 512*virtio_blk_req_hdr.sector byte payload and 1 byte status follows */
    103 
    104 
    105 /*
    106  * ld_virtiovar:
    107  */
    108 struct virtio_blk_req {
    109 	struct virtio_blk_req_hdr	vr_hdr;
    110 	uint8_t				vr_status;
    111 	struct buf			*vr_bp;
    112 	bus_dmamap_t			vr_cmdsts;
    113 	bus_dmamap_t			vr_payload;
    114 };
    115 
    116 struct ld_virtio_softc {
    117 	struct ld_softc		sc_ld;
    118 	device_t		sc_dev;
    119 
    120 	struct virtio_softc	*sc_virtio;
    121 	struct virtqueue	sc_vq;
    122 
    123 	struct virtio_blk_req	*sc_reqs;
    124 	bus_dma_segment_t	sc_reqs_seg;
    125 
    126 	int			sc_readonly;
    127 };
    128 
    129 static int	ld_virtio_match(device_t, cfdata_t, void *);
    130 static void	ld_virtio_attach(device_t, device_t, void *);
    131 static int	ld_virtio_detach(device_t, int);
    132 
    133 CFATTACH_DECL_NEW(ld_virtio, sizeof(struct ld_virtio_softc),
    134     ld_virtio_match, ld_virtio_attach, ld_virtio_detach, NULL);
    135 
    136 static int
    137 ld_virtio_match(device_t parent, cfdata_t match, void *aux)
    138 {
    139 	struct virtio_softc *va = aux;
    140 
    141 	if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_BLOCK)
    142 		return 1;
    143 
    144 	return 0;
    145 }
    146 
    147 static int ld_virtio_vq_done(struct virtqueue *);
    148 static int ld_virtio_dump(struct ld_softc *, void *, int, int);
    149 static int ld_virtio_start(struct ld_softc *, struct buf *);
    150 
    151 static int
    152 ld_virtio_alloc_reqs(struct ld_virtio_softc *sc, int qsize)
    153 {
    154 	int allocsize, r, rsegs, i;
    155 	struct ld_softc *ld = &sc->sc_ld;
    156 	void *vaddr;
    157 
    158 	allocsize = sizeof(struct virtio_blk_req) * qsize;
    159 	r = bus_dmamem_alloc(sc->sc_virtio->sc_dmat, allocsize, 0, 0,
    160 			     &sc->sc_reqs_seg, 1, &rsegs, BUS_DMA_NOWAIT);
    161 	if (r != 0) {
    162 		aprint_error_dev(sc->sc_dev,
    163 				 "DMA memory allocation failed, size %d, "
    164 				 "error code %d\n", allocsize, r);
    165 		goto err_none;
    166 	}
    167 	r = bus_dmamem_map(sc->sc_virtio->sc_dmat,
    168 			   &sc->sc_reqs_seg, 1, allocsize,
    169 			   &vaddr, BUS_DMA_NOWAIT);
    170 	if (r != 0) {
    171 		aprint_error_dev(sc->sc_dev,
    172 				 "DMA memory map failed, "
    173 				 "error code %d\n", r);
    174 		goto err_dmamem_alloc;
    175 	}
    176 	sc->sc_reqs = vaddr;
    177 	memset(vaddr, 0, allocsize);
    178 	for (i = 0; i < qsize; i++) {
    179 		struct virtio_blk_req *vr = &sc->sc_reqs[i];
    180 		r = bus_dmamap_create(sc->sc_virtio->sc_dmat,
    181 				      offsetof(struct virtio_blk_req, vr_bp),
    182 				      1,
    183 				      offsetof(struct virtio_blk_req, vr_bp),
    184 				      0,
    185 				      BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW,
    186 				      &vr->vr_cmdsts);
    187 		if (r != 0) {
    188 			aprint_error_dev(sc->sc_dev,
    189 					 "command dmamap creation failed, "
    190 					 "error code %d\n", r);
    191 			goto err_reqs;
    192 		}
    193 		r = bus_dmamap_load(sc->sc_virtio->sc_dmat, vr->vr_cmdsts,
    194 				    &vr->vr_hdr,
    195 				    offsetof(struct virtio_blk_req, vr_bp),
    196 				    NULL, BUS_DMA_NOWAIT);
    197 		if (r != 0) {
    198 			aprint_error_dev(sc->sc_dev,
    199 					 "command dmamap load failed, "
    200 					 "error code %d\n", r);
    201 			goto err_reqs;
    202 		}
    203 		r = bus_dmamap_create(sc->sc_virtio->sc_dmat,
    204 				      ld->sc_maxxfer,
    205 				      (ld->sc_maxxfer / NBPG) +
    206 				      VIRTIO_BLK_MIN_SEGMENTS,
    207 				      ld->sc_maxxfer,
    208 				      0,
    209 				      BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW,
    210 				      &vr->vr_payload);
    211 		if (r != 0) {
    212 			aprint_error_dev(sc->sc_dev,
    213 					 "payload dmamap creation failed, "
    214 					 "error code %d\n", r);
    215 			goto err_reqs;
    216 		}
    217 	}
    218 	return 0;
    219 
    220 err_reqs:
    221 	for (i = 0; i < qsize; i++) {
    222 		struct virtio_blk_req *vr = &sc->sc_reqs[i];
    223 		if (vr->vr_cmdsts) {
    224 			bus_dmamap_destroy(sc->sc_virtio->sc_dmat,
    225 					   vr->vr_cmdsts);
    226 			vr->vr_cmdsts = 0;
    227 		}
    228 		if (vr->vr_payload) {
    229 			bus_dmamap_destroy(sc->sc_virtio->sc_dmat,
    230 					   vr->vr_payload);
    231 			vr->vr_payload = 0;
    232 		}
    233 	}
    234 	bus_dmamem_unmap(sc->sc_virtio->sc_dmat, sc->sc_reqs, allocsize);
    235 err_dmamem_alloc:
    236 	bus_dmamem_free(sc->sc_virtio->sc_dmat, &sc->sc_reqs_seg, 1);
    237 err_none:
    238 	return -1;
    239 }
    240 
    241 static void
    242 ld_virtio_attach(device_t parent, device_t self, void *aux)
    243 {
    244 	struct ld_virtio_softc *sc = device_private(self);
    245 	struct ld_softc *ld = &sc->sc_ld;
    246 	struct virtio_softc *vsc = device_private(parent);
    247 	uint32_t features;
    248 	char buf[256];
    249 	int qsize, maxxfersize, maxnsegs;
    250 
    251 	if (vsc->sc_child != NULL) {
    252 		aprint_normal(": child already attached for %s; "
    253 			      "something wrong...\n", device_xname(parent));
    254 		return;
    255 	}
    256 
    257 	sc->sc_dev = self;
    258 	sc->sc_virtio = vsc;
    259 
    260 	vsc->sc_child = self;
    261 	vsc->sc_ipl = IPL_BIO;
    262 	vsc->sc_vqs = &sc->sc_vq;
    263 	vsc->sc_nvqs = 1;
    264 	vsc->sc_config_change = NULL;
    265 	vsc->sc_intrhand = virtio_vq_intr;
    266 	vsc->sc_flags = 0;
    267 
    268 	features = virtio_negotiate_features(vsc,
    269 					     (VIRTIO_BLK_F_SIZE_MAX |
    270 					      VIRTIO_BLK_F_SEG_MAX |
    271 					      VIRTIO_BLK_F_GEOMETRY |
    272 					      VIRTIO_BLK_F_RO |
    273 					      VIRTIO_BLK_F_BLK_SIZE));
    274 	if (features & VIRTIO_BLK_F_RO)
    275 		sc->sc_readonly = 1;
    276 	else
    277 		sc->sc_readonly = 0;
    278 
    279 	snprintb(buf, sizeof(buf), VIRTIO_BLK_FLAG_BITS, features);
    280 	aprint_normal(": Features: %s\n", buf);
    281 	aprint_naive("\n");
    282 	if (features & VIRTIO_BLK_F_BLK_SIZE) {
    283 		ld->sc_secsize = virtio_read_device_config_4(vsc,
    284 					VIRTIO_BLK_CONFIG_BLK_SIZE);
    285 	} else
    286 		ld->sc_secsize = 512;
    287 
    288 	/* At least genfs_io assumes maxxfer == MAXPHYS. */
    289 	if (features & VIRTIO_BLK_F_SIZE_MAX) {
    290 		maxxfersize = virtio_read_device_config_4(vsc,
    291 		    VIRTIO_BLK_CONFIG_SIZE_MAX);
    292 		if (maxxfersize < MAXPHYS) {
    293 			aprint_error_dev(sc->sc_dev,
    294 			    "Too small SIZE_MAX %dK minimum is %dK\n",
    295 			    maxxfersize / 1024, MAXPHYS / 1024);
    296 			// goto err;
    297 			maxxfersize = MAXPHYS;
    298 		} else if (maxxfersize > MAXPHYS) {
    299 			aprint_normal_dev(sc->sc_dev,
    300 			    "Clip SEG_MAX from %dK to %dK\n",
    301 			    maxxfersize / 1024,
    302 			    MAXPHYS / 1024);
    303 			maxxfersize = MAXPHYS;
    304 		}
    305 	} else
    306 		maxxfersize = MAXPHYS;
    307 
    308 	if (features & VIRTIO_BLK_F_SEG_MAX) {
    309 		maxnsegs = virtio_read_device_config_4(vsc,
    310 		    VIRTIO_BLK_CONFIG_SEG_MAX);
    311 		if (maxnsegs < VIRTIO_BLK_MIN_SEGMENTS) {
    312 			aprint_error_dev(sc->sc_dev,
    313 			    "Too small SEG_MAX %d minimum is %d\n",
    314 			    maxnsegs, VIRTIO_BLK_MIN_SEGMENTS);
    315 			maxnsegs = maxxfersize / NBPG;
    316 			// goto err;
    317 		}
    318 	} else
    319 		maxnsegs = maxxfersize / NBPG;
    320 
    321 	/* 2 for the minimum size */
    322 	maxnsegs += VIRTIO_BLK_MIN_SEGMENTS;
    323 
    324 	if (virtio_alloc_vq(vsc, &sc->sc_vq, 0, maxxfersize, maxnsegs,
    325 	    "I/O request") != 0) {
    326 		goto err;
    327 	}
    328 	qsize = sc->sc_vq.vq_num;
    329 	sc->sc_vq.vq_done = ld_virtio_vq_done;
    330 
    331 	ld->sc_dv = self;
    332 	ld->sc_secperunit = virtio_read_device_config_8(vsc,
    333 				VIRTIO_BLK_CONFIG_CAPACITY);
    334 	ld->sc_maxxfer = maxxfersize;
    335 	if (features & VIRTIO_BLK_F_GEOMETRY) {
    336 		ld->sc_ncylinders = virtio_read_device_config_2(vsc,
    337 					VIRTIO_BLK_CONFIG_GEOMETRY_C);
    338 		ld->sc_nheads     = virtio_read_device_config_1(vsc,
    339 					VIRTIO_BLK_CONFIG_GEOMETRY_H);
    340 		ld->sc_nsectors   = virtio_read_device_config_1(vsc,
    341 					VIRTIO_BLK_CONFIG_GEOMETRY_S);
    342 	}
    343 	ld->sc_maxqueuecnt = qsize;
    344 
    345 	if (ld_virtio_alloc_reqs(sc, qsize) < 0)
    346 		goto err;
    347 
    348 	ld->sc_dump = ld_virtio_dump;
    349 	ld->sc_flush = NULL;
    350 	ld->sc_start = ld_virtio_start;
    351 
    352 	ld->sc_flags = LDF_ENABLED;
    353 	ldattach(ld);
    354 
    355 	return;
    356 
    357 err:
    358 	vsc->sc_child = (void*)1;
    359 	return;
    360 }
    361 
    362 static int
    363 ld_virtio_start(struct ld_softc *ld, struct buf *bp)
    364 {
    365 	/* splbio */
    366 	struct ld_virtio_softc *sc = device_private(ld->sc_dv);
    367 	struct virtio_softc *vsc = sc->sc_virtio;
    368 	struct virtqueue *vq = &sc->sc_vq;
    369 	struct virtio_blk_req *vr;
    370 	int r;
    371 	int isread = (bp->b_flags & B_READ);
    372 	int slot;
    373 
    374 	if (sc->sc_readonly && !isread)
    375 		return EIO;
    376 
    377 	r = virtio_enqueue_prep(vsc, vq, &slot);
    378 	if (r != 0)
    379 		return r;
    380 
    381 	vr = &sc->sc_reqs[slot];
    382 	KASSERT(vr->vr_bp == NULL);
    383 
    384 	r = bus_dmamap_load(vsc->sc_dmat, vr->vr_payload,
    385 			    bp->b_data, bp->b_bcount, NULL,
    386 			    ((isread?BUS_DMA_READ:BUS_DMA_WRITE)
    387 			     |BUS_DMA_NOWAIT));
    388 	if (r != 0) {
    389 		aprint_error_dev(sc->sc_dev,
    390 		    "payload dmamap failed, error code %d\n", r);
    391 		virtio_enqueue_abort(vsc, vq, slot);
    392 		return r;
    393 	}
    394 
    395 	r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs +
    396 	    VIRTIO_BLK_MIN_SEGMENTS);
    397 	if (r != 0) {
    398 		virtio_enqueue_abort(vsc, vq, slot);
    399 		bus_dmamap_unload(vsc->sc_dmat, vr->vr_payload);
    400 		return r;
    401 	}
    402 
    403 	vr->vr_bp = bp;
    404 	vr->vr_hdr.type = isread?VIRTIO_BLK_T_IN:VIRTIO_BLK_T_OUT;
    405 	vr->vr_hdr.ioprio = 0;
    406 	vr->vr_hdr.sector = bp->b_rawblkno * sc->sc_ld.sc_secsize / 512;
    407 
    408 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
    409 			0, sizeof(struct virtio_blk_req_hdr),
    410 			BUS_DMASYNC_PREWRITE);
    411 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_payload,
    412 			0, bp->b_bcount,
    413 			isread?BUS_DMASYNC_PREREAD:BUS_DMASYNC_PREWRITE);
    414 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
    415 			offsetof(struct virtio_blk_req, vr_status),
    416 			sizeof(uint8_t),
    417 			BUS_DMASYNC_PREREAD);
    418 
    419 	virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
    420 			 0, sizeof(struct virtio_blk_req_hdr),
    421 			 true);
    422 	virtio_enqueue(vsc, vq, slot, vr->vr_payload, !isread);
    423 	virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
    424 			 offsetof(struct virtio_blk_req, vr_status),
    425 			 sizeof(uint8_t),
    426 			 false);
    427 	virtio_enqueue_commit(vsc, vq, slot, true);
    428 
    429 	return 0;
    430 }
    431 
    432 static void
    433 ld_virtio_vq_done1(struct ld_virtio_softc *sc, struct virtio_softc *vsc,
    434 		   struct virtqueue *vq, int slot)
    435 {
    436 	struct virtio_blk_req *vr = &sc->sc_reqs[slot];
    437 	struct buf *bp = vr->vr_bp;
    438 
    439 	vr->vr_bp = NULL;
    440 
    441 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
    442 			0, sizeof(struct virtio_blk_req_hdr),
    443 			BUS_DMASYNC_POSTWRITE);
    444 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_payload,
    445 			0, bp->b_bcount,
    446 			(bp->b_flags & B_READ)?BUS_DMASYNC_POSTREAD
    447 					      :BUS_DMASYNC_POSTWRITE);
    448 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
    449 			sizeof(struct virtio_blk_req_hdr), sizeof(uint8_t),
    450 			BUS_DMASYNC_POSTREAD);
    451 
    452 	if (vr->vr_status != VIRTIO_BLK_S_OK) {
    453 		bp->b_error = EIO;
    454 		bp->b_resid = bp->b_bcount;
    455 	} else {
    456 		bp->b_error = 0;
    457 		bp->b_resid = 0;
    458 	}
    459 
    460 	virtio_dequeue_commit(vsc, vq, slot);
    461 
    462 	lddone(&sc->sc_ld, bp);
    463 }
    464 
    465 static int
    466 ld_virtio_vq_done(struct virtqueue *vq)
    467 {
    468 	struct virtio_softc *vsc = vq->vq_owner;
    469 	struct ld_virtio_softc *sc = device_private(vsc->sc_child);
    470 	int r = 0;
    471 	int slot;
    472 
    473 again:
    474 	if (virtio_dequeue(vsc, vq, &slot, NULL))
    475 		return r;
    476 	r = 1;
    477 
    478 	ld_virtio_vq_done1(sc, vsc, vq, slot);
    479 	goto again;
    480 }
    481 
    482 static int
    483 ld_virtio_dump(struct ld_softc *ld, void *data, int blkno, int blkcnt)
    484 {
    485 	struct ld_virtio_softc *sc = device_private(ld->sc_dv);
    486 	struct virtio_softc *vsc = sc->sc_virtio;
    487 	struct virtqueue *vq = &sc->sc_vq;
    488 	struct virtio_blk_req *vr;
    489 	int slot, r;
    490 
    491 	if (sc->sc_readonly)
    492 		return EIO;
    493 
    494 	r = virtio_enqueue_prep(vsc, vq, &slot);
    495 	if (r != 0) {
    496 		if (r == EAGAIN) { /* no free slot; dequeue first */
    497 			delay(100);
    498 			ld_virtio_vq_done(vq);
    499 			r = virtio_enqueue_prep(vsc, vq, &slot);
    500 			if (r != 0)
    501 				return r;
    502 		}
    503 		return r;
    504 	}
    505 	vr = &sc->sc_reqs[slot];
    506 	r = bus_dmamap_load(vsc->sc_dmat, vr->vr_payload,
    507 			    data, blkcnt*ld->sc_secsize, NULL,
    508 			    BUS_DMA_WRITE|BUS_DMA_NOWAIT);
    509 	if (r != 0)
    510 		return r;
    511 
    512 	r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs +
    513 	    VIRTIO_BLK_MIN_SEGMENTS);
    514 	if (r != 0) {
    515 		bus_dmamap_unload(vsc->sc_dmat, vr->vr_payload);
    516 		return r;
    517 	}
    518 
    519 	vr->vr_bp = (void*)0xdeadbeef;
    520 	vr->vr_hdr.type = VIRTIO_BLK_T_OUT;
    521 	vr->vr_hdr.ioprio = 0;
    522 	vr->vr_hdr.sector = (daddr_t) blkno * ld->sc_secsize / 512;
    523 
    524 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
    525 			0, sizeof(struct virtio_blk_req_hdr),
    526 			BUS_DMASYNC_PREWRITE);
    527 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_payload,
    528 			0, blkcnt*ld->sc_secsize,
    529 			BUS_DMASYNC_PREWRITE);
    530 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
    531 			offsetof(struct virtio_blk_req, vr_status),
    532 			sizeof(uint8_t),
    533 			BUS_DMASYNC_PREREAD);
    534 
    535 	virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
    536 			 0, sizeof(struct virtio_blk_req_hdr),
    537 			 true);
    538 	virtio_enqueue(vsc, vq, slot, vr->vr_payload, true);
    539 	virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
    540 			 offsetof(struct virtio_blk_req, vr_status),
    541 			 sizeof(uint8_t),
    542 			 false);
    543 	virtio_enqueue_commit(vsc, vq, slot, true);
    544 
    545 	for ( ; ; ) {
    546 		int dslot;
    547 
    548 		r = virtio_dequeue(vsc, vq, &dslot, NULL);
    549 		if (r != 0)
    550 			continue;
    551 		if (dslot != slot) {
    552 			ld_virtio_vq_done1(sc, vsc, vq, dslot);
    553 			continue;
    554 		} else
    555 			break;
    556 	}
    557 
    558 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
    559 			0, sizeof(struct virtio_blk_req_hdr),
    560 			BUS_DMASYNC_POSTWRITE);
    561 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_payload,
    562 			0, blkcnt*ld->sc_secsize,
    563 			BUS_DMASYNC_POSTWRITE);
    564 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
    565 			offsetof(struct virtio_blk_req, vr_status),
    566 			sizeof(uint8_t),
    567 			BUS_DMASYNC_POSTREAD);
    568 	if (vr->vr_status == VIRTIO_BLK_S_OK)
    569 		r = 0;
    570 	else
    571 		r = EIO;
    572 	virtio_dequeue_commit(vsc, vq, slot);
    573 
    574 	return r;
    575 }
    576 
    577 static int
    578 ld_virtio_detach(device_t self, int flags)
    579 {
    580 	struct ld_virtio_softc *sc = device_private(self);
    581 	struct ld_softc *ld = &sc->sc_ld;
    582 	bus_dma_tag_t dmat = sc->sc_virtio->sc_dmat;
    583 	int r, i, qsize;
    584 
    585 	qsize = sc->sc_vq.vq_num;
    586 	r = ldbegindetach(ld, flags);
    587 	if (r != 0)
    588 		return r;
    589 	virtio_reset(sc->sc_virtio);
    590 	virtio_free_vq(sc->sc_virtio, &sc->sc_vq);
    591 
    592 	for (i = 0; i < qsize; i++) {
    593 		bus_dmamap_destroy(dmat,
    594 				   sc->sc_reqs[i].vr_cmdsts);
    595 		bus_dmamap_destroy(dmat,
    596 				   sc->sc_reqs[i].vr_payload);
    597 	}
    598 	bus_dmamem_unmap(dmat, sc->sc_reqs,
    599 			 sizeof(struct virtio_blk_req) * qsize);
    600 	bus_dmamem_free(dmat, &sc->sc_reqs_seg, 1);
    601 
    602 	ldenddetach(ld);
    603 
    604 	return 0;
    605 }
    606