Home | History | Annotate | Line # | Download | only in pci
ld_virtio.c revision 1.6.4.6
      1  1.6.4.6    skrll /*	$NetBSD: ld_virtio.c,v 1.6.4.6 2017/08/28 17:52:06 skrll Exp $	*/
      2      1.1  hannken 
      3      1.1  hannken /*
      4      1.1  hannken  * Copyright (c) 2010 Minoura Makoto.
      5      1.1  hannken  * All rights reserved.
      6      1.1  hannken  *
      7      1.1  hannken  * Redistribution and use in source and binary forms, with or without
      8      1.1  hannken  * modification, are permitted provided that the following conditions
      9      1.1  hannken  * are met:
     10      1.1  hannken  * 1. Redistributions of source code must retain the above copyright
     11      1.1  hannken  *    notice, this list of conditions and the following disclaimer.
     12      1.1  hannken  * 2. Redistributions in binary form must reproduce the above copyright
     13      1.1  hannken  *    notice, this list of conditions and the following disclaimer in the
     14      1.1  hannken  *    documentation and/or other materials provided with the distribution.
     15      1.1  hannken  *
     16      1.1  hannken  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17      1.1  hannken  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18      1.1  hannken  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19      1.1  hannken  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20      1.1  hannken  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     21      1.1  hannken  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     22      1.1  hannken  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     23      1.1  hannken  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     24      1.1  hannken  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     25      1.1  hannken  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26      1.1  hannken  */
     27      1.1  hannken 
     28      1.1  hannken #include <sys/cdefs.h>
     29  1.6.4.6    skrll __KERNEL_RCSID(0, "$NetBSD: ld_virtio.c,v 1.6.4.6 2017/08/28 17:52:06 skrll Exp $");
     30      1.1  hannken 
     31      1.1  hannken #include <sys/param.h>
     32      1.1  hannken #include <sys/systm.h>
     33      1.1  hannken #include <sys/kernel.h>
     34      1.1  hannken #include <sys/buf.h>
     35  1.6.4.4    skrll #include <sys/bufq.h>
     36      1.1  hannken #include <sys/bus.h>
     37      1.1  hannken #include <sys/device.h>
     38      1.1  hannken #include <sys/disk.h>
     39      1.1  hannken #include <sys/mutex.h>
     40  1.6.4.4    skrll #include <sys/module.h>
     41      1.1  hannken 
     42      1.1  hannken #include <dev/pci/pcidevs.h>
     43      1.1  hannken #include <dev/pci/pcireg.h>
     44      1.1  hannken #include <dev/pci/pcivar.h>
     45      1.1  hannken 
     46      1.1  hannken #include <dev/ldvar.h>
     47      1.1  hannken #include <dev/pci/virtioreg.h>
     48      1.1  hannken #include <dev/pci/virtiovar.h>
     49      1.1  hannken 
     50  1.6.4.4    skrll #include "ioconf.h"
     51  1.6.4.4    skrll 
     52      1.1  hannken /*
     53      1.1  hannken  * ld_virtioreg:
     54      1.1  hannken  */
     55      1.1  hannken /* Configuration registers */
     56      1.1  hannken #define VIRTIO_BLK_CONFIG_CAPACITY	0 /* 64bit */
     57      1.1  hannken #define VIRTIO_BLK_CONFIG_SIZE_MAX	8 /* 32bit */
     58      1.1  hannken #define VIRTIO_BLK_CONFIG_SEG_MAX	12 /* 32bit */
     59      1.1  hannken #define VIRTIO_BLK_CONFIG_GEOMETRY_C	16 /* 16bit */
     60      1.1  hannken #define VIRTIO_BLK_CONFIG_GEOMETRY_H	18 /* 8bit */
     61      1.1  hannken #define VIRTIO_BLK_CONFIG_GEOMETRY_S	19 /* 8bit */
     62      1.1  hannken #define VIRTIO_BLK_CONFIG_BLK_SIZE	20 /* 32bit */
     63      1.1  hannken 
     64      1.1  hannken /* Feature bits */
     65      1.1  hannken #define VIRTIO_BLK_F_BARRIER	(1<<0)
     66      1.1  hannken #define VIRTIO_BLK_F_SIZE_MAX	(1<<1)
     67      1.1  hannken #define VIRTIO_BLK_F_SEG_MAX	(1<<2)
     68      1.1  hannken #define VIRTIO_BLK_F_GEOMETRY	(1<<4)
     69      1.1  hannken #define VIRTIO_BLK_F_RO		(1<<5)
     70      1.1  hannken #define VIRTIO_BLK_F_BLK_SIZE	(1<<6)
     71      1.1  hannken #define VIRTIO_BLK_F_SCSI	(1<<7)
     72      1.1  hannken #define VIRTIO_BLK_F_FLUSH	(1<<9)
     73      1.1  hannken 
     74  1.6.4.4    skrll /*
     75  1.6.4.2    skrll  * Each block request uses at least two segments - one for the header
     76  1.6.4.2    skrll  * and one for the status.
     77  1.6.4.4    skrll */
     78  1.6.4.2    skrll #define	VIRTIO_BLK_MIN_SEGMENTS	2
     79  1.6.4.2    skrll 
     80  1.6.4.2    skrll #define VIRTIO_BLK_FLAG_BITS \
     81  1.6.4.2    skrll 	VIRTIO_COMMON_FLAG_BITS \
     82  1.6.4.2    skrll 	"\x0a""FLUSH" \
     83  1.6.4.2    skrll 	"\x08""SCSI" \
     84  1.6.4.2    skrll 	"\x07""BLK_SIZE" \
     85  1.6.4.2    skrll 	"\x06""RO" \
     86  1.6.4.2    skrll 	"\x05""GEOMETRY" \
     87  1.6.4.2    skrll 	"\x03""SEG_MAX" \
     88  1.6.4.2    skrll 	"\x02""SIZE_MAX" \
     89  1.6.4.2    skrll 	"\x01""BARRIER"
     90  1.6.4.2    skrll 
     91      1.1  hannken /* Command */
     92      1.1  hannken #define VIRTIO_BLK_T_IN		0
     93      1.1  hannken #define VIRTIO_BLK_T_OUT	1
     94      1.1  hannken #define VIRTIO_BLK_T_BARRIER	0x80000000
     95      1.1  hannken 
     96      1.1  hannken /* Status */
     97      1.1  hannken #define VIRTIO_BLK_S_OK		0
     98      1.1  hannken #define VIRTIO_BLK_S_IOERR	1
     99      1.1  hannken 
    100      1.1  hannken /* Request header structure */
    101      1.1  hannken struct virtio_blk_req_hdr {
    102      1.1  hannken 	uint32_t	type;	/* VIRTIO_BLK_T_* */
    103      1.1  hannken 	uint32_t	ioprio;
    104      1.1  hannken 	uint64_t	sector;
    105      1.1  hannken } __packed;
    106      1.1  hannken /* 512*virtio_blk_req_hdr.sector byte payload and 1 byte status follows */
    107      1.1  hannken 
    108      1.1  hannken 
    109      1.1  hannken /*
    110      1.1  hannken  * ld_virtiovar:
    111      1.1  hannken  */
    112      1.1  hannken struct virtio_blk_req {
    113      1.1  hannken 	struct virtio_blk_req_hdr	vr_hdr;
    114      1.1  hannken 	uint8_t				vr_status;
    115      1.1  hannken 	struct buf			*vr_bp;
    116      1.1  hannken 	bus_dmamap_t			vr_cmdsts;
    117      1.1  hannken 	bus_dmamap_t			vr_payload;
    118      1.1  hannken };
    119      1.1  hannken 
    120      1.1  hannken struct ld_virtio_softc {
    121      1.1  hannken 	struct ld_softc		sc_ld;
    122      1.1  hannken 	device_t		sc_dev;
    123      1.1  hannken 
    124      1.1  hannken 	struct virtio_softc	*sc_virtio;
    125  1.6.4.2    skrll 	struct virtqueue	sc_vq;
    126      1.1  hannken 
    127      1.1  hannken 	struct virtio_blk_req	*sc_reqs;
    128  1.6.4.2    skrll 	bus_dma_segment_t	sc_reqs_seg;
    129      1.1  hannken 
    130      1.1  hannken 	int			sc_readonly;
    131      1.1  hannken };
    132      1.1  hannken 
    133      1.1  hannken static int	ld_virtio_match(device_t, cfdata_t, void *);
    134      1.1  hannken static void	ld_virtio_attach(device_t, device_t, void *);
    135      1.1  hannken static int	ld_virtio_detach(device_t, int);
    136      1.1  hannken 
    137      1.1  hannken CFATTACH_DECL_NEW(ld_virtio, sizeof(struct ld_virtio_softc),
    138      1.1  hannken     ld_virtio_match, ld_virtio_attach, ld_virtio_detach, NULL);
    139      1.1  hannken 
    140      1.1  hannken static int
    141      1.1  hannken ld_virtio_match(device_t parent, cfdata_t match, void *aux)
    142      1.1  hannken {
    143  1.6.4.6    skrll 	struct virtio_attach_args *va = aux;
    144      1.1  hannken 
    145      1.1  hannken 	if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_BLOCK)
    146      1.1  hannken 		return 1;
    147      1.1  hannken 
    148      1.1  hannken 	return 0;
    149      1.1  hannken }
    150      1.1  hannken 
    151      1.1  hannken static int ld_virtio_vq_done(struct virtqueue *);
    152      1.1  hannken static int ld_virtio_dump(struct ld_softc *, void *, int, int);
    153      1.1  hannken static int ld_virtio_start(struct ld_softc *, struct buf *);
    154      1.1  hannken 
    155      1.1  hannken static int
    156      1.1  hannken ld_virtio_alloc_reqs(struct ld_virtio_softc *sc, int qsize)
    157      1.1  hannken {
    158      1.1  hannken 	int allocsize, r, rsegs, i;
    159      1.1  hannken 	struct ld_softc *ld = &sc->sc_ld;
    160      1.1  hannken 	void *vaddr;
    161      1.1  hannken 
    162      1.1  hannken 	allocsize = sizeof(struct virtio_blk_req) * qsize;
    163  1.6.4.6    skrll 	r = bus_dmamem_alloc(virtio_dmat(sc->sc_virtio), allocsize, 0, 0,
    164  1.6.4.2    skrll 			     &sc->sc_reqs_seg, 1, &rsegs, BUS_DMA_NOWAIT);
    165      1.1  hannken 	if (r != 0) {
    166      1.1  hannken 		aprint_error_dev(sc->sc_dev,
    167      1.1  hannken 				 "DMA memory allocation failed, size %d, "
    168      1.1  hannken 				 "error code %d\n", allocsize, r);
    169      1.1  hannken 		goto err_none;
    170      1.1  hannken 	}
    171  1.6.4.6    skrll 	r = bus_dmamem_map(virtio_dmat(sc->sc_virtio),
    172  1.6.4.2    skrll 			   &sc->sc_reqs_seg, 1, allocsize,
    173      1.1  hannken 			   &vaddr, BUS_DMA_NOWAIT);
    174      1.1  hannken 	if (r != 0) {
    175      1.1  hannken 		aprint_error_dev(sc->sc_dev,
    176      1.1  hannken 				 "DMA memory map failed, "
    177      1.1  hannken 				 "error code %d\n", r);
    178      1.1  hannken 		goto err_dmamem_alloc;
    179      1.1  hannken 	}
    180      1.1  hannken 	sc->sc_reqs = vaddr;
    181      1.1  hannken 	memset(vaddr, 0, allocsize);
    182      1.1  hannken 	for (i = 0; i < qsize; i++) {
    183      1.1  hannken 		struct virtio_blk_req *vr = &sc->sc_reqs[i];
    184  1.6.4.6    skrll 		r = bus_dmamap_create(virtio_dmat(sc->sc_virtio),
    185      1.1  hannken 				      offsetof(struct virtio_blk_req, vr_bp),
    186      1.1  hannken 				      1,
    187      1.1  hannken 				      offsetof(struct virtio_blk_req, vr_bp),
    188      1.1  hannken 				      0,
    189      1.1  hannken 				      BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW,
    190      1.1  hannken 				      &vr->vr_cmdsts);
    191      1.1  hannken 		if (r != 0) {
    192      1.1  hannken 			aprint_error_dev(sc->sc_dev,
    193      1.1  hannken 					 "command dmamap creation failed, "
    194      1.1  hannken 					 "error code %d\n", r);
    195      1.1  hannken 			goto err_reqs;
    196      1.1  hannken 		}
    197  1.6.4.6    skrll 		r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), vr->vr_cmdsts,
    198      1.1  hannken 				    &vr->vr_hdr,
    199      1.1  hannken 				    offsetof(struct virtio_blk_req, vr_bp),
    200      1.1  hannken 				    NULL, BUS_DMA_NOWAIT);
    201      1.1  hannken 		if (r != 0) {
    202      1.1  hannken 			aprint_error_dev(sc->sc_dev,
    203      1.1  hannken 					 "command dmamap load failed, "
    204      1.1  hannken 					 "error code %d\n", r);
    205      1.1  hannken 			goto err_reqs;
    206      1.1  hannken 		}
    207  1.6.4.6    skrll 		r = bus_dmamap_create(virtio_dmat(sc->sc_virtio),
    208      1.1  hannken 				      ld->sc_maxxfer,
    209  1.6.4.2    skrll 				      (ld->sc_maxxfer / NBPG) +
    210  1.6.4.2    skrll 				      VIRTIO_BLK_MIN_SEGMENTS,
    211      1.2  hannken 				      ld->sc_maxxfer,
    212      1.1  hannken 				      0,
    213      1.1  hannken 				      BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW,
    214      1.1  hannken 				      &vr->vr_payload);
    215      1.1  hannken 		if (r != 0) {
    216      1.1  hannken 			aprint_error_dev(sc->sc_dev,
    217      1.1  hannken 					 "payload dmamap creation failed, "
    218      1.1  hannken 					 "error code %d\n", r);
    219      1.1  hannken 			goto err_reqs;
    220      1.1  hannken 		}
    221      1.1  hannken 	}
    222      1.1  hannken 	return 0;
    223      1.1  hannken 
    224      1.1  hannken err_reqs:
    225      1.1  hannken 	for (i = 0; i < qsize; i++) {
    226      1.1  hannken 		struct virtio_blk_req *vr = &sc->sc_reqs[i];
    227      1.1  hannken 		if (vr->vr_cmdsts) {
    228  1.6.4.6    skrll 			bus_dmamap_destroy(virtio_dmat(sc->sc_virtio),
    229      1.1  hannken 					   vr->vr_cmdsts);
    230      1.1  hannken 			vr->vr_cmdsts = 0;
    231      1.1  hannken 		}
    232      1.1  hannken 		if (vr->vr_payload) {
    233  1.6.4.6    skrll 			bus_dmamap_destroy(virtio_dmat(sc->sc_virtio),
    234      1.1  hannken 					   vr->vr_payload);
    235      1.1  hannken 			vr->vr_payload = 0;
    236      1.1  hannken 		}
    237      1.1  hannken 	}
    238  1.6.4.6    skrll 	bus_dmamem_unmap(virtio_dmat(sc->sc_virtio), sc->sc_reqs, allocsize);
    239      1.1  hannken err_dmamem_alloc:
    240  1.6.4.6    skrll 	bus_dmamem_free(virtio_dmat(sc->sc_virtio), &sc->sc_reqs_seg, 1);
    241      1.1  hannken err_none:
    242      1.1  hannken 	return -1;
    243      1.1  hannken }
    244      1.1  hannken 
    245      1.1  hannken static void
    246      1.1  hannken ld_virtio_attach(device_t parent, device_t self, void *aux)
    247      1.1  hannken {
    248      1.1  hannken 	struct ld_virtio_softc *sc = device_private(self);
    249      1.1  hannken 	struct ld_softc *ld = &sc->sc_ld;
    250      1.1  hannken 	struct virtio_softc *vsc = device_private(parent);
    251      1.1  hannken 	uint32_t features;
    252  1.6.4.2    skrll 	int qsize, maxxfersize, maxnsegs;
    253      1.1  hannken 
    254  1.6.4.6    skrll 	if (virtio_child(vsc) != NULL) {
    255      1.1  hannken 		aprint_normal(": child already attached for %s; "
    256  1.6.4.3    skrll 			      "something wrong...\n", device_xname(parent));
    257      1.1  hannken 		return;
    258      1.1  hannken 	}
    259      1.1  hannken 
    260      1.1  hannken 	sc->sc_dev = self;
    261      1.1  hannken 	sc->sc_virtio = vsc;
    262      1.1  hannken 
    263  1.6.4.6    skrll 	virtio_child_attach_start(vsc, self, IPL_BIO, &sc->sc_vq,
    264  1.6.4.6    skrll 	    NULL, virtio_vq_intr, 0,
    265  1.6.4.6    skrll 	    (VIRTIO_BLK_F_SIZE_MAX | VIRTIO_BLK_F_SEG_MAX |
    266  1.6.4.6    skrll 	     VIRTIO_BLK_F_GEOMETRY | VIRTIO_BLK_F_RO | VIRTIO_BLK_F_BLK_SIZE),
    267  1.6.4.6    skrll 	    VIRTIO_BLK_FLAG_BITS);
    268  1.6.4.6    skrll 
    269  1.6.4.6    skrll 	features = virtio_features(vsc);
    270  1.6.4.6    skrll 
    271      1.1  hannken 	if (features & VIRTIO_BLK_F_RO)
    272      1.1  hannken 		sc->sc_readonly = 1;
    273      1.1  hannken 	else
    274      1.1  hannken 		sc->sc_readonly = 0;
    275      1.1  hannken 
    276      1.3  hannken 	if (features & VIRTIO_BLK_F_BLK_SIZE) {
    277      1.3  hannken 		ld->sc_secsize = virtio_read_device_config_4(vsc,
    278      1.3  hannken 					VIRTIO_BLK_CONFIG_BLK_SIZE);
    279  1.6.4.2    skrll 	} else
    280  1.6.4.2    skrll 		ld->sc_secsize = 512;
    281  1.6.4.2    skrll 
    282  1.6.4.2    skrll 	/* At least genfs_io assumes maxxfer == MAXPHYS. */
    283  1.6.4.2    skrll 	if (features & VIRTIO_BLK_F_SIZE_MAX) {
    284      1.1  hannken 		maxxfersize = virtio_read_device_config_4(vsc,
    285  1.6.4.2    skrll 		    VIRTIO_BLK_CONFIG_SIZE_MAX);
    286  1.6.4.2    skrll 		if (maxxfersize < MAXPHYS) {
    287  1.6.4.2    skrll 			aprint_error_dev(sc->sc_dev,
    288  1.6.4.2    skrll 			    "Too small SIZE_MAX %dK minimum is %dK\n",
    289  1.6.4.2    skrll 			    maxxfersize / 1024, MAXPHYS / 1024);
    290  1.6.4.2    skrll 			// goto err;
    291      1.1  hannken 			maxxfersize = MAXPHYS;
    292  1.6.4.2    skrll 		} else if (maxxfersize > MAXPHYS) {
    293  1.6.4.2    skrll 			aprint_normal_dev(sc->sc_dev,
    294  1.6.4.2    skrll 			    "Clip SEG_MAX from %dK to %dK\n",
    295  1.6.4.2    skrll 			    maxxfersize / 1024,
    296  1.6.4.2    skrll 			    MAXPHYS / 1024);
    297  1.6.4.2    skrll 			maxxfersize = MAXPHYS;
    298  1.6.4.2    skrll 		}
    299  1.6.4.2    skrll 	} else
    300  1.6.4.2    skrll 		maxxfersize = MAXPHYS;
    301  1.6.4.2    skrll 
    302  1.6.4.2    skrll 	if (features & VIRTIO_BLK_F_SEG_MAX) {
    303  1.6.4.2    skrll 		maxnsegs = virtio_read_device_config_4(vsc,
    304  1.6.4.2    skrll 		    VIRTIO_BLK_CONFIG_SEG_MAX);
    305  1.6.4.2    skrll 		if (maxnsegs < VIRTIO_BLK_MIN_SEGMENTS) {
    306  1.6.4.2    skrll 			aprint_error_dev(sc->sc_dev,
    307  1.6.4.2    skrll 			    "Too small SEG_MAX %d minimum is %d\n",
    308  1.6.4.2    skrll 			    maxnsegs, VIRTIO_BLK_MIN_SEGMENTS);
    309  1.6.4.2    skrll 			maxnsegs = maxxfersize / NBPG;
    310  1.6.4.2    skrll 			// goto err;
    311  1.6.4.2    skrll 		}
    312  1.6.4.2    skrll 	} else
    313  1.6.4.2    skrll 		maxnsegs = maxxfersize / NBPG;
    314      1.1  hannken 
    315  1.6.4.2    skrll 	/* 2 for the minimum size */
    316  1.6.4.2    skrll 	maxnsegs += VIRTIO_BLK_MIN_SEGMENTS;
    317  1.6.4.2    skrll 
    318  1.6.4.2    skrll 	if (virtio_alloc_vq(vsc, &sc->sc_vq, 0, maxxfersize, maxnsegs,
    319  1.6.4.2    skrll 	    "I/O request") != 0) {
    320      1.1  hannken 		goto err;
    321      1.1  hannken 	}
    322  1.6.4.2    skrll 	qsize = sc->sc_vq.vq_num;
    323  1.6.4.2    skrll 	sc->sc_vq.vq_done = ld_virtio_vq_done;
    324      1.1  hannken 
    325  1.6.4.6    skrll 	if (virtio_child_attach_finish(vsc) != 0)
    326  1.6.4.6    skrll 		goto err;
    327  1.6.4.6    skrll 
    328      1.1  hannken 	ld->sc_dv = self;
    329      1.1  hannken 	ld->sc_secperunit = virtio_read_device_config_8(vsc,
    330      1.1  hannken 				VIRTIO_BLK_CONFIG_CAPACITY);
    331      1.1  hannken 	ld->sc_maxxfer = maxxfersize;
    332      1.1  hannken 	if (features & VIRTIO_BLK_F_GEOMETRY) {
    333      1.1  hannken 		ld->sc_ncylinders = virtio_read_device_config_2(vsc,
    334      1.1  hannken 					VIRTIO_BLK_CONFIG_GEOMETRY_C);
    335      1.1  hannken 		ld->sc_nheads     = virtio_read_device_config_1(vsc,
    336      1.1  hannken 					VIRTIO_BLK_CONFIG_GEOMETRY_H);
    337      1.1  hannken 		ld->sc_nsectors   = virtio_read_device_config_1(vsc,
    338      1.1  hannken 					VIRTIO_BLK_CONFIG_GEOMETRY_S);
    339      1.1  hannken 	}
    340      1.1  hannken 	ld->sc_maxqueuecnt = qsize;
    341      1.1  hannken 
    342      1.1  hannken 	if (ld_virtio_alloc_reqs(sc, qsize) < 0)
    343      1.1  hannken 		goto err;
    344      1.1  hannken 
    345      1.1  hannken 	ld->sc_dump = ld_virtio_dump;
    346      1.1  hannken 	ld->sc_start = ld_virtio_start;
    347      1.1  hannken 
    348  1.6.4.6    skrll 	ld->sc_flags = LDF_ENABLED | LDF_MPSAFE;
    349  1.6.4.4    skrll 	ldattach(ld, BUFQ_DISK_DEFAULT_STRAT);
    350      1.1  hannken 
    351      1.1  hannken 	return;
    352      1.1  hannken 
    353      1.1  hannken err:
    354  1.6.4.6    skrll 	virtio_child_attach_failed(vsc);
    355      1.1  hannken 	return;
    356      1.1  hannken }
    357      1.1  hannken 
    358      1.1  hannken static int
    359      1.1  hannken ld_virtio_start(struct ld_softc *ld, struct buf *bp)
    360      1.1  hannken {
    361      1.1  hannken 	/* splbio */
    362      1.1  hannken 	struct ld_virtio_softc *sc = device_private(ld->sc_dv);
    363      1.1  hannken 	struct virtio_softc *vsc = sc->sc_virtio;
    364  1.6.4.2    skrll 	struct virtqueue *vq = &sc->sc_vq;
    365      1.1  hannken 	struct virtio_blk_req *vr;
    366      1.1  hannken 	int r;
    367      1.1  hannken 	int isread = (bp->b_flags & B_READ);
    368      1.1  hannken 	int slot;
    369      1.1  hannken 
    370      1.1  hannken 	if (sc->sc_readonly && !isread)
    371      1.1  hannken 		return EIO;
    372      1.1  hannken 
    373      1.1  hannken 	r = virtio_enqueue_prep(vsc, vq, &slot);
    374      1.1  hannken 	if (r != 0)
    375      1.1  hannken 		return r;
    376  1.6.4.2    skrll 
    377      1.1  hannken 	vr = &sc->sc_reqs[slot];
    378  1.6.4.2    skrll 	KASSERT(vr->vr_bp == NULL);
    379  1.6.4.2    skrll 
    380  1.6.4.6    skrll 	r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload,
    381      1.1  hannken 			    bp->b_data, bp->b_bcount, NULL,
    382      1.1  hannken 			    ((isread?BUS_DMA_READ:BUS_DMA_WRITE)
    383      1.1  hannken 			     |BUS_DMA_NOWAIT));
    384  1.6.4.2    skrll 	if (r != 0) {
    385  1.6.4.2    skrll 		aprint_error_dev(sc->sc_dev,
    386  1.6.4.2    skrll 		    "payload dmamap failed, error code %d\n", r);
    387  1.6.4.2    skrll 		virtio_enqueue_abort(vsc, vq, slot);
    388      1.1  hannken 		return r;
    389  1.6.4.2    skrll 	}
    390      1.1  hannken 
    391  1.6.4.2    skrll 	r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs +
    392  1.6.4.2    skrll 	    VIRTIO_BLK_MIN_SEGMENTS);
    393      1.1  hannken 	if (r != 0) {
    394  1.6.4.6    skrll 		bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload);
    395      1.1  hannken 		return r;
    396      1.1  hannken 	}
    397      1.1  hannken 
    398      1.1  hannken 	vr->vr_bp = bp;
    399      1.1  hannken 	vr->vr_hdr.type = isread?VIRTIO_BLK_T_IN:VIRTIO_BLK_T_OUT;
    400      1.1  hannken 	vr->vr_hdr.ioprio = 0;
    401      1.1  hannken 	vr->vr_hdr.sector = bp->b_rawblkno * sc->sc_ld.sc_secsize / 512;
    402      1.1  hannken 
    403  1.6.4.6    skrll 	bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
    404      1.1  hannken 			0, sizeof(struct virtio_blk_req_hdr),
    405      1.1  hannken 			BUS_DMASYNC_PREWRITE);
    406  1.6.4.6    skrll 	bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload,
    407      1.1  hannken 			0, bp->b_bcount,
    408      1.1  hannken 			isread?BUS_DMASYNC_PREREAD:BUS_DMASYNC_PREWRITE);
    409  1.6.4.6    skrll 	bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
    410      1.1  hannken 			offsetof(struct virtio_blk_req, vr_status),
    411      1.1  hannken 			sizeof(uint8_t),
    412      1.1  hannken 			BUS_DMASYNC_PREREAD);
    413      1.1  hannken 
    414      1.1  hannken 	virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
    415      1.1  hannken 			 0, sizeof(struct virtio_blk_req_hdr),
    416      1.1  hannken 			 true);
    417      1.1  hannken 	virtio_enqueue(vsc, vq, slot, vr->vr_payload, !isread);
    418      1.1  hannken 	virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
    419      1.1  hannken 			 offsetof(struct virtio_blk_req, vr_status),
    420      1.1  hannken 			 sizeof(uint8_t),
    421      1.1  hannken 			 false);
    422      1.1  hannken 	virtio_enqueue_commit(vsc, vq, slot, true);
    423      1.1  hannken 
    424      1.1  hannken 	return 0;
    425      1.1  hannken }
    426      1.1  hannken 
    427      1.1  hannken static void
    428      1.1  hannken ld_virtio_vq_done1(struct ld_virtio_softc *sc, struct virtio_softc *vsc,
    429      1.1  hannken 		   struct virtqueue *vq, int slot)
    430      1.1  hannken {
    431      1.1  hannken 	struct virtio_blk_req *vr = &sc->sc_reqs[slot];
    432      1.1  hannken 	struct buf *bp = vr->vr_bp;
    433      1.1  hannken 
    434  1.6.4.2    skrll 	vr->vr_bp = NULL;
    435  1.6.4.2    skrll 
    436  1.6.4.6    skrll 	bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
    437      1.1  hannken 			0, sizeof(struct virtio_blk_req_hdr),
    438      1.1  hannken 			BUS_DMASYNC_POSTWRITE);
    439  1.6.4.6    skrll 	bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload,
    440      1.1  hannken 			0, bp->b_bcount,
    441      1.1  hannken 			(bp->b_flags & B_READ)?BUS_DMASYNC_POSTREAD
    442      1.1  hannken 					      :BUS_DMASYNC_POSTWRITE);
    443  1.6.4.6    skrll 	bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
    444      1.1  hannken 			sizeof(struct virtio_blk_req_hdr), sizeof(uint8_t),
    445      1.1  hannken 			BUS_DMASYNC_POSTREAD);
    446      1.1  hannken 
    447      1.1  hannken 	if (vr->vr_status != VIRTIO_BLK_S_OK) {
    448      1.1  hannken 		bp->b_error = EIO;
    449      1.1  hannken 		bp->b_resid = bp->b_bcount;
    450      1.1  hannken 	} else {
    451      1.1  hannken 		bp->b_error = 0;
    452      1.1  hannken 		bp->b_resid = 0;
    453      1.1  hannken 	}
    454      1.1  hannken 
    455      1.1  hannken 	virtio_dequeue_commit(vsc, vq, slot);
    456      1.1  hannken 
    457      1.1  hannken 	lddone(&sc->sc_ld, bp);
    458      1.1  hannken }
    459      1.1  hannken 
    460      1.1  hannken static int
    461      1.1  hannken ld_virtio_vq_done(struct virtqueue *vq)
    462      1.1  hannken {
    463      1.1  hannken 	struct virtio_softc *vsc = vq->vq_owner;
    464  1.6.4.6    skrll 	struct ld_virtio_softc *sc = device_private(virtio_child(vsc));
    465      1.1  hannken 	int r = 0;
    466      1.1  hannken 	int slot;
    467      1.1  hannken 
    468      1.1  hannken again:
    469      1.1  hannken 	if (virtio_dequeue(vsc, vq, &slot, NULL))
    470      1.1  hannken 		return r;
    471      1.1  hannken 	r = 1;
    472      1.1  hannken 
    473      1.1  hannken 	ld_virtio_vq_done1(sc, vsc, vq, slot);
    474      1.1  hannken 	goto again;
    475      1.1  hannken }
    476      1.1  hannken 
    477      1.1  hannken static int
    478      1.1  hannken ld_virtio_dump(struct ld_softc *ld, void *data, int blkno, int blkcnt)
    479      1.1  hannken {
    480      1.1  hannken 	struct ld_virtio_softc *sc = device_private(ld->sc_dv);
    481      1.1  hannken 	struct virtio_softc *vsc = sc->sc_virtio;
    482  1.6.4.2    skrll 	struct virtqueue *vq = &sc->sc_vq;
    483      1.1  hannken 	struct virtio_blk_req *vr;
    484      1.1  hannken 	int slot, r;
    485      1.1  hannken 
    486      1.1  hannken 	if (sc->sc_readonly)
    487      1.1  hannken 		return EIO;
    488      1.1  hannken 
    489      1.1  hannken 	r = virtio_enqueue_prep(vsc, vq, &slot);
    490      1.1  hannken 	if (r != 0) {
    491      1.1  hannken 		if (r == EAGAIN) { /* no free slot; dequeue first */
    492      1.1  hannken 			delay(100);
    493      1.1  hannken 			ld_virtio_vq_done(vq);
    494      1.1  hannken 			r = virtio_enqueue_prep(vsc, vq, &slot);
    495      1.1  hannken 			if (r != 0)
    496      1.1  hannken 				return r;
    497      1.1  hannken 		}
    498      1.1  hannken 		return r;
    499      1.1  hannken 	}
    500      1.1  hannken 	vr = &sc->sc_reqs[slot];
    501  1.6.4.6    skrll 	r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload,
    502      1.1  hannken 			    data, blkcnt*ld->sc_secsize, NULL,
    503      1.1  hannken 			    BUS_DMA_WRITE|BUS_DMA_NOWAIT);
    504      1.1  hannken 	if (r != 0)
    505      1.1  hannken 		return r;
    506      1.1  hannken 
    507  1.6.4.4    skrll 	r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs +
    508  1.6.4.2    skrll 	    VIRTIO_BLK_MIN_SEGMENTS);
    509      1.1  hannken 	if (r != 0) {
    510  1.6.4.6    skrll 		bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload);
    511      1.1  hannken 		return r;
    512      1.1  hannken 	}
    513      1.1  hannken 
    514      1.1  hannken 	vr->vr_bp = (void*)0xdeadbeef;
    515      1.1  hannken 	vr->vr_hdr.type = VIRTIO_BLK_T_OUT;
    516      1.1  hannken 	vr->vr_hdr.ioprio = 0;
    517      1.1  hannken 	vr->vr_hdr.sector = (daddr_t) blkno * ld->sc_secsize / 512;
    518      1.1  hannken 
    519  1.6.4.6    skrll 	bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
    520      1.1  hannken 			0, sizeof(struct virtio_blk_req_hdr),
    521      1.1  hannken 			BUS_DMASYNC_PREWRITE);
    522  1.6.4.6    skrll 	bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload,
    523      1.1  hannken 			0, blkcnt*ld->sc_secsize,
    524      1.1  hannken 			BUS_DMASYNC_PREWRITE);
    525  1.6.4.6    skrll 	bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
    526      1.1  hannken 			offsetof(struct virtio_blk_req, vr_status),
    527      1.1  hannken 			sizeof(uint8_t),
    528      1.1  hannken 			BUS_DMASYNC_PREREAD);
    529      1.1  hannken 
    530      1.1  hannken 	virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
    531      1.1  hannken 			 0, sizeof(struct virtio_blk_req_hdr),
    532      1.1  hannken 			 true);
    533      1.1  hannken 	virtio_enqueue(vsc, vq, slot, vr->vr_payload, true);
    534      1.1  hannken 	virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
    535      1.1  hannken 			 offsetof(struct virtio_blk_req, vr_status),
    536      1.1  hannken 			 sizeof(uint8_t),
    537      1.1  hannken 			 false);
    538      1.1  hannken 	virtio_enqueue_commit(vsc, vq, slot, true);
    539      1.1  hannken 
    540      1.1  hannken 	for ( ; ; ) {
    541      1.1  hannken 		int dslot;
    542      1.1  hannken 
    543      1.1  hannken 		r = virtio_dequeue(vsc, vq, &dslot, NULL);
    544      1.1  hannken 		if (r != 0)
    545      1.1  hannken 			continue;
    546      1.1  hannken 		if (dslot != slot) {
    547      1.1  hannken 			ld_virtio_vq_done1(sc, vsc, vq, dslot);
    548      1.1  hannken 			continue;
    549      1.1  hannken 		} else
    550      1.1  hannken 			break;
    551      1.1  hannken 	}
    552  1.6.4.4    skrll 
    553  1.6.4.6    skrll 	bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
    554      1.1  hannken 			0, sizeof(struct virtio_blk_req_hdr),
    555      1.1  hannken 			BUS_DMASYNC_POSTWRITE);
    556  1.6.4.6    skrll 	bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload,
    557      1.1  hannken 			0, blkcnt*ld->sc_secsize,
    558      1.1  hannken 			BUS_DMASYNC_POSTWRITE);
    559  1.6.4.6    skrll 	bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
    560      1.1  hannken 			offsetof(struct virtio_blk_req, vr_status),
    561      1.1  hannken 			sizeof(uint8_t),
    562      1.1  hannken 			BUS_DMASYNC_POSTREAD);
    563      1.1  hannken 	if (vr->vr_status == VIRTIO_BLK_S_OK)
    564      1.1  hannken 		r = 0;
    565      1.1  hannken 	else
    566      1.1  hannken 		r = EIO;
    567      1.1  hannken 	virtio_dequeue_commit(vsc, vq, slot);
    568      1.1  hannken 
    569      1.1  hannken 	return r;
    570      1.1  hannken }
    571      1.1  hannken 
    572      1.1  hannken static int
    573      1.1  hannken ld_virtio_detach(device_t self, int flags)
    574      1.1  hannken {
    575      1.1  hannken 	struct ld_virtio_softc *sc = device_private(self);
    576      1.1  hannken 	struct ld_softc *ld = &sc->sc_ld;
    577  1.6.4.6    skrll 	bus_dma_tag_t dmat = virtio_dmat(sc->sc_virtio);
    578      1.1  hannken 	int r, i, qsize;
    579      1.1  hannken 
    580  1.6.4.2    skrll 	qsize = sc->sc_vq.vq_num;
    581      1.1  hannken 	r = ldbegindetach(ld, flags);
    582      1.1  hannken 	if (r != 0)
    583      1.1  hannken 		return r;
    584      1.1  hannken 	virtio_reset(sc->sc_virtio);
    585  1.6.4.2    skrll 	virtio_free_vq(sc->sc_virtio, &sc->sc_vq);
    586      1.1  hannken 
    587      1.1  hannken 	for (i = 0; i < qsize; i++) {
    588      1.1  hannken 		bus_dmamap_destroy(dmat,
    589      1.1  hannken 				   sc->sc_reqs[i].vr_cmdsts);
    590      1.1  hannken 		bus_dmamap_destroy(dmat,
    591      1.1  hannken 				   sc->sc_reqs[i].vr_payload);
    592      1.1  hannken 	}
    593      1.1  hannken 	bus_dmamem_unmap(dmat, sc->sc_reqs,
    594      1.1  hannken 			 sizeof(struct virtio_blk_req) * qsize);
    595  1.6.4.2    skrll 	bus_dmamem_free(dmat, &sc->sc_reqs_seg, 1);
    596      1.1  hannken 
    597      1.1  hannken 	ldenddetach(ld);
    598      1.1  hannken 
    599  1.6.4.6    skrll 	virtio_child_detach(sc->sc_virtio);
    600  1.6.4.6    skrll 
    601      1.1  hannken 	return 0;
    602      1.1  hannken }
    603  1.6.4.4    skrll 
    604  1.6.4.4    skrll MODULE(MODULE_CLASS_DRIVER, ld_virtio, "ld,virtio");
    605  1.6.4.4    skrll 
    606  1.6.4.4    skrll #ifdef _MODULE
    607  1.6.4.4    skrll /*
    608  1.6.4.4    skrll  * XXX Don't allow ioconf.c to redefine the "struct cfdriver ld_cd"
    609  1.6.4.4    skrll  * XXX it will be defined in the common-code module
    610  1.6.4.4    skrll  */
    611  1.6.4.4    skrll #undef  CFDRIVER_DECL
    612  1.6.4.4    skrll #define CFDRIVER_DECL(name, class, attr)
    613  1.6.4.4    skrll #include "ioconf.c"
    614  1.6.4.4    skrll #endif
    615  1.6.4.4    skrll 
    616  1.6.4.4    skrll static int
    617  1.6.4.4    skrll ld_virtio_modcmd(modcmd_t cmd, void *opaque)
    618  1.6.4.4    skrll {
    619  1.6.4.4    skrll #ifdef _MODULE
    620  1.6.4.4    skrll 	/*
    621  1.6.4.4    skrll 	 * We ignore the cfdriver_vec[] that ioconf provides, since
    622  1.6.4.4    skrll 	 * the cfdrivers are attached already.
    623  1.6.4.4    skrll 	 */
    624  1.6.4.4    skrll 	static struct cfdriver * const no_cfdriver_vec[] = { NULL };
    625  1.6.4.4    skrll #endif
    626  1.6.4.4    skrll 	int error = 0;
    627  1.6.4.4    skrll 
    628  1.6.4.4    skrll #ifdef _MODULE
    629  1.6.4.4    skrll 	switch (cmd) {
    630  1.6.4.4    skrll 	case MODULE_CMD_INIT:
    631  1.6.4.4    skrll 		error = config_init_component(no_cfdriver_vec,
    632  1.6.4.4    skrll 		    cfattach_ioconf_ld_virtio, cfdata_ioconf_ld_virtio);
    633  1.6.4.4    skrll 		break;
    634  1.6.4.4    skrll 	case MODULE_CMD_FINI:
    635  1.6.4.4    skrll 		error = config_fini_component(no_cfdriver_vec,
    636  1.6.4.4    skrll 		    cfattach_ioconf_ld_virtio, cfdata_ioconf_ld_virtio);
    637  1.6.4.4    skrll 		break;
    638  1.6.4.4    skrll 	default:
    639  1.6.4.4    skrll 		error = ENOTTY;
    640  1.6.4.4    skrll 		break;
    641  1.6.4.4    skrll 	}
    642  1.6.4.4    skrll #endif
    643  1.6.4.4    skrll 
    644  1.6.4.4    skrll 	return error;
    645  1.6.4.4    skrll }
    646