Home | History | Annotate | Line # | Download | only in ic
nvme.c revision 1.30.2.1
      1  1.30.2.1    martin /*	$NetBSD: nvme.c,v 1.30.2.1 2018/03/17 08:11:18 martin Exp $	*/
      2       1.1    nonaka /*	$OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */
      3       1.1    nonaka 
      4       1.1    nonaka /*
      5       1.1    nonaka  * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org>
      6       1.1    nonaka  *
      7       1.1    nonaka  * Permission to use, copy, modify, and distribute this software for any
      8       1.1    nonaka  * purpose with or without fee is hereby granted, provided that the above
      9       1.1    nonaka  * copyright notice and this permission notice appear in all copies.
     10       1.1    nonaka  *
     11       1.1    nonaka  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
     12       1.1    nonaka  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     13       1.1    nonaka  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     14       1.1    nonaka  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     15       1.1    nonaka  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     16       1.1    nonaka  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     17       1.1    nonaka  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     18       1.1    nonaka  */
     19       1.1    nonaka 
     20       1.1    nonaka #include <sys/cdefs.h>
     21  1.30.2.1    martin __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.30.2.1 2018/03/17 08:11:18 martin Exp $");
     22       1.1    nonaka 
     23       1.1    nonaka #include <sys/param.h>
     24       1.1    nonaka #include <sys/systm.h>
     25       1.1    nonaka #include <sys/kernel.h>
     26       1.1    nonaka #include <sys/atomic.h>
     27       1.1    nonaka #include <sys/bus.h>
     28       1.1    nonaka #include <sys/buf.h>
     29       1.3    nonaka #include <sys/conf.h>
     30       1.1    nonaka #include <sys/device.h>
     31       1.1    nonaka #include <sys/kmem.h>
     32       1.1    nonaka #include <sys/once.h>
     33       1.3    nonaka #include <sys/proc.h>
     34       1.1    nonaka #include <sys/queue.h>
     35       1.1    nonaka #include <sys/mutex.h>
     36       1.1    nonaka 
     37       1.3    nonaka #include <uvm/uvm_extern.h>
     38       1.3    nonaka 
     39       1.1    nonaka #include <dev/ic/nvmereg.h>
     40       1.1    nonaka #include <dev/ic/nvmevar.h>
     41       1.3    nonaka #include <dev/ic/nvmeio.h>
     42       1.1    nonaka 
     43      1.22  jdolecek int nvme_adminq_size = 32;
     44       1.9  jdolecek int nvme_ioq_size = 1024;
     45       1.1    nonaka 
     46       1.1    nonaka static int	nvme_print(void *, const char *);
     47       1.1    nonaka 
     48       1.1    nonaka static int	nvme_ready(struct nvme_softc *, uint32_t);
     49       1.1    nonaka static int	nvme_enable(struct nvme_softc *, u_int);
     50       1.1    nonaka static int	nvme_disable(struct nvme_softc *);
     51       1.1    nonaka static int	nvme_shutdown(struct nvme_softc *);
     52       1.1    nonaka 
     53       1.1    nonaka #ifdef NVME_DEBUG
     54       1.1    nonaka static void	nvme_dumpregs(struct nvme_softc *);
     55       1.1    nonaka #endif
     56       1.1    nonaka static int	nvme_identify(struct nvme_softc *, u_int);
     57       1.1    nonaka static void	nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *,
     58       1.1    nonaka 		    void *);
     59       1.1    nonaka 
     60      1.20  jdolecek static int	nvme_ccbs_alloc(struct nvme_queue *, uint16_t);
     61       1.1    nonaka static void	nvme_ccbs_free(struct nvme_queue *);
     62       1.1    nonaka 
     63       1.1    nonaka static struct nvme_ccb *
     64       1.1    nonaka 		nvme_ccb_get(struct nvme_queue *);
     65       1.1    nonaka static void	nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *);
     66       1.1    nonaka 
     67       1.1    nonaka static int	nvme_poll(struct nvme_softc *, struct nvme_queue *,
     68       1.1    nonaka 		    struct nvme_ccb *, void (*)(struct nvme_queue *,
     69       1.7  jdolecek 		    struct nvme_ccb *, void *), int);
     70       1.1    nonaka static void	nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *);
     71       1.1    nonaka static void	nvme_poll_done(struct nvme_queue *, struct nvme_ccb *,
     72       1.1    nonaka 		    struct nvme_cqe *);
     73       1.1    nonaka static void	nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *);
     74       1.1    nonaka static void	nvme_empty_done(struct nvme_queue *, struct nvme_ccb *,
     75       1.1    nonaka 		    struct nvme_cqe *);
     76       1.1    nonaka 
     77       1.1    nonaka static struct nvme_queue *
     78       1.1    nonaka 		nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int);
     79       1.1    nonaka static int	nvme_q_create(struct nvme_softc *, struct nvme_queue *);
     80       1.1    nonaka static int	nvme_q_delete(struct nvme_softc *, struct nvme_queue *);
     81       1.1    nonaka static void	nvme_q_submit(struct nvme_softc *, struct nvme_queue *,
     82       1.1    nonaka 		    struct nvme_ccb *, void (*)(struct nvme_queue *,
     83       1.1    nonaka 		    struct nvme_ccb *, void *));
     84       1.1    nonaka static int	nvme_q_complete(struct nvme_softc *, struct nvme_queue *q);
     85       1.1    nonaka static void	nvme_q_free(struct nvme_softc *, struct nvme_queue *);
     86       1.1    nonaka 
     87      1.19  jdolecek static struct nvme_dmamem *
     88      1.19  jdolecek 		nvme_dmamem_alloc(struct nvme_softc *, size_t);
     89       1.1    nonaka static void	nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *);
     90      1.19  jdolecek static void	nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *,
     91      1.19  jdolecek 		    int);
     92       1.1    nonaka 
     93       1.1    nonaka static void	nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *,
     94       1.1    nonaka 		    void *);
     95       1.1    nonaka static void	nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *,
     96       1.1    nonaka 		    struct nvme_cqe *);
     97       1.1    nonaka static void	nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *,
     98       1.1    nonaka 		    void *);
     99       1.1    nonaka static void	nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *,
    100       1.1    nonaka 		    struct nvme_cqe *);
    101      1.25  jdolecek static void	nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *,
    102      1.25  jdolecek 		    void *);
    103      1.25  jdolecek static void	nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *,
    104      1.25  jdolecek 		    struct nvme_cqe *);
    105       1.1    nonaka 
    106       1.3    nonaka static void	nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *,
    107       1.3    nonaka 		    void *);
    108       1.3    nonaka static void	nvme_pt_done(struct nvme_queue *, struct nvme_ccb *,
    109       1.3    nonaka 		    struct nvme_cqe *);
    110       1.3    nonaka static int	nvme_command_passthrough(struct nvme_softc *,
    111       1.3    nonaka 		    struct nvme_pt_command *, uint16_t, struct lwp *, bool);
    112       1.3    nonaka 
    113      1.23    nonaka static int	nvme_get_number_of_queues(struct nvme_softc *, u_int *);
    114      1.23    nonaka 
    115       1.7  jdolecek #define NVME_TIMO_QOP		5	/* queue create and delete timeout */
    116       1.7  jdolecek #define NVME_TIMO_IDENT		10	/* probe identify timeout */
    117       1.7  jdolecek #define NVME_TIMO_PT		-1	/* passthrough cmd timeout */
    118      1.13  jdolecek #define NVME_TIMO_SY		60	/* sync cache timeout */
    119       1.7  jdolecek 
    120       1.1    nonaka #define nvme_read4(_s, _r) \
    121       1.1    nonaka 	bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r))
    122       1.1    nonaka #define nvme_write4(_s, _r, _v) \
    123       1.1    nonaka 	bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v))
    124      1.28    nonaka /*
    125      1.28    nonaka  * Some controllers, at least Apple NVMe, always require split
    126      1.28    nonaka  * transfers, so don't use bus_space_{read,write}_8() on LP64.
    127      1.28    nonaka  */
    128       1.1    nonaka static inline uint64_t
    129       1.1    nonaka nvme_read8(struct nvme_softc *sc, bus_size_t r)
    130       1.1    nonaka {
    131       1.1    nonaka 	uint64_t v;
    132       1.1    nonaka 	uint32_t *a = (uint32_t *)&v;
    133       1.1    nonaka 
    134       1.1    nonaka #if _BYTE_ORDER == _LITTLE_ENDIAN
    135       1.1    nonaka 	a[0] = nvme_read4(sc, r);
    136       1.1    nonaka 	a[1] = nvme_read4(sc, r + 4);
    137       1.1    nonaka #else /* _BYTE_ORDER == _LITTLE_ENDIAN */
    138       1.1    nonaka 	a[1] = nvme_read4(sc, r);
    139       1.1    nonaka 	a[0] = nvme_read4(sc, r + 4);
    140       1.1    nonaka #endif
    141       1.1    nonaka 
    142       1.1    nonaka 	return v;
    143       1.1    nonaka }
    144       1.1    nonaka 
    145       1.1    nonaka static inline void
    146       1.1    nonaka nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v)
    147       1.1    nonaka {
    148       1.1    nonaka 	uint32_t *a = (uint32_t *)&v;
    149       1.1    nonaka 
    150       1.1    nonaka #if _BYTE_ORDER == _LITTLE_ENDIAN
    151       1.1    nonaka 	nvme_write4(sc, r, a[0]);
    152       1.1    nonaka 	nvme_write4(sc, r + 4, a[1]);
    153       1.1    nonaka #else /* _BYTE_ORDER == _LITTLE_ENDIAN */
    154       1.1    nonaka 	nvme_write4(sc, r, a[1]);
    155       1.1    nonaka 	nvme_write4(sc, r + 4, a[0]);
    156       1.1    nonaka #endif
    157       1.1    nonaka }
    158       1.1    nonaka #define nvme_barrier(_s, _r, _l, _f) \
    159       1.1    nonaka 	bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f))
    160       1.1    nonaka 
    161       1.1    nonaka #ifdef NVME_DEBUG
    162       1.6  jdolecek static __used void
    163       1.1    nonaka nvme_dumpregs(struct nvme_softc *sc)
    164       1.1    nonaka {
    165       1.1    nonaka 	uint64_t r8;
    166       1.1    nonaka 	uint32_t r4;
    167       1.1    nonaka 
    168       1.1    nonaka #define	DEVNAME(_sc) device_xname((_sc)->sc_dev)
    169       1.1    nonaka 	r8 = nvme_read8(sc, NVME_CAP);
    170       1.8  jdolecek 	printf("%s: cap  0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP));
    171       1.1    nonaka 	printf("%s:  mpsmax %u (%u)\n", DEVNAME(sc),
    172       1.1    nonaka 	    (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8)));
    173       1.1    nonaka 	printf("%s:  mpsmin %u (%u)\n", DEVNAME(sc),
    174       1.1    nonaka 	    (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8)));
    175       1.8  jdolecek 	printf("%s:  css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8));
    176       1.8  jdolecek 	printf("%s:  nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8));
    177       1.8  jdolecek 	printf("%s:  dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8));
    178       1.8  jdolecek 	printf("%s:  to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8));
    179       1.8  jdolecek 	printf("%s:  ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8));
    180       1.8  jdolecek 	printf("%s:  cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8));
    181       1.8  jdolecek 	printf("%s:  mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8));
    182       1.1    nonaka 
    183       1.1    nonaka 	printf("%s: vs   0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS));
    184       1.1    nonaka 
    185       1.1    nonaka 	r4 = nvme_read4(sc, NVME_CC);
    186       1.1    nonaka 	printf("%s: cc   0x%04x\n", DEVNAME(sc), r4);
    187       1.8  jdolecek 	printf("%s:  iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4),
    188       1.8  jdolecek 	    (1 << NVME_CC_IOCQES_R(r4)));
    189       1.8  jdolecek 	printf("%s:  iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4),
    190       1.8  jdolecek 	    (1 << NVME_CC_IOSQES_R(r4)));
    191       1.1    nonaka 	printf("%s:  shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4));
    192       1.1    nonaka 	printf("%s:  ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4));
    193       1.8  jdolecek 	printf("%s:  mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4),
    194       1.8  jdolecek 	    (1 << NVME_CC_MPS_R(r4)));
    195       1.1    nonaka 	printf("%s:  css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4));
    196       1.6  jdolecek 	printf("%s:  en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0);
    197       1.1    nonaka 
    198       1.8  jdolecek 	r4 = nvme_read4(sc, NVME_CSTS);
    199       1.8  jdolecek 	printf("%s: csts 0x%08x\n", DEVNAME(sc), r4);
    200       1.8  jdolecek 	printf("%s:  rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY);
    201       1.8  jdolecek 	printf("%s:  cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS);
    202       1.8  jdolecek 	printf("%s:  shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK);
    203       1.8  jdolecek 
    204       1.8  jdolecek 	r4 = nvme_read4(sc, NVME_AQA);
    205       1.8  jdolecek 	printf("%s: aqa  0x%08x\n", DEVNAME(sc), r4);
    206       1.8  jdolecek 	printf("%s:  acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4));
    207       1.8  jdolecek 	printf("%s:  asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4));
    208       1.8  jdolecek 
    209       1.8  jdolecek 	printf("%s: asq  0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ));
    210       1.8  jdolecek 	printf("%s: acq  0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ));
    211       1.1    nonaka #undef	DEVNAME
    212       1.1    nonaka }
    213       1.1    nonaka #endif	/* NVME_DEBUG */
    214       1.1    nonaka 
    215       1.1    nonaka static int
    216       1.1    nonaka nvme_ready(struct nvme_softc *sc, uint32_t rdy)
    217       1.1    nonaka {
    218       1.1    nonaka 	u_int i = 0;
    219       1.8  jdolecek 	uint32_t cc;
    220       1.8  jdolecek 
    221       1.8  jdolecek 	cc = nvme_read4(sc, NVME_CC);
    222       1.8  jdolecek 	if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) {
    223       1.8  jdolecek 		aprint_error_dev(sc->sc_dev,
    224       1.8  jdolecek 		    "controller enabled status expected %d, found to be %d\n",
    225       1.8  jdolecek 		    (rdy != 0), ((cc & NVME_CC_EN) != 0));
    226       1.8  jdolecek 		return ENXIO;
    227       1.8  jdolecek 	}
    228       1.1    nonaka 
    229       1.1    nonaka 	while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) {
    230       1.1    nonaka 		if (i++ > sc->sc_rdy_to)
    231       1.8  jdolecek 			return ENXIO;
    232       1.1    nonaka 
    233       1.1    nonaka 		delay(1000);
    234       1.1    nonaka 		nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ);
    235       1.1    nonaka 	}
    236       1.1    nonaka 
    237       1.1    nonaka 	return 0;
    238       1.1    nonaka }
    239       1.1    nonaka 
    240       1.1    nonaka static int
    241       1.1    nonaka nvme_enable(struct nvme_softc *sc, u_int mps)
    242       1.1    nonaka {
    243       1.8  jdolecek 	uint32_t cc, csts;
    244       1.1    nonaka 
    245       1.1    nonaka 	cc = nvme_read4(sc, NVME_CC);
    246       1.8  jdolecek 	csts = nvme_read4(sc, NVME_CSTS);
    247       1.8  jdolecek 
    248       1.7  jdolecek 	if (ISSET(cc, NVME_CC_EN)) {
    249       1.7  jdolecek 		aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n");
    250       1.8  jdolecek 
    251       1.8  jdolecek 		if (ISSET(csts, NVME_CSTS_RDY))
    252       1.8  jdolecek 			return 1;
    253       1.8  jdolecek 
    254       1.8  jdolecek 		goto waitready;
    255       1.7  jdolecek 	}
    256       1.1    nonaka 
    257       1.1    nonaka 	nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem));
    258       1.1    nonaka 	nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE);
    259       1.8  jdolecek 	delay(5000);
    260       1.1    nonaka 	nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem));
    261       1.1    nonaka 	nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE);
    262       1.8  jdolecek 	delay(5000);
    263       1.8  jdolecek 
    264       1.8  jdolecek 	nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) |
    265       1.8  jdolecek 	    NVME_AQA_ASQS(sc->sc_admin_q->q_entries));
    266       1.8  jdolecek 	nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE);
    267       1.8  jdolecek 	delay(5000);
    268       1.1    nonaka 
    269       1.1    nonaka 	CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK |
    270       1.1    nonaka 	    NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK);
    271       1.1    nonaka 	SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1));
    272       1.1    nonaka 	SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE));
    273       1.1    nonaka 	SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM));
    274       1.1    nonaka 	SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR));
    275       1.1    nonaka 	SET(cc, NVME_CC_MPS(mps));
    276       1.1    nonaka 	SET(cc, NVME_CC_EN);
    277       1.1    nonaka 
    278       1.1    nonaka 	nvme_write4(sc, NVME_CC, cc);
    279       1.1    nonaka 	nvme_barrier(sc, 0, sc->sc_ios,
    280       1.1    nonaka 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
    281       1.8  jdolecek 	delay(5000);
    282       1.1    nonaka 
    283       1.8  jdolecek     waitready:
    284       1.1    nonaka 	return nvme_ready(sc, NVME_CSTS_RDY);
    285       1.1    nonaka }
    286       1.1    nonaka 
    287       1.1    nonaka static int
    288       1.1    nonaka nvme_disable(struct nvme_softc *sc)
    289       1.1    nonaka {
    290       1.1    nonaka 	uint32_t cc, csts;
    291       1.1    nonaka 
    292       1.1    nonaka 	cc = nvme_read4(sc, NVME_CC);
    293       1.8  jdolecek 	csts = nvme_read4(sc, NVME_CSTS);
    294       1.8  jdolecek 
    295       1.8  jdolecek 	if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY))
    296       1.8  jdolecek 		nvme_ready(sc, NVME_CSTS_RDY);
    297       1.1    nonaka 
    298       1.1    nonaka 	CLR(cc, NVME_CC_EN);
    299       1.1    nonaka 
    300       1.1    nonaka 	nvme_write4(sc, NVME_CC, cc);
    301       1.8  jdolecek 	nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ);
    302       1.8  jdolecek 
    303       1.8  jdolecek 	delay(5000);
    304       1.1    nonaka 
    305       1.1    nonaka 	return nvme_ready(sc, 0);
    306       1.1    nonaka }
    307       1.1    nonaka 
    308       1.1    nonaka int
    309       1.1    nonaka nvme_attach(struct nvme_softc *sc)
    310       1.1    nonaka {
    311       1.1    nonaka 	uint64_t cap;
    312       1.1    nonaka 	uint32_t reg;
    313       1.1    nonaka 	u_int dstrd;
    314       1.1    nonaka 	u_int mps = PAGE_SHIFT;
    315      1.23    nonaka 	u_int ioq_allocated;
    316      1.20  jdolecek 	uint16_t adminq_entries = nvme_adminq_size;
    317      1.20  jdolecek 	uint16_t ioq_entries = nvme_ioq_size;
    318       1.1    nonaka 	int i;
    319       1.1    nonaka 
    320       1.1    nonaka 	reg = nvme_read4(sc, NVME_VS);
    321       1.1    nonaka 	if (reg == 0xffffffff) {
    322       1.1    nonaka 		aprint_error_dev(sc->sc_dev, "invalid mapping\n");
    323       1.1    nonaka 		return 1;
    324       1.1    nonaka 	}
    325       1.1    nonaka 
    326      1.27    nonaka 	if (NVME_VS_TER(reg) == 0)
    327      1.27    nonaka 		aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg),
    328      1.27    nonaka 		    NVME_VS_MNR(reg));
    329      1.27    nonaka 	else
    330      1.27    nonaka 		aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg),
    331      1.27    nonaka 		    NVME_VS_MNR(reg), NVME_VS_TER(reg));
    332       1.1    nonaka 
    333       1.1    nonaka 	cap = nvme_read8(sc, NVME_CAP);
    334       1.1    nonaka 	dstrd = NVME_CAP_DSTRD(cap);
    335       1.1    nonaka 	if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) {
    336       1.1    nonaka 		aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u "
    337       1.1    nonaka 		    "is greater than CPU page size %u\n",
    338       1.1    nonaka 		    1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT);
    339       1.1    nonaka 		return 1;
    340       1.1    nonaka 	}
    341       1.1    nonaka 	if (NVME_CAP_MPSMAX(cap) < mps)
    342       1.1    nonaka 		mps = NVME_CAP_MPSMAX(cap);
    343      1.15    nonaka 	if (ioq_entries > NVME_CAP_MQES(cap))
    344      1.15    nonaka 		ioq_entries = NVME_CAP_MQES(cap);
    345       1.1    nonaka 
    346       1.8  jdolecek 	/* set initial values to be used for admin queue during probe */
    347       1.1    nonaka 	sc->sc_rdy_to = NVME_CAP_TO(cap);
    348       1.1    nonaka 	sc->sc_mps = 1 << mps;
    349       1.1    nonaka 	sc->sc_mdts = MAXPHYS;
    350       1.1    nonaka 	sc->sc_max_sgl = 2;
    351       1.1    nonaka 
    352       1.1    nonaka 	if (nvme_disable(sc) != 0) {
    353       1.1    nonaka 		aprint_error_dev(sc->sc_dev, "unable to disable controller\n");
    354       1.1    nonaka 		return 1;
    355       1.1    nonaka 	}
    356       1.1    nonaka 
    357       1.1    nonaka 	sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd);
    358       1.1    nonaka 	if (sc->sc_admin_q == NULL) {
    359       1.1    nonaka 		aprint_error_dev(sc->sc_dev,
    360       1.1    nonaka 		    "unable to allocate admin queue\n");
    361       1.1    nonaka 		return 1;
    362       1.1    nonaka 	}
    363       1.1    nonaka 	if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q))
    364       1.1    nonaka 		goto free_admin_q;
    365       1.1    nonaka 
    366       1.1    nonaka 	if (nvme_enable(sc, mps) != 0) {
    367       1.1    nonaka 		aprint_error_dev(sc->sc_dev, "unable to enable controller\n");
    368       1.1    nonaka 		goto disestablish_admin_q;
    369       1.1    nonaka 	}
    370       1.1    nonaka 
    371       1.1    nonaka 	if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) {
    372       1.1    nonaka 		aprint_error_dev(sc->sc_dev, "unable to identify controller\n");
    373       1.1    nonaka 		goto disable;
    374       1.1    nonaka 	}
    375       1.1    nonaka 
    376       1.1    nonaka 	/* we know how big things are now */
    377       1.1    nonaka 	sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps;
    378       1.1    nonaka 
    379       1.1    nonaka 	/* reallocate ccbs of admin queue with new max sgl. */
    380       1.1    nonaka 	nvme_ccbs_free(sc->sc_admin_q);
    381       1.1    nonaka 	nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries);
    382       1.1    nonaka 
    383      1.23    nonaka 	if (sc->sc_use_mq) {
    384      1.23    nonaka 		/* Limit the number of queues to the number allocated in HW */
    385      1.23    nonaka 		if (nvme_get_number_of_queues(sc, &ioq_allocated) != 0) {
    386      1.23    nonaka 			aprint_error_dev(sc->sc_dev,
    387      1.23    nonaka 			    "unable to get number of queues\n");
    388      1.23    nonaka 			goto disable;
    389      1.23    nonaka 		}
    390      1.23    nonaka 		if (sc->sc_nq > ioq_allocated)
    391      1.23    nonaka 			sc->sc_nq = ioq_allocated;
    392      1.23    nonaka 	}
    393      1.23    nonaka 
    394       1.1    nonaka 	sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP);
    395       1.1    nonaka 	for (i = 0; i < sc->sc_nq; i++) {
    396       1.1    nonaka 		sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd);
    397       1.1    nonaka 		if (sc->sc_q[i] == NULL) {
    398       1.1    nonaka 			aprint_error_dev(sc->sc_dev,
    399       1.1    nonaka 			    "unable to allocate io queue\n");
    400       1.1    nonaka 			goto free_q;
    401       1.1    nonaka 		}
    402       1.1    nonaka 		if (nvme_q_create(sc, sc->sc_q[i]) != 0) {
    403       1.1    nonaka 			aprint_error_dev(sc->sc_dev,
    404       1.1    nonaka 			    "unable to create io queue\n");
    405       1.1    nonaka 			nvme_q_free(sc, sc->sc_q[i]);
    406       1.1    nonaka 			goto free_q;
    407       1.1    nonaka 		}
    408       1.1    nonaka 	}
    409       1.1    nonaka 
    410       1.1    nonaka 	if (!sc->sc_use_mq)
    411       1.1    nonaka 		nvme_write4(sc, NVME_INTMC, 1);
    412       1.1    nonaka 
    413       1.9  jdolecek 	/* probe subdevices */
    414       1.1    nonaka 	sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn,
    415       1.1    nonaka 	    KM_SLEEP);
    416      1.14  pgoyette 	nvme_rescan(sc->sc_dev, "nvme", &i);
    417       1.1    nonaka 
    418       1.1    nonaka 	return 0;
    419       1.1    nonaka 
    420       1.1    nonaka free_q:
    421       1.1    nonaka 	while (--i >= 0) {
    422       1.1    nonaka 		nvme_q_delete(sc, sc->sc_q[i]);
    423       1.1    nonaka 		nvme_q_free(sc, sc->sc_q[i]);
    424       1.1    nonaka 	}
    425       1.1    nonaka disable:
    426       1.1    nonaka 	nvme_disable(sc);
    427       1.1    nonaka disestablish_admin_q:
    428       1.1    nonaka 	sc->sc_intr_disestablish(sc, NVME_ADMIN_Q);
    429       1.1    nonaka free_admin_q:
    430       1.1    nonaka 	nvme_q_free(sc, sc->sc_admin_q);
    431       1.1    nonaka 
    432       1.1    nonaka 	return 1;
    433       1.1    nonaka }
    434       1.1    nonaka 
    435      1.14  pgoyette int
    436      1.14  pgoyette nvme_rescan(device_t self, const char *attr, const int *flags)
    437      1.14  pgoyette {
    438      1.14  pgoyette 	struct nvme_softc *sc = device_private(self);
    439      1.14  pgoyette 	struct nvme_attach_args naa;
    440      1.15    nonaka 	uint64_t cap;
    441      1.15    nonaka 	int ioq_entries = nvme_ioq_size;
    442      1.15    nonaka 	int i;
    443      1.15    nonaka 
    444      1.15    nonaka 	cap = nvme_read8(sc, NVME_CAP);
    445      1.15    nonaka 	if (ioq_entries > NVME_CAP_MQES(cap))
    446      1.15    nonaka 		ioq_entries = NVME_CAP_MQES(cap);
    447      1.14  pgoyette 
    448      1.14  pgoyette 	for (i = 0; i < sc->sc_nn; i++) {
    449      1.14  pgoyette 		if (sc->sc_namespaces[i].dev)
    450      1.14  pgoyette 			continue;
    451      1.14  pgoyette 		memset(&naa, 0, sizeof(naa));
    452      1.14  pgoyette 		naa.naa_nsid = i + 1;
    453      1.21  jdolecek 		naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq;
    454      1.21  jdolecek 		naa.naa_maxphys = sc->sc_mdts;
    455      1.14  pgoyette 		sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa,
    456      1.14  pgoyette 		    nvme_print);
    457      1.14  pgoyette 	}
    458      1.14  pgoyette 	return 0;
    459      1.14  pgoyette }
    460      1.14  pgoyette 
    461       1.1    nonaka static int
    462       1.1    nonaka nvme_print(void *aux, const char *pnp)
    463       1.1    nonaka {
    464       1.1    nonaka 	struct nvme_attach_args *naa = aux;
    465       1.1    nonaka 
    466       1.1    nonaka 	if (pnp)
    467       1.1    nonaka 		aprint_normal("at %s", pnp);
    468       1.1    nonaka 
    469       1.1    nonaka 	if (naa->naa_nsid > 0)
    470       1.1    nonaka 		aprint_normal(" nsid %d", naa->naa_nsid);
    471       1.1    nonaka 
    472       1.1    nonaka 	return UNCONF;
    473       1.1    nonaka }
    474       1.1    nonaka 
    475       1.1    nonaka int
    476       1.1    nonaka nvme_detach(struct nvme_softc *sc, int flags)
    477       1.1    nonaka {
    478       1.1    nonaka 	int i, error;
    479       1.1    nonaka 
    480       1.1    nonaka 	error = config_detach_children(sc->sc_dev, flags);
    481       1.1    nonaka 	if (error)
    482       1.1    nonaka 		return error;
    483       1.1    nonaka 
    484       1.1    nonaka 	error = nvme_shutdown(sc);
    485       1.1    nonaka 	if (error)
    486       1.1    nonaka 		return error;
    487       1.1    nonaka 
    488       1.9  jdolecek 	/* from now on we are committed to detach, following will never fail */
    489       1.1    nonaka 	for (i = 0; i < sc->sc_nq; i++)
    490       1.1    nonaka 		nvme_q_free(sc, sc->sc_q[i]);
    491       1.1    nonaka 	kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq);
    492       1.1    nonaka 	nvme_q_free(sc, sc->sc_admin_q);
    493       1.1    nonaka 
    494       1.1    nonaka 	return 0;
    495       1.1    nonaka }
    496       1.1    nonaka 
    497       1.1    nonaka static int
    498       1.1    nonaka nvme_shutdown(struct nvme_softc *sc)
    499       1.1    nonaka {
    500       1.1    nonaka 	uint32_t cc, csts;
    501       1.1    nonaka 	bool disabled = false;
    502       1.1    nonaka 	int i;
    503       1.1    nonaka 
    504       1.1    nonaka 	if (!sc->sc_use_mq)
    505       1.1    nonaka 		nvme_write4(sc, NVME_INTMS, 1);
    506       1.1    nonaka 
    507       1.1    nonaka 	for (i = 0; i < sc->sc_nq; i++) {
    508       1.1    nonaka 		if (nvme_q_delete(sc, sc->sc_q[i]) != 0) {
    509       1.1    nonaka 			aprint_error_dev(sc->sc_dev,
    510       1.1    nonaka 			    "unable to delete io queue %d, disabling\n", i + 1);
    511       1.1    nonaka 			disabled = true;
    512       1.1    nonaka 		}
    513       1.1    nonaka 	}
    514       1.1    nonaka 	sc->sc_intr_disestablish(sc, NVME_ADMIN_Q);
    515       1.1    nonaka 	if (disabled)
    516       1.1    nonaka 		goto disable;
    517       1.1    nonaka 
    518       1.1    nonaka 	cc = nvme_read4(sc, NVME_CC);
    519       1.1    nonaka 	CLR(cc, NVME_CC_SHN_MASK);
    520       1.1    nonaka 	SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL));
    521       1.1    nonaka 	nvme_write4(sc, NVME_CC, cc);
    522       1.1    nonaka 
    523       1.1    nonaka 	for (i = 0; i < 4000; i++) {
    524       1.1    nonaka 		nvme_barrier(sc, 0, sc->sc_ios,
    525       1.1    nonaka 		    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
    526       1.1    nonaka 		csts = nvme_read4(sc, NVME_CSTS);
    527       1.1    nonaka 		if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE)
    528       1.1    nonaka 			return 0;
    529       1.1    nonaka 
    530       1.1    nonaka 		delay(1000);
    531       1.1    nonaka 	}
    532       1.1    nonaka 
    533       1.1    nonaka 	aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n");
    534       1.1    nonaka 
    535       1.1    nonaka disable:
    536       1.1    nonaka 	nvme_disable(sc);
    537       1.1    nonaka 	return 0;
    538       1.1    nonaka }
    539       1.1    nonaka 
    540       1.1    nonaka void
    541       1.1    nonaka nvme_childdet(device_t self, device_t child)
    542       1.1    nonaka {
    543       1.1    nonaka 	struct nvme_softc *sc = device_private(self);
    544       1.1    nonaka 	int i;
    545       1.1    nonaka 
    546       1.1    nonaka 	for (i = 0; i < sc->sc_nn; i++) {
    547       1.1    nonaka 		if (sc->sc_namespaces[i].dev == child) {
    548       1.1    nonaka 			/* Already freed ns->ident. */
    549       1.1    nonaka 			sc->sc_namespaces[i].dev = NULL;
    550       1.1    nonaka 			break;
    551       1.1    nonaka 		}
    552       1.1    nonaka 	}
    553       1.1    nonaka }
    554       1.1    nonaka 
    555       1.1    nonaka int
    556       1.1    nonaka nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid)
    557       1.1    nonaka {
    558       1.1    nonaka 	struct nvme_sqe sqe;
    559       1.1    nonaka 	struct nvm_identify_namespace *identify;
    560      1.19  jdolecek 	struct nvme_dmamem *mem;
    561       1.1    nonaka 	struct nvme_ccb *ccb;
    562       1.1    nonaka 	struct nvme_namespace *ns;
    563      1.19  jdolecek 	int rv;
    564       1.1    nonaka 
    565       1.1    nonaka 	KASSERT(nsid > 0);
    566       1.1    nonaka 
    567       1.1    nonaka 	ccb = nvme_ccb_get(sc->sc_admin_q);
    568      1.11  jdolecek 	KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
    569       1.1    nonaka 
    570      1.19  jdolecek 	mem = nvme_dmamem_alloc(sc, sizeof(*identify));
    571      1.19  jdolecek 	if (mem == NULL)
    572      1.19  jdolecek 		return ENOMEM;
    573       1.1    nonaka 
    574       1.1    nonaka 	memset(&sqe, 0, sizeof(sqe));
    575       1.1    nonaka 	sqe.opcode = NVM_ADMIN_IDENTIFY;
    576       1.1    nonaka 	htolem32(&sqe.nsid, nsid);
    577       1.1    nonaka 	htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem));
    578       1.1    nonaka 	htolem32(&sqe.cdw10, 0);
    579       1.1    nonaka 
    580       1.1    nonaka 	ccb->ccb_done = nvme_empty_done;
    581       1.1    nonaka 	ccb->ccb_cookie = &sqe;
    582       1.1    nonaka 
    583       1.1    nonaka 	nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD);
    584      1.19  jdolecek 	rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT);
    585       1.1    nonaka 	nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD);
    586       1.1    nonaka 
    587       1.1    nonaka 	nvme_ccb_put(sc->sc_admin_q, ccb);
    588       1.1    nonaka 
    589      1.19  jdolecek 	if (rv != 0) {
    590      1.19  jdolecek 		rv = EIO;
    591       1.1    nonaka 		goto done;
    592       1.1    nonaka 	}
    593       1.1    nonaka 
    594       1.1    nonaka 	/* commit */
    595       1.1    nonaka 
    596       1.1    nonaka 	identify = kmem_zalloc(sizeof(*identify), KM_SLEEP);
    597      1.19  jdolecek 	*identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem));
    598      1.19  jdolecek 	//memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify));
    599       1.1    nonaka 
    600       1.1    nonaka 	ns = nvme_ns_get(sc, nsid);
    601       1.1    nonaka 	KASSERT(ns);
    602       1.1    nonaka 	ns->ident = identify;
    603       1.1    nonaka 
    604       1.1    nonaka done:
    605      1.19  jdolecek 	nvme_dmamem_free(sc, mem);
    606       1.1    nonaka 
    607      1.19  jdolecek 	return rv;
    608       1.1    nonaka }
    609       1.1    nonaka 
    610       1.1    nonaka int
    611      1.11  jdolecek nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie,
    612      1.11  jdolecek     struct buf *bp, void *data, size_t datasize,
    613      1.11  jdolecek     int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done)
    614       1.1    nonaka {
    615       1.1    nonaka 	struct nvme_queue *q = nvme_get_q(sc);
    616       1.1    nonaka 	struct nvme_ccb *ccb;
    617       1.1    nonaka 	bus_dmamap_t dmap;
    618       1.1    nonaka 	int i, error;
    619       1.1    nonaka 
    620       1.1    nonaka 	ccb = nvme_ccb_get(q);
    621       1.1    nonaka 	if (ccb == NULL)
    622       1.1    nonaka 		return EAGAIN;
    623       1.1    nonaka 
    624       1.1    nonaka 	ccb->ccb_done = nvme_ns_io_done;
    625      1.11  jdolecek 	ccb->ccb_cookie = cookie;
    626      1.11  jdolecek 
    627      1.11  jdolecek 	/* namespace context */
    628      1.11  jdolecek 	ccb->nnc_nsid = nsid;
    629      1.11  jdolecek 	ccb->nnc_flags = flags;
    630      1.11  jdolecek 	ccb->nnc_buf = bp;
    631      1.11  jdolecek 	ccb->nnc_datasize = datasize;
    632      1.11  jdolecek 	ccb->nnc_secsize = secsize;
    633      1.11  jdolecek 	ccb->nnc_blkno = blkno;
    634      1.11  jdolecek 	ccb->nnc_done = nnc_done;
    635       1.1    nonaka 
    636       1.1    nonaka 	dmap = ccb->ccb_dmamap;
    637      1.11  jdolecek 	error = bus_dmamap_load(sc->sc_dmat, dmap, data,
    638      1.11  jdolecek 	    datasize, NULL,
    639      1.11  jdolecek 	    (ISSET(flags, NVME_NS_CTX_F_POLL) ?
    640       1.1    nonaka 	      BUS_DMA_NOWAIT : BUS_DMA_WAITOK) |
    641      1.11  jdolecek 	    (ISSET(flags, NVME_NS_CTX_F_READ) ?
    642       1.1    nonaka 	      BUS_DMA_READ : BUS_DMA_WRITE));
    643       1.1    nonaka 	if (error) {
    644       1.1    nonaka 		nvme_ccb_put(q, ccb);
    645       1.1    nonaka 		return error;
    646       1.1    nonaka 	}
    647       1.1    nonaka 
    648       1.1    nonaka 	bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
    649      1.11  jdolecek 	    ISSET(flags, NVME_NS_CTX_F_READ) ?
    650       1.1    nonaka 	    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
    651       1.1    nonaka 
    652       1.1    nonaka 	if (dmap->dm_nsegs > 2) {
    653       1.1    nonaka 		for (i = 1; i < dmap->dm_nsegs; i++) {
    654       1.1    nonaka 			htolem64(&ccb->ccb_prpl[i - 1],
    655       1.1    nonaka 			    dmap->dm_segs[i].ds_addr);
    656       1.1    nonaka 		}
    657       1.1    nonaka 		bus_dmamap_sync(sc->sc_dmat,
    658       1.1    nonaka 		    NVME_DMA_MAP(q->q_ccb_prpls),
    659       1.1    nonaka 		    ccb->ccb_prpl_off,
    660      1.16    nonaka 		    sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
    661       1.1    nonaka 		    BUS_DMASYNC_PREWRITE);
    662       1.1    nonaka 	}
    663       1.1    nonaka 
    664      1.11  jdolecek 	if (ISSET(flags, NVME_NS_CTX_F_POLL)) {
    665       1.7  jdolecek 		if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0)
    666       1.1    nonaka 			return EIO;
    667       1.1    nonaka 		return 0;
    668       1.1    nonaka 	}
    669       1.1    nonaka 
    670       1.1    nonaka 	nvme_q_submit(sc, q, ccb, nvme_ns_io_fill);
    671       1.1    nonaka 	return 0;
    672       1.1    nonaka }
    673       1.1    nonaka 
    674       1.1    nonaka static void
    675       1.1    nonaka nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
    676       1.1    nonaka {
    677       1.1    nonaka 	struct nvme_sqe_io *sqe = slot;
    678       1.1    nonaka 	bus_dmamap_t dmap = ccb->ccb_dmamap;
    679       1.1    nonaka 
    680      1.11  jdolecek 	sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ?
    681       1.1    nonaka 	    NVM_CMD_READ : NVM_CMD_WRITE;
    682      1.11  jdolecek 	htolem32(&sqe->nsid, ccb->nnc_nsid);
    683       1.1    nonaka 
    684       1.1    nonaka 	htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr);
    685       1.1    nonaka 	switch (dmap->dm_nsegs) {
    686       1.1    nonaka 	case 1:
    687       1.1    nonaka 		break;
    688       1.1    nonaka 	case 2:
    689       1.1    nonaka 		htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr);
    690       1.1    nonaka 		break;
    691       1.1    nonaka 	default:
    692       1.1    nonaka 		/* the prp list is already set up and synced */
    693       1.1    nonaka 		htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva);
    694       1.1    nonaka 		break;
    695       1.1    nonaka 	}
    696       1.1    nonaka 
    697      1.11  jdolecek 	htolem64(&sqe->slba, ccb->nnc_blkno);
    698      1.11  jdolecek 
    699      1.26  jdolecek 	if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA))
    700      1.26  jdolecek 		htolem16(&sqe->ioflags, NVM_SQE_IO_FUA);
    701      1.26  jdolecek 
    702      1.11  jdolecek 	/* guaranteed by upper layers, but check just in case */
    703      1.11  jdolecek 	KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0);
    704      1.11  jdolecek 	htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1);
    705       1.1    nonaka }
    706       1.1    nonaka 
    707       1.1    nonaka static void
    708       1.1    nonaka nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb,
    709       1.1    nonaka     struct nvme_cqe *cqe)
    710       1.1    nonaka {
    711       1.1    nonaka 	struct nvme_softc *sc = q->q_sc;
    712       1.1    nonaka 	bus_dmamap_t dmap = ccb->ccb_dmamap;
    713      1.11  jdolecek 	void *nnc_cookie = ccb->ccb_cookie;
    714      1.11  jdolecek 	nvme_nnc_done nnc_done = ccb->nnc_done;
    715      1.11  jdolecek 	struct buf *bp = ccb->nnc_buf;
    716       1.1    nonaka 
    717       1.1    nonaka 	if (dmap->dm_nsegs > 2) {
    718       1.1    nonaka 		bus_dmamap_sync(sc->sc_dmat,
    719       1.1    nonaka 		    NVME_DMA_MAP(q->q_ccb_prpls),
    720       1.1    nonaka 		    ccb->ccb_prpl_off,
    721      1.16    nonaka 		    sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
    722       1.1    nonaka 		    BUS_DMASYNC_POSTWRITE);
    723       1.1    nonaka 	}
    724       1.1    nonaka 
    725       1.1    nonaka 	bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
    726      1.11  jdolecek 	    ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ?
    727       1.1    nonaka 	    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
    728       1.1    nonaka 
    729       1.1    nonaka 	bus_dmamap_unload(sc->sc_dmat, dmap);
    730       1.1    nonaka 	nvme_ccb_put(q, ccb);
    731       1.1    nonaka 
    732      1.25  jdolecek 	nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0));
    733      1.25  jdolecek }
    734      1.25  jdolecek 
    735      1.25  jdolecek /*
    736      1.25  jdolecek  * If there is no volatile write cache, it makes no sense to issue
    737      1.25  jdolecek  * flush commands or query for the status.
    738      1.25  jdolecek  */
    739      1.25  jdolecek bool
    740      1.25  jdolecek nvme_has_volatile_write_cache(struct nvme_softc *sc)
    741      1.25  jdolecek {
    742      1.25  jdolecek 	/* sc_identify is filled during attachment */
    743      1.25  jdolecek 	return  ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0);
    744       1.1    nonaka }
    745       1.1    nonaka 
    746       1.1    nonaka int
    747      1.11  jdolecek nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie,
    748      1.11  jdolecek     int flags, nvme_nnc_done nnc_done)
    749       1.1    nonaka {
    750       1.1    nonaka 	struct nvme_queue *q = nvme_get_q(sc);
    751       1.1    nonaka 	struct nvme_ccb *ccb;
    752       1.1    nonaka 
    753       1.1    nonaka 	ccb = nvme_ccb_get(q);
    754       1.1    nonaka 	if (ccb == NULL)
    755       1.1    nonaka 		return EAGAIN;
    756       1.1    nonaka 
    757       1.1    nonaka 	ccb->ccb_done = nvme_ns_sync_done;
    758      1.11  jdolecek 	ccb->ccb_cookie = cookie;
    759       1.1    nonaka 
    760      1.11  jdolecek 	/* namespace context */
    761      1.11  jdolecek 	ccb->nnc_nsid = nsid;
    762      1.11  jdolecek 	ccb->nnc_flags = flags;
    763      1.11  jdolecek 	ccb->nnc_done = nnc_done;
    764      1.11  jdolecek 
    765      1.11  jdolecek 	if (ISSET(flags, NVME_NS_CTX_F_POLL)) {
    766       1.7  jdolecek 		if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0)
    767       1.1    nonaka 			return EIO;
    768       1.1    nonaka 		return 0;
    769       1.1    nonaka 	}
    770       1.1    nonaka 
    771       1.1    nonaka 	nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill);
    772       1.1    nonaka 	return 0;
    773       1.1    nonaka }
    774       1.1    nonaka 
    775       1.1    nonaka static void
    776       1.1    nonaka nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
    777       1.1    nonaka {
    778       1.1    nonaka 	struct nvme_sqe *sqe = slot;
    779       1.1    nonaka 
    780       1.1    nonaka 	sqe->opcode = NVM_CMD_FLUSH;
    781      1.11  jdolecek 	htolem32(&sqe->nsid, ccb->nnc_nsid);
    782       1.1    nonaka }
    783       1.1    nonaka 
    784       1.1    nonaka static void
    785       1.1    nonaka nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb,
    786       1.1    nonaka     struct nvme_cqe *cqe)
    787       1.1    nonaka {
    788      1.11  jdolecek 	void *cookie = ccb->ccb_cookie;
    789      1.11  jdolecek 	nvme_nnc_done nnc_done = ccb->nnc_done;
    790       1.1    nonaka 
    791       1.1    nonaka 	nvme_ccb_put(q, ccb);
    792       1.1    nonaka 
    793      1.25  jdolecek 	nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0));
    794      1.25  jdolecek }
    795      1.25  jdolecek 
    796      1.25  jdolecek /*
    797      1.25  jdolecek  * Get status of volatile write cache. Always asynchronous.
    798      1.25  jdolecek  */
    799      1.25  jdolecek int
    800      1.25  jdolecek nvme_admin_getcache(struct nvme_softc *sc, void *cookie, nvme_nnc_done nnc_done)
    801      1.25  jdolecek {
    802      1.25  jdolecek 	struct nvme_ccb *ccb;
    803      1.25  jdolecek 	struct nvme_queue *q = sc->sc_admin_q;
    804      1.25  jdolecek 
    805      1.25  jdolecek 	ccb = nvme_ccb_get(q);
    806      1.25  jdolecek 	if (ccb == NULL)
    807      1.25  jdolecek 		return EAGAIN;
    808      1.25  jdolecek 
    809      1.25  jdolecek 	ccb->ccb_done = nvme_getcache_done;
    810      1.25  jdolecek 	ccb->ccb_cookie = cookie;
    811      1.25  jdolecek 
    812      1.25  jdolecek 	/* namespace context */
    813      1.25  jdolecek 	ccb->nnc_flags = 0;
    814      1.25  jdolecek 	ccb->nnc_done = nnc_done;
    815      1.25  jdolecek 
    816      1.25  jdolecek 	nvme_q_submit(sc, q, ccb, nvme_getcache_fill);
    817      1.25  jdolecek 	return 0;
    818      1.25  jdolecek }
    819      1.25  jdolecek 
    820      1.25  jdolecek static void
    821      1.25  jdolecek nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
    822      1.25  jdolecek {
    823      1.25  jdolecek 	struct nvme_sqe *sqe = slot;
    824      1.25  jdolecek 
    825      1.25  jdolecek 	sqe->opcode = NVM_ADMIN_GET_FEATURES;
    826      1.25  jdolecek 	sqe->cdw10 = NVM_FEATURE_VOLATILE_WRITE_CACHE;
    827      1.25  jdolecek }
    828      1.25  jdolecek 
    829      1.25  jdolecek static void
    830      1.25  jdolecek nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb,
    831      1.25  jdolecek     struct nvme_cqe *cqe)
    832      1.25  jdolecek {
    833      1.25  jdolecek 	void *cookie = ccb->ccb_cookie;
    834      1.25  jdolecek 	nvme_nnc_done nnc_done = ccb->nnc_done;
    835      1.25  jdolecek 
    836      1.25  jdolecek 	nvme_ccb_put(q, ccb);
    837      1.25  jdolecek 
    838      1.25  jdolecek 	nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0));
    839       1.1    nonaka }
    840       1.1    nonaka 
    841       1.1    nonaka void
    842       1.1    nonaka nvme_ns_free(struct nvme_softc *sc, uint16_t nsid)
    843       1.1    nonaka {
    844       1.1    nonaka 	struct nvme_namespace *ns;
    845       1.1    nonaka 	struct nvm_identify_namespace *identify;
    846       1.1    nonaka 
    847       1.1    nonaka 	ns = nvme_ns_get(sc, nsid);
    848       1.1    nonaka 	KASSERT(ns);
    849       1.1    nonaka 
    850       1.1    nonaka 	identify = ns->ident;
    851       1.1    nonaka 	ns->ident = NULL;
    852       1.1    nonaka 	if (identify != NULL)
    853       1.1    nonaka 		kmem_free(identify, sizeof(*identify));
    854       1.1    nonaka }
    855       1.1    nonaka 
    856       1.1    nonaka static void
    857       1.3    nonaka nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
    858       1.3    nonaka {
    859       1.3    nonaka 	struct nvme_softc *sc = q->q_sc;
    860       1.3    nonaka 	struct nvme_sqe *sqe = slot;
    861       1.3    nonaka 	struct nvme_pt_command *pt = ccb->ccb_cookie;
    862       1.3    nonaka 	bus_dmamap_t dmap = ccb->ccb_dmamap;
    863       1.3    nonaka 	int i;
    864       1.3    nonaka 
    865       1.3    nonaka 	sqe->opcode = pt->cmd.opcode;
    866       1.3    nonaka 	htolem32(&sqe->nsid, pt->cmd.nsid);
    867       1.3    nonaka 
    868       1.3    nonaka 	if (pt->buf != NULL && pt->len > 0) {
    869       1.3    nonaka 		htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr);
    870       1.3    nonaka 		switch (dmap->dm_nsegs) {
    871       1.3    nonaka 		case 1:
    872       1.3    nonaka 			break;
    873       1.3    nonaka 		case 2:
    874       1.3    nonaka 			htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr);
    875       1.3    nonaka 			break;
    876       1.3    nonaka 		default:
    877       1.3    nonaka 			for (i = 1; i < dmap->dm_nsegs; i++) {
    878       1.3    nonaka 				htolem64(&ccb->ccb_prpl[i - 1],
    879       1.3    nonaka 				    dmap->dm_segs[i].ds_addr);
    880       1.3    nonaka 			}
    881       1.3    nonaka 			bus_dmamap_sync(sc->sc_dmat,
    882       1.3    nonaka 			    NVME_DMA_MAP(q->q_ccb_prpls),
    883       1.3    nonaka 			    ccb->ccb_prpl_off,
    884      1.16    nonaka 			    sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
    885       1.3    nonaka 			    BUS_DMASYNC_PREWRITE);
    886       1.3    nonaka 			htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva);
    887       1.3    nonaka 			break;
    888       1.3    nonaka 		}
    889       1.3    nonaka 	}
    890       1.3    nonaka 
    891       1.3    nonaka 	htolem32(&sqe->cdw10, pt->cmd.cdw10);
    892       1.3    nonaka 	htolem32(&sqe->cdw11, pt->cmd.cdw11);
    893       1.3    nonaka 	htolem32(&sqe->cdw12, pt->cmd.cdw12);
    894       1.3    nonaka 	htolem32(&sqe->cdw13, pt->cmd.cdw13);
    895       1.3    nonaka 	htolem32(&sqe->cdw14, pt->cmd.cdw14);
    896       1.3    nonaka 	htolem32(&sqe->cdw15, pt->cmd.cdw15);
    897       1.3    nonaka }
    898       1.3    nonaka 
    899       1.3    nonaka static void
    900       1.3    nonaka nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe)
    901       1.3    nonaka {
    902       1.3    nonaka 	struct nvme_softc *sc = q->q_sc;
    903       1.3    nonaka 	struct nvme_pt_command *pt = ccb->ccb_cookie;
    904       1.3    nonaka 	bus_dmamap_t dmap = ccb->ccb_dmamap;
    905       1.3    nonaka 
    906       1.3    nonaka 	if (pt->buf != NULL && pt->len > 0) {
    907       1.3    nonaka 		if (dmap->dm_nsegs > 2) {
    908       1.3    nonaka 			bus_dmamap_sync(sc->sc_dmat,
    909       1.3    nonaka 			    NVME_DMA_MAP(q->q_ccb_prpls),
    910       1.3    nonaka 			    ccb->ccb_prpl_off,
    911      1.16    nonaka 			    sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
    912       1.3    nonaka 			    BUS_DMASYNC_POSTWRITE);
    913       1.3    nonaka 		}
    914       1.3    nonaka 
    915       1.3    nonaka 		bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
    916       1.3    nonaka 		    pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
    917       1.3    nonaka 		bus_dmamap_unload(sc->sc_dmat, dmap);
    918       1.3    nonaka 	}
    919       1.3    nonaka 
    920      1.23    nonaka 	pt->cpl.cdw0 = lemtoh32(&cqe->cdw0);
    921      1.23    nonaka 	pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE;
    922       1.3    nonaka }
    923       1.3    nonaka 
    924       1.3    nonaka static int
    925       1.3    nonaka nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt,
    926       1.3    nonaka     uint16_t nsid, struct lwp *l, bool is_adminq)
    927       1.3    nonaka {
    928       1.3    nonaka 	struct nvme_queue *q;
    929       1.3    nonaka 	struct nvme_ccb *ccb;
    930       1.3    nonaka 	void *buf = NULL;
    931       1.3    nonaka 	int error;
    932       1.3    nonaka 
    933       1.9  jdolecek 	/* limit command size to maximum data transfer size */
    934       1.3    nonaka 	if ((pt->buf == NULL && pt->len > 0) ||
    935       1.9  jdolecek 	    (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts)))
    936       1.3    nonaka 		return EINVAL;
    937       1.3    nonaka 
    938       1.3    nonaka 	q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc);
    939       1.3    nonaka 	ccb = nvme_ccb_get(q);
    940       1.3    nonaka 	if (ccb == NULL)
    941       1.3    nonaka 		return EBUSY;
    942       1.3    nonaka 
    943       1.9  jdolecek 	if (pt->buf != NULL) {
    944       1.9  jdolecek 		KASSERT(pt->len > 0);
    945       1.3    nonaka 		buf = kmem_alloc(pt->len, KM_SLEEP);
    946       1.3    nonaka 		if (!pt->is_read) {
    947       1.3    nonaka 			error = copyin(pt->buf, buf, pt->len);
    948       1.3    nonaka 			if (error)
    949       1.3    nonaka 				goto kmem_free;
    950       1.3    nonaka 		}
    951       1.3    nonaka 		error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf,
    952       1.3    nonaka 		    pt->len, NULL,
    953       1.3    nonaka 		    BUS_DMA_WAITOK |
    954       1.3    nonaka 		      (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE));
    955       1.3    nonaka 		if (error)
    956       1.3    nonaka 			goto kmem_free;
    957       1.3    nonaka 		bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap,
    958       1.3    nonaka 		    0, ccb->ccb_dmamap->dm_mapsize,
    959       1.3    nonaka 		    pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
    960       1.3    nonaka 	}
    961       1.3    nonaka 
    962       1.3    nonaka 	ccb->ccb_done = nvme_pt_done;
    963       1.3    nonaka 	ccb->ccb_cookie = pt;
    964       1.3    nonaka 
    965       1.3    nonaka 	pt->cmd.nsid = nsid;
    966       1.7  jdolecek 	if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) {
    967       1.3    nonaka 		error = EIO;
    968       1.3    nonaka 		goto out;
    969       1.3    nonaka 	}
    970       1.3    nonaka 
    971       1.3    nonaka 	error = 0;
    972       1.3    nonaka out:
    973       1.3    nonaka 	if (buf != NULL) {
    974       1.3    nonaka 		if (error == 0 && pt->is_read)
    975       1.3    nonaka 			error = copyout(buf, pt->buf, pt->len);
    976       1.3    nonaka kmem_free:
    977       1.3    nonaka 		kmem_free(buf, pt->len);
    978       1.3    nonaka 	}
    979       1.3    nonaka 	nvme_ccb_put(q, ccb);
    980       1.3    nonaka 	return error;
    981       1.3    nonaka }
    982       1.3    nonaka 
    983       1.3    nonaka static void
    984       1.1    nonaka nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb,
    985       1.1    nonaka     void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *))
    986       1.1    nonaka {
    987       1.1    nonaka 	struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem);
    988       1.1    nonaka 	uint32_t tail;
    989       1.1    nonaka 
    990       1.1    nonaka 	mutex_enter(&q->q_sq_mtx);
    991       1.1    nonaka 	tail = q->q_sq_tail;
    992       1.1    nonaka 	if (++q->q_sq_tail >= q->q_entries)
    993       1.1    nonaka 		q->q_sq_tail = 0;
    994       1.1    nonaka 
    995       1.1    nonaka 	sqe += tail;
    996       1.1    nonaka 
    997       1.1    nonaka 	bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem),
    998       1.1    nonaka 	    sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE);
    999       1.1    nonaka 	memset(sqe, 0, sizeof(*sqe));
   1000       1.1    nonaka 	(*fill)(q, ccb, sqe);
   1001       1.1    nonaka 	sqe->cid = ccb->ccb_id;
   1002       1.1    nonaka 	bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem),
   1003       1.1    nonaka 	    sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE);
   1004       1.1    nonaka 
   1005       1.1    nonaka 	nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail);
   1006       1.1    nonaka 	mutex_exit(&q->q_sq_mtx);
   1007       1.1    nonaka }
   1008       1.1    nonaka 
   1009       1.1    nonaka struct nvme_poll_state {
   1010       1.1    nonaka 	struct nvme_sqe s;
   1011       1.1    nonaka 	struct nvme_cqe c;
   1012       1.1    nonaka };
   1013       1.1    nonaka 
   1014       1.1    nonaka static int
   1015       1.1    nonaka nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb,
   1016       1.7  jdolecek     void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec)
   1017       1.1    nonaka {
   1018       1.1    nonaka 	struct nvme_poll_state state;
   1019       1.1    nonaka 	void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *);
   1020       1.1    nonaka 	void *cookie;
   1021       1.1    nonaka 	uint16_t flags;
   1022       1.7  jdolecek 	int step = 10;
   1023       1.7  jdolecek 	int maxloop = timo_sec * 1000000 / step;
   1024       1.7  jdolecek 	int error = 0;
   1025       1.1    nonaka 
   1026       1.1    nonaka 	memset(&state, 0, sizeof(state));
   1027       1.1    nonaka 	(*fill)(q, ccb, &state.s);
   1028       1.1    nonaka 
   1029       1.1    nonaka 	done = ccb->ccb_done;
   1030       1.1    nonaka 	cookie = ccb->ccb_cookie;
   1031       1.1    nonaka 
   1032       1.1    nonaka 	ccb->ccb_done = nvme_poll_done;
   1033       1.1    nonaka 	ccb->ccb_cookie = &state;
   1034       1.1    nonaka 
   1035       1.1    nonaka 	nvme_q_submit(sc, q, ccb, nvme_poll_fill);
   1036       1.1    nonaka 	while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) {
   1037       1.1    nonaka 		if (nvme_q_complete(sc, q) == 0)
   1038       1.7  jdolecek 			delay(step);
   1039       1.1    nonaka 
   1040       1.7  jdolecek 		if (timo_sec >= 0 && --maxloop <= 0) {
   1041       1.7  jdolecek 			error = ETIMEDOUT;
   1042       1.7  jdolecek 			break;
   1043       1.7  jdolecek 		}
   1044       1.1    nonaka 	}
   1045       1.1    nonaka 
   1046       1.1    nonaka 	ccb->ccb_cookie = cookie;
   1047       1.1    nonaka 	done(q, ccb, &state.c);
   1048       1.1    nonaka 
   1049       1.7  jdolecek 	if (error == 0) {
   1050       1.7  jdolecek 		flags = lemtoh16(&state.c.flags);
   1051       1.7  jdolecek 		return flags & ~NVME_CQE_PHASE;
   1052       1.7  jdolecek 	} else {
   1053       1.7  jdolecek 		return 1;
   1054       1.7  jdolecek 	}
   1055       1.1    nonaka }
   1056       1.1    nonaka 
   1057       1.1    nonaka static void
   1058       1.1    nonaka nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
   1059       1.1    nonaka {
   1060       1.1    nonaka 	struct nvme_sqe *sqe = slot;
   1061       1.1    nonaka 	struct nvme_poll_state *state = ccb->ccb_cookie;
   1062       1.1    nonaka 
   1063       1.1    nonaka 	*sqe = state->s;
   1064       1.1    nonaka }
   1065       1.1    nonaka 
   1066       1.1    nonaka static void
   1067       1.1    nonaka nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb,
   1068       1.1    nonaka     struct nvme_cqe *cqe)
   1069       1.1    nonaka {
   1070       1.1    nonaka 	struct nvme_poll_state *state = ccb->ccb_cookie;
   1071       1.1    nonaka 
   1072       1.1    nonaka 	SET(cqe->flags, htole16(NVME_CQE_PHASE));
   1073       1.1    nonaka 	state->c = *cqe;
   1074       1.1    nonaka }
   1075       1.1    nonaka 
   1076       1.1    nonaka static void
   1077       1.1    nonaka nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
   1078       1.1    nonaka {
   1079       1.1    nonaka 	struct nvme_sqe *src = ccb->ccb_cookie;
   1080       1.1    nonaka 	struct nvme_sqe *dst = slot;
   1081       1.1    nonaka 
   1082       1.1    nonaka 	*dst = *src;
   1083       1.1    nonaka }
   1084       1.1    nonaka 
   1085       1.1    nonaka static void
   1086       1.1    nonaka nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb,
   1087       1.1    nonaka     struct nvme_cqe *cqe)
   1088       1.1    nonaka {
   1089       1.1    nonaka }
   1090       1.1    nonaka 
   1091       1.1    nonaka static int
   1092       1.1    nonaka nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q)
   1093       1.1    nonaka {
   1094       1.1    nonaka 	struct nvme_ccb *ccb;
   1095       1.1    nonaka 	struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe;
   1096       1.1    nonaka 	uint16_t flags;
   1097       1.1    nonaka 	int rv = 0;
   1098       1.1    nonaka 
   1099       1.9  jdolecek 	mutex_enter(&q->q_cq_mtx);
   1100       1.1    nonaka 
   1101       1.1    nonaka 	nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD);
   1102       1.1    nonaka 	for (;;) {
   1103       1.9  jdolecek 		cqe = &ring[q->q_cq_head];
   1104       1.1    nonaka 		flags = lemtoh16(&cqe->flags);
   1105       1.1    nonaka 		if ((flags & NVME_CQE_PHASE) != q->q_cq_phase)
   1106       1.1    nonaka 			break;
   1107       1.1    nonaka 
   1108       1.1    nonaka 		ccb = &q->q_ccbs[cqe->cid];
   1109       1.1    nonaka 
   1110       1.9  jdolecek 		if (++q->q_cq_head >= q->q_entries) {
   1111       1.9  jdolecek 			q->q_cq_head = 0;
   1112       1.1    nonaka 			q->q_cq_phase ^= NVME_CQE_PHASE;
   1113       1.1    nonaka 		}
   1114       1.1    nonaka 
   1115      1.18  jdolecek #ifdef DEBUG
   1116      1.18  jdolecek 		/*
   1117      1.18  jdolecek 		 * If we get spurious completion notification, something
   1118      1.18  jdolecek 		 * is seriously hosed up. Very likely DMA to some random
   1119      1.18  jdolecek 		 * memory place happened, so just bail out.
   1120      1.18  jdolecek 		 */
   1121      1.18  jdolecek 		if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) {
   1122      1.18  jdolecek 			panic("%s: invalid ccb detected",
   1123      1.18  jdolecek 			    device_xname(sc->sc_dev));
   1124      1.18  jdolecek 			/* NOTREACHED */
   1125      1.18  jdolecek 		}
   1126      1.18  jdolecek #endif
   1127      1.20  jdolecek 
   1128      1.20  jdolecek 		rv++;
   1129       1.9  jdolecek 
   1130       1.9  jdolecek 		/*
   1131      1.10  jdolecek 		 * Unlock the mutex before calling the ccb_done callback
   1132       1.9  jdolecek 		 * and re-lock afterwards. The callback triggers lddone()
   1133       1.9  jdolecek 		 * which schedules another i/o, and also calls nvme_ccb_put().
   1134       1.9  jdolecek 		 * Unlock/relock avoids possibility of deadlock.
   1135       1.9  jdolecek 		 */
   1136       1.9  jdolecek 		mutex_exit(&q->q_cq_mtx);
   1137       1.9  jdolecek 		ccb->ccb_done(q, ccb, cqe);
   1138       1.9  jdolecek 		mutex_enter(&q->q_cq_mtx);
   1139       1.1    nonaka 	}
   1140       1.1    nonaka 	nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD);
   1141       1.1    nonaka 
   1142       1.1    nonaka 	if (rv)
   1143       1.9  jdolecek 		nvme_write4(sc, q->q_cqhdbl, q->q_cq_head);
   1144       1.9  jdolecek 
   1145       1.1    nonaka 	mutex_exit(&q->q_cq_mtx);
   1146       1.1    nonaka 
   1147       1.1    nonaka 	return rv;
   1148       1.1    nonaka }
   1149       1.1    nonaka 
   1150       1.1    nonaka static int
   1151       1.1    nonaka nvme_identify(struct nvme_softc *sc, u_int mps)
   1152       1.1    nonaka {
   1153       1.1    nonaka 	char sn[41], mn[81], fr[17];
   1154       1.1    nonaka 	struct nvm_identify_controller *identify;
   1155      1.19  jdolecek 	struct nvme_dmamem *mem;
   1156       1.1    nonaka 	struct nvme_ccb *ccb;
   1157       1.1    nonaka 	u_int mdts;
   1158      1.19  jdolecek 	int rv = 1;
   1159       1.1    nonaka 
   1160       1.1    nonaka 	ccb = nvme_ccb_get(sc->sc_admin_q);
   1161      1.11  jdolecek 	KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
   1162       1.1    nonaka 
   1163      1.19  jdolecek 	mem = nvme_dmamem_alloc(sc, sizeof(*identify));
   1164      1.19  jdolecek 	if (mem == NULL)
   1165      1.19  jdolecek 		return 1;
   1166       1.1    nonaka 
   1167       1.1    nonaka 	ccb->ccb_done = nvme_empty_done;
   1168      1.19  jdolecek 	ccb->ccb_cookie = mem;
   1169       1.1    nonaka 
   1170       1.1    nonaka 	nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD);
   1171      1.19  jdolecek 	rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify,
   1172       1.7  jdolecek 	    NVME_TIMO_IDENT);
   1173       1.1    nonaka 	nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD);
   1174       1.1    nonaka 
   1175       1.1    nonaka 	nvme_ccb_put(sc->sc_admin_q, ccb);
   1176       1.1    nonaka 
   1177      1.19  jdolecek 	if (rv != 0)
   1178       1.1    nonaka 		goto done;
   1179       1.1    nonaka 
   1180       1.1    nonaka 	identify = NVME_DMA_KVA(mem);
   1181       1.1    nonaka 
   1182       1.2  christos 	strnvisx(sn, sizeof(sn), (const char *)identify->sn,
   1183       1.2  christos 	    sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
   1184       1.2  christos 	strnvisx(mn, sizeof(mn), (const char *)identify->mn,
   1185       1.2  christos 	    sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
   1186       1.2  christos 	strnvisx(fr, sizeof(fr), (const char *)identify->fr,
   1187       1.2  christos 	    sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
   1188       1.1    nonaka 	aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr,
   1189       1.1    nonaka 	    sn);
   1190       1.1    nonaka 
   1191       1.1    nonaka 	if (identify->mdts > 0) {
   1192       1.1    nonaka 		mdts = (1 << identify->mdts) * (1 << mps);
   1193       1.1    nonaka 		if (mdts < sc->sc_mdts)
   1194       1.1    nonaka 			sc->sc_mdts = mdts;
   1195       1.1    nonaka 	}
   1196       1.1    nonaka 
   1197       1.1    nonaka 	sc->sc_nn = lemtoh32(&identify->nn);
   1198       1.1    nonaka 
   1199       1.1    nonaka 	memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify));
   1200       1.1    nonaka 
   1201       1.1    nonaka done:
   1202      1.19  jdolecek 	nvme_dmamem_free(sc, mem);
   1203       1.1    nonaka 
   1204      1.19  jdolecek 	return rv;
   1205       1.1    nonaka }
   1206       1.1    nonaka 
   1207       1.1    nonaka static int
   1208       1.1    nonaka nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q)
   1209       1.1    nonaka {
   1210       1.1    nonaka 	struct nvme_sqe_q sqe;
   1211       1.1    nonaka 	struct nvme_ccb *ccb;
   1212       1.1    nonaka 	int rv;
   1213       1.1    nonaka 
   1214       1.9  jdolecek 	if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0)
   1215       1.1    nonaka 		return 1;
   1216       1.1    nonaka 
   1217       1.1    nonaka 	ccb = nvme_ccb_get(sc->sc_admin_q);
   1218       1.1    nonaka 	KASSERT(ccb != NULL);
   1219       1.1    nonaka 
   1220       1.1    nonaka 	ccb->ccb_done = nvme_empty_done;
   1221       1.1    nonaka 	ccb->ccb_cookie = &sqe;
   1222       1.1    nonaka 
   1223       1.1    nonaka 	memset(&sqe, 0, sizeof(sqe));
   1224       1.1    nonaka 	sqe.opcode = NVM_ADMIN_ADD_IOCQ;
   1225       1.1    nonaka 	htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem));
   1226       1.1    nonaka 	htolem16(&sqe.qsize, q->q_entries - 1);
   1227       1.1    nonaka 	htolem16(&sqe.qid, q->q_id);
   1228       1.1    nonaka 	sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC;
   1229       1.1    nonaka 	if (sc->sc_use_mq)
   1230       1.1    nonaka 		htolem16(&sqe.cqid, q->q_id);	/* qid == vector */
   1231       1.1    nonaka 
   1232       1.7  jdolecek 	rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
   1233       1.1    nonaka 	if (rv != 0)
   1234       1.1    nonaka 		goto fail;
   1235       1.1    nonaka 
   1236       1.1    nonaka 	ccb->ccb_done = nvme_empty_done;
   1237       1.1    nonaka 	ccb->ccb_cookie = &sqe;
   1238       1.1    nonaka 
   1239       1.1    nonaka 	memset(&sqe, 0, sizeof(sqe));
   1240       1.1    nonaka 	sqe.opcode = NVM_ADMIN_ADD_IOSQ;
   1241       1.1    nonaka 	htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem));
   1242       1.1    nonaka 	htolem16(&sqe.qsize, q->q_entries - 1);
   1243       1.1    nonaka 	htolem16(&sqe.qid, q->q_id);
   1244       1.1    nonaka 	htolem16(&sqe.cqid, q->q_id);
   1245       1.1    nonaka 	sqe.qflags = NVM_SQE_Q_PC;
   1246       1.1    nonaka 
   1247       1.7  jdolecek 	rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
   1248       1.1    nonaka 	if (rv != 0)
   1249       1.1    nonaka 		goto fail;
   1250       1.1    nonaka 
   1251       1.1    nonaka fail:
   1252       1.1    nonaka 	nvme_ccb_put(sc->sc_admin_q, ccb);
   1253       1.1    nonaka 	return rv;
   1254       1.1    nonaka }
   1255       1.1    nonaka 
   1256       1.1    nonaka static int
   1257       1.1    nonaka nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q)
   1258       1.1    nonaka {
   1259       1.1    nonaka 	struct nvme_sqe_q sqe;
   1260       1.1    nonaka 	struct nvme_ccb *ccb;
   1261       1.1    nonaka 	int rv;
   1262       1.1    nonaka 
   1263       1.1    nonaka 	ccb = nvme_ccb_get(sc->sc_admin_q);
   1264       1.1    nonaka 	KASSERT(ccb != NULL);
   1265       1.1    nonaka 
   1266       1.1    nonaka 	ccb->ccb_done = nvme_empty_done;
   1267       1.1    nonaka 	ccb->ccb_cookie = &sqe;
   1268       1.1    nonaka 
   1269       1.1    nonaka 	memset(&sqe, 0, sizeof(sqe));
   1270       1.1    nonaka 	sqe.opcode = NVM_ADMIN_DEL_IOSQ;
   1271       1.1    nonaka 	htolem16(&sqe.qid, q->q_id);
   1272       1.1    nonaka 
   1273       1.7  jdolecek 	rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
   1274       1.1    nonaka 	if (rv != 0)
   1275       1.1    nonaka 		goto fail;
   1276       1.1    nonaka 
   1277       1.1    nonaka 	ccb->ccb_done = nvme_empty_done;
   1278       1.1    nonaka 	ccb->ccb_cookie = &sqe;
   1279       1.1    nonaka 
   1280       1.1    nonaka 	memset(&sqe, 0, sizeof(sqe));
   1281       1.1    nonaka 	sqe.opcode = NVM_ADMIN_DEL_IOCQ;
   1282       1.1    nonaka 	htolem16(&sqe.qid, q->q_id);
   1283       1.1    nonaka 
   1284       1.7  jdolecek 	rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
   1285       1.1    nonaka 	if (rv != 0)
   1286       1.1    nonaka 		goto fail;
   1287       1.1    nonaka 
   1288       1.1    nonaka fail:
   1289       1.1    nonaka 	nvme_ccb_put(sc->sc_admin_q, ccb);
   1290       1.1    nonaka 
   1291       1.1    nonaka 	if (rv == 0 && sc->sc_use_mq) {
   1292       1.1    nonaka 		if (sc->sc_intr_disestablish(sc, q->q_id))
   1293       1.1    nonaka 			rv = 1;
   1294       1.1    nonaka 	}
   1295       1.1    nonaka 
   1296       1.1    nonaka 	return rv;
   1297       1.1    nonaka }
   1298       1.1    nonaka 
   1299       1.1    nonaka static void
   1300       1.1    nonaka nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
   1301       1.1    nonaka {
   1302       1.1    nonaka 	struct nvme_sqe *sqe = slot;
   1303       1.1    nonaka 	struct nvme_dmamem *mem = ccb->ccb_cookie;
   1304       1.1    nonaka 
   1305       1.1    nonaka 	sqe->opcode = NVM_ADMIN_IDENTIFY;
   1306      1.19  jdolecek 	htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem));
   1307       1.1    nonaka 	htolem32(&sqe->cdw10, 1);
   1308       1.1    nonaka }
   1309       1.1    nonaka 
   1310       1.1    nonaka static int
   1311      1.23    nonaka nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap)
   1312      1.23    nonaka {
   1313      1.23    nonaka 	struct nvme_pt_command pt;
   1314      1.23    nonaka 	struct nvme_ccb *ccb;
   1315      1.23    nonaka 	uint16_t ncqa, nsqa;
   1316      1.23    nonaka 	int rv;
   1317      1.23    nonaka 
   1318      1.23    nonaka 	ccb = nvme_ccb_get(sc->sc_admin_q);
   1319      1.23    nonaka 	KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
   1320      1.23    nonaka 
   1321      1.23    nonaka 	memset(&pt, 0, sizeof(pt));
   1322      1.23    nonaka 	pt.cmd.opcode = NVM_ADMIN_GET_FEATURES;
   1323      1.25  jdolecek 	pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES;
   1324      1.23    nonaka 
   1325      1.23    nonaka 	ccb->ccb_done = nvme_pt_done;
   1326      1.23    nonaka 	ccb->ccb_cookie = &pt;
   1327      1.23    nonaka 
   1328      1.23    nonaka 	rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP);
   1329      1.23    nonaka 
   1330      1.23    nonaka 	nvme_ccb_put(sc->sc_admin_q, ccb);
   1331      1.23    nonaka 
   1332      1.23    nonaka 	if (rv != 0) {
   1333      1.23    nonaka 		*nqap = 0;
   1334      1.23    nonaka 		return EIO;
   1335      1.23    nonaka 	}
   1336      1.23    nonaka 
   1337      1.23    nonaka 	ncqa = pt.cpl.cdw0 >> 16;
   1338      1.23    nonaka 	nsqa = pt.cpl.cdw0 & 0xffff;
   1339      1.23    nonaka 	*nqap = MIN(ncqa, nsqa) + 1;
   1340      1.23    nonaka 
   1341      1.23    nonaka 	return 0;
   1342      1.23    nonaka }
   1343      1.23    nonaka 
   1344      1.23    nonaka static int
   1345      1.20  jdolecek nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs)
   1346       1.1    nonaka {
   1347       1.1    nonaka 	struct nvme_softc *sc = q->q_sc;
   1348       1.1    nonaka 	struct nvme_ccb *ccb;
   1349       1.1    nonaka 	bus_addr_t off;
   1350       1.1    nonaka 	uint64_t *prpl;
   1351       1.1    nonaka 	u_int i;
   1352       1.1    nonaka 
   1353       1.1    nonaka 	mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO);
   1354       1.1    nonaka 	SIMPLEQ_INIT(&q->q_ccb_list);
   1355       1.1    nonaka 
   1356       1.1    nonaka 	q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP);
   1357       1.1    nonaka 
   1358       1.1    nonaka 	q->q_nccbs = nccbs;
   1359      1.19  jdolecek 	q->q_ccb_prpls = nvme_dmamem_alloc(sc,
   1360      1.19  jdolecek 	    sizeof(*prpl) * sc->sc_max_sgl * nccbs);
   1361       1.1    nonaka 
   1362       1.1    nonaka 	prpl = NVME_DMA_KVA(q->q_ccb_prpls);
   1363       1.1    nonaka 	off = 0;
   1364       1.1    nonaka 
   1365       1.1    nonaka 	for (i = 0; i < nccbs; i++) {
   1366       1.1    nonaka 		ccb = &q->q_ccbs[i];
   1367       1.1    nonaka 
   1368       1.1    nonaka 		if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts,
   1369       1.1    nonaka 		    sc->sc_max_sgl + 1 /* we get a free prp in the sqe */,
   1370       1.1    nonaka 		    sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
   1371       1.1    nonaka 		    &ccb->ccb_dmamap) != 0)
   1372       1.1    nonaka 			goto free_maps;
   1373       1.1    nonaka 
   1374       1.1    nonaka 		ccb->ccb_id = i;
   1375       1.1    nonaka 		ccb->ccb_prpl = prpl;
   1376       1.1    nonaka 		ccb->ccb_prpl_off = off;
   1377       1.1    nonaka 		ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off;
   1378       1.1    nonaka 
   1379       1.1    nonaka 		SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry);
   1380       1.1    nonaka 
   1381       1.1    nonaka 		prpl += sc->sc_max_sgl;
   1382       1.1    nonaka 		off += sizeof(*prpl) * sc->sc_max_sgl;
   1383       1.1    nonaka 	}
   1384       1.1    nonaka 
   1385       1.1    nonaka 	return 0;
   1386       1.1    nonaka 
   1387       1.1    nonaka free_maps:
   1388       1.1    nonaka 	nvme_ccbs_free(q);
   1389       1.1    nonaka 	return 1;
   1390       1.1    nonaka }
   1391       1.1    nonaka 
   1392       1.1    nonaka static struct nvme_ccb *
   1393       1.1    nonaka nvme_ccb_get(struct nvme_queue *q)
   1394       1.1    nonaka {
   1395      1.20  jdolecek 	struct nvme_ccb *ccb = NULL;
   1396       1.1    nonaka 
   1397       1.1    nonaka 	mutex_enter(&q->q_ccb_mtx);
   1398  1.30.2.1    martin 	ccb = SIMPLEQ_FIRST(&q->q_ccb_list);
   1399  1.30.2.1    martin 	if (ccb != NULL) {
   1400       1.1    nonaka 		SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry);
   1401      1.18  jdolecek #ifdef DEBUG
   1402      1.18  jdolecek 		ccb->ccb_cookie = NULL;
   1403      1.18  jdolecek #endif
   1404      1.18  jdolecek 	}
   1405       1.1    nonaka 	mutex_exit(&q->q_ccb_mtx);
   1406       1.1    nonaka 
   1407       1.1    nonaka 	return ccb;
   1408       1.1    nonaka }
   1409       1.1    nonaka 
   1410       1.1    nonaka static void
   1411       1.1    nonaka nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb)
   1412       1.1    nonaka {
   1413       1.1    nonaka 
   1414       1.1    nonaka 	mutex_enter(&q->q_ccb_mtx);
   1415      1.18  jdolecek #ifdef DEBUG
   1416      1.18  jdolecek 	ccb->ccb_cookie = (void *)NVME_CCB_FREE;
   1417      1.18  jdolecek #endif
   1418       1.1    nonaka 	SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry);
   1419       1.1    nonaka 	mutex_exit(&q->q_ccb_mtx);
   1420       1.1    nonaka }
   1421       1.1    nonaka 
   1422       1.1    nonaka static void
   1423       1.1    nonaka nvme_ccbs_free(struct nvme_queue *q)
   1424       1.1    nonaka {
   1425       1.1    nonaka 	struct nvme_softc *sc = q->q_sc;
   1426       1.1    nonaka 	struct nvme_ccb *ccb;
   1427       1.1    nonaka 
   1428       1.1    nonaka 	mutex_enter(&q->q_ccb_mtx);
   1429       1.1    nonaka 	while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) {
   1430       1.1    nonaka 		SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry);
   1431       1.1    nonaka 		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
   1432       1.1    nonaka 	}
   1433       1.1    nonaka 	mutex_exit(&q->q_ccb_mtx);
   1434       1.1    nonaka 
   1435      1.19  jdolecek 	nvme_dmamem_free(sc, q->q_ccb_prpls);
   1436       1.1    nonaka 	kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs);
   1437       1.1    nonaka 	q->q_ccbs = NULL;
   1438       1.1    nonaka 	mutex_destroy(&q->q_ccb_mtx);
   1439       1.1    nonaka }
   1440       1.1    nonaka 
   1441       1.1    nonaka static struct nvme_queue *
   1442       1.1    nonaka nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd)
   1443       1.1    nonaka {
   1444       1.1    nonaka 	struct nvme_queue *q;
   1445       1.1    nonaka 
   1446       1.1    nonaka 	q = kmem_alloc(sizeof(*q), KM_SLEEP);
   1447       1.1    nonaka 	q->q_sc = sc;
   1448      1.19  jdolecek 	q->q_sq_dmamem = nvme_dmamem_alloc(sc,
   1449      1.19  jdolecek 	    sizeof(struct nvme_sqe) * entries);
   1450      1.19  jdolecek 	if (q->q_sq_dmamem == NULL)
   1451       1.1    nonaka 		goto free;
   1452       1.1    nonaka 
   1453      1.19  jdolecek 	q->q_cq_dmamem = nvme_dmamem_alloc(sc,
   1454      1.19  jdolecek 	    sizeof(struct nvme_cqe) * entries);
   1455      1.19  jdolecek 	if (q->q_cq_dmamem == NULL)
   1456       1.1    nonaka 		goto free_sq;
   1457       1.1    nonaka 
   1458       1.1    nonaka 	memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem));
   1459       1.1    nonaka 	memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem));
   1460       1.1    nonaka 
   1461       1.1    nonaka 	mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO);
   1462       1.1    nonaka 	mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO);
   1463       1.1    nonaka 	q->q_sqtdbl = NVME_SQTDBL(id, dstrd);
   1464       1.1    nonaka 	q->q_cqhdbl = NVME_CQHDBL(id, dstrd);
   1465       1.1    nonaka 	q->q_id = id;
   1466       1.1    nonaka 	q->q_entries = entries;
   1467       1.1    nonaka 	q->q_sq_tail = 0;
   1468       1.1    nonaka 	q->q_cq_head = 0;
   1469       1.1    nonaka 	q->q_cq_phase = NVME_CQE_PHASE;
   1470       1.1    nonaka 
   1471       1.1    nonaka 	nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE);
   1472       1.1    nonaka 	nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD);
   1473       1.1    nonaka 
   1474      1.20  jdolecek 	/*
   1475      1.20  jdolecek 	 * Due to definition of full and empty queue (queue is empty
   1476      1.20  jdolecek 	 * when head == tail, full when tail is one less then head),
   1477      1.20  jdolecek 	 * we can actually only have (entries - 1) in-flight commands.
   1478      1.20  jdolecek 	 */
   1479      1.20  jdolecek 	if (nvme_ccbs_alloc(q, entries - 1) != 0) {
   1480       1.1    nonaka 		aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n");
   1481       1.1    nonaka 		goto free_cq;
   1482       1.1    nonaka 	}
   1483       1.1    nonaka 
   1484       1.1    nonaka 	return q;
   1485       1.1    nonaka 
   1486       1.1    nonaka free_cq:
   1487      1.19  jdolecek 	nvme_dmamem_free(sc, q->q_cq_dmamem);
   1488       1.1    nonaka free_sq:
   1489      1.19  jdolecek 	nvme_dmamem_free(sc, q->q_sq_dmamem);
   1490       1.1    nonaka free:
   1491       1.1    nonaka 	kmem_free(q, sizeof(*q));
   1492       1.1    nonaka 
   1493       1.1    nonaka 	return NULL;
   1494       1.1    nonaka }
   1495       1.1    nonaka 
   1496       1.1    nonaka static void
   1497       1.1    nonaka nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q)
   1498       1.1    nonaka {
   1499       1.1    nonaka 	nvme_ccbs_free(q);
   1500       1.9  jdolecek 	mutex_destroy(&q->q_sq_mtx);
   1501       1.9  jdolecek 	mutex_destroy(&q->q_cq_mtx);
   1502       1.1    nonaka 	nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD);
   1503       1.1    nonaka 	nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE);
   1504      1.19  jdolecek 	nvme_dmamem_free(sc, q->q_cq_dmamem);
   1505      1.19  jdolecek 	nvme_dmamem_free(sc, q->q_sq_dmamem);
   1506       1.1    nonaka 	kmem_free(q, sizeof(*q));
   1507       1.1    nonaka }
   1508       1.1    nonaka 
   1509       1.1    nonaka int
   1510       1.1    nonaka nvme_intr(void *xsc)
   1511       1.1    nonaka {
   1512       1.1    nonaka 	struct nvme_softc *sc = xsc;
   1513       1.1    nonaka 
   1514      1.10  jdolecek 	/*
   1515      1.10  jdolecek 	 * INTx is level triggered, controller deasserts the interrupt only
   1516      1.10  jdolecek 	 * when we advance command queue head via write to the doorbell.
   1517      1.17  jdolecek 	 * Tell the controller to block the interrupts while we process
   1518      1.17  jdolecek 	 * the queue(s).
   1519      1.10  jdolecek 	 */
   1520      1.17  jdolecek 	nvme_write4(sc, NVME_INTMS, 1);
   1521      1.17  jdolecek 
   1522      1.17  jdolecek 	softint_schedule(sc->sc_softih[0]);
   1523      1.17  jdolecek 
   1524      1.17  jdolecek 	/* don't know, might not have been for us */
   1525      1.17  jdolecek 	return 1;
   1526      1.17  jdolecek }
   1527      1.17  jdolecek 
   1528      1.17  jdolecek void
   1529      1.17  jdolecek nvme_softintr_intx(void *xq)
   1530      1.17  jdolecek {
   1531      1.17  jdolecek 	struct nvme_queue *q = xq;
   1532      1.17  jdolecek 	struct nvme_softc *sc = q->q_sc;
   1533      1.17  jdolecek 
   1534      1.17  jdolecek 	nvme_q_complete(sc, sc->sc_admin_q);
   1535       1.1    nonaka 	if (sc->sc_q != NULL)
   1536      1.17  jdolecek 	        nvme_q_complete(sc, sc->sc_q[0]);
   1537       1.1    nonaka 
   1538      1.17  jdolecek 	/*
   1539      1.17  jdolecek 	 * Processing done, tell controller to issue interrupts again. There
   1540      1.17  jdolecek 	 * is no race, as NVMe spec requires the controller to maintain state,
   1541      1.17  jdolecek 	 * and assert the interrupt whenever there are unacknowledged
   1542      1.17  jdolecek 	 * completion queue entries.
   1543      1.17  jdolecek 	 */
   1544      1.17  jdolecek 	nvme_write4(sc, NVME_INTMC, 1);
   1545       1.1    nonaka }
   1546       1.1    nonaka 
   1547       1.1    nonaka int
   1548       1.9  jdolecek nvme_intr_msi(void *xq)
   1549       1.1    nonaka {
   1550       1.1    nonaka 	struct nvme_queue *q = xq;
   1551       1.1    nonaka 
   1552       1.9  jdolecek 	KASSERT(q && q->q_sc && q->q_sc->sc_softih
   1553       1.9  jdolecek 	    && q->q_sc->sc_softih[q->q_id]);
   1554       1.1    nonaka 
   1555      1.17  jdolecek 	/*
   1556      1.17  jdolecek 	 * MSI/MSI-X are edge triggered, so can handover processing to softint
   1557      1.17  jdolecek 	 * without masking the interrupt.
   1558      1.17  jdolecek 	 */
   1559       1.9  jdolecek 	softint_schedule(q->q_sc->sc_softih[q->q_id]);
   1560       1.1    nonaka 
   1561       1.9  jdolecek 	return 1;
   1562       1.1    nonaka }
   1563       1.1    nonaka 
   1564       1.9  jdolecek void
   1565       1.9  jdolecek nvme_softintr_msi(void *xq)
   1566       1.1    nonaka {
   1567       1.1    nonaka 	struct nvme_queue *q = xq;
   1568       1.9  jdolecek 	struct nvme_softc *sc = q->q_sc;
   1569       1.1    nonaka 
   1570       1.9  jdolecek 	nvme_q_complete(sc, q);
   1571       1.1    nonaka }
   1572       1.1    nonaka 
   1573      1.19  jdolecek static struct nvme_dmamem *
   1574      1.19  jdolecek nvme_dmamem_alloc(struct nvme_softc *sc, size_t size)
   1575       1.1    nonaka {
   1576      1.19  jdolecek 	struct nvme_dmamem *ndm;
   1577       1.1    nonaka 	int nsegs;
   1578       1.1    nonaka 
   1579      1.19  jdolecek 	ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP);
   1580      1.19  jdolecek 	if (ndm == NULL)
   1581      1.19  jdolecek 		return NULL;
   1582      1.19  jdolecek 
   1583       1.1    nonaka 	ndm->ndm_size = size;
   1584       1.1    nonaka 
   1585       1.1    nonaka 	if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
   1586       1.1    nonaka 	    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0)
   1587       1.1    nonaka 		goto ndmfree;
   1588       1.1    nonaka 
   1589       1.1    nonaka 	if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg,
   1590       1.1    nonaka 	    1, &nsegs, BUS_DMA_WAITOK) != 0)
   1591       1.1    nonaka 		goto destroy;
   1592       1.1    nonaka 
   1593       1.1    nonaka 	if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size,
   1594       1.1    nonaka 	    &ndm->ndm_kva, BUS_DMA_WAITOK) != 0)
   1595       1.1    nonaka 		goto free;
   1596       1.1    nonaka 	memset(ndm->ndm_kva, 0, size);
   1597       1.1    nonaka 
   1598       1.1    nonaka 	if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size,
   1599       1.1    nonaka 	    NULL, BUS_DMA_WAITOK) != 0)
   1600       1.1    nonaka 		goto unmap;
   1601       1.1    nonaka 
   1602      1.19  jdolecek 	return ndm;
   1603       1.1    nonaka 
   1604       1.1    nonaka unmap:
   1605       1.1    nonaka 	bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size);
   1606       1.1    nonaka free:
   1607       1.1    nonaka 	bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1);
   1608       1.1    nonaka destroy:
   1609       1.1    nonaka 	bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map);
   1610       1.1    nonaka ndmfree:
   1611      1.19  jdolecek 	kmem_free(ndm, sizeof(*ndm));
   1612      1.19  jdolecek 	return NULL;
   1613      1.19  jdolecek }
   1614      1.19  jdolecek 
   1615      1.19  jdolecek static void
   1616      1.19  jdolecek nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops)
   1617      1.19  jdolecek {
   1618      1.19  jdolecek 	bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem),
   1619      1.19  jdolecek 	    0, NVME_DMA_LEN(mem), ops);
   1620       1.1    nonaka }
   1621       1.1    nonaka 
   1622       1.1    nonaka void
   1623       1.1    nonaka nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm)
   1624       1.1    nonaka {
   1625       1.1    nonaka 	bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map);
   1626       1.1    nonaka 	bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size);
   1627       1.1    nonaka 	bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1);
   1628       1.1    nonaka 	bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map);
   1629      1.19  jdolecek 	kmem_free(ndm, sizeof(*ndm));
   1630       1.1    nonaka }
   1631       1.3    nonaka 
   1632       1.3    nonaka /*
   1633       1.3    nonaka  * ioctl
   1634       1.3    nonaka  */
   1635       1.3    nonaka 
   1636       1.3    nonaka dev_type_open(nvmeopen);
   1637       1.3    nonaka dev_type_close(nvmeclose);
   1638       1.3    nonaka dev_type_ioctl(nvmeioctl);
   1639       1.3    nonaka 
   1640       1.3    nonaka const struct cdevsw nvme_cdevsw = {
   1641       1.3    nonaka 	.d_open = nvmeopen,
   1642       1.3    nonaka 	.d_close = nvmeclose,
   1643       1.3    nonaka 	.d_read = noread,
   1644       1.3    nonaka 	.d_write = nowrite,
   1645       1.3    nonaka 	.d_ioctl = nvmeioctl,
   1646       1.3    nonaka 	.d_stop = nostop,
   1647       1.3    nonaka 	.d_tty = notty,
   1648       1.3    nonaka 	.d_poll = nopoll,
   1649       1.3    nonaka 	.d_mmap = nommap,
   1650       1.3    nonaka 	.d_kqfilter = nokqfilter,
   1651       1.3    nonaka 	.d_discard = nodiscard,
   1652       1.3    nonaka 	.d_flag = D_OTHER,
   1653       1.3    nonaka };
   1654       1.3    nonaka 
   1655       1.3    nonaka extern struct cfdriver nvme_cd;
   1656       1.3    nonaka 
   1657       1.3    nonaka /*
   1658       1.3    nonaka  * Accept an open operation on the control device.
   1659       1.3    nonaka  */
   1660       1.3    nonaka int
   1661       1.3    nonaka nvmeopen(dev_t dev, int flag, int mode, struct lwp *l)
   1662       1.3    nonaka {
   1663       1.3    nonaka 	struct nvme_softc *sc;
   1664       1.3    nonaka 	int unit = minor(dev) / 0x10000;
   1665       1.3    nonaka 	int nsid = minor(dev) & 0xffff;
   1666       1.3    nonaka 	int nsidx;
   1667       1.3    nonaka 
   1668       1.3    nonaka 	if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL)
   1669       1.3    nonaka 		return ENXIO;
   1670       1.3    nonaka 	if ((sc->sc_flags & NVME_F_ATTACHED) == 0)
   1671       1.3    nonaka 		return ENXIO;
   1672       1.3    nonaka 
   1673       1.5    nonaka 	if (nsid == 0) {
   1674       1.5    nonaka 		/* controller */
   1675       1.5    nonaka 		if (ISSET(sc->sc_flags, NVME_F_OPEN))
   1676       1.5    nonaka 			return EBUSY;
   1677       1.5    nonaka 		SET(sc->sc_flags, NVME_F_OPEN);
   1678       1.5    nonaka 	} else {
   1679       1.5    nonaka 		/* namespace */
   1680       1.5    nonaka 		nsidx = nsid - 1;
   1681       1.5    nonaka 		if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL)
   1682       1.5    nonaka 			return ENXIO;
   1683       1.5    nonaka 		if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN))
   1684       1.5    nonaka 			return EBUSY;
   1685       1.5    nonaka 		SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN);
   1686       1.5    nonaka 	}
   1687       1.3    nonaka 	return 0;
   1688       1.3    nonaka }
   1689       1.3    nonaka 
   1690       1.3    nonaka /*
   1691       1.3    nonaka  * Accept the last close on the control device.
   1692       1.3    nonaka  */
   1693       1.3    nonaka int
   1694       1.5    nonaka nvmeclose(dev_t dev, int flag, int mode, struct lwp *l)
   1695       1.3    nonaka {
   1696       1.3    nonaka 	struct nvme_softc *sc;
   1697       1.3    nonaka 	int unit = minor(dev) / 0x10000;
   1698       1.3    nonaka 	int nsid = minor(dev) & 0xffff;
   1699       1.3    nonaka 	int nsidx;
   1700       1.3    nonaka 
   1701       1.3    nonaka 	sc = device_lookup_private(&nvme_cd, unit);
   1702       1.3    nonaka 	if (sc == NULL)
   1703       1.3    nonaka 		return ENXIO;
   1704       1.3    nonaka 
   1705       1.5    nonaka 	if (nsid == 0) {
   1706       1.5    nonaka 		/* controller */
   1707       1.5    nonaka 		CLR(sc->sc_flags, NVME_F_OPEN);
   1708       1.5    nonaka 	} else {
   1709       1.5    nonaka 		/* namespace */
   1710       1.5    nonaka 		nsidx = nsid - 1;
   1711       1.5    nonaka 		if (nsidx >= sc->sc_nn)
   1712       1.5    nonaka 			return ENXIO;
   1713       1.5    nonaka 		CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN);
   1714       1.5    nonaka 	}
   1715       1.3    nonaka 
   1716       1.3    nonaka 	return 0;
   1717       1.3    nonaka }
   1718       1.3    nonaka 
   1719       1.3    nonaka /*
   1720       1.3    nonaka  * Handle control operations.
   1721       1.3    nonaka  */
   1722       1.3    nonaka int
   1723       1.5    nonaka nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
   1724       1.3    nonaka {
   1725       1.3    nonaka 	struct nvme_softc *sc;
   1726       1.3    nonaka 	int unit = minor(dev) / 0x10000;
   1727       1.3    nonaka 	int nsid = minor(dev) & 0xffff;
   1728       1.5    nonaka 	struct nvme_pt_command *pt;
   1729       1.3    nonaka 
   1730       1.3    nonaka 	sc = device_lookup_private(&nvme_cd, unit);
   1731       1.3    nonaka 	if (sc == NULL)
   1732       1.3    nonaka 		return ENXIO;
   1733       1.3    nonaka 
   1734       1.3    nonaka 	switch (cmd) {
   1735       1.3    nonaka 	case NVME_PASSTHROUGH_CMD:
   1736       1.5    nonaka 		pt = data;
   1737       1.5    nonaka 		return nvme_command_passthrough(sc, data,
   1738       1.5    nonaka 		    nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0);
   1739       1.3    nonaka 	}
   1740       1.3    nonaka 
   1741       1.3    nonaka 	return ENOTTY;
   1742       1.3    nonaka }
   1743