nvme.c revision 1.2.2.5
1 1.2.2.5 skrll /* $NetBSD: nvme.c,v 1.2.2.5 2016/12/05 10:55:01 skrll Exp $ */ 2 1.2.2.2 skrll /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 1.2.2.2 skrll 4 1.2.2.2 skrll /* 5 1.2.2.2 skrll * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 1.2.2.2 skrll * 7 1.2.2.2 skrll * Permission to use, copy, modify, and distribute this software for any 8 1.2.2.2 skrll * purpose with or without fee is hereby granted, provided that the above 9 1.2.2.2 skrll * copyright notice and this permission notice appear in all copies. 10 1.2.2.2 skrll * 11 1.2.2.2 skrll * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 1.2.2.2 skrll * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 1.2.2.2 skrll * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 1.2.2.2 skrll * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 1.2.2.2 skrll * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 1.2.2.2 skrll * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 1.2.2.2 skrll * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 1.2.2.2 skrll */ 19 1.2.2.2 skrll 20 1.2.2.2 skrll #include <sys/cdefs.h> 21 1.2.2.5 skrll __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.2.2.5 2016/12/05 10:55:01 skrll Exp $"); 22 1.2.2.2 skrll 23 1.2.2.2 skrll #include <sys/param.h> 24 1.2.2.2 skrll #include <sys/systm.h> 25 1.2.2.2 skrll #include <sys/kernel.h> 26 1.2.2.2 skrll #include <sys/atomic.h> 27 1.2.2.2 skrll #include <sys/bus.h> 28 1.2.2.2 skrll #include <sys/buf.h> 29 1.2.2.3 skrll #include <sys/conf.h> 30 1.2.2.2 skrll #include <sys/device.h> 31 1.2.2.2 skrll #include <sys/kmem.h> 32 1.2.2.2 skrll #include <sys/once.h> 33 1.2.2.3 skrll #include <sys/proc.h> 34 1.2.2.2 skrll #include <sys/queue.h> 35 1.2.2.2 skrll #include <sys/mutex.h> 36 1.2.2.2 skrll 37 1.2.2.3 skrll #include <uvm/uvm_extern.h> 38 1.2.2.3 skrll 39 1.2.2.2 skrll #include <dev/ic/nvmereg.h> 40 1.2.2.2 skrll #include <dev/ic/nvmevar.h> 41 1.2.2.3 skrll #include <dev/ic/nvmeio.h> 42 1.2.2.2 skrll 43 1.2.2.5 skrll int nvme_adminq_size = 32; 44 1.2.2.4 skrll int nvme_ioq_size = 1024; 45 1.2.2.2 skrll 46 1.2.2.2 skrll static int nvme_print(void *, const char *); 47 1.2.2.2 skrll 48 1.2.2.2 skrll static int nvme_ready(struct nvme_softc *, uint32_t); 49 1.2.2.2 skrll static int nvme_enable(struct nvme_softc *, u_int); 50 1.2.2.2 skrll static int nvme_disable(struct nvme_softc *); 51 1.2.2.2 skrll static int nvme_shutdown(struct nvme_softc *); 52 1.2.2.2 skrll 53 1.2.2.2 skrll static void nvme_version(struct nvme_softc *, uint32_t); 54 1.2.2.2 skrll #ifdef NVME_DEBUG 55 1.2.2.2 skrll static void nvme_dumpregs(struct nvme_softc *); 56 1.2.2.2 skrll #endif 57 1.2.2.2 skrll static int nvme_identify(struct nvme_softc *, u_int); 58 1.2.2.2 skrll static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 59 1.2.2.2 skrll void *); 60 1.2.2.2 skrll 61 1.2.2.5 skrll static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 62 1.2.2.2 skrll static void nvme_ccbs_free(struct nvme_queue *); 63 1.2.2.2 skrll 64 1.2.2.2 skrll static struct nvme_ccb * 65 1.2.2.2 skrll nvme_ccb_get(struct nvme_queue *); 66 1.2.2.2 skrll static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 67 1.2.2.2 skrll 68 1.2.2.2 skrll static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 69 1.2.2.2 skrll struct nvme_ccb *, void (*)(struct nvme_queue *, 70 1.2.2.4 skrll struct nvme_ccb *, void *), int); 71 1.2.2.2 skrll static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 72 1.2.2.2 skrll static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 73 1.2.2.2 skrll struct nvme_cqe *); 74 1.2.2.2 skrll static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 1.2.2.2 skrll static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 76 1.2.2.2 skrll struct nvme_cqe *); 77 1.2.2.2 skrll 78 1.2.2.2 skrll static struct nvme_queue * 79 1.2.2.2 skrll nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 80 1.2.2.2 skrll static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 81 1.2.2.2 skrll static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 82 1.2.2.2 skrll static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 83 1.2.2.2 skrll struct nvme_ccb *, void (*)(struct nvme_queue *, 84 1.2.2.2 skrll struct nvme_ccb *, void *)); 85 1.2.2.2 skrll static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 86 1.2.2.2 skrll static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 87 1.2.2.2 skrll 88 1.2.2.5 skrll static struct nvme_dmamem * 89 1.2.2.5 skrll nvme_dmamem_alloc(struct nvme_softc *, size_t); 90 1.2.2.2 skrll static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 91 1.2.2.5 skrll static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 92 1.2.2.5 skrll int); 93 1.2.2.2 skrll 94 1.2.2.2 skrll static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 95 1.2.2.2 skrll void *); 96 1.2.2.2 skrll static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 97 1.2.2.2 skrll struct nvme_cqe *); 98 1.2.2.2 skrll static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 99 1.2.2.2 skrll void *); 100 1.2.2.2 skrll static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 101 1.2.2.2 skrll struct nvme_cqe *); 102 1.2.2.2 skrll 103 1.2.2.3 skrll static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 104 1.2.2.3 skrll void *); 105 1.2.2.3 skrll static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 106 1.2.2.3 skrll struct nvme_cqe *); 107 1.2.2.3 skrll static int nvme_command_passthrough(struct nvme_softc *, 108 1.2.2.3 skrll struct nvme_pt_command *, uint16_t, struct lwp *, bool); 109 1.2.2.3 skrll 110 1.2.2.4 skrll #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 111 1.2.2.4 skrll #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 112 1.2.2.4 skrll #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 113 1.2.2.4 skrll #define NVME_TIMO_SY 60 /* sync cache timeout */ 114 1.2.2.4 skrll 115 1.2.2.2 skrll #define nvme_read4(_s, _r) \ 116 1.2.2.2 skrll bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 117 1.2.2.2 skrll #define nvme_write4(_s, _r, _v) \ 118 1.2.2.2 skrll bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 119 1.2.2.2 skrll #ifdef __LP64__ 120 1.2.2.2 skrll #define nvme_read8(_s, _r) \ 121 1.2.2.2 skrll bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r)) 122 1.2.2.2 skrll #define nvme_write8(_s, _r, _v) \ 123 1.2.2.2 skrll bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 124 1.2.2.2 skrll #else /* __LP64__ */ 125 1.2.2.2 skrll static inline uint64_t 126 1.2.2.2 skrll nvme_read8(struct nvme_softc *sc, bus_size_t r) 127 1.2.2.2 skrll { 128 1.2.2.2 skrll uint64_t v; 129 1.2.2.2 skrll uint32_t *a = (uint32_t *)&v; 130 1.2.2.2 skrll 131 1.2.2.2 skrll #if _BYTE_ORDER == _LITTLE_ENDIAN 132 1.2.2.2 skrll a[0] = nvme_read4(sc, r); 133 1.2.2.2 skrll a[1] = nvme_read4(sc, r + 4); 134 1.2.2.2 skrll #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 135 1.2.2.2 skrll a[1] = nvme_read4(sc, r); 136 1.2.2.2 skrll a[0] = nvme_read4(sc, r + 4); 137 1.2.2.2 skrll #endif 138 1.2.2.2 skrll 139 1.2.2.2 skrll return v; 140 1.2.2.2 skrll } 141 1.2.2.2 skrll 142 1.2.2.2 skrll static inline void 143 1.2.2.2 skrll nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 144 1.2.2.2 skrll { 145 1.2.2.2 skrll uint32_t *a = (uint32_t *)&v; 146 1.2.2.2 skrll 147 1.2.2.2 skrll #if _BYTE_ORDER == _LITTLE_ENDIAN 148 1.2.2.2 skrll nvme_write4(sc, r, a[0]); 149 1.2.2.2 skrll nvme_write4(sc, r + 4, a[1]); 150 1.2.2.2 skrll #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 151 1.2.2.2 skrll nvme_write4(sc, r, a[1]); 152 1.2.2.2 skrll nvme_write4(sc, r + 4, a[0]); 153 1.2.2.2 skrll #endif 154 1.2.2.2 skrll } 155 1.2.2.2 skrll #endif /* __LP64__ */ 156 1.2.2.2 skrll #define nvme_barrier(_s, _r, _l, _f) \ 157 1.2.2.2 skrll bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 158 1.2.2.2 skrll 159 1.2.2.2 skrll static void 160 1.2.2.2 skrll nvme_version(struct nvme_softc *sc, uint32_t ver) 161 1.2.2.2 skrll { 162 1.2.2.2 skrll const char *v = NULL; 163 1.2.2.2 skrll 164 1.2.2.2 skrll switch (ver) { 165 1.2.2.2 skrll case NVME_VS_1_0: 166 1.2.2.2 skrll v = "1.0"; 167 1.2.2.2 skrll break; 168 1.2.2.2 skrll case NVME_VS_1_1: 169 1.2.2.2 skrll v = "1.1"; 170 1.2.2.2 skrll break; 171 1.2.2.2 skrll case NVME_VS_1_2: 172 1.2.2.2 skrll v = "1.2"; 173 1.2.2.2 skrll break; 174 1.2.2.2 skrll default: 175 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unknown version 0x%08x\n", ver); 176 1.2.2.2 skrll return; 177 1.2.2.2 skrll } 178 1.2.2.2 skrll 179 1.2.2.2 skrll aprint_normal_dev(sc->sc_dev, "NVMe %s\n", v); 180 1.2.2.2 skrll } 181 1.2.2.2 skrll 182 1.2.2.2 skrll #ifdef NVME_DEBUG 183 1.2.2.4 skrll static __used void 184 1.2.2.2 skrll nvme_dumpregs(struct nvme_softc *sc) 185 1.2.2.2 skrll { 186 1.2.2.2 skrll uint64_t r8; 187 1.2.2.2 skrll uint32_t r4; 188 1.2.2.2 skrll 189 1.2.2.2 skrll #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 190 1.2.2.2 skrll r8 = nvme_read8(sc, NVME_CAP); 191 1.2.2.4 skrll printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 192 1.2.2.2 skrll printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 193 1.2.2.2 skrll (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 194 1.2.2.2 skrll printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 195 1.2.2.2 skrll (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 196 1.2.2.4 skrll printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 197 1.2.2.4 skrll printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 198 1.2.2.4 skrll printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 199 1.2.2.4 skrll printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 200 1.2.2.4 skrll printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 201 1.2.2.4 skrll printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 202 1.2.2.4 skrll printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 203 1.2.2.2 skrll 204 1.2.2.2 skrll printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 205 1.2.2.2 skrll 206 1.2.2.2 skrll r4 = nvme_read4(sc, NVME_CC); 207 1.2.2.2 skrll printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 208 1.2.2.4 skrll printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 209 1.2.2.4 skrll (1 << NVME_CC_IOCQES_R(r4))); 210 1.2.2.4 skrll printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 211 1.2.2.4 skrll (1 << NVME_CC_IOSQES_R(r4))); 212 1.2.2.2 skrll printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 213 1.2.2.2 skrll printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 214 1.2.2.4 skrll printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 215 1.2.2.4 skrll (1 << NVME_CC_MPS_R(r4))); 216 1.2.2.2 skrll printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 217 1.2.2.4 skrll printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 218 1.2.2.2 skrll 219 1.2.2.4 skrll r4 = nvme_read4(sc, NVME_CSTS); 220 1.2.2.4 skrll printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 221 1.2.2.4 skrll printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 222 1.2.2.4 skrll printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 223 1.2.2.4 skrll printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 224 1.2.2.4 skrll 225 1.2.2.4 skrll r4 = nvme_read4(sc, NVME_AQA); 226 1.2.2.4 skrll printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 227 1.2.2.4 skrll printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 228 1.2.2.4 skrll printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 229 1.2.2.4 skrll 230 1.2.2.4 skrll printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 231 1.2.2.4 skrll printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 232 1.2.2.2 skrll #undef DEVNAME 233 1.2.2.2 skrll } 234 1.2.2.2 skrll #endif /* NVME_DEBUG */ 235 1.2.2.2 skrll 236 1.2.2.2 skrll static int 237 1.2.2.2 skrll nvme_ready(struct nvme_softc *sc, uint32_t rdy) 238 1.2.2.2 skrll { 239 1.2.2.2 skrll u_int i = 0; 240 1.2.2.4 skrll uint32_t cc; 241 1.2.2.4 skrll 242 1.2.2.4 skrll cc = nvme_read4(sc, NVME_CC); 243 1.2.2.4 skrll if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) { 244 1.2.2.4 skrll aprint_error_dev(sc->sc_dev, 245 1.2.2.4 skrll "controller enabled status expected %d, found to be %d\n", 246 1.2.2.4 skrll (rdy != 0), ((cc & NVME_CC_EN) != 0)); 247 1.2.2.4 skrll return ENXIO; 248 1.2.2.4 skrll } 249 1.2.2.2 skrll 250 1.2.2.2 skrll while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 251 1.2.2.2 skrll if (i++ > sc->sc_rdy_to) 252 1.2.2.4 skrll return ENXIO; 253 1.2.2.2 skrll 254 1.2.2.2 skrll delay(1000); 255 1.2.2.2 skrll nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 256 1.2.2.2 skrll } 257 1.2.2.2 skrll 258 1.2.2.2 skrll return 0; 259 1.2.2.2 skrll } 260 1.2.2.2 skrll 261 1.2.2.2 skrll static int 262 1.2.2.2 skrll nvme_enable(struct nvme_softc *sc, u_int mps) 263 1.2.2.2 skrll { 264 1.2.2.4 skrll uint32_t cc, csts; 265 1.2.2.2 skrll 266 1.2.2.2 skrll cc = nvme_read4(sc, NVME_CC); 267 1.2.2.4 skrll csts = nvme_read4(sc, NVME_CSTS); 268 1.2.2.2 skrll 269 1.2.2.4 skrll if (ISSET(cc, NVME_CC_EN)) { 270 1.2.2.4 skrll aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 271 1.2.2.4 skrll 272 1.2.2.4 skrll if (ISSET(csts, NVME_CSTS_RDY)) 273 1.2.2.4 skrll return 1; 274 1.2.2.4 skrll 275 1.2.2.4 skrll goto waitready; 276 1.2.2.4 skrll } 277 1.2.2.2 skrll 278 1.2.2.2 skrll nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 279 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 280 1.2.2.4 skrll delay(5000); 281 1.2.2.2 skrll nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 282 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 283 1.2.2.4 skrll delay(5000); 284 1.2.2.4 skrll 285 1.2.2.4 skrll nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 286 1.2.2.4 skrll NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 287 1.2.2.4 skrll nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 288 1.2.2.4 skrll delay(5000); 289 1.2.2.2 skrll 290 1.2.2.2 skrll CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 291 1.2.2.2 skrll NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 292 1.2.2.2 skrll SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 293 1.2.2.2 skrll SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 294 1.2.2.2 skrll SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 295 1.2.2.2 skrll SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 296 1.2.2.2 skrll SET(cc, NVME_CC_MPS(mps)); 297 1.2.2.2 skrll SET(cc, NVME_CC_EN); 298 1.2.2.2 skrll 299 1.2.2.2 skrll nvme_write4(sc, NVME_CC, cc); 300 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, 301 1.2.2.2 skrll BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 302 1.2.2.4 skrll delay(5000); 303 1.2.2.2 skrll 304 1.2.2.4 skrll waitready: 305 1.2.2.2 skrll return nvme_ready(sc, NVME_CSTS_RDY); 306 1.2.2.2 skrll } 307 1.2.2.2 skrll 308 1.2.2.2 skrll static int 309 1.2.2.2 skrll nvme_disable(struct nvme_softc *sc) 310 1.2.2.2 skrll { 311 1.2.2.2 skrll uint32_t cc, csts; 312 1.2.2.2 skrll 313 1.2.2.2 skrll cc = nvme_read4(sc, NVME_CC); 314 1.2.2.4 skrll csts = nvme_read4(sc, NVME_CSTS); 315 1.2.2.4 skrll 316 1.2.2.4 skrll if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY)) 317 1.2.2.4 skrll nvme_ready(sc, NVME_CSTS_RDY); 318 1.2.2.2 skrll 319 1.2.2.2 skrll CLR(cc, NVME_CC_EN); 320 1.2.2.2 skrll 321 1.2.2.2 skrll nvme_write4(sc, NVME_CC, cc); 322 1.2.2.4 skrll nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 323 1.2.2.4 skrll 324 1.2.2.4 skrll delay(5000); 325 1.2.2.2 skrll 326 1.2.2.2 skrll return nvme_ready(sc, 0); 327 1.2.2.2 skrll } 328 1.2.2.2 skrll 329 1.2.2.2 skrll int 330 1.2.2.2 skrll nvme_attach(struct nvme_softc *sc) 331 1.2.2.2 skrll { 332 1.2.2.2 skrll uint64_t cap; 333 1.2.2.2 skrll uint32_t reg; 334 1.2.2.2 skrll u_int dstrd; 335 1.2.2.2 skrll u_int mps = PAGE_SHIFT; 336 1.2.2.5 skrll uint16_t adminq_entries = nvme_adminq_size; 337 1.2.2.5 skrll uint16_t ioq_entries = nvme_ioq_size; 338 1.2.2.2 skrll int i; 339 1.2.2.2 skrll 340 1.2.2.2 skrll reg = nvme_read4(sc, NVME_VS); 341 1.2.2.2 skrll if (reg == 0xffffffff) { 342 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 343 1.2.2.2 skrll return 1; 344 1.2.2.2 skrll } 345 1.2.2.2 skrll 346 1.2.2.2 skrll nvme_version(sc, reg); 347 1.2.2.2 skrll 348 1.2.2.2 skrll cap = nvme_read8(sc, NVME_CAP); 349 1.2.2.2 skrll dstrd = NVME_CAP_DSTRD(cap); 350 1.2.2.2 skrll if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 351 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 352 1.2.2.2 skrll "is greater than CPU page size %u\n", 353 1.2.2.2 skrll 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 354 1.2.2.2 skrll return 1; 355 1.2.2.2 skrll } 356 1.2.2.2 skrll if (NVME_CAP_MPSMAX(cap) < mps) 357 1.2.2.2 skrll mps = NVME_CAP_MPSMAX(cap); 358 1.2.2.5 skrll if (ioq_entries > NVME_CAP_MQES(cap)) 359 1.2.2.5 skrll ioq_entries = NVME_CAP_MQES(cap); 360 1.2.2.2 skrll 361 1.2.2.4 skrll /* set initial values to be used for admin queue during probe */ 362 1.2.2.2 skrll sc->sc_rdy_to = NVME_CAP_TO(cap); 363 1.2.2.2 skrll sc->sc_mps = 1 << mps; 364 1.2.2.2 skrll sc->sc_mdts = MAXPHYS; 365 1.2.2.2 skrll sc->sc_max_sgl = 2; 366 1.2.2.2 skrll 367 1.2.2.2 skrll if (nvme_disable(sc) != 0) { 368 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 369 1.2.2.2 skrll return 1; 370 1.2.2.2 skrll } 371 1.2.2.2 skrll 372 1.2.2.2 skrll sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 373 1.2.2.2 skrll if (sc->sc_admin_q == NULL) { 374 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, 375 1.2.2.2 skrll "unable to allocate admin queue\n"); 376 1.2.2.2 skrll return 1; 377 1.2.2.2 skrll } 378 1.2.2.2 skrll if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 379 1.2.2.2 skrll goto free_admin_q; 380 1.2.2.2 skrll 381 1.2.2.2 skrll if (nvme_enable(sc, mps) != 0) { 382 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 383 1.2.2.2 skrll goto disestablish_admin_q; 384 1.2.2.2 skrll } 385 1.2.2.2 skrll 386 1.2.2.2 skrll if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 387 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 388 1.2.2.2 skrll goto disable; 389 1.2.2.2 skrll } 390 1.2.2.2 skrll 391 1.2.2.2 skrll /* we know how big things are now */ 392 1.2.2.2 skrll sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 393 1.2.2.2 skrll 394 1.2.2.2 skrll /* reallocate ccbs of admin queue with new max sgl. */ 395 1.2.2.2 skrll nvme_ccbs_free(sc->sc_admin_q); 396 1.2.2.2 skrll nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 397 1.2.2.2 skrll 398 1.2.2.2 skrll sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 399 1.2.2.2 skrll if (sc->sc_q == NULL) { 400 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n"); 401 1.2.2.2 skrll goto disable; 402 1.2.2.2 skrll } 403 1.2.2.2 skrll for (i = 0; i < sc->sc_nq; i++) { 404 1.2.2.2 skrll sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 405 1.2.2.2 skrll if (sc->sc_q[i] == NULL) { 406 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, 407 1.2.2.2 skrll "unable to allocate io queue\n"); 408 1.2.2.2 skrll goto free_q; 409 1.2.2.2 skrll } 410 1.2.2.2 skrll if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 411 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, 412 1.2.2.2 skrll "unable to create io queue\n"); 413 1.2.2.2 skrll nvme_q_free(sc, sc->sc_q[i]); 414 1.2.2.2 skrll goto free_q; 415 1.2.2.2 skrll } 416 1.2.2.2 skrll } 417 1.2.2.2 skrll 418 1.2.2.2 skrll if (!sc->sc_use_mq) 419 1.2.2.2 skrll nvme_write4(sc, NVME_INTMC, 1); 420 1.2.2.2 skrll 421 1.2.2.4 skrll /* probe subdevices */ 422 1.2.2.2 skrll sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 423 1.2.2.2 skrll KM_SLEEP); 424 1.2.2.4 skrll if (sc->sc_namespaces == NULL) 425 1.2.2.4 skrll goto free_q; 426 1.2.2.4 skrll nvme_rescan(sc->sc_dev, "nvme", &i); 427 1.2.2.2 skrll 428 1.2.2.2 skrll return 0; 429 1.2.2.2 skrll 430 1.2.2.2 skrll free_q: 431 1.2.2.2 skrll while (--i >= 0) { 432 1.2.2.2 skrll nvme_q_delete(sc, sc->sc_q[i]); 433 1.2.2.2 skrll nvme_q_free(sc, sc->sc_q[i]); 434 1.2.2.2 skrll } 435 1.2.2.2 skrll disable: 436 1.2.2.2 skrll nvme_disable(sc); 437 1.2.2.2 skrll disestablish_admin_q: 438 1.2.2.2 skrll sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 439 1.2.2.2 skrll free_admin_q: 440 1.2.2.2 skrll nvme_q_free(sc, sc->sc_admin_q); 441 1.2.2.2 skrll 442 1.2.2.2 skrll return 1; 443 1.2.2.2 skrll } 444 1.2.2.2 skrll 445 1.2.2.4 skrll int 446 1.2.2.4 skrll nvme_rescan(device_t self, const char *attr, const int *flags) 447 1.2.2.4 skrll { 448 1.2.2.4 skrll struct nvme_softc *sc = device_private(self); 449 1.2.2.4 skrll struct nvme_attach_args naa; 450 1.2.2.5 skrll uint64_t cap; 451 1.2.2.5 skrll int ioq_entries = nvme_ioq_size; 452 1.2.2.5 skrll int i; 453 1.2.2.5 skrll 454 1.2.2.5 skrll cap = nvme_read8(sc, NVME_CAP); 455 1.2.2.5 skrll if (ioq_entries > NVME_CAP_MQES(cap)) 456 1.2.2.5 skrll ioq_entries = NVME_CAP_MQES(cap); 457 1.2.2.4 skrll 458 1.2.2.4 skrll for (i = 0; i < sc->sc_nn; i++) { 459 1.2.2.4 skrll if (sc->sc_namespaces[i].dev) 460 1.2.2.4 skrll continue; 461 1.2.2.4 skrll memset(&naa, 0, sizeof(naa)); 462 1.2.2.4 skrll naa.naa_nsid = i + 1; 463 1.2.2.5 skrll naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 464 1.2.2.5 skrll naa.naa_maxphys = sc->sc_mdts; 465 1.2.2.4 skrll sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 466 1.2.2.4 skrll nvme_print); 467 1.2.2.4 skrll } 468 1.2.2.4 skrll return 0; 469 1.2.2.4 skrll } 470 1.2.2.4 skrll 471 1.2.2.2 skrll static int 472 1.2.2.2 skrll nvme_print(void *aux, const char *pnp) 473 1.2.2.2 skrll { 474 1.2.2.2 skrll struct nvme_attach_args *naa = aux; 475 1.2.2.2 skrll 476 1.2.2.2 skrll if (pnp) 477 1.2.2.2 skrll aprint_normal("at %s", pnp); 478 1.2.2.2 skrll 479 1.2.2.2 skrll if (naa->naa_nsid > 0) 480 1.2.2.2 skrll aprint_normal(" nsid %d", naa->naa_nsid); 481 1.2.2.2 skrll 482 1.2.2.2 skrll return UNCONF; 483 1.2.2.2 skrll } 484 1.2.2.2 skrll 485 1.2.2.2 skrll int 486 1.2.2.2 skrll nvme_detach(struct nvme_softc *sc, int flags) 487 1.2.2.2 skrll { 488 1.2.2.2 skrll int i, error; 489 1.2.2.2 skrll 490 1.2.2.2 skrll error = config_detach_children(sc->sc_dev, flags); 491 1.2.2.2 skrll if (error) 492 1.2.2.2 skrll return error; 493 1.2.2.2 skrll 494 1.2.2.2 skrll error = nvme_shutdown(sc); 495 1.2.2.2 skrll if (error) 496 1.2.2.2 skrll return error; 497 1.2.2.2 skrll 498 1.2.2.4 skrll /* from now on we are committed to detach, following will never fail */ 499 1.2.2.2 skrll for (i = 0; i < sc->sc_nq; i++) 500 1.2.2.2 skrll nvme_q_free(sc, sc->sc_q[i]); 501 1.2.2.2 skrll kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 502 1.2.2.2 skrll nvme_q_free(sc, sc->sc_admin_q); 503 1.2.2.2 skrll 504 1.2.2.2 skrll return 0; 505 1.2.2.2 skrll } 506 1.2.2.2 skrll 507 1.2.2.2 skrll static int 508 1.2.2.2 skrll nvme_shutdown(struct nvme_softc *sc) 509 1.2.2.2 skrll { 510 1.2.2.2 skrll uint32_t cc, csts; 511 1.2.2.2 skrll bool disabled = false; 512 1.2.2.2 skrll int i; 513 1.2.2.2 skrll 514 1.2.2.2 skrll if (!sc->sc_use_mq) 515 1.2.2.2 skrll nvme_write4(sc, NVME_INTMS, 1); 516 1.2.2.2 skrll 517 1.2.2.2 skrll for (i = 0; i < sc->sc_nq; i++) { 518 1.2.2.2 skrll if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 519 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, 520 1.2.2.2 skrll "unable to delete io queue %d, disabling\n", i + 1); 521 1.2.2.2 skrll disabled = true; 522 1.2.2.2 skrll } 523 1.2.2.2 skrll } 524 1.2.2.2 skrll sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 525 1.2.2.2 skrll if (disabled) 526 1.2.2.2 skrll goto disable; 527 1.2.2.2 skrll 528 1.2.2.2 skrll cc = nvme_read4(sc, NVME_CC); 529 1.2.2.2 skrll CLR(cc, NVME_CC_SHN_MASK); 530 1.2.2.2 skrll SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 531 1.2.2.2 skrll nvme_write4(sc, NVME_CC, cc); 532 1.2.2.2 skrll 533 1.2.2.2 skrll for (i = 0; i < 4000; i++) { 534 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, 535 1.2.2.2 skrll BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 536 1.2.2.2 skrll csts = nvme_read4(sc, NVME_CSTS); 537 1.2.2.2 skrll if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 538 1.2.2.2 skrll return 0; 539 1.2.2.2 skrll 540 1.2.2.2 skrll delay(1000); 541 1.2.2.2 skrll } 542 1.2.2.2 skrll 543 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 544 1.2.2.2 skrll 545 1.2.2.2 skrll disable: 546 1.2.2.2 skrll nvme_disable(sc); 547 1.2.2.2 skrll return 0; 548 1.2.2.2 skrll } 549 1.2.2.2 skrll 550 1.2.2.2 skrll void 551 1.2.2.2 skrll nvme_childdet(device_t self, device_t child) 552 1.2.2.2 skrll { 553 1.2.2.2 skrll struct nvme_softc *sc = device_private(self); 554 1.2.2.2 skrll int i; 555 1.2.2.2 skrll 556 1.2.2.2 skrll for (i = 0; i < sc->sc_nn; i++) { 557 1.2.2.2 skrll if (sc->sc_namespaces[i].dev == child) { 558 1.2.2.2 skrll /* Already freed ns->ident. */ 559 1.2.2.2 skrll sc->sc_namespaces[i].dev = NULL; 560 1.2.2.2 skrll break; 561 1.2.2.2 skrll } 562 1.2.2.2 skrll } 563 1.2.2.2 skrll } 564 1.2.2.2 skrll 565 1.2.2.2 skrll int 566 1.2.2.2 skrll nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 567 1.2.2.2 skrll { 568 1.2.2.2 skrll struct nvme_sqe sqe; 569 1.2.2.2 skrll struct nvm_identify_namespace *identify; 570 1.2.2.5 skrll struct nvme_dmamem *mem; 571 1.2.2.2 skrll struct nvme_ccb *ccb; 572 1.2.2.2 skrll struct nvme_namespace *ns; 573 1.2.2.5 skrll int rv; 574 1.2.2.2 skrll 575 1.2.2.2 skrll KASSERT(nsid > 0); 576 1.2.2.2 skrll 577 1.2.2.2 skrll ccb = nvme_ccb_get(sc->sc_admin_q); 578 1.2.2.4 skrll KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 579 1.2.2.2 skrll 580 1.2.2.5 skrll mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 581 1.2.2.5 skrll if (mem == NULL) 582 1.2.2.5 skrll return ENOMEM; 583 1.2.2.2 skrll 584 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 585 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_IDENTIFY; 586 1.2.2.2 skrll htolem32(&sqe.nsid, nsid); 587 1.2.2.2 skrll htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 588 1.2.2.2 skrll htolem32(&sqe.cdw10, 0); 589 1.2.2.2 skrll 590 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 591 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 592 1.2.2.2 skrll 593 1.2.2.2 skrll nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 594 1.2.2.5 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 595 1.2.2.2 skrll nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 596 1.2.2.2 skrll 597 1.2.2.2 skrll nvme_ccb_put(sc->sc_admin_q, ccb); 598 1.2.2.2 skrll 599 1.2.2.5 skrll if (rv != 0) { 600 1.2.2.5 skrll rv = EIO; 601 1.2.2.2 skrll goto done; 602 1.2.2.2 skrll } 603 1.2.2.2 skrll 604 1.2.2.2 skrll /* commit */ 605 1.2.2.2 skrll 606 1.2.2.2 skrll identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 607 1.2.2.5 skrll *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 608 1.2.2.5 skrll //memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 609 1.2.2.2 skrll 610 1.2.2.2 skrll ns = nvme_ns_get(sc, nsid); 611 1.2.2.2 skrll KASSERT(ns); 612 1.2.2.2 skrll ns->ident = identify; 613 1.2.2.2 skrll 614 1.2.2.2 skrll done: 615 1.2.2.5 skrll nvme_dmamem_free(sc, mem); 616 1.2.2.2 skrll 617 1.2.2.5 skrll return rv; 618 1.2.2.2 skrll } 619 1.2.2.2 skrll 620 1.2.2.2 skrll int 621 1.2.2.4 skrll nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 622 1.2.2.4 skrll struct buf *bp, void *data, size_t datasize, 623 1.2.2.4 skrll int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 624 1.2.2.2 skrll { 625 1.2.2.2 skrll struct nvme_queue *q = nvme_get_q(sc); 626 1.2.2.2 skrll struct nvme_ccb *ccb; 627 1.2.2.2 skrll bus_dmamap_t dmap; 628 1.2.2.2 skrll int i, error; 629 1.2.2.2 skrll 630 1.2.2.2 skrll ccb = nvme_ccb_get(q); 631 1.2.2.2 skrll if (ccb == NULL) 632 1.2.2.2 skrll return EAGAIN; 633 1.2.2.2 skrll 634 1.2.2.2 skrll ccb->ccb_done = nvme_ns_io_done; 635 1.2.2.4 skrll ccb->ccb_cookie = cookie; 636 1.2.2.4 skrll 637 1.2.2.4 skrll /* namespace context */ 638 1.2.2.4 skrll ccb->nnc_nsid = nsid; 639 1.2.2.4 skrll ccb->nnc_flags = flags; 640 1.2.2.4 skrll ccb->nnc_buf = bp; 641 1.2.2.4 skrll ccb->nnc_datasize = datasize; 642 1.2.2.4 skrll ccb->nnc_secsize = secsize; 643 1.2.2.4 skrll ccb->nnc_blkno = blkno; 644 1.2.2.4 skrll ccb->nnc_done = nnc_done; 645 1.2.2.2 skrll 646 1.2.2.2 skrll dmap = ccb->ccb_dmamap; 647 1.2.2.4 skrll error = bus_dmamap_load(sc->sc_dmat, dmap, data, 648 1.2.2.4 skrll datasize, NULL, 649 1.2.2.4 skrll (ISSET(flags, NVME_NS_CTX_F_POLL) ? 650 1.2.2.2 skrll BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 651 1.2.2.4 skrll (ISSET(flags, NVME_NS_CTX_F_READ) ? 652 1.2.2.2 skrll BUS_DMA_READ : BUS_DMA_WRITE)); 653 1.2.2.2 skrll if (error) { 654 1.2.2.2 skrll nvme_ccb_put(q, ccb); 655 1.2.2.2 skrll return error; 656 1.2.2.2 skrll } 657 1.2.2.2 skrll 658 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 659 1.2.2.4 skrll ISSET(flags, NVME_NS_CTX_F_READ) ? 660 1.2.2.2 skrll BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 661 1.2.2.2 skrll 662 1.2.2.2 skrll if (dmap->dm_nsegs > 2) { 663 1.2.2.2 skrll for (i = 1; i < dmap->dm_nsegs; i++) { 664 1.2.2.2 skrll htolem64(&ccb->ccb_prpl[i - 1], 665 1.2.2.2 skrll dmap->dm_segs[i].ds_addr); 666 1.2.2.2 skrll } 667 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, 668 1.2.2.2 skrll NVME_DMA_MAP(q->q_ccb_prpls), 669 1.2.2.2 skrll ccb->ccb_prpl_off, 670 1.2.2.5 skrll sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 671 1.2.2.2 skrll BUS_DMASYNC_PREWRITE); 672 1.2.2.2 skrll } 673 1.2.2.2 skrll 674 1.2.2.4 skrll if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 675 1.2.2.4 skrll if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 676 1.2.2.2 skrll return EIO; 677 1.2.2.2 skrll return 0; 678 1.2.2.2 skrll } 679 1.2.2.2 skrll 680 1.2.2.2 skrll nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 681 1.2.2.2 skrll return 0; 682 1.2.2.2 skrll } 683 1.2.2.2 skrll 684 1.2.2.2 skrll static void 685 1.2.2.2 skrll nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 686 1.2.2.2 skrll { 687 1.2.2.2 skrll struct nvme_sqe_io *sqe = slot; 688 1.2.2.2 skrll bus_dmamap_t dmap = ccb->ccb_dmamap; 689 1.2.2.2 skrll 690 1.2.2.4 skrll sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 691 1.2.2.2 skrll NVM_CMD_READ : NVM_CMD_WRITE; 692 1.2.2.4 skrll htolem32(&sqe->nsid, ccb->nnc_nsid); 693 1.2.2.2 skrll 694 1.2.2.2 skrll htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 695 1.2.2.2 skrll switch (dmap->dm_nsegs) { 696 1.2.2.2 skrll case 1: 697 1.2.2.2 skrll break; 698 1.2.2.2 skrll case 2: 699 1.2.2.2 skrll htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 700 1.2.2.2 skrll break; 701 1.2.2.2 skrll default: 702 1.2.2.2 skrll /* the prp list is already set up and synced */ 703 1.2.2.2 skrll htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 704 1.2.2.2 skrll break; 705 1.2.2.2 skrll } 706 1.2.2.2 skrll 707 1.2.2.4 skrll htolem64(&sqe->slba, ccb->nnc_blkno); 708 1.2.2.4 skrll 709 1.2.2.4 skrll /* guaranteed by upper layers, but check just in case */ 710 1.2.2.4 skrll KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 711 1.2.2.4 skrll htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 712 1.2.2.2 skrll } 713 1.2.2.2 skrll 714 1.2.2.2 skrll static void 715 1.2.2.2 skrll nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 716 1.2.2.2 skrll struct nvme_cqe *cqe) 717 1.2.2.2 skrll { 718 1.2.2.2 skrll struct nvme_softc *sc = q->q_sc; 719 1.2.2.2 skrll bus_dmamap_t dmap = ccb->ccb_dmamap; 720 1.2.2.4 skrll void *nnc_cookie = ccb->ccb_cookie; 721 1.2.2.4 skrll nvme_nnc_done nnc_done = ccb->nnc_done; 722 1.2.2.4 skrll struct buf *bp = ccb->nnc_buf; 723 1.2.2.2 skrll 724 1.2.2.2 skrll if (dmap->dm_nsegs > 2) { 725 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, 726 1.2.2.2 skrll NVME_DMA_MAP(q->q_ccb_prpls), 727 1.2.2.2 skrll ccb->ccb_prpl_off, 728 1.2.2.5 skrll sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 729 1.2.2.2 skrll BUS_DMASYNC_POSTWRITE); 730 1.2.2.2 skrll } 731 1.2.2.2 skrll 732 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 733 1.2.2.4 skrll ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 734 1.2.2.2 skrll BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 735 1.2.2.2 skrll 736 1.2.2.2 skrll bus_dmamap_unload(sc->sc_dmat, dmap); 737 1.2.2.2 skrll nvme_ccb_put(q, ccb); 738 1.2.2.2 skrll 739 1.2.2.4 skrll nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags)); 740 1.2.2.2 skrll } 741 1.2.2.2 skrll 742 1.2.2.2 skrll int 743 1.2.2.4 skrll nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie, 744 1.2.2.4 skrll int flags, nvme_nnc_done nnc_done) 745 1.2.2.2 skrll { 746 1.2.2.2 skrll struct nvme_queue *q = nvme_get_q(sc); 747 1.2.2.2 skrll struct nvme_ccb *ccb; 748 1.2.2.2 skrll 749 1.2.2.2 skrll ccb = nvme_ccb_get(q); 750 1.2.2.2 skrll if (ccb == NULL) 751 1.2.2.2 skrll return EAGAIN; 752 1.2.2.2 skrll 753 1.2.2.2 skrll ccb->ccb_done = nvme_ns_sync_done; 754 1.2.2.4 skrll ccb->ccb_cookie = cookie; 755 1.2.2.4 skrll 756 1.2.2.4 skrll /* namespace context */ 757 1.2.2.4 skrll ccb->nnc_nsid = nsid; 758 1.2.2.4 skrll ccb->nnc_flags = flags; 759 1.2.2.4 skrll ccb->nnc_done = nnc_done; 760 1.2.2.2 skrll 761 1.2.2.4 skrll if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 762 1.2.2.4 skrll if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 763 1.2.2.2 skrll return EIO; 764 1.2.2.2 skrll return 0; 765 1.2.2.2 skrll } 766 1.2.2.2 skrll 767 1.2.2.2 skrll nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 768 1.2.2.2 skrll return 0; 769 1.2.2.2 skrll } 770 1.2.2.2 skrll 771 1.2.2.2 skrll static void 772 1.2.2.2 skrll nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 773 1.2.2.2 skrll { 774 1.2.2.2 skrll struct nvme_sqe *sqe = slot; 775 1.2.2.2 skrll 776 1.2.2.2 skrll sqe->opcode = NVM_CMD_FLUSH; 777 1.2.2.4 skrll htolem32(&sqe->nsid, ccb->nnc_nsid); 778 1.2.2.2 skrll } 779 1.2.2.2 skrll 780 1.2.2.2 skrll static void 781 1.2.2.2 skrll nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 782 1.2.2.2 skrll struct nvme_cqe *cqe) 783 1.2.2.2 skrll { 784 1.2.2.4 skrll void *cookie = ccb->ccb_cookie; 785 1.2.2.4 skrll nvme_nnc_done nnc_done = ccb->nnc_done; 786 1.2.2.2 skrll 787 1.2.2.2 skrll nvme_ccb_put(q, ccb); 788 1.2.2.2 skrll 789 1.2.2.4 skrll nnc_done(cookie, NULL, lemtoh16(&cqe->flags)); 790 1.2.2.2 skrll } 791 1.2.2.2 skrll 792 1.2.2.2 skrll void 793 1.2.2.2 skrll nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 794 1.2.2.2 skrll { 795 1.2.2.2 skrll struct nvme_namespace *ns; 796 1.2.2.2 skrll struct nvm_identify_namespace *identify; 797 1.2.2.2 skrll 798 1.2.2.2 skrll ns = nvme_ns_get(sc, nsid); 799 1.2.2.2 skrll KASSERT(ns); 800 1.2.2.2 skrll 801 1.2.2.2 skrll identify = ns->ident; 802 1.2.2.2 skrll ns->ident = NULL; 803 1.2.2.2 skrll if (identify != NULL) 804 1.2.2.2 skrll kmem_free(identify, sizeof(*identify)); 805 1.2.2.2 skrll } 806 1.2.2.2 skrll 807 1.2.2.2 skrll static void 808 1.2.2.3 skrll nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 809 1.2.2.3 skrll { 810 1.2.2.3 skrll struct nvme_softc *sc = q->q_sc; 811 1.2.2.3 skrll struct nvme_sqe *sqe = slot; 812 1.2.2.3 skrll struct nvme_pt_command *pt = ccb->ccb_cookie; 813 1.2.2.3 skrll bus_dmamap_t dmap = ccb->ccb_dmamap; 814 1.2.2.3 skrll int i; 815 1.2.2.3 skrll 816 1.2.2.3 skrll sqe->opcode = pt->cmd.opcode; 817 1.2.2.3 skrll htolem32(&sqe->nsid, pt->cmd.nsid); 818 1.2.2.3 skrll 819 1.2.2.3 skrll if (pt->buf != NULL && pt->len > 0) { 820 1.2.2.3 skrll htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 821 1.2.2.3 skrll switch (dmap->dm_nsegs) { 822 1.2.2.3 skrll case 1: 823 1.2.2.3 skrll break; 824 1.2.2.3 skrll case 2: 825 1.2.2.3 skrll htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 826 1.2.2.3 skrll break; 827 1.2.2.3 skrll default: 828 1.2.2.3 skrll for (i = 1; i < dmap->dm_nsegs; i++) { 829 1.2.2.3 skrll htolem64(&ccb->ccb_prpl[i - 1], 830 1.2.2.3 skrll dmap->dm_segs[i].ds_addr); 831 1.2.2.3 skrll } 832 1.2.2.3 skrll bus_dmamap_sync(sc->sc_dmat, 833 1.2.2.3 skrll NVME_DMA_MAP(q->q_ccb_prpls), 834 1.2.2.3 skrll ccb->ccb_prpl_off, 835 1.2.2.5 skrll sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 836 1.2.2.3 skrll BUS_DMASYNC_PREWRITE); 837 1.2.2.3 skrll htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 838 1.2.2.3 skrll break; 839 1.2.2.3 skrll } 840 1.2.2.3 skrll } 841 1.2.2.3 skrll 842 1.2.2.3 skrll htolem32(&sqe->cdw10, pt->cmd.cdw10); 843 1.2.2.3 skrll htolem32(&sqe->cdw11, pt->cmd.cdw11); 844 1.2.2.3 skrll htolem32(&sqe->cdw12, pt->cmd.cdw12); 845 1.2.2.3 skrll htolem32(&sqe->cdw13, pt->cmd.cdw13); 846 1.2.2.3 skrll htolem32(&sqe->cdw14, pt->cmd.cdw14); 847 1.2.2.3 skrll htolem32(&sqe->cdw15, pt->cmd.cdw15); 848 1.2.2.3 skrll } 849 1.2.2.3 skrll 850 1.2.2.3 skrll static void 851 1.2.2.3 skrll nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 852 1.2.2.3 skrll { 853 1.2.2.3 skrll struct nvme_softc *sc = q->q_sc; 854 1.2.2.3 skrll struct nvme_pt_command *pt = ccb->ccb_cookie; 855 1.2.2.3 skrll bus_dmamap_t dmap = ccb->ccb_dmamap; 856 1.2.2.3 skrll 857 1.2.2.3 skrll if (pt->buf != NULL && pt->len > 0) { 858 1.2.2.3 skrll if (dmap->dm_nsegs > 2) { 859 1.2.2.3 skrll bus_dmamap_sync(sc->sc_dmat, 860 1.2.2.3 skrll NVME_DMA_MAP(q->q_ccb_prpls), 861 1.2.2.3 skrll ccb->ccb_prpl_off, 862 1.2.2.5 skrll sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 863 1.2.2.3 skrll BUS_DMASYNC_POSTWRITE); 864 1.2.2.3 skrll } 865 1.2.2.3 skrll 866 1.2.2.3 skrll bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 867 1.2.2.3 skrll pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 868 1.2.2.3 skrll bus_dmamap_unload(sc->sc_dmat, dmap); 869 1.2.2.3 skrll } 870 1.2.2.3 skrll 871 1.2.2.3 skrll pt->cpl.cdw0 = cqe->cdw0; 872 1.2.2.3 skrll pt->cpl.flags = cqe->flags & ~NVME_CQE_PHASE; 873 1.2.2.3 skrll } 874 1.2.2.3 skrll 875 1.2.2.3 skrll static int 876 1.2.2.3 skrll nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 877 1.2.2.3 skrll uint16_t nsid, struct lwp *l, bool is_adminq) 878 1.2.2.3 skrll { 879 1.2.2.3 skrll struct nvme_queue *q; 880 1.2.2.3 skrll struct nvme_ccb *ccb; 881 1.2.2.3 skrll void *buf = NULL; 882 1.2.2.3 skrll int error; 883 1.2.2.3 skrll 884 1.2.2.4 skrll /* limit command size to maximum data transfer size */ 885 1.2.2.3 skrll if ((pt->buf == NULL && pt->len > 0) || 886 1.2.2.4 skrll (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 887 1.2.2.3 skrll return EINVAL; 888 1.2.2.3 skrll 889 1.2.2.3 skrll q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 890 1.2.2.3 skrll ccb = nvme_ccb_get(q); 891 1.2.2.3 skrll if (ccb == NULL) 892 1.2.2.3 skrll return EBUSY; 893 1.2.2.3 skrll 894 1.2.2.4 skrll if (pt->buf != NULL) { 895 1.2.2.4 skrll KASSERT(pt->len > 0); 896 1.2.2.3 skrll buf = kmem_alloc(pt->len, KM_SLEEP); 897 1.2.2.3 skrll if (buf == NULL) { 898 1.2.2.3 skrll error = ENOMEM; 899 1.2.2.3 skrll goto ccb_put; 900 1.2.2.3 skrll } 901 1.2.2.3 skrll if (!pt->is_read) { 902 1.2.2.3 skrll error = copyin(pt->buf, buf, pt->len); 903 1.2.2.3 skrll if (error) 904 1.2.2.3 skrll goto kmem_free; 905 1.2.2.3 skrll } 906 1.2.2.3 skrll error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 907 1.2.2.3 skrll pt->len, NULL, 908 1.2.2.3 skrll BUS_DMA_WAITOK | 909 1.2.2.3 skrll (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 910 1.2.2.3 skrll if (error) 911 1.2.2.3 skrll goto kmem_free; 912 1.2.2.3 skrll bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 913 1.2.2.3 skrll 0, ccb->ccb_dmamap->dm_mapsize, 914 1.2.2.3 skrll pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 915 1.2.2.3 skrll } 916 1.2.2.3 skrll 917 1.2.2.3 skrll ccb->ccb_done = nvme_pt_done; 918 1.2.2.3 skrll ccb->ccb_cookie = pt; 919 1.2.2.3 skrll 920 1.2.2.3 skrll pt->cmd.nsid = nsid; 921 1.2.2.4 skrll if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) { 922 1.2.2.3 skrll error = EIO; 923 1.2.2.3 skrll goto out; 924 1.2.2.3 skrll } 925 1.2.2.3 skrll 926 1.2.2.3 skrll error = 0; 927 1.2.2.3 skrll out: 928 1.2.2.3 skrll if (buf != NULL) { 929 1.2.2.3 skrll if (error == 0 && pt->is_read) 930 1.2.2.3 skrll error = copyout(buf, pt->buf, pt->len); 931 1.2.2.3 skrll kmem_free: 932 1.2.2.3 skrll kmem_free(buf, pt->len); 933 1.2.2.3 skrll } 934 1.2.2.3 skrll ccb_put: 935 1.2.2.3 skrll nvme_ccb_put(q, ccb); 936 1.2.2.3 skrll return error; 937 1.2.2.3 skrll } 938 1.2.2.3 skrll 939 1.2.2.3 skrll static void 940 1.2.2.2 skrll nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 941 1.2.2.2 skrll void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 942 1.2.2.2 skrll { 943 1.2.2.2 skrll struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 944 1.2.2.2 skrll uint32_t tail; 945 1.2.2.2 skrll 946 1.2.2.2 skrll mutex_enter(&q->q_sq_mtx); 947 1.2.2.2 skrll tail = q->q_sq_tail; 948 1.2.2.2 skrll if (++q->q_sq_tail >= q->q_entries) 949 1.2.2.2 skrll q->q_sq_tail = 0; 950 1.2.2.2 skrll 951 1.2.2.2 skrll sqe += tail; 952 1.2.2.2 skrll 953 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 954 1.2.2.2 skrll sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 955 1.2.2.2 skrll memset(sqe, 0, sizeof(*sqe)); 956 1.2.2.2 skrll (*fill)(q, ccb, sqe); 957 1.2.2.2 skrll sqe->cid = ccb->ccb_id; 958 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 959 1.2.2.2 skrll sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 960 1.2.2.2 skrll 961 1.2.2.2 skrll nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 962 1.2.2.2 skrll mutex_exit(&q->q_sq_mtx); 963 1.2.2.2 skrll } 964 1.2.2.2 skrll 965 1.2.2.2 skrll struct nvme_poll_state { 966 1.2.2.2 skrll struct nvme_sqe s; 967 1.2.2.2 skrll struct nvme_cqe c; 968 1.2.2.2 skrll }; 969 1.2.2.2 skrll 970 1.2.2.2 skrll static int 971 1.2.2.2 skrll nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 972 1.2.2.4 skrll void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 973 1.2.2.2 skrll { 974 1.2.2.2 skrll struct nvme_poll_state state; 975 1.2.2.2 skrll void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 976 1.2.2.2 skrll void *cookie; 977 1.2.2.2 skrll uint16_t flags; 978 1.2.2.4 skrll int step = 10; 979 1.2.2.4 skrll int maxloop = timo_sec * 1000000 / step; 980 1.2.2.4 skrll int error = 0; 981 1.2.2.2 skrll 982 1.2.2.2 skrll memset(&state, 0, sizeof(state)); 983 1.2.2.2 skrll (*fill)(q, ccb, &state.s); 984 1.2.2.2 skrll 985 1.2.2.2 skrll done = ccb->ccb_done; 986 1.2.2.2 skrll cookie = ccb->ccb_cookie; 987 1.2.2.2 skrll 988 1.2.2.2 skrll ccb->ccb_done = nvme_poll_done; 989 1.2.2.2 skrll ccb->ccb_cookie = &state; 990 1.2.2.2 skrll 991 1.2.2.2 skrll nvme_q_submit(sc, q, ccb, nvme_poll_fill); 992 1.2.2.2 skrll while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 993 1.2.2.2 skrll if (nvme_q_complete(sc, q) == 0) 994 1.2.2.4 skrll delay(step); 995 1.2.2.2 skrll 996 1.2.2.4 skrll if (timo_sec >= 0 && --maxloop <= 0) { 997 1.2.2.4 skrll error = ETIMEDOUT; 998 1.2.2.4 skrll break; 999 1.2.2.4 skrll } 1000 1.2.2.2 skrll } 1001 1.2.2.2 skrll 1002 1.2.2.2 skrll ccb->ccb_cookie = cookie; 1003 1.2.2.2 skrll done(q, ccb, &state.c); 1004 1.2.2.2 skrll 1005 1.2.2.4 skrll if (error == 0) { 1006 1.2.2.4 skrll flags = lemtoh16(&state.c.flags); 1007 1.2.2.4 skrll return flags & ~NVME_CQE_PHASE; 1008 1.2.2.4 skrll } else { 1009 1.2.2.4 skrll return 1; 1010 1.2.2.4 skrll } 1011 1.2.2.2 skrll } 1012 1.2.2.2 skrll 1013 1.2.2.2 skrll static void 1014 1.2.2.2 skrll nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1015 1.2.2.2 skrll { 1016 1.2.2.2 skrll struct nvme_sqe *sqe = slot; 1017 1.2.2.2 skrll struct nvme_poll_state *state = ccb->ccb_cookie; 1018 1.2.2.2 skrll 1019 1.2.2.2 skrll *sqe = state->s; 1020 1.2.2.2 skrll } 1021 1.2.2.2 skrll 1022 1.2.2.2 skrll static void 1023 1.2.2.2 skrll nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1024 1.2.2.2 skrll struct nvme_cqe *cqe) 1025 1.2.2.2 skrll { 1026 1.2.2.2 skrll struct nvme_poll_state *state = ccb->ccb_cookie; 1027 1.2.2.2 skrll 1028 1.2.2.2 skrll SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1029 1.2.2.2 skrll state->c = *cqe; 1030 1.2.2.2 skrll } 1031 1.2.2.2 skrll 1032 1.2.2.2 skrll static void 1033 1.2.2.2 skrll nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1034 1.2.2.2 skrll { 1035 1.2.2.2 skrll struct nvme_sqe *src = ccb->ccb_cookie; 1036 1.2.2.2 skrll struct nvme_sqe *dst = slot; 1037 1.2.2.2 skrll 1038 1.2.2.2 skrll *dst = *src; 1039 1.2.2.2 skrll } 1040 1.2.2.2 skrll 1041 1.2.2.2 skrll static void 1042 1.2.2.2 skrll nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1043 1.2.2.2 skrll struct nvme_cqe *cqe) 1044 1.2.2.2 skrll { 1045 1.2.2.2 skrll } 1046 1.2.2.2 skrll 1047 1.2.2.2 skrll static int 1048 1.2.2.2 skrll nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1049 1.2.2.2 skrll { 1050 1.2.2.2 skrll struct nvme_ccb *ccb; 1051 1.2.2.2 skrll struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1052 1.2.2.2 skrll uint16_t flags; 1053 1.2.2.2 skrll int rv = 0; 1054 1.2.2.2 skrll 1055 1.2.2.4 skrll mutex_enter(&q->q_cq_mtx); 1056 1.2.2.2 skrll 1057 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1058 1.2.2.2 skrll for (;;) { 1059 1.2.2.4 skrll cqe = &ring[q->q_cq_head]; 1060 1.2.2.2 skrll flags = lemtoh16(&cqe->flags); 1061 1.2.2.2 skrll if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1062 1.2.2.2 skrll break; 1063 1.2.2.2 skrll 1064 1.2.2.2 skrll ccb = &q->q_ccbs[cqe->cid]; 1065 1.2.2.2 skrll 1066 1.2.2.4 skrll if (++q->q_cq_head >= q->q_entries) { 1067 1.2.2.4 skrll q->q_cq_head = 0; 1068 1.2.2.2 skrll q->q_cq_phase ^= NVME_CQE_PHASE; 1069 1.2.2.2 skrll } 1070 1.2.2.2 skrll 1071 1.2.2.5 skrll #ifdef DEBUG 1072 1.2.2.5 skrll /* 1073 1.2.2.5 skrll * If we get spurious completion notification, something 1074 1.2.2.5 skrll * is seriously hosed up. Very likely DMA to some random 1075 1.2.2.5 skrll * memory place happened, so just bail out. 1076 1.2.2.5 skrll */ 1077 1.2.2.5 skrll if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1078 1.2.2.5 skrll panic("%s: invalid ccb detected", 1079 1.2.2.5 skrll device_xname(sc->sc_dev)); 1080 1.2.2.5 skrll /* NOTREACHED */ 1081 1.2.2.5 skrll } 1082 1.2.2.5 skrll #endif 1083 1.2.2.5 skrll 1084 1.2.2.5 skrll rv++; 1085 1.2.2.4 skrll 1086 1.2.2.4 skrll /* 1087 1.2.2.4 skrll * Unlock the mutex before calling the ccb_done callback 1088 1.2.2.4 skrll * and re-lock afterwards. The callback triggers lddone() 1089 1.2.2.4 skrll * which schedules another i/o, and also calls nvme_ccb_put(). 1090 1.2.2.4 skrll * Unlock/relock avoids possibility of deadlock. 1091 1.2.2.4 skrll */ 1092 1.2.2.4 skrll mutex_exit(&q->q_cq_mtx); 1093 1.2.2.4 skrll ccb->ccb_done(q, ccb, cqe); 1094 1.2.2.4 skrll mutex_enter(&q->q_cq_mtx); 1095 1.2.2.2 skrll } 1096 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1097 1.2.2.2 skrll 1098 1.2.2.2 skrll if (rv) 1099 1.2.2.4 skrll nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1100 1.2.2.4 skrll 1101 1.2.2.2 skrll mutex_exit(&q->q_cq_mtx); 1102 1.2.2.2 skrll 1103 1.2.2.5 skrll if (rv) { 1104 1.2.2.5 skrll mutex_enter(&q->q_ccb_mtx); 1105 1.2.2.5 skrll q->q_nccbs_avail += rv; 1106 1.2.2.5 skrll mutex_exit(&q->q_ccb_mtx); 1107 1.2.2.5 skrll } 1108 1.2.2.5 skrll 1109 1.2.2.2 skrll return rv; 1110 1.2.2.2 skrll } 1111 1.2.2.2 skrll 1112 1.2.2.2 skrll static int 1113 1.2.2.2 skrll nvme_identify(struct nvme_softc *sc, u_int mps) 1114 1.2.2.2 skrll { 1115 1.2.2.2 skrll char sn[41], mn[81], fr[17]; 1116 1.2.2.2 skrll struct nvm_identify_controller *identify; 1117 1.2.2.5 skrll struct nvme_dmamem *mem; 1118 1.2.2.2 skrll struct nvme_ccb *ccb; 1119 1.2.2.2 skrll u_int mdts; 1120 1.2.2.5 skrll int rv = 1; 1121 1.2.2.2 skrll 1122 1.2.2.2 skrll ccb = nvme_ccb_get(sc->sc_admin_q); 1123 1.2.2.4 skrll KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1124 1.2.2.2 skrll 1125 1.2.2.5 skrll mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1126 1.2.2.5 skrll if (mem == NULL) 1127 1.2.2.5 skrll return 1; 1128 1.2.2.2 skrll 1129 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 1130 1.2.2.5 skrll ccb->ccb_cookie = mem; 1131 1.2.2.2 skrll 1132 1.2.2.2 skrll nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1133 1.2.2.5 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1134 1.2.2.4 skrll NVME_TIMO_IDENT); 1135 1.2.2.2 skrll nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1136 1.2.2.2 skrll 1137 1.2.2.2 skrll nvme_ccb_put(sc->sc_admin_q, ccb); 1138 1.2.2.2 skrll 1139 1.2.2.5 skrll if (rv != 0) 1140 1.2.2.2 skrll goto done; 1141 1.2.2.2 skrll 1142 1.2.2.2 skrll identify = NVME_DMA_KVA(mem); 1143 1.2.2.2 skrll 1144 1.2.2.2 skrll strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1145 1.2.2.2 skrll sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1146 1.2.2.2 skrll strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1147 1.2.2.2 skrll sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1148 1.2.2.2 skrll strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1149 1.2.2.2 skrll sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1150 1.2.2.2 skrll aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1151 1.2.2.2 skrll sn); 1152 1.2.2.2 skrll 1153 1.2.2.2 skrll if (identify->mdts > 0) { 1154 1.2.2.2 skrll mdts = (1 << identify->mdts) * (1 << mps); 1155 1.2.2.2 skrll if (mdts < sc->sc_mdts) 1156 1.2.2.2 skrll sc->sc_mdts = mdts; 1157 1.2.2.2 skrll } 1158 1.2.2.2 skrll 1159 1.2.2.2 skrll sc->sc_nn = lemtoh32(&identify->nn); 1160 1.2.2.2 skrll 1161 1.2.2.2 skrll memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1162 1.2.2.2 skrll 1163 1.2.2.2 skrll done: 1164 1.2.2.5 skrll nvme_dmamem_free(sc, mem); 1165 1.2.2.2 skrll 1166 1.2.2.5 skrll return rv; 1167 1.2.2.2 skrll } 1168 1.2.2.2 skrll 1169 1.2.2.2 skrll static int 1170 1.2.2.2 skrll nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1171 1.2.2.2 skrll { 1172 1.2.2.2 skrll struct nvme_sqe_q sqe; 1173 1.2.2.2 skrll struct nvme_ccb *ccb; 1174 1.2.2.2 skrll int rv; 1175 1.2.2.2 skrll 1176 1.2.2.4 skrll if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1177 1.2.2.2 skrll return 1; 1178 1.2.2.2 skrll 1179 1.2.2.2 skrll ccb = nvme_ccb_get(sc->sc_admin_q); 1180 1.2.2.2 skrll KASSERT(ccb != NULL); 1181 1.2.2.2 skrll 1182 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 1183 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 1184 1.2.2.2 skrll 1185 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 1186 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1187 1.2.2.2 skrll htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1188 1.2.2.2 skrll htolem16(&sqe.qsize, q->q_entries - 1); 1189 1.2.2.2 skrll htolem16(&sqe.qid, q->q_id); 1190 1.2.2.2 skrll sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1191 1.2.2.2 skrll if (sc->sc_use_mq) 1192 1.2.2.2 skrll htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1193 1.2.2.2 skrll 1194 1.2.2.4 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1195 1.2.2.2 skrll if (rv != 0) 1196 1.2.2.2 skrll goto fail; 1197 1.2.2.2 skrll 1198 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 1199 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 1200 1.2.2.2 skrll 1201 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 1202 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1203 1.2.2.2 skrll htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1204 1.2.2.2 skrll htolem16(&sqe.qsize, q->q_entries - 1); 1205 1.2.2.2 skrll htolem16(&sqe.qid, q->q_id); 1206 1.2.2.2 skrll htolem16(&sqe.cqid, q->q_id); 1207 1.2.2.2 skrll sqe.qflags = NVM_SQE_Q_PC; 1208 1.2.2.2 skrll 1209 1.2.2.4 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1210 1.2.2.2 skrll if (rv != 0) 1211 1.2.2.2 skrll goto fail; 1212 1.2.2.2 skrll 1213 1.2.2.2 skrll fail: 1214 1.2.2.2 skrll nvme_ccb_put(sc->sc_admin_q, ccb); 1215 1.2.2.2 skrll return rv; 1216 1.2.2.2 skrll } 1217 1.2.2.2 skrll 1218 1.2.2.2 skrll static int 1219 1.2.2.2 skrll nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1220 1.2.2.2 skrll { 1221 1.2.2.2 skrll struct nvme_sqe_q sqe; 1222 1.2.2.2 skrll struct nvme_ccb *ccb; 1223 1.2.2.2 skrll int rv; 1224 1.2.2.2 skrll 1225 1.2.2.2 skrll ccb = nvme_ccb_get(sc->sc_admin_q); 1226 1.2.2.2 skrll KASSERT(ccb != NULL); 1227 1.2.2.2 skrll 1228 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 1229 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 1230 1.2.2.2 skrll 1231 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 1232 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1233 1.2.2.2 skrll htolem16(&sqe.qid, q->q_id); 1234 1.2.2.2 skrll 1235 1.2.2.4 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1236 1.2.2.2 skrll if (rv != 0) 1237 1.2.2.2 skrll goto fail; 1238 1.2.2.2 skrll 1239 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 1240 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 1241 1.2.2.2 skrll 1242 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 1243 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1244 1.2.2.2 skrll htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1245 1.2.2.2 skrll htolem16(&sqe.qid, q->q_id); 1246 1.2.2.2 skrll 1247 1.2.2.4 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1248 1.2.2.2 skrll if (rv != 0) 1249 1.2.2.2 skrll goto fail; 1250 1.2.2.2 skrll 1251 1.2.2.2 skrll fail: 1252 1.2.2.2 skrll nvme_ccb_put(sc->sc_admin_q, ccb); 1253 1.2.2.2 skrll 1254 1.2.2.2 skrll if (rv == 0 && sc->sc_use_mq) { 1255 1.2.2.2 skrll if (sc->sc_intr_disestablish(sc, q->q_id)) 1256 1.2.2.2 skrll rv = 1; 1257 1.2.2.2 skrll } 1258 1.2.2.2 skrll 1259 1.2.2.2 skrll return rv; 1260 1.2.2.2 skrll } 1261 1.2.2.2 skrll 1262 1.2.2.2 skrll static void 1263 1.2.2.2 skrll nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1264 1.2.2.2 skrll { 1265 1.2.2.2 skrll struct nvme_sqe *sqe = slot; 1266 1.2.2.2 skrll struct nvme_dmamem *mem = ccb->ccb_cookie; 1267 1.2.2.2 skrll 1268 1.2.2.2 skrll sqe->opcode = NVM_ADMIN_IDENTIFY; 1269 1.2.2.5 skrll htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1270 1.2.2.2 skrll htolem32(&sqe->cdw10, 1); 1271 1.2.2.2 skrll } 1272 1.2.2.2 skrll 1273 1.2.2.2 skrll static int 1274 1.2.2.5 skrll nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1275 1.2.2.2 skrll { 1276 1.2.2.2 skrll struct nvme_softc *sc = q->q_sc; 1277 1.2.2.2 skrll struct nvme_ccb *ccb; 1278 1.2.2.2 skrll bus_addr_t off; 1279 1.2.2.2 skrll uint64_t *prpl; 1280 1.2.2.2 skrll u_int i; 1281 1.2.2.2 skrll 1282 1.2.2.2 skrll mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1283 1.2.2.2 skrll SIMPLEQ_INIT(&q->q_ccb_list); 1284 1.2.2.2 skrll 1285 1.2.2.2 skrll q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1286 1.2.2.2 skrll if (q->q_ccbs == NULL) 1287 1.2.2.2 skrll return 1; 1288 1.2.2.2 skrll 1289 1.2.2.2 skrll q->q_nccbs = nccbs; 1290 1.2.2.5 skrll q->q_nccbs_avail = nccbs; 1291 1.2.2.5 skrll q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1292 1.2.2.5 skrll sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1293 1.2.2.2 skrll 1294 1.2.2.2 skrll prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1295 1.2.2.2 skrll off = 0; 1296 1.2.2.2 skrll 1297 1.2.2.2 skrll for (i = 0; i < nccbs; i++) { 1298 1.2.2.2 skrll ccb = &q->q_ccbs[i]; 1299 1.2.2.2 skrll 1300 1.2.2.2 skrll if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1301 1.2.2.2 skrll sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1302 1.2.2.2 skrll sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1303 1.2.2.2 skrll &ccb->ccb_dmamap) != 0) 1304 1.2.2.2 skrll goto free_maps; 1305 1.2.2.2 skrll 1306 1.2.2.2 skrll ccb->ccb_id = i; 1307 1.2.2.2 skrll ccb->ccb_prpl = prpl; 1308 1.2.2.2 skrll ccb->ccb_prpl_off = off; 1309 1.2.2.2 skrll ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1310 1.2.2.2 skrll 1311 1.2.2.2 skrll SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1312 1.2.2.2 skrll 1313 1.2.2.2 skrll prpl += sc->sc_max_sgl; 1314 1.2.2.2 skrll off += sizeof(*prpl) * sc->sc_max_sgl; 1315 1.2.2.2 skrll } 1316 1.2.2.2 skrll 1317 1.2.2.2 skrll return 0; 1318 1.2.2.2 skrll 1319 1.2.2.2 skrll free_maps: 1320 1.2.2.2 skrll nvme_ccbs_free(q); 1321 1.2.2.2 skrll return 1; 1322 1.2.2.2 skrll } 1323 1.2.2.2 skrll 1324 1.2.2.2 skrll static struct nvme_ccb * 1325 1.2.2.2 skrll nvme_ccb_get(struct nvme_queue *q) 1326 1.2.2.2 skrll { 1327 1.2.2.5 skrll struct nvme_ccb *ccb = NULL; 1328 1.2.2.2 skrll 1329 1.2.2.2 skrll mutex_enter(&q->q_ccb_mtx); 1330 1.2.2.5 skrll if (q->q_nccbs_avail > 0) { 1331 1.2.2.5 skrll ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1332 1.2.2.5 skrll KASSERT(ccb != NULL); 1333 1.2.2.5 skrll q->q_nccbs_avail--; 1334 1.2.2.5 skrll 1335 1.2.2.2 skrll SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1336 1.2.2.5 skrll #ifdef DEBUG 1337 1.2.2.5 skrll ccb->ccb_cookie = NULL; 1338 1.2.2.5 skrll #endif 1339 1.2.2.5 skrll } 1340 1.2.2.2 skrll mutex_exit(&q->q_ccb_mtx); 1341 1.2.2.2 skrll 1342 1.2.2.2 skrll return ccb; 1343 1.2.2.2 skrll } 1344 1.2.2.2 skrll 1345 1.2.2.2 skrll static void 1346 1.2.2.2 skrll nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1347 1.2.2.2 skrll { 1348 1.2.2.2 skrll 1349 1.2.2.2 skrll mutex_enter(&q->q_ccb_mtx); 1350 1.2.2.5 skrll #ifdef DEBUG 1351 1.2.2.5 skrll ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1352 1.2.2.5 skrll #endif 1353 1.2.2.2 skrll SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1354 1.2.2.2 skrll mutex_exit(&q->q_ccb_mtx); 1355 1.2.2.2 skrll } 1356 1.2.2.2 skrll 1357 1.2.2.2 skrll static void 1358 1.2.2.2 skrll nvme_ccbs_free(struct nvme_queue *q) 1359 1.2.2.2 skrll { 1360 1.2.2.2 skrll struct nvme_softc *sc = q->q_sc; 1361 1.2.2.2 skrll struct nvme_ccb *ccb; 1362 1.2.2.2 skrll 1363 1.2.2.2 skrll mutex_enter(&q->q_ccb_mtx); 1364 1.2.2.2 skrll while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1365 1.2.2.2 skrll SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1366 1.2.2.2 skrll bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1367 1.2.2.2 skrll } 1368 1.2.2.2 skrll mutex_exit(&q->q_ccb_mtx); 1369 1.2.2.2 skrll 1370 1.2.2.5 skrll nvme_dmamem_free(sc, q->q_ccb_prpls); 1371 1.2.2.2 skrll kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1372 1.2.2.2 skrll q->q_ccbs = NULL; 1373 1.2.2.2 skrll mutex_destroy(&q->q_ccb_mtx); 1374 1.2.2.2 skrll } 1375 1.2.2.2 skrll 1376 1.2.2.2 skrll static struct nvme_queue * 1377 1.2.2.2 skrll nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1378 1.2.2.2 skrll { 1379 1.2.2.2 skrll struct nvme_queue *q; 1380 1.2.2.2 skrll 1381 1.2.2.2 skrll q = kmem_alloc(sizeof(*q), KM_SLEEP); 1382 1.2.2.2 skrll if (q == NULL) 1383 1.2.2.2 skrll return NULL; 1384 1.2.2.2 skrll 1385 1.2.2.2 skrll q->q_sc = sc; 1386 1.2.2.5 skrll q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1387 1.2.2.5 skrll sizeof(struct nvme_sqe) * entries); 1388 1.2.2.5 skrll if (q->q_sq_dmamem == NULL) 1389 1.2.2.2 skrll goto free; 1390 1.2.2.2 skrll 1391 1.2.2.5 skrll q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1392 1.2.2.5 skrll sizeof(struct nvme_cqe) * entries); 1393 1.2.2.5 skrll if (q->q_cq_dmamem == NULL) 1394 1.2.2.2 skrll goto free_sq; 1395 1.2.2.2 skrll 1396 1.2.2.2 skrll memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1397 1.2.2.2 skrll memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1398 1.2.2.2 skrll 1399 1.2.2.2 skrll mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1400 1.2.2.2 skrll mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1401 1.2.2.2 skrll q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1402 1.2.2.2 skrll q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1403 1.2.2.2 skrll q->q_id = id; 1404 1.2.2.2 skrll q->q_entries = entries; 1405 1.2.2.2 skrll q->q_sq_tail = 0; 1406 1.2.2.2 skrll q->q_cq_head = 0; 1407 1.2.2.2 skrll q->q_cq_phase = NVME_CQE_PHASE; 1408 1.2.2.2 skrll 1409 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1410 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1411 1.2.2.2 skrll 1412 1.2.2.5 skrll /* 1413 1.2.2.5 skrll * Due to definition of full and empty queue (queue is empty 1414 1.2.2.5 skrll * when head == tail, full when tail is one less then head), 1415 1.2.2.5 skrll * we can actually only have (entries - 1) in-flight commands. 1416 1.2.2.5 skrll */ 1417 1.2.2.5 skrll if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1418 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1419 1.2.2.2 skrll goto free_cq; 1420 1.2.2.2 skrll } 1421 1.2.2.2 skrll 1422 1.2.2.2 skrll return q; 1423 1.2.2.2 skrll 1424 1.2.2.2 skrll free_cq: 1425 1.2.2.5 skrll nvme_dmamem_free(sc, q->q_cq_dmamem); 1426 1.2.2.2 skrll free_sq: 1427 1.2.2.5 skrll nvme_dmamem_free(sc, q->q_sq_dmamem); 1428 1.2.2.2 skrll free: 1429 1.2.2.2 skrll kmem_free(q, sizeof(*q)); 1430 1.2.2.2 skrll 1431 1.2.2.2 skrll return NULL; 1432 1.2.2.2 skrll } 1433 1.2.2.2 skrll 1434 1.2.2.2 skrll static void 1435 1.2.2.2 skrll nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1436 1.2.2.2 skrll { 1437 1.2.2.2 skrll nvme_ccbs_free(q); 1438 1.2.2.4 skrll mutex_destroy(&q->q_sq_mtx); 1439 1.2.2.4 skrll mutex_destroy(&q->q_cq_mtx); 1440 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1441 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1442 1.2.2.5 skrll nvme_dmamem_free(sc, q->q_cq_dmamem); 1443 1.2.2.5 skrll nvme_dmamem_free(sc, q->q_sq_dmamem); 1444 1.2.2.2 skrll kmem_free(q, sizeof(*q)); 1445 1.2.2.2 skrll } 1446 1.2.2.2 skrll 1447 1.2.2.2 skrll int 1448 1.2.2.2 skrll nvme_intr(void *xsc) 1449 1.2.2.2 skrll { 1450 1.2.2.2 skrll struct nvme_softc *sc = xsc; 1451 1.2.2.2 skrll 1452 1.2.2.4 skrll /* 1453 1.2.2.4 skrll * INTx is level triggered, controller deasserts the interrupt only 1454 1.2.2.4 skrll * when we advance command queue head via write to the doorbell. 1455 1.2.2.5 skrll * Tell the controller to block the interrupts while we process 1456 1.2.2.5 skrll * the queue(s). 1457 1.2.2.4 skrll */ 1458 1.2.2.5 skrll nvme_write4(sc, NVME_INTMS, 1); 1459 1.2.2.5 skrll 1460 1.2.2.5 skrll softint_schedule(sc->sc_softih[0]); 1461 1.2.2.5 skrll 1462 1.2.2.5 skrll /* don't know, might not have been for us */ 1463 1.2.2.5 skrll return 1; 1464 1.2.2.5 skrll } 1465 1.2.2.5 skrll 1466 1.2.2.5 skrll void 1467 1.2.2.5 skrll nvme_softintr_intx(void *xq) 1468 1.2.2.5 skrll { 1469 1.2.2.5 skrll struct nvme_queue *q = xq; 1470 1.2.2.5 skrll struct nvme_softc *sc = q->q_sc; 1471 1.2.2.5 skrll 1472 1.2.2.5 skrll nvme_q_complete(sc, sc->sc_admin_q); 1473 1.2.2.2 skrll if (sc->sc_q != NULL) 1474 1.2.2.5 skrll nvme_q_complete(sc, sc->sc_q[0]); 1475 1.2.2.2 skrll 1476 1.2.2.5 skrll /* 1477 1.2.2.5 skrll * Processing done, tell controller to issue interrupts again. There 1478 1.2.2.5 skrll * is no race, as NVMe spec requires the controller to maintain state, 1479 1.2.2.5 skrll * and assert the interrupt whenever there are unacknowledged 1480 1.2.2.5 skrll * completion queue entries. 1481 1.2.2.5 skrll */ 1482 1.2.2.5 skrll nvme_write4(sc, NVME_INTMC, 1); 1483 1.2.2.2 skrll } 1484 1.2.2.2 skrll 1485 1.2.2.2 skrll int 1486 1.2.2.4 skrll nvme_intr_msi(void *xq) 1487 1.2.2.2 skrll { 1488 1.2.2.2 skrll struct nvme_queue *q = xq; 1489 1.2.2.2 skrll 1490 1.2.2.4 skrll KASSERT(q && q->q_sc && q->q_sc->sc_softih 1491 1.2.2.4 skrll && q->q_sc->sc_softih[q->q_id]); 1492 1.2.2.2 skrll 1493 1.2.2.5 skrll /* 1494 1.2.2.5 skrll * MSI/MSI-X are edge triggered, so can handover processing to softint 1495 1.2.2.5 skrll * without masking the interrupt. 1496 1.2.2.5 skrll */ 1497 1.2.2.4 skrll softint_schedule(q->q_sc->sc_softih[q->q_id]); 1498 1.2.2.2 skrll 1499 1.2.2.4 skrll return 1; 1500 1.2.2.2 skrll } 1501 1.2.2.2 skrll 1502 1.2.2.4 skrll void 1503 1.2.2.4 skrll nvme_softintr_msi(void *xq) 1504 1.2.2.2 skrll { 1505 1.2.2.2 skrll struct nvme_queue *q = xq; 1506 1.2.2.4 skrll struct nvme_softc *sc = q->q_sc; 1507 1.2.2.2 skrll 1508 1.2.2.4 skrll nvme_q_complete(sc, q); 1509 1.2.2.2 skrll } 1510 1.2.2.2 skrll 1511 1.2.2.5 skrll static struct nvme_dmamem * 1512 1.2.2.5 skrll nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1513 1.2.2.2 skrll { 1514 1.2.2.5 skrll struct nvme_dmamem *ndm; 1515 1.2.2.2 skrll int nsegs; 1516 1.2.2.2 skrll 1517 1.2.2.5 skrll ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1518 1.2.2.5 skrll if (ndm == NULL) 1519 1.2.2.5 skrll return NULL; 1520 1.2.2.5 skrll 1521 1.2.2.2 skrll ndm->ndm_size = size; 1522 1.2.2.2 skrll 1523 1.2.2.2 skrll if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1524 1.2.2.2 skrll BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1525 1.2.2.2 skrll goto ndmfree; 1526 1.2.2.2 skrll 1527 1.2.2.2 skrll if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1528 1.2.2.2 skrll 1, &nsegs, BUS_DMA_WAITOK) != 0) 1529 1.2.2.2 skrll goto destroy; 1530 1.2.2.2 skrll 1531 1.2.2.2 skrll if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1532 1.2.2.2 skrll &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1533 1.2.2.2 skrll goto free; 1534 1.2.2.2 skrll memset(ndm->ndm_kva, 0, size); 1535 1.2.2.2 skrll 1536 1.2.2.2 skrll if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1537 1.2.2.2 skrll NULL, BUS_DMA_WAITOK) != 0) 1538 1.2.2.2 skrll goto unmap; 1539 1.2.2.2 skrll 1540 1.2.2.5 skrll return ndm; 1541 1.2.2.2 skrll 1542 1.2.2.2 skrll unmap: 1543 1.2.2.2 skrll bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1544 1.2.2.2 skrll free: 1545 1.2.2.2 skrll bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1546 1.2.2.2 skrll destroy: 1547 1.2.2.2 skrll bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1548 1.2.2.2 skrll ndmfree: 1549 1.2.2.5 skrll kmem_free(ndm, sizeof(*ndm)); 1550 1.2.2.5 skrll return NULL; 1551 1.2.2.5 skrll } 1552 1.2.2.5 skrll 1553 1.2.2.5 skrll static void 1554 1.2.2.5 skrll nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1555 1.2.2.5 skrll { 1556 1.2.2.5 skrll bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1557 1.2.2.5 skrll 0, NVME_DMA_LEN(mem), ops); 1558 1.2.2.2 skrll } 1559 1.2.2.2 skrll 1560 1.2.2.2 skrll void 1561 1.2.2.2 skrll nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1562 1.2.2.2 skrll { 1563 1.2.2.2 skrll bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1564 1.2.2.2 skrll bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1565 1.2.2.2 skrll bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1566 1.2.2.2 skrll bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1567 1.2.2.5 skrll kmem_free(ndm, sizeof(*ndm)); 1568 1.2.2.2 skrll } 1569 1.2.2.3 skrll 1570 1.2.2.3 skrll /* 1571 1.2.2.3 skrll * ioctl 1572 1.2.2.3 skrll */ 1573 1.2.2.3 skrll 1574 1.2.2.3 skrll dev_type_open(nvmeopen); 1575 1.2.2.3 skrll dev_type_close(nvmeclose); 1576 1.2.2.3 skrll dev_type_ioctl(nvmeioctl); 1577 1.2.2.3 skrll 1578 1.2.2.3 skrll const struct cdevsw nvme_cdevsw = { 1579 1.2.2.3 skrll .d_open = nvmeopen, 1580 1.2.2.3 skrll .d_close = nvmeclose, 1581 1.2.2.3 skrll .d_read = noread, 1582 1.2.2.3 skrll .d_write = nowrite, 1583 1.2.2.3 skrll .d_ioctl = nvmeioctl, 1584 1.2.2.3 skrll .d_stop = nostop, 1585 1.2.2.3 skrll .d_tty = notty, 1586 1.2.2.3 skrll .d_poll = nopoll, 1587 1.2.2.3 skrll .d_mmap = nommap, 1588 1.2.2.3 skrll .d_kqfilter = nokqfilter, 1589 1.2.2.3 skrll .d_discard = nodiscard, 1590 1.2.2.3 skrll .d_flag = D_OTHER, 1591 1.2.2.3 skrll }; 1592 1.2.2.3 skrll 1593 1.2.2.3 skrll extern struct cfdriver nvme_cd; 1594 1.2.2.3 skrll 1595 1.2.2.3 skrll /* 1596 1.2.2.3 skrll * Accept an open operation on the control device. 1597 1.2.2.3 skrll */ 1598 1.2.2.3 skrll int 1599 1.2.2.3 skrll nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1600 1.2.2.3 skrll { 1601 1.2.2.3 skrll struct nvme_softc *sc; 1602 1.2.2.3 skrll int unit = minor(dev) / 0x10000; 1603 1.2.2.3 skrll int nsid = minor(dev) & 0xffff; 1604 1.2.2.3 skrll int nsidx; 1605 1.2.2.3 skrll 1606 1.2.2.3 skrll if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1607 1.2.2.3 skrll return ENXIO; 1608 1.2.2.3 skrll if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1609 1.2.2.3 skrll return ENXIO; 1610 1.2.2.3 skrll 1611 1.2.2.4 skrll if (nsid == 0) { 1612 1.2.2.4 skrll /* controller */ 1613 1.2.2.4 skrll if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1614 1.2.2.4 skrll return EBUSY; 1615 1.2.2.4 skrll SET(sc->sc_flags, NVME_F_OPEN); 1616 1.2.2.4 skrll } else { 1617 1.2.2.4 skrll /* namespace */ 1618 1.2.2.4 skrll nsidx = nsid - 1; 1619 1.2.2.4 skrll if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1620 1.2.2.4 skrll return ENXIO; 1621 1.2.2.4 skrll if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1622 1.2.2.4 skrll return EBUSY; 1623 1.2.2.4 skrll SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1624 1.2.2.4 skrll } 1625 1.2.2.3 skrll return 0; 1626 1.2.2.3 skrll } 1627 1.2.2.3 skrll 1628 1.2.2.3 skrll /* 1629 1.2.2.3 skrll * Accept the last close on the control device. 1630 1.2.2.3 skrll */ 1631 1.2.2.3 skrll int 1632 1.2.2.4 skrll nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1633 1.2.2.3 skrll { 1634 1.2.2.3 skrll struct nvme_softc *sc; 1635 1.2.2.3 skrll int unit = minor(dev) / 0x10000; 1636 1.2.2.3 skrll int nsid = minor(dev) & 0xffff; 1637 1.2.2.3 skrll int nsidx; 1638 1.2.2.3 skrll 1639 1.2.2.3 skrll sc = device_lookup_private(&nvme_cd, unit); 1640 1.2.2.3 skrll if (sc == NULL) 1641 1.2.2.3 skrll return ENXIO; 1642 1.2.2.3 skrll 1643 1.2.2.4 skrll if (nsid == 0) { 1644 1.2.2.4 skrll /* controller */ 1645 1.2.2.4 skrll CLR(sc->sc_flags, NVME_F_OPEN); 1646 1.2.2.4 skrll } else { 1647 1.2.2.4 skrll /* namespace */ 1648 1.2.2.4 skrll nsidx = nsid - 1; 1649 1.2.2.4 skrll if (nsidx >= sc->sc_nn) 1650 1.2.2.4 skrll return ENXIO; 1651 1.2.2.4 skrll CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1652 1.2.2.4 skrll } 1653 1.2.2.3 skrll 1654 1.2.2.3 skrll return 0; 1655 1.2.2.3 skrll } 1656 1.2.2.3 skrll 1657 1.2.2.3 skrll /* 1658 1.2.2.3 skrll * Handle control operations. 1659 1.2.2.3 skrll */ 1660 1.2.2.3 skrll int 1661 1.2.2.4 skrll nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1662 1.2.2.3 skrll { 1663 1.2.2.3 skrll struct nvme_softc *sc; 1664 1.2.2.3 skrll int unit = minor(dev) / 0x10000; 1665 1.2.2.3 skrll int nsid = minor(dev) & 0xffff; 1666 1.2.2.4 skrll struct nvme_pt_command *pt; 1667 1.2.2.3 skrll 1668 1.2.2.3 skrll sc = device_lookup_private(&nvme_cd, unit); 1669 1.2.2.3 skrll if (sc == NULL) 1670 1.2.2.3 skrll return ENXIO; 1671 1.2.2.3 skrll 1672 1.2.2.3 skrll switch (cmd) { 1673 1.2.2.3 skrll case NVME_PASSTHROUGH_CMD: 1674 1.2.2.4 skrll pt = data; 1675 1.2.2.4 skrll return nvme_command_passthrough(sc, data, 1676 1.2.2.4 skrll nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1677 1.2.2.3 skrll } 1678 1.2.2.3 skrll 1679 1.2.2.3 skrll return ENOTTY; 1680 1.2.2.3 skrll } 1681
Indexes created Thu Oct 23 22:10:10 GMT 2025