nvme.c revision 1.2.2.3
1 1.2.2.3 skrll /* $NetBSD: nvme.c,v 1.2.2.3 2016/07/09 20:25:02 skrll Exp $ */ 2 1.2.2.2 skrll /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 1.2.2.2 skrll 4 1.2.2.2 skrll /* 5 1.2.2.2 skrll * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 1.2.2.2 skrll * 7 1.2.2.2 skrll * Permission to use, copy, modify, and distribute this software for any 8 1.2.2.2 skrll * purpose with or without fee is hereby granted, provided that the above 9 1.2.2.2 skrll * copyright notice and this permission notice appear in all copies. 10 1.2.2.2 skrll * 11 1.2.2.2 skrll * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 1.2.2.2 skrll * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 1.2.2.2 skrll * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 1.2.2.2 skrll * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 1.2.2.2 skrll * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 1.2.2.2 skrll * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 1.2.2.2 skrll * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 1.2.2.2 skrll */ 19 1.2.2.2 skrll 20 1.2.2.2 skrll #include <sys/cdefs.h> 21 1.2.2.3 skrll __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.2.2.3 2016/07/09 20:25:02 skrll Exp $"); 22 1.2.2.2 skrll 23 1.2.2.2 skrll #include <sys/param.h> 24 1.2.2.2 skrll #include <sys/systm.h> 25 1.2.2.2 skrll #include <sys/kernel.h> 26 1.2.2.2 skrll #include <sys/atomic.h> 27 1.2.2.2 skrll #include <sys/bus.h> 28 1.2.2.2 skrll #include <sys/buf.h> 29 1.2.2.3 skrll #include <sys/conf.h> 30 1.2.2.2 skrll #include <sys/device.h> 31 1.2.2.2 skrll #include <sys/kmem.h> 32 1.2.2.2 skrll #include <sys/once.h> 33 1.2.2.3 skrll #include <sys/proc.h> 34 1.2.2.2 skrll #include <sys/queue.h> 35 1.2.2.2 skrll #include <sys/mutex.h> 36 1.2.2.2 skrll 37 1.2.2.3 skrll #include <uvm/uvm_extern.h> 38 1.2.2.3 skrll 39 1.2.2.2 skrll #include <dev/ic/nvmereg.h> 40 1.2.2.2 skrll #include <dev/ic/nvmevar.h> 41 1.2.2.3 skrll #include <dev/ic/nvmeio.h> 42 1.2.2.2 skrll 43 1.2.2.2 skrll int nvme_adminq_size = 128; 44 1.2.2.2 skrll int nvme_ioq_size = 128; 45 1.2.2.2 skrll 46 1.2.2.2 skrll static int nvme_print(void *, const char *); 47 1.2.2.2 skrll 48 1.2.2.2 skrll static int nvme_ready(struct nvme_softc *, uint32_t); 49 1.2.2.2 skrll static int nvme_enable(struct nvme_softc *, u_int); 50 1.2.2.2 skrll static int nvme_disable(struct nvme_softc *); 51 1.2.2.2 skrll static int nvme_shutdown(struct nvme_softc *); 52 1.2.2.2 skrll 53 1.2.2.2 skrll static void nvme_version(struct nvme_softc *, uint32_t); 54 1.2.2.2 skrll #ifdef NVME_DEBUG 55 1.2.2.2 skrll static void nvme_dumpregs(struct nvme_softc *); 56 1.2.2.2 skrll #endif 57 1.2.2.2 skrll static int nvme_identify(struct nvme_softc *, u_int); 58 1.2.2.2 skrll static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 59 1.2.2.2 skrll void *); 60 1.2.2.2 skrll 61 1.2.2.2 skrll static int nvme_ccbs_alloc(struct nvme_queue *, u_int); 62 1.2.2.2 skrll static void nvme_ccbs_free(struct nvme_queue *); 63 1.2.2.2 skrll 64 1.2.2.2 skrll static struct nvme_ccb * 65 1.2.2.2 skrll nvme_ccb_get(struct nvme_queue *); 66 1.2.2.2 skrll static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 67 1.2.2.2 skrll 68 1.2.2.2 skrll static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 69 1.2.2.2 skrll struct nvme_ccb *, void (*)(struct nvme_queue *, 70 1.2.2.2 skrll struct nvme_ccb *, void *)); 71 1.2.2.2 skrll static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 72 1.2.2.2 skrll static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 73 1.2.2.2 skrll struct nvme_cqe *); 74 1.2.2.2 skrll static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 1.2.2.2 skrll static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 76 1.2.2.2 skrll struct nvme_cqe *); 77 1.2.2.2 skrll 78 1.2.2.2 skrll static struct nvme_queue * 79 1.2.2.2 skrll nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 80 1.2.2.2 skrll static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 81 1.2.2.2 skrll static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 82 1.2.2.2 skrll static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 83 1.2.2.2 skrll struct nvme_ccb *, void (*)(struct nvme_queue *, 84 1.2.2.2 skrll struct nvme_ccb *, void *)); 85 1.2.2.2 skrll static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 86 1.2.2.2 skrll static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 87 1.2.2.2 skrll 88 1.2.2.2 skrll static struct nvme_dmamem * 89 1.2.2.2 skrll nvme_dmamem_alloc(struct nvme_softc *, size_t); 90 1.2.2.2 skrll static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 91 1.2.2.2 skrll static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 92 1.2.2.2 skrll int); 93 1.2.2.2 skrll 94 1.2.2.2 skrll static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 95 1.2.2.2 skrll void *); 96 1.2.2.2 skrll static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 97 1.2.2.2 skrll struct nvme_cqe *); 98 1.2.2.2 skrll static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 99 1.2.2.2 skrll void *); 100 1.2.2.2 skrll static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 101 1.2.2.2 skrll struct nvme_cqe *); 102 1.2.2.2 skrll 103 1.2.2.3 skrll static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 104 1.2.2.3 skrll void *); 105 1.2.2.3 skrll static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 106 1.2.2.3 skrll struct nvme_cqe *); 107 1.2.2.3 skrll static int nvme_command_passthrough(struct nvme_softc *, 108 1.2.2.3 skrll struct nvme_pt_command *, uint16_t, struct lwp *, bool); 109 1.2.2.3 skrll 110 1.2.2.2 skrll #define nvme_read4(_s, _r) \ 111 1.2.2.2 skrll bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 112 1.2.2.2 skrll #define nvme_write4(_s, _r, _v) \ 113 1.2.2.2 skrll bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 114 1.2.2.2 skrll #ifdef __LP64__ 115 1.2.2.2 skrll #define nvme_read8(_s, _r) \ 116 1.2.2.2 skrll bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r)) 117 1.2.2.2 skrll #define nvme_write8(_s, _r, _v) \ 118 1.2.2.2 skrll bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 119 1.2.2.2 skrll #else /* __LP64__ */ 120 1.2.2.2 skrll static inline uint64_t 121 1.2.2.2 skrll nvme_read8(struct nvme_softc *sc, bus_size_t r) 122 1.2.2.2 skrll { 123 1.2.2.2 skrll uint64_t v; 124 1.2.2.2 skrll uint32_t *a = (uint32_t *)&v; 125 1.2.2.2 skrll 126 1.2.2.2 skrll #if _BYTE_ORDER == _LITTLE_ENDIAN 127 1.2.2.2 skrll a[0] = nvme_read4(sc, r); 128 1.2.2.2 skrll a[1] = nvme_read4(sc, r + 4); 129 1.2.2.2 skrll #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 130 1.2.2.2 skrll a[1] = nvme_read4(sc, r); 131 1.2.2.2 skrll a[0] = nvme_read4(sc, r + 4); 132 1.2.2.2 skrll #endif 133 1.2.2.2 skrll 134 1.2.2.2 skrll return v; 135 1.2.2.2 skrll } 136 1.2.2.2 skrll 137 1.2.2.2 skrll static inline void 138 1.2.2.2 skrll nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 139 1.2.2.2 skrll { 140 1.2.2.2 skrll uint32_t *a = (uint32_t *)&v; 141 1.2.2.2 skrll 142 1.2.2.2 skrll #if _BYTE_ORDER == _LITTLE_ENDIAN 143 1.2.2.2 skrll nvme_write4(sc, r, a[0]); 144 1.2.2.2 skrll nvme_write4(sc, r + 4, a[1]); 145 1.2.2.2 skrll #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 146 1.2.2.2 skrll nvme_write4(sc, r, a[1]); 147 1.2.2.2 skrll nvme_write4(sc, r + 4, a[0]); 148 1.2.2.2 skrll #endif 149 1.2.2.2 skrll } 150 1.2.2.2 skrll #endif /* __LP64__ */ 151 1.2.2.2 skrll #define nvme_barrier(_s, _r, _l, _f) \ 152 1.2.2.2 skrll bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 153 1.2.2.2 skrll 154 1.2.2.2 skrll pool_cache_t nvme_ns_ctx_cache; 155 1.2.2.2 skrll ONCE_DECL(nvme_init_once); 156 1.2.2.2 skrll 157 1.2.2.2 skrll static int 158 1.2.2.2 skrll nvme_init(void) 159 1.2.2.2 skrll { 160 1.2.2.2 skrll nvme_ns_ctx_cache = pool_cache_init(sizeof(struct nvme_ns_context), 161 1.2.2.2 skrll 0, 0, 0, "nvme_ns_ctx", NULL, IPL_BIO, NULL, NULL, NULL); 162 1.2.2.2 skrll KASSERT(nvme_ns_ctx_cache != NULL); 163 1.2.2.2 skrll return 0; 164 1.2.2.2 skrll } 165 1.2.2.2 skrll 166 1.2.2.2 skrll static void 167 1.2.2.2 skrll nvme_version(struct nvme_softc *sc, uint32_t ver) 168 1.2.2.2 skrll { 169 1.2.2.2 skrll const char *v = NULL; 170 1.2.2.2 skrll 171 1.2.2.2 skrll switch (ver) { 172 1.2.2.2 skrll case NVME_VS_1_0: 173 1.2.2.2 skrll v = "1.0"; 174 1.2.2.2 skrll break; 175 1.2.2.2 skrll case NVME_VS_1_1: 176 1.2.2.2 skrll v = "1.1"; 177 1.2.2.2 skrll break; 178 1.2.2.2 skrll case NVME_VS_1_2: 179 1.2.2.2 skrll v = "1.2"; 180 1.2.2.2 skrll break; 181 1.2.2.2 skrll default: 182 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unknown version 0x%08x\n", ver); 183 1.2.2.2 skrll return; 184 1.2.2.2 skrll } 185 1.2.2.2 skrll 186 1.2.2.2 skrll aprint_normal_dev(sc->sc_dev, "NVMe %s\n", v); 187 1.2.2.2 skrll } 188 1.2.2.2 skrll 189 1.2.2.2 skrll #ifdef NVME_DEBUG 190 1.2.2.2 skrll static void 191 1.2.2.2 skrll nvme_dumpregs(struct nvme_softc *sc) 192 1.2.2.2 skrll { 193 1.2.2.2 skrll uint64_t r8; 194 1.2.2.2 skrll uint32_t r4; 195 1.2.2.2 skrll 196 1.2.2.2 skrll #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 197 1.2.2.2 skrll r8 = nvme_read8(sc, NVME_CAP); 198 1.2.2.2 skrll printf("%s: cap 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 199 1.2.2.2 skrll printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 200 1.2.2.2 skrll (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 201 1.2.2.2 skrll printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 202 1.2.2.2 skrll (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 203 1.2.2.2 skrll printf("%s: css %llu\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 204 1.2.2.2 skrll printf("%s: nssrs %llu\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 205 1.2.2.2 skrll printf("%s: dstrd %u\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 206 1.2.2.2 skrll printf("%s: to %llu msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 207 1.2.2.2 skrll printf("%s: ams %llu\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 208 1.2.2.2 skrll printf("%s: cqr %llu\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 209 1.2.2.2 skrll printf("%s: mqes %llu\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 210 1.2.2.2 skrll 211 1.2.2.2 skrll printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 212 1.2.2.2 skrll 213 1.2.2.2 skrll r4 = nvme_read4(sc, NVME_CC); 214 1.2.2.2 skrll printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 215 1.2.2.2 skrll printf("%s: iocqes %u\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4)); 216 1.2.2.2 skrll printf("%s: iosqes %u\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4)); 217 1.2.2.2 skrll printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 218 1.2.2.2 skrll printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 219 1.2.2.2 skrll printf("%s: mps %u\n", DEVNAME(sc), NVME_CC_MPS_R(r4)); 220 1.2.2.2 skrll printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 221 1.2.2.2 skrll printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN)); 222 1.2.2.2 skrll 223 1.2.2.2 skrll printf("%s: csts 0x%08x\n", DEVNAME(sc), nvme_read4(sc, NVME_CSTS)); 224 1.2.2.2 skrll printf("%s: aqa 0x%08x\n", DEVNAME(sc), nvme_read4(sc, NVME_AQA)); 225 1.2.2.2 skrll printf("%s: asq 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 226 1.2.2.2 skrll printf("%s: acq 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 227 1.2.2.2 skrll #undef DEVNAME 228 1.2.2.2 skrll } 229 1.2.2.2 skrll #endif /* NVME_DEBUG */ 230 1.2.2.2 skrll 231 1.2.2.2 skrll static int 232 1.2.2.2 skrll nvme_ready(struct nvme_softc *sc, uint32_t rdy) 233 1.2.2.2 skrll { 234 1.2.2.2 skrll u_int i = 0; 235 1.2.2.2 skrll 236 1.2.2.2 skrll while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 237 1.2.2.2 skrll if (i++ > sc->sc_rdy_to) 238 1.2.2.2 skrll return 1; 239 1.2.2.2 skrll 240 1.2.2.2 skrll delay(1000); 241 1.2.2.2 skrll nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 242 1.2.2.2 skrll } 243 1.2.2.2 skrll 244 1.2.2.2 skrll return 0; 245 1.2.2.2 skrll } 246 1.2.2.2 skrll 247 1.2.2.2 skrll static int 248 1.2.2.2 skrll nvme_enable(struct nvme_softc *sc, u_int mps) 249 1.2.2.2 skrll { 250 1.2.2.2 skrll uint32_t cc; 251 1.2.2.2 skrll 252 1.2.2.2 skrll cc = nvme_read4(sc, NVME_CC); 253 1.2.2.2 skrll if (ISSET(cc, NVME_CC_EN)) 254 1.2.2.2 skrll return nvme_ready(sc, NVME_CSTS_RDY); 255 1.2.2.2 skrll 256 1.2.2.2 skrll nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 257 1.2.2.2 skrll NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 258 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 259 1.2.2.2 skrll 260 1.2.2.2 skrll nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 261 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 262 1.2.2.2 skrll nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 263 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 264 1.2.2.2 skrll 265 1.2.2.2 skrll CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 266 1.2.2.2 skrll NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 267 1.2.2.2 skrll SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 268 1.2.2.2 skrll SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 269 1.2.2.2 skrll SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 270 1.2.2.2 skrll SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 271 1.2.2.2 skrll SET(cc, NVME_CC_MPS(mps)); 272 1.2.2.2 skrll SET(cc, NVME_CC_EN); 273 1.2.2.2 skrll 274 1.2.2.2 skrll nvme_write4(sc, NVME_CC, cc); 275 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, 276 1.2.2.2 skrll BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 277 1.2.2.2 skrll 278 1.2.2.2 skrll return nvme_ready(sc, NVME_CSTS_RDY); 279 1.2.2.2 skrll } 280 1.2.2.2 skrll 281 1.2.2.2 skrll static int 282 1.2.2.2 skrll nvme_disable(struct nvme_softc *sc) 283 1.2.2.2 skrll { 284 1.2.2.2 skrll uint32_t cc, csts; 285 1.2.2.2 skrll 286 1.2.2.2 skrll cc = nvme_read4(sc, NVME_CC); 287 1.2.2.2 skrll if (ISSET(cc, NVME_CC_EN)) { 288 1.2.2.2 skrll csts = nvme_read4(sc, NVME_CSTS); 289 1.2.2.2 skrll if (!ISSET(csts, NVME_CSTS_CFS) && 290 1.2.2.2 skrll nvme_ready(sc, NVME_CSTS_RDY) != 0) 291 1.2.2.2 skrll return 1; 292 1.2.2.2 skrll } 293 1.2.2.2 skrll 294 1.2.2.2 skrll CLR(cc, NVME_CC_EN); 295 1.2.2.2 skrll 296 1.2.2.2 skrll nvme_write4(sc, NVME_CC, cc); 297 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, 298 1.2.2.2 skrll BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 299 1.2.2.2 skrll 300 1.2.2.2 skrll return nvme_ready(sc, 0); 301 1.2.2.2 skrll } 302 1.2.2.2 skrll 303 1.2.2.2 skrll int 304 1.2.2.2 skrll nvme_attach(struct nvme_softc *sc) 305 1.2.2.2 skrll { 306 1.2.2.2 skrll struct nvme_attach_args naa; 307 1.2.2.2 skrll uint64_t cap; 308 1.2.2.2 skrll uint32_t reg; 309 1.2.2.2 skrll u_int dstrd; 310 1.2.2.2 skrll u_int mps = PAGE_SHIFT; 311 1.2.2.2 skrll int adminq_entries = nvme_adminq_size; 312 1.2.2.2 skrll int ioq_entries = nvme_ioq_size; 313 1.2.2.2 skrll int i; 314 1.2.2.2 skrll 315 1.2.2.2 skrll RUN_ONCE(&nvme_init_once, nvme_init); 316 1.2.2.2 skrll 317 1.2.2.2 skrll reg = nvme_read4(sc, NVME_VS); 318 1.2.2.2 skrll if (reg == 0xffffffff) { 319 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 320 1.2.2.2 skrll return 1; 321 1.2.2.2 skrll } 322 1.2.2.2 skrll 323 1.2.2.2 skrll nvme_version(sc, reg); 324 1.2.2.2 skrll 325 1.2.2.2 skrll cap = nvme_read8(sc, NVME_CAP); 326 1.2.2.2 skrll dstrd = NVME_CAP_DSTRD(cap); 327 1.2.2.2 skrll if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 328 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 329 1.2.2.2 skrll "is greater than CPU page size %u\n", 330 1.2.2.2 skrll 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 331 1.2.2.2 skrll return 1; 332 1.2.2.2 skrll } 333 1.2.2.2 skrll if (NVME_CAP_MPSMAX(cap) < mps) 334 1.2.2.2 skrll mps = NVME_CAP_MPSMAX(cap); 335 1.2.2.2 skrll 336 1.2.2.2 skrll sc->sc_rdy_to = NVME_CAP_TO(cap); 337 1.2.2.2 skrll sc->sc_mps = 1 << mps; 338 1.2.2.2 skrll sc->sc_mdts = MAXPHYS; 339 1.2.2.2 skrll sc->sc_max_sgl = 2; 340 1.2.2.2 skrll 341 1.2.2.2 skrll if (nvme_disable(sc) != 0) { 342 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 343 1.2.2.2 skrll return 1; 344 1.2.2.2 skrll } 345 1.2.2.2 skrll 346 1.2.2.2 skrll sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 347 1.2.2.2 skrll if (sc->sc_admin_q == NULL) { 348 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, 349 1.2.2.2 skrll "unable to allocate admin queue\n"); 350 1.2.2.2 skrll return 1; 351 1.2.2.2 skrll } 352 1.2.2.2 skrll if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 353 1.2.2.2 skrll goto free_admin_q; 354 1.2.2.2 skrll 355 1.2.2.2 skrll if (nvme_enable(sc, mps) != 0) { 356 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 357 1.2.2.2 skrll goto disestablish_admin_q; 358 1.2.2.2 skrll } 359 1.2.2.2 skrll 360 1.2.2.2 skrll if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 361 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 362 1.2.2.2 skrll goto disable; 363 1.2.2.2 skrll } 364 1.2.2.2 skrll 365 1.2.2.2 skrll /* we know how big things are now */ 366 1.2.2.2 skrll sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 367 1.2.2.2 skrll 368 1.2.2.2 skrll /* reallocate ccbs of admin queue with new max sgl. */ 369 1.2.2.2 skrll nvme_ccbs_free(sc->sc_admin_q); 370 1.2.2.2 skrll nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 371 1.2.2.2 skrll 372 1.2.2.2 skrll sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 373 1.2.2.2 skrll if (sc->sc_q == NULL) { 374 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n"); 375 1.2.2.2 skrll goto disable; 376 1.2.2.2 skrll } 377 1.2.2.2 skrll for (i = 0; i < sc->sc_nq; i++) { 378 1.2.2.2 skrll sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 379 1.2.2.2 skrll if (sc->sc_q[i] == NULL) { 380 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, 381 1.2.2.2 skrll "unable to allocate io queue\n"); 382 1.2.2.2 skrll goto free_q; 383 1.2.2.2 skrll } 384 1.2.2.2 skrll if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 385 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, 386 1.2.2.2 skrll "unable to create io queue\n"); 387 1.2.2.2 skrll nvme_q_free(sc, sc->sc_q[i]); 388 1.2.2.2 skrll goto free_q; 389 1.2.2.2 skrll } 390 1.2.2.2 skrll } 391 1.2.2.2 skrll 392 1.2.2.2 skrll if (!sc->sc_use_mq) 393 1.2.2.2 skrll nvme_write4(sc, NVME_INTMC, 1); 394 1.2.2.2 skrll 395 1.2.2.2 skrll sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 396 1.2.2.2 skrll KM_SLEEP); 397 1.2.2.2 skrll for (i = 0; i < sc->sc_nn; i++) { 398 1.2.2.2 skrll memset(&naa, 0, sizeof(naa)); 399 1.2.2.2 skrll naa.naa_nsid = i + 1; 400 1.2.2.2 skrll naa.naa_qentries = ioq_entries; 401 1.2.2.2 skrll sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 402 1.2.2.2 skrll nvme_print); 403 1.2.2.2 skrll } 404 1.2.2.2 skrll 405 1.2.2.2 skrll return 0; 406 1.2.2.2 skrll 407 1.2.2.2 skrll free_q: 408 1.2.2.2 skrll while (--i >= 0) { 409 1.2.2.2 skrll nvme_q_delete(sc, sc->sc_q[i]); 410 1.2.2.2 skrll nvme_q_free(sc, sc->sc_q[i]); 411 1.2.2.2 skrll } 412 1.2.2.2 skrll disable: 413 1.2.2.2 skrll nvme_disable(sc); 414 1.2.2.2 skrll disestablish_admin_q: 415 1.2.2.2 skrll sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 416 1.2.2.2 skrll free_admin_q: 417 1.2.2.2 skrll nvme_q_free(sc, sc->sc_admin_q); 418 1.2.2.2 skrll 419 1.2.2.2 skrll return 1; 420 1.2.2.2 skrll } 421 1.2.2.2 skrll 422 1.2.2.2 skrll static int 423 1.2.2.2 skrll nvme_print(void *aux, const char *pnp) 424 1.2.2.2 skrll { 425 1.2.2.2 skrll struct nvme_attach_args *naa = aux; 426 1.2.2.2 skrll 427 1.2.2.2 skrll if (pnp) 428 1.2.2.2 skrll aprint_normal("at %s", pnp); 429 1.2.2.2 skrll 430 1.2.2.2 skrll if (naa->naa_nsid > 0) 431 1.2.2.2 skrll aprint_normal(" nsid %d", naa->naa_nsid); 432 1.2.2.2 skrll 433 1.2.2.2 skrll return UNCONF; 434 1.2.2.2 skrll } 435 1.2.2.2 skrll 436 1.2.2.2 skrll int 437 1.2.2.2 skrll nvme_detach(struct nvme_softc *sc, int flags) 438 1.2.2.2 skrll { 439 1.2.2.2 skrll int i, error; 440 1.2.2.2 skrll 441 1.2.2.2 skrll error = config_detach_children(sc->sc_dev, flags); 442 1.2.2.2 skrll if (error) 443 1.2.2.2 skrll return error; 444 1.2.2.2 skrll 445 1.2.2.2 skrll error = nvme_shutdown(sc); 446 1.2.2.2 skrll if (error) 447 1.2.2.2 skrll return error; 448 1.2.2.2 skrll 449 1.2.2.2 skrll for (i = 0; i < sc->sc_nq; i++) 450 1.2.2.2 skrll nvme_q_free(sc, sc->sc_q[i]); 451 1.2.2.2 skrll kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 452 1.2.2.2 skrll nvme_q_free(sc, sc->sc_admin_q); 453 1.2.2.2 skrll 454 1.2.2.2 skrll return 0; 455 1.2.2.2 skrll } 456 1.2.2.2 skrll 457 1.2.2.2 skrll static int 458 1.2.2.2 skrll nvme_shutdown(struct nvme_softc *sc) 459 1.2.2.2 skrll { 460 1.2.2.2 skrll uint32_t cc, csts; 461 1.2.2.2 skrll bool disabled = false; 462 1.2.2.2 skrll int i; 463 1.2.2.2 skrll 464 1.2.2.2 skrll if (!sc->sc_use_mq) 465 1.2.2.2 skrll nvme_write4(sc, NVME_INTMS, 1); 466 1.2.2.2 skrll 467 1.2.2.2 skrll for (i = 0; i < sc->sc_nq; i++) { 468 1.2.2.2 skrll if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 469 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, 470 1.2.2.2 skrll "unable to delete io queue %d, disabling\n", i + 1); 471 1.2.2.2 skrll disabled = true; 472 1.2.2.2 skrll } 473 1.2.2.2 skrll } 474 1.2.2.2 skrll sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 475 1.2.2.2 skrll if (disabled) 476 1.2.2.2 skrll goto disable; 477 1.2.2.2 skrll 478 1.2.2.2 skrll cc = nvme_read4(sc, NVME_CC); 479 1.2.2.2 skrll CLR(cc, NVME_CC_SHN_MASK); 480 1.2.2.2 skrll SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 481 1.2.2.2 skrll nvme_write4(sc, NVME_CC, cc); 482 1.2.2.2 skrll 483 1.2.2.2 skrll for (i = 0; i < 4000; i++) { 484 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, 485 1.2.2.2 skrll BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 486 1.2.2.2 skrll csts = nvme_read4(sc, NVME_CSTS); 487 1.2.2.2 skrll if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 488 1.2.2.2 skrll return 0; 489 1.2.2.2 skrll 490 1.2.2.2 skrll delay(1000); 491 1.2.2.2 skrll } 492 1.2.2.2 skrll 493 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 494 1.2.2.2 skrll 495 1.2.2.2 skrll disable: 496 1.2.2.2 skrll nvme_disable(sc); 497 1.2.2.2 skrll return 0; 498 1.2.2.2 skrll } 499 1.2.2.2 skrll 500 1.2.2.2 skrll void 501 1.2.2.2 skrll nvme_childdet(device_t self, device_t child) 502 1.2.2.2 skrll { 503 1.2.2.2 skrll struct nvme_softc *sc = device_private(self); 504 1.2.2.2 skrll int i; 505 1.2.2.2 skrll 506 1.2.2.2 skrll for (i = 0; i < sc->sc_nn; i++) { 507 1.2.2.2 skrll if (sc->sc_namespaces[i].dev == child) { 508 1.2.2.2 skrll /* Already freed ns->ident. */ 509 1.2.2.2 skrll sc->sc_namespaces[i].dev = NULL; 510 1.2.2.2 skrll break; 511 1.2.2.2 skrll } 512 1.2.2.2 skrll } 513 1.2.2.2 skrll } 514 1.2.2.2 skrll 515 1.2.2.2 skrll int 516 1.2.2.2 skrll nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 517 1.2.2.2 skrll { 518 1.2.2.2 skrll struct nvme_sqe sqe; 519 1.2.2.2 skrll struct nvm_identify_namespace *identify; 520 1.2.2.2 skrll struct nvme_dmamem *mem; 521 1.2.2.2 skrll struct nvme_ccb *ccb; 522 1.2.2.2 skrll struct nvme_namespace *ns; 523 1.2.2.2 skrll int rv; 524 1.2.2.2 skrll 525 1.2.2.2 skrll KASSERT(nsid > 0); 526 1.2.2.2 skrll 527 1.2.2.2 skrll ccb = nvme_ccb_get(sc->sc_admin_q); 528 1.2.2.2 skrll KASSERT(ccb != NULL); 529 1.2.2.2 skrll 530 1.2.2.2 skrll mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 531 1.2.2.2 skrll if (mem == NULL) 532 1.2.2.2 skrll return ENOMEM; 533 1.2.2.2 skrll 534 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 535 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_IDENTIFY; 536 1.2.2.2 skrll htolem32(&sqe.nsid, nsid); 537 1.2.2.2 skrll htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 538 1.2.2.2 skrll htolem32(&sqe.cdw10, 0); 539 1.2.2.2 skrll 540 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 541 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 542 1.2.2.2 skrll 543 1.2.2.2 skrll nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 544 1.2.2.2 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 545 1.2.2.2 skrll nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 546 1.2.2.2 skrll 547 1.2.2.2 skrll nvme_ccb_put(sc->sc_admin_q, ccb); 548 1.2.2.2 skrll 549 1.2.2.2 skrll if (rv != 0) { 550 1.2.2.2 skrll rv = EIO; 551 1.2.2.2 skrll goto done; 552 1.2.2.2 skrll } 553 1.2.2.2 skrll 554 1.2.2.2 skrll /* commit */ 555 1.2.2.2 skrll 556 1.2.2.2 skrll identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 557 1.2.2.2 skrll memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 558 1.2.2.2 skrll 559 1.2.2.2 skrll ns = nvme_ns_get(sc, nsid); 560 1.2.2.2 skrll KASSERT(ns); 561 1.2.2.2 skrll ns->ident = identify; 562 1.2.2.2 skrll 563 1.2.2.2 skrll done: 564 1.2.2.2 skrll nvme_dmamem_free(sc, mem); 565 1.2.2.2 skrll 566 1.2.2.2 skrll return rv; 567 1.2.2.2 skrll } 568 1.2.2.2 skrll 569 1.2.2.2 skrll int 570 1.2.2.2 skrll nvme_ns_dobio(struct nvme_softc *sc, struct nvme_ns_context *ctx) 571 1.2.2.2 skrll { 572 1.2.2.2 skrll struct nvme_queue *q = nvme_get_q(sc); 573 1.2.2.2 skrll struct nvme_ccb *ccb; 574 1.2.2.2 skrll bus_dmamap_t dmap; 575 1.2.2.2 skrll int i, error; 576 1.2.2.2 skrll 577 1.2.2.2 skrll ccb = nvme_ccb_get(q); 578 1.2.2.2 skrll if (ccb == NULL) 579 1.2.2.2 skrll return EAGAIN; 580 1.2.2.2 skrll 581 1.2.2.2 skrll ccb->ccb_done = nvme_ns_io_done; 582 1.2.2.2 skrll ccb->ccb_cookie = ctx; 583 1.2.2.2 skrll 584 1.2.2.2 skrll dmap = ccb->ccb_dmamap; 585 1.2.2.2 skrll error = bus_dmamap_load(sc->sc_dmat, dmap, ctx->nnc_data, 586 1.2.2.2 skrll ctx->nnc_datasize, NULL, 587 1.2.2.2 skrll (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL) ? 588 1.2.2.2 skrll BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 589 1.2.2.2 skrll (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 590 1.2.2.2 skrll BUS_DMA_READ : BUS_DMA_WRITE)); 591 1.2.2.2 skrll if (error) { 592 1.2.2.2 skrll nvme_ccb_put(q, ccb); 593 1.2.2.2 skrll return error; 594 1.2.2.2 skrll } 595 1.2.2.2 skrll 596 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 597 1.2.2.2 skrll ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 598 1.2.2.2 skrll BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 599 1.2.2.2 skrll 600 1.2.2.2 skrll if (dmap->dm_nsegs > 2) { 601 1.2.2.2 skrll for (i = 1; i < dmap->dm_nsegs; i++) { 602 1.2.2.2 skrll htolem64(&ccb->ccb_prpl[i - 1], 603 1.2.2.2 skrll dmap->dm_segs[i].ds_addr); 604 1.2.2.2 skrll } 605 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, 606 1.2.2.2 skrll NVME_DMA_MAP(q->q_ccb_prpls), 607 1.2.2.2 skrll ccb->ccb_prpl_off, 608 1.2.2.2 skrll sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 609 1.2.2.2 skrll BUS_DMASYNC_PREWRITE); 610 1.2.2.2 skrll } 611 1.2.2.2 skrll 612 1.2.2.2 skrll if (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL)) { 613 1.2.2.2 skrll if (nvme_poll(sc, q, ccb, nvme_ns_io_fill) != 0) 614 1.2.2.2 skrll return EIO; 615 1.2.2.2 skrll return 0; 616 1.2.2.2 skrll } 617 1.2.2.2 skrll 618 1.2.2.2 skrll nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 619 1.2.2.2 skrll return 0; 620 1.2.2.2 skrll } 621 1.2.2.2 skrll 622 1.2.2.2 skrll static void 623 1.2.2.2 skrll nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 624 1.2.2.2 skrll { 625 1.2.2.2 skrll struct nvme_sqe_io *sqe = slot; 626 1.2.2.2 skrll struct nvme_ns_context *ctx = ccb->ccb_cookie; 627 1.2.2.2 skrll bus_dmamap_t dmap = ccb->ccb_dmamap; 628 1.2.2.2 skrll 629 1.2.2.2 skrll sqe->opcode = ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 630 1.2.2.2 skrll NVM_CMD_READ : NVM_CMD_WRITE; 631 1.2.2.2 skrll htolem32(&sqe->nsid, ctx->nnc_nsid); 632 1.2.2.2 skrll 633 1.2.2.2 skrll htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 634 1.2.2.2 skrll switch (dmap->dm_nsegs) { 635 1.2.2.2 skrll case 1: 636 1.2.2.2 skrll break; 637 1.2.2.2 skrll case 2: 638 1.2.2.2 skrll htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 639 1.2.2.2 skrll break; 640 1.2.2.2 skrll default: 641 1.2.2.2 skrll /* the prp list is already set up and synced */ 642 1.2.2.2 skrll htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 643 1.2.2.2 skrll break; 644 1.2.2.2 skrll } 645 1.2.2.2 skrll 646 1.2.2.2 skrll htolem64(&sqe->slba, ctx->nnc_blkno); 647 1.2.2.2 skrll htolem16(&sqe->nlb, (ctx->nnc_datasize / ctx->nnc_secsize) - 1); 648 1.2.2.2 skrll } 649 1.2.2.2 skrll 650 1.2.2.2 skrll static void 651 1.2.2.2 skrll nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 652 1.2.2.2 skrll struct nvme_cqe *cqe) 653 1.2.2.2 skrll { 654 1.2.2.2 skrll struct nvme_softc *sc = q->q_sc; 655 1.2.2.2 skrll struct nvme_ns_context *ctx = ccb->ccb_cookie; 656 1.2.2.2 skrll bus_dmamap_t dmap = ccb->ccb_dmamap; 657 1.2.2.2 skrll uint16_t flags; 658 1.2.2.2 skrll 659 1.2.2.2 skrll if (dmap->dm_nsegs > 2) { 660 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, 661 1.2.2.2 skrll NVME_DMA_MAP(q->q_ccb_prpls), 662 1.2.2.2 skrll ccb->ccb_prpl_off, 663 1.2.2.2 skrll sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 664 1.2.2.2 skrll BUS_DMASYNC_POSTWRITE); 665 1.2.2.2 skrll } 666 1.2.2.2 skrll 667 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 668 1.2.2.2 skrll ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 669 1.2.2.2 skrll BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 670 1.2.2.2 skrll 671 1.2.2.2 skrll bus_dmamap_unload(sc->sc_dmat, dmap); 672 1.2.2.2 skrll nvme_ccb_put(q, ccb); 673 1.2.2.2 skrll 674 1.2.2.2 skrll flags = lemtoh16(&cqe->flags); 675 1.2.2.2 skrll 676 1.2.2.2 skrll ctx->nnc_status = flags; 677 1.2.2.2 skrll (*ctx->nnc_done)(ctx); 678 1.2.2.2 skrll } 679 1.2.2.2 skrll 680 1.2.2.2 skrll int 681 1.2.2.2 skrll nvme_ns_sync(struct nvme_softc *sc, struct nvme_ns_context *ctx) 682 1.2.2.2 skrll { 683 1.2.2.2 skrll struct nvme_queue *q = nvme_get_q(sc); 684 1.2.2.2 skrll struct nvme_ccb *ccb; 685 1.2.2.2 skrll 686 1.2.2.2 skrll ccb = nvme_ccb_get(q); 687 1.2.2.2 skrll if (ccb == NULL) 688 1.2.2.2 skrll return EAGAIN; 689 1.2.2.2 skrll 690 1.2.2.2 skrll ccb->ccb_done = nvme_ns_sync_done; 691 1.2.2.2 skrll ccb->ccb_cookie = ctx; 692 1.2.2.2 skrll 693 1.2.2.2 skrll if (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL)) { 694 1.2.2.2 skrll if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill) != 0) 695 1.2.2.2 skrll return EIO; 696 1.2.2.2 skrll return 0; 697 1.2.2.2 skrll } 698 1.2.2.2 skrll 699 1.2.2.2 skrll nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 700 1.2.2.2 skrll return 0; 701 1.2.2.2 skrll } 702 1.2.2.2 skrll 703 1.2.2.2 skrll static void 704 1.2.2.2 skrll nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 705 1.2.2.2 skrll { 706 1.2.2.2 skrll struct nvme_sqe *sqe = slot; 707 1.2.2.2 skrll struct nvme_ns_context *ctx = ccb->ccb_cookie; 708 1.2.2.2 skrll 709 1.2.2.2 skrll sqe->opcode = NVM_CMD_FLUSH; 710 1.2.2.2 skrll htolem32(&sqe->nsid, ctx->nnc_nsid); 711 1.2.2.2 skrll } 712 1.2.2.2 skrll 713 1.2.2.2 skrll static void 714 1.2.2.2 skrll nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 715 1.2.2.2 skrll struct nvme_cqe *cqe) 716 1.2.2.2 skrll { 717 1.2.2.2 skrll struct nvme_ns_context *ctx = ccb->ccb_cookie; 718 1.2.2.2 skrll uint16_t flags; 719 1.2.2.2 skrll 720 1.2.2.2 skrll nvme_ccb_put(q, ccb); 721 1.2.2.2 skrll 722 1.2.2.2 skrll flags = lemtoh16(&cqe->flags); 723 1.2.2.2 skrll 724 1.2.2.2 skrll ctx->nnc_status = flags; 725 1.2.2.2 skrll (*ctx->nnc_done)(ctx); 726 1.2.2.2 skrll } 727 1.2.2.2 skrll 728 1.2.2.2 skrll void 729 1.2.2.2 skrll nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 730 1.2.2.2 skrll { 731 1.2.2.2 skrll struct nvme_namespace *ns; 732 1.2.2.2 skrll struct nvm_identify_namespace *identify; 733 1.2.2.2 skrll 734 1.2.2.2 skrll ns = nvme_ns_get(sc, nsid); 735 1.2.2.2 skrll KASSERT(ns); 736 1.2.2.2 skrll 737 1.2.2.2 skrll identify = ns->ident; 738 1.2.2.2 skrll ns->ident = NULL; 739 1.2.2.2 skrll if (identify != NULL) 740 1.2.2.2 skrll kmem_free(identify, sizeof(*identify)); 741 1.2.2.2 skrll } 742 1.2.2.2 skrll 743 1.2.2.2 skrll static void 744 1.2.2.3 skrll nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 745 1.2.2.3 skrll { 746 1.2.2.3 skrll struct nvme_softc *sc = q->q_sc; 747 1.2.2.3 skrll struct nvme_sqe *sqe = slot; 748 1.2.2.3 skrll struct nvme_pt_command *pt = ccb->ccb_cookie; 749 1.2.2.3 skrll bus_dmamap_t dmap = ccb->ccb_dmamap; 750 1.2.2.3 skrll int i; 751 1.2.2.3 skrll 752 1.2.2.3 skrll sqe->opcode = pt->cmd.opcode; 753 1.2.2.3 skrll htolem32(&sqe->nsid, pt->cmd.nsid); 754 1.2.2.3 skrll 755 1.2.2.3 skrll if (pt->buf != NULL && pt->len > 0) { 756 1.2.2.3 skrll htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 757 1.2.2.3 skrll switch (dmap->dm_nsegs) { 758 1.2.2.3 skrll case 1: 759 1.2.2.3 skrll break; 760 1.2.2.3 skrll case 2: 761 1.2.2.3 skrll htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 762 1.2.2.3 skrll break; 763 1.2.2.3 skrll default: 764 1.2.2.3 skrll for (i = 1; i < dmap->dm_nsegs; i++) { 765 1.2.2.3 skrll htolem64(&ccb->ccb_prpl[i - 1], 766 1.2.2.3 skrll dmap->dm_segs[i].ds_addr); 767 1.2.2.3 skrll } 768 1.2.2.3 skrll bus_dmamap_sync(sc->sc_dmat, 769 1.2.2.3 skrll NVME_DMA_MAP(q->q_ccb_prpls), 770 1.2.2.3 skrll ccb->ccb_prpl_off, 771 1.2.2.3 skrll sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 772 1.2.2.3 skrll BUS_DMASYNC_PREWRITE); 773 1.2.2.3 skrll htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 774 1.2.2.3 skrll break; 775 1.2.2.3 skrll } 776 1.2.2.3 skrll } 777 1.2.2.3 skrll 778 1.2.2.3 skrll htolem32(&sqe->cdw10, pt->cmd.cdw10); 779 1.2.2.3 skrll htolem32(&sqe->cdw11, pt->cmd.cdw11); 780 1.2.2.3 skrll htolem32(&sqe->cdw12, pt->cmd.cdw12); 781 1.2.2.3 skrll htolem32(&sqe->cdw13, pt->cmd.cdw13); 782 1.2.2.3 skrll htolem32(&sqe->cdw14, pt->cmd.cdw14); 783 1.2.2.3 skrll htolem32(&sqe->cdw15, pt->cmd.cdw15); 784 1.2.2.3 skrll } 785 1.2.2.3 skrll 786 1.2.2.3 skrll static void 787 1.2.2.3 skrll nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 788 1.2.2.3 skrll { 789 1.2.2.3 skrll struct nvme_softc *sc = q->q_sc; 790 1.2.2.3 skrll struct nvme_pt_command *pt = ccb->ccb_cookie; 791 1.2.2.3 skrll bus_dmamap_t dmap = ccb->ccb_dmamap; 792 1.2.2.3 skrll 793 1.2.2.3 skrll if (pt->buf != NULL && pt->len > 0) { 794 1.2.2.3 skrll if (dmap->dm_nsegs > 2) { 795 1.2.2.3 skrll bus_dmamap_sync(sc->sc_dmat, 796 1.2.2.3 skrll NVME_DMA_MAP(q->q_ccb_prpls), 797 1.2.2.3 skrll ccb->ccb_prpl_off, 798 1.2.2.3 skrll sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 799 1.2.2.3 skrll BUS_DMASYNC_POSTWRITE); 800 1.2.2.3 skrll } 801 1.2.2.3 skrll 802 1.2.2.3 skrll bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 803 1.2.2.3 skrll pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 804 1.2.2.3 skrll bus_dmamap_unload(sc->sc_dmat, dmap); 805 1.2.2.3 skrll } 806 1.2.2.3 skrll 807 1.2.2.3 skrll pt->cpl.cdw0 = cqe->cdw0; 808 1.2.2.3 skrll pt->cpl.flags = cqe->flags & ~NVME_CQE_PHASE; 809 1.2.2.3 skrll } 810 1.2.2.3 skrll 811 1.2.2.3 skrll static int 812 1.2.2.3 skrll nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 813 1.2.2.3 skrll uint16_t nsid, struct lwp *l, bool is_adminq) 814 1.2.2.3 skrll { 815 1.2.2.3 skrll struct nvme_queue *q; 816 1.2.2.3 skrll struct nvme_ccb *ccb; 817 1.2.2.3 skrll void *buf = NULL; 818 1.2.2.3 skrll int error; 819 1.2.2.3 skrll 820 1.2.2.3 skrll if ((pt->buf == NULL && pt->len > 0) || 821 1.2.2.3 skrll (pt->buf != NULL && pt->len == 0)) 822 1.2.2.3 skrll return EINVAL; 823 1.2.2.3 skrll 824 1.2.2.3 skrll q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 825 1.2.2.3 skrll ccb = nvme_ccb_get(q); 826 1.2.2.3 skrll if (ccb == NULL) 827 1.2.2.3 skrll return EBUSY; 828 1.2.2.3 skrll 829 1.2.2.3 skrll if (pt->buf != NULL && pt->len > 0) { 830 1.2.2.3 skrll buf = kmem_alloc(pt->len, KM_SLEEP); 831 1.2.2.3 skrll if (buf == NULL) { 832 1.2.2.3 skrll error = ENOMEM; 833 1.2.2.3 skrll goto ccb_put; 834 1.2.2.3 skrll } 835 1.2.2.3 skrll if (!pt->is_read) { 836 1.2.2.3 skrll error = copyin(pt->buf, buf, pt->len); 837 1.2.2.3 skrll if (error) 838 1.2.2.3 skrll goto kmem_free; 839 1.2.2.3 skrll } 840 1.2.2.3 skrll error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 841 1.2.2.3 skrll pt->len, NULL, 842 1.2.2.3 skrll BUS_DMA_WAITOK | 843 1.2.2.3 skrll (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 844 1.2.2.3 skrll if (error) 845 1.2.2.3 skrll goto kmem_free; 846 1.2.2.3 skrll bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 847 1.2.2.3 skrll 0, ccb->ccb_dmamap->dm_mapsize, 848 1.2.2.3 skrll pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 849 1.2.2.3 skrll } 850 1.2.2.3 skrll 851 1.2.2.3 skrll ccb->ccb_done = nvme_pt_done; 852 1.2.2.3 skrll ccb->ccb_cookie = pt; 853 1.2.2.3 skrll 854 1.2.2.3 skrll pt->cmd.nsid = nsid; 855 1.2.2.3 skrll if (nvme_poll(sc, q, ccb, nvme_pt_fill)) { 856 1.2.2.3 skrll error = EIO; 857 1.2.2.3 skrll goto out; 858 1.2.2.3 skrll } 859 1.2.2.3 skrll 860 1.2.2.3 skrll error = 0; 861 1.2.2.3 skrll out: 862 1.2.2.3 skrll if (buf != NULL) { 863 1.2.2.3 skrll if (error == 0 && pt->is_read) 864 1.2.2.3 skrll error = copyout(buf, pt->buf, pt->len); 865 1.2.2.3 skrll kmem_free: 866 1.2.2.3 skrll kmem_free(buf, pt->len); 867 1.2.2.3 skrll } 868 1.2.2.3 skrll ccb_put: 869 1.2.2.3 skrll nvme_ccb_put(q, ccb); 870 1.2.2.3 skrll return error; 871 1.2.2.3 skrll } 872 1.2.2.3 skrll 873 1.2.2.3 skrll static void 874 1.2.2.2 skrll nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 875 1.2.2.2 skrll void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 876 1.2.2.2 skrll { 877 1.2.2.2 skrll struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 878 1.2.2.2 skrll uint32_t tail; 879 1.2.2.2 skrll 880 1.2.2.2 skrll mutex_enter(&q->q_sq_mtx); 881 1.2.2.2 skrll tail = q->q_sq_tail; 882 1.2.2.2 skrll if (++q->q_sq_tail >= q->q_entries) 883 1.2.2.2 skrll q->q_sq_tail = 0; 884 1.2.2.2 skrll 885 1.2.2.2 skrll sqe += tail; 886 1.2.2.2 skrll 887 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 888 1.2.2.2 skrll sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 889 1.2.2.2 skrll memset(sqe, 0, sizeof(*sqe)); 890 1.2.2.2 skrll (*fill)(q, ccb, sqe); 891 1.2.2.2 skrll sqe->cid = ccb->ccb_id; 892 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 893 1.2.2.2 skrll sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 894 1.2.2.2 skrll 895 1.2.2.2 skrll nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 896 1.2.2.2 skrll mutex_exit(&q->q_sq_mtx); 897 1.2.2.2 skrll } 898 1.2.2.2 skrll 899 1.2.2.2 skrll struct nvme_poll_state { 900 1.2.2.2 skrll struct nvme_sqe s; 901 1.2.2.2 skrll struct nvme_cqe c; 902 1.2.2.2 skrll }; 903 1.2.2.2 skrll 904 1.2.2.2 skrll static int 905 1.2.2.2 skrll nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 906 1.2.2.2 skrll void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 907 1.2.2.2 skrll { 908 1.2.2.2 skrll struct nvme_poll_state state; 909 1.2.2.2 skrll void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 910 1.2.2.2 skrll void *cookie; 911 1.2.2.2 skrll uint16_t flags; 912 1.2.2.2 skrll 913 1.2.2.2 skrll memset(&state, 0, sizeof(state)); 914 1.2.2.2 skrll (*fill)(q, ccb, &state.s); 915 1.2.2.2 skrll 916 1.2.2.2 skrll done = ccb->ccb_done; 917 1.2.2.2 skrll cookie = ccb->ccb_cookie; 918 1.2.2.2 skrll 919 1.2.2.2 skrll ccb->ccb_done = nvme_poll_done; 920 1.2.2.2 skrll ccb->ccb_cookie = &state; 921 1.2.2.2 skrll 922 1.2.2.2 skrll nvme_q_submit(sc, q, ccb, nvme_poll_fill); 923 1.2.2.2 skrll while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 924 1.2.2.2 skrll if (nvme_q_complete(sc, q) == 0) 925 1.2.2.2 skrll delay(10); 926 1.2.2.2 skrll 927 1.2.2.2 skrll /* XXX no timeout? */ 928 1.2.2.2 skrll } 929 1.2.2.2 skrll 930 1.2.2.2 skrll ccb->ccb_cookie = cookie; 931 1.2.2.2 skrll done(q, ccb, &state.c); 932 1.2.2.2 skrll 933 1.2.2.2 skrll flags = lemtoh16(&state.c.flags); 934 1.2.2.2 skrll 935 1.2.2.2 skrll return flags & ~NVME_CQE_PHASE; 936 1.2.2.2 skrll } 937 1.2.2.2 skrll 938 1.2.2.2 skrll static void 939 1.2.2.2 skrll nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 940 1.2.2.2 skrll { 941 1.2.2.2 skrll struct nvme_sqe *sqe = slot; 942 1.2.2.2 skrll struct nvme_poll_state *state = ccb->ccb_cookie; 943 1.2.2.2 skrll 944 1.2.2.2 skrll *sqe = state->s; 945 1.2.2.2 skrll } 946 1.2.2.2 skrll 947 1.2.2.2 skrll static void 948 1.2.2.2 skrll nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 949 1.2.2.2 skrll struct nvme_cqe *cqe) 950 1.2.2.2 skrll { 951 1.2.2.2 skrll struct nvme_poll_state *state = ccb->ccb_cookie; 952 1.2.2.2 skrll 953 1.2.2.2 skrll SET(cqe->flags, htole16(NVME_CQE_PHASE)); 954 1.2.2.2 skrll state->c = *cqe; 955 1.2.2.2 skrll } 956 1.2.2.2 skrll 957 1.2.2.2 skrll static void 958 1.2.2.2 skrll nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 959 1.2.2.2 skrll { 960 1.2.2.2 skrll struct nvme_sqe *src = ccb->ccb_cookie; 961 1.2.2.2 skrll struct nvme_sqe *dst = slot; 962 1.2.2.2 skrll 963 1.2.2.2 skrll *dst = *src; 964 1.2.2.2 skrll } 965 1.2.2.2 skrll 966 1.2.2.2 skrll static void 967 1.2.2.2 skrll nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 968 1.2.2.2 skrll struct nvme_cqe *cqe) 969 1.2.2.2 skrll { 970 1.2.2.2 skrll } 971 1.2.2.2 skrll 972 1.2.2.2 skrll static int 973 1.2.2.2 skrll nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 974 1.2.2.2 skrll { 975 1.2.2.2 skrll struct nvme_ccb *ccb; 976 1.2.2.2 skrll struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 977 1.2.2.2 skrll uint32_t head; 978 1.2.2.2 skrll uint16_t flags; 979 1.2.2.2 skrll int rv = 0; 980 1.2.2.2 skrll 981 1.2.2.2 skrll if (!mutex_tryenter(&q->q_cq_mtx)) 982 1.2.2.2 skrll return -1; 983 1.2.2.2 skrll 984 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 985 1.2.2.2 skrll head = q->q_cq_head; 986 1.2.2.2 skrll for (;;) { 987 1.2.2.2 skrll cqe = &ring[head]; 988 1.2.2.2 skrll flags = lemtoh16(&cqe->flags); 989 1.2.2.2 skrll if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 990 1.2.2.2 skrll break; 991 1.2.2.2 skrll 992 1.2.2.2 skrll ccb = &q->q_ccbs[cqe->cid]; 993 1.2.2.2 skrll ccb->ccb_done(q, ccb, cqe); 994 1.2.2.2 skrll 995 1.2.2.2 skrll if (++head >= q->q_entries) { 996 1.2.2.2 skrll head = 0; 997 1.2.2.2 skrll q->q_cq_phase ^= NVME_CQE_PHASE; 998 1.2.2.2 skrll } 999 1.2.2.2 skrll 1000 1.2.2.2 skrll rv = 1; 1001 1.2.2.2 skrll } 1002 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1003 1.2.2.2 skrll 1004 1.2.2.2 skrll if (rv) 1005 1.2.2.2 skrll nvme_write4(sc, q->q_cqhdbl, q->q_cq_head = head); 1006 1.2.2.2 skrll mutex_exit(&q->q_cq_mtx); 1007 1.2.2.2 skrll 1008 1.2.2.2 skrll return rv; 1009 1.2.2.2 skrll } 1010 1.2.2.2 skrll 1011 1.2.2.2 skrll static int 1012 1.2.2.2 skrll nvme_identify(struct nvme_softc *sc, u_int mps) 1013 1.2.2.2 skrll { 1014 1.2.2.2 skrll char sn[41], mn[81], fr[17]; 1015 1.2.2.2 skrll struct nvm_identify_controller *identify; 1016 1.2.2.2 skrll struct nvme_dmamem *mem; 1017 1.2.2.2 skrll struct nvme_ccb *ccb; 1018 1.2.2.2 skrll u_int mdts; 1019 1.2.2.2 skrll int rv = 1; 1020 1.2.2.2 skrll 1021 1.2.2.2 skrll ccb = nvme_ccb_get(sc->sc_admin_q); 1022 1.2.2.2 skrll if (ccb == NULL) 1023 1.2.2.2 skrll panic("%s: nvme_ccb_get returned NULL", __func__); 1024 1.2.2.2 skrll 1025 1.2.2.2 skrll mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1026 1.2.2.2 skrll if (mem == NULL) 1027 1.2.2.2 skrll return 1; 1028 1.2.2.2 skrll 1029 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 1030 1.2.2.2 skrll ccb->ccb_cookie = mem; 1031 1.2.2.2 skrll 1032 1.2.2.2 skrll nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1033 1.2.2.2 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify); 1034 1.2.2.2 skrll nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1035 1.2.2.2 skrll 1036 1.2.2.2 skrll nvme_ccb_put(sc->sc_admin_q, ccb); 1037 1.2.2.2 skrll 1038 1.2.2.2 skrll if (rv != 0) 1039 1.2.2.2 skrll goto done; 1040 1.2.2.2 skrll 1041 1.2.2.2 skrll identify = NVME_DMA_KVA(mem); 1042 1.2.2.2 skrll 1043 1.2.2.2 skrll strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1044 1.2.2.2 skrll sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1045 1.2.2.2 skrll strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1046 1.2.2.2 skrll sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1047 1.2.2.2 skrll strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1048 1.2.2.2 skrll sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1049 1.2.2.2 skrll aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1050 1.2.2.2 skrll sn); 1051 1.2.2.2 skrll 1052 1.2.2.2 skrll if (identify->mdts > 0) { 1053 1.2.2.2 skrll mdts = (1 << identify->mdts) * (1 << mps); 1054 1.2.2.2 skrll if (mdts < sc->sc_mdts) 1055 1.2.2.2 skrll sc->sc_mdts = mdts; 1056 1.2.2.2 skrll } 1057 1.2.2.2 skrll 1058 1.2.2.2 skrll sc->sc_nn = lemtoh32(&identify->nn); 1059 1.2.2.2 skrll 1060 1.2.2.2 skrll memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1061 1.2.2.2 skrll 1062 1.2.2.2 skrll done: 1063 1.2.2.2 skrll nvme_dmamem_free(sc, mem); 1064 1.2.2.2 skrll 1065 1.2.2.2 skrll return rv; 1066 1.2.2.2 skrll } 1067 1.2.2.2 skrll 1068 1.2.2.2 skrll static int 1069 1.2.2.2 skrll nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1070 1.2.2.2 skrll { 1071 1.2.2.2 skrll struct nvme_sqe_q sqe; 1072 1.2.2.2 skrll struct nvme_ccb *ccb; 1073 1.2.2.2 skrll int rv; 1074 1.2.2.2 skrll 1075 1.2.2.2 skrll if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q)) 1076 1.2.2.2 skrll return 1; 1077 1.2.2.2 skrll 1078 1.2.2.2 skrll ccb = nvme_ccb_get(sc->sc_admin_q); 1079 1.2.2.2 skrll KASSERT(ccb != NULL); 1080 1.2.2.2 skrll 1081 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 1082 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 1083 1.2.2.2 skrll 1084 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 1085 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1086 1.2.2.2 skrll htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1087 1.2.2.2 skrll htolem16(&sqe.qsize, q->q_entries - 1); 1088 1.2.2.2 skrll htolem16(&sqe.qid, q->q_id); 1089 1.2.2.2 skrll sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1090 1.2.2.2 skrll if (sc->sc_use_mq) 1091 1.2.2.2 skrll htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1092 1.2.2.2 skrll 1093 1.2.2.2 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1094 1.2.2.2 skrll if (rv != 0) 1095 1.2.2.2 skrll goto fail; 1096 1.2.2.2 skrll 1097 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 1098 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 1099 1.2.2.2 skrll 1100 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 1101 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1102 1.2.2.2 skrll htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1103 1.2.2.2 skrll htolem16(&sqe.qsize, q->q_entries - 1); 1104 1.2.2.2 skrll htolem16(&sqe.qid, q->q_id); 1105 1.2.2.2 skrll htolem16(&sqe.cqid, q->q_id); 1106 1.2.2.2 skrll sqe.qflags = NVM_SQE_Q_PC; 1107 1.2.2.2 skrll 1108 1.2.2.2 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1109 1.2.2.2 skrll if (rv != 0) 1110 1.2.2.2 skrll goto fail; 1111 1.2.2.2 skrll 1112 1.2.2.2 skrll fail: 1113 1.2.2.2 skrll nvme_ccb_put(sc->sc_admin_q, ccb); 1114 1.2.2.2 skrll return rv; 1115 1.2.2.2 skrll } 1116 1.2.2.2 skrll 1117 1.2.2.2 skrll static int 1118 1.2.2.2 skrll nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1119 1.2.2.2 skrll { 1120 1.2.2.2 skrll struct nvme_sqe_q sqe; 1121 1.2.2.2 skrll struct nvme_ccb *ccb; 1122 1.2.2.2 skrll int rv; 1123 1.2.2.2 skrll 1124 1.2.2.2 skrll ccb = nvme_ccb_get(sc->sc_admin_q); 1125 1.2.2.2 skrll KASSERT(ccb != NULL); 1126 1.2.2.2 skrll 1127 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 1128 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 1129 1.2.2.2 skrll 1130 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 1131 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1132 1.2.2.2 skrll htolem16(&sqe.qid, q->q_id); 1133 1.2.2.2 skrll 1134 1.2.2.2 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1135 1.2.2.2 skrll if (rv != 0) 1136 1.2.2.2 skrll goto fail; 1137 1.2.2.2 skrll 1138 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 1139 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 1140 1.2.2.2 skrll 1141 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 1142 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1143 1.2.2.2 skrll htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1144 1.2.2.2 skrll htolem16(&sqe.qid, q->q_id); 1145 1.2.2.2 skrll 1146 1.2.2.2 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1147 1.2.2.2 skrll if (rv != 0) 1148 1.2.2.2 skrll goto fail; 1149 1.2.2.2 skrll 1150 1.2.2.2 skrll fail: 1151 1.2.2.2 skrll nvme_ccb_put(sc->sc_admin_q, ccb); 1152 1.2.2.2 skrll 1153 1.2.2.2 skrll if (rv == 0 && sc->sc_use_mq) { 1154 1.2.2.2 skrll if (sc->sc_intr_disestablish(sc, q->q_id)) 1155 1.2.2.2 skrll rv = 1; 1156 1.2.2.2 skrll } 1157 1.2.2.2 skrll 1158 1.2.2.2 skrll return rv; 1159 1.2.2.2 skrll } 1160 1.2.2.2 skrll 1161 1.2.2.2 skrll static void 1162 1.2.2.2 skrll nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1163 1.2.2.2 skrll { 1164 1.2.2.2 skrll struct nvme_sqe *sqe = slot; 1165 1.2.2.2 skrll struct nvme_dmamem *mem = ccb->ccb_cookie; 1166 1.2.2.2 skrll 1167 1.2.2.2 skrll sqe->opcode = NVM_ADMIN_IDENTIFY; 1168 1.2.2.2 skrll htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1169 1.2.2.2 skrll htolem32(&sqe->cdw10, 1); 1170 1.2.2.2 skrll } 1171 1.2.2.2 skrll 1172 1.2.2.2 skrll static int 1173 1.2.2.2 skrll nvme_ccbs_alloc(struct nvme_queue *q, u_int nccbs) 1174 1.2.2.2 skrll { 1175 1.2.2.2 skrll struct nvme_softc *sc = q->q_sc; 1176 1.2.2.2 skrll struct nvme_ccb *ccb; 1177 1.2.2.2 skrll bus_addr_t off; 1178 1.2.2.2 skrll uint64_t *prpl; 1179 1.2.2.2 skrll u_int i; 1180 1.2.2.2 skrll 1181 1.2.2.2 skrll mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1182 1.2.2.2 skrll SIMPLEQ_INIT(&q->q_ccb_list); 1183 1.2.2.2 skrll 1184 1.2.2.2 skrll q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1185 1.2.2.2 skrll if (q->q_ccbs == NULL) 1186 1.2.2.2 skrll return 1; 1187 1.2.2.2 skrll 1188 1.2.2.2 skrll q->q_nccbs = nccbs; 1189 1.2.2.2 skrll q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1190 1.2.2.2 skrll sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1191 1.2.2.2 skrll 1192 1.2.2.2 skrll prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1193 1.2.2.2 skrll off = 0; 1194 1.2.2.2 skrll 1195 1.2.2.2 skrll for (i = 0; i < nccbs; i++) { 1196 1.2.2.2 skrll ccb = &q->q_ccbs[i]; 1197 1.2.2.2 skrll 1198 1.2.2.2 skrll if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1199 1.2.2.2 skrll sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1200 1.2.2.2 skrll sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1201 1.2.2.2 skrll &ccb->ccb_dmamap) != 0) 1202 1.2.2.2 skrll goto free_maps; 1203 1.2.2.2 skrll 1204 1.2.2.2 skrll ccb->ccb_id = i; 1205 1.2.2.2 skrll ccb->ccb_prpl = prpl; 1206 1.2.2.2 skrll ccb->ccb_prpl_off = off; 1207 1.2.2.2 skrll ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1208 1.2.2.2 skrll 1209 1.2.2.2 skrll SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1210 1.2.2.2 skrll 1211 1.2.2.2 skrll prpl += sc->sc_max_sgl; 1212 1.2.2.2 skrll off += sizeof(*prpl) * sc->sc_max_sgl; 1213 1.2.2.2 skrll } 1214 1.2.2.2 skrll 1215 1.2.2.2 skrll return 0; 1216 1.2.2.2 skrll 1217 1.2.2.2 skrll free_maps: 1218 1.2.2.2 skrll nvme_ccbs_free(q); 1219 1.2.2.2 skrll return 1; 1220 1.2.2.2 skrll } 1221 1.2.2.2 skrll 1222 1.2.2.2 skrll static struct nvme_ccb * 1223 1.2.2.2 skrll nvme_ccb_get(struct nvme_queue *q) 1224 1.2.2.2 skrll { 1225 1.2.2.2 skrll struct nvme_ccb *ccb; 1226 1.2.2.2 skrll 1227 1.2.2.2 skrll mutex_enter(&q->q_ccb_mtx); 1228 1.2.2.2 skrll ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1229 1.2.2.2 skrll if (ccb != NULL) 1230 1.2.2.2 skrll SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1231 1.2.2.2 skrll mutex_exit(&q->q_ccb_mtx); 1232 1.2.2.2 skrll 1233 1.2.2.2 skrll return ccb; 1234 1.2.2.2 skrll } 1235 1.2.2.2 skrll 1236 1.2.2.2 skrll static void 1237 1.2.2.2 skrll nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1238 1.2.2.2 skrll { 1239 1.2.2.2 skrll 1240 1.2.2.2 skrll mutex_enter(&q->q_ccb_mtx); 1241 1.2.2.2 skrll SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1242 1.2.2.2 skrll mutex_exit(&q->q_ccb_mtx); 1243 1.2.2.2 skrll } 1244 1.2.2.2 skrll 1245 1.2.2.2 skrll static void 1246 1.2.2.2 skrll nvme_ccbs_free(struct nvme_queue *q) 1247 1.2.2.2 skrll { 1248 1.2.2.2 skrll struct nvme_softc *sc = q->q_sc; 1249 1.2.2.2 skrll struct nvme_ccb *ccb; 1250 1.2.2.2 skrll 1251 1.2.2.2 skrll mutex_enter(&q->q_ccb_mtx); 1252 1.2.2.2 skrll while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1253 1.2.2.2 skrll SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1254 1.2.2.2 skrll bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1255 1.2.2.2 skrll } 1256 1.2.2.2 skrll mutex_exit(&q->q_ccb_mtx); 1257 1.2.2.2 skrll 1258 1.2.2.2 skrll nvme_dmamem_free(sc, q->q_ccb_prpls); 1259 1.2.2.2 skrll kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1260 1.2.2.2 skrll q->q_ccbs = NULL; 1261 1.2.2.2 skrll mutex_destroy(&q->q_ccb_mtx); 1262 1.2.2.2 skrll } 1263 1.2.2.2 skrll 1264 1.2.2.2 skrll static struct nvme_queue * 1265 1.2.2.2 skrll nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1266 1.2.2.2 skrll { 1267 1.2.2.2 skrll struct nvme_queue *q; 1268 1.2.2.2 skrll 1269 1.2.2.2 skrll q = kmem_alloc(sizeof(*q), KM_SLEEP); 1270 1.2.2.2 skrll if (q == NULL) 1271 1.2.2.2 skrll return NULL; 1272 1.2.2.2 skrll 1273 1.2.2.2 skrll q->q_sc = sc; 1274 1.2.2.2 skrll q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1275 1.2.2.2 skrll sizeof(struct nvme_sqe) * entries); 1276 1.2.2.2 skrll if (q->q_sq_dmamem == NULL) 1277 1.2.2.2 skrll goto free; 1278 1.2.2.2 skrll 1279 1.2.2.2 skrll q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1280 1.2.2.2 skrll sizeof(struct nvme_cqe) * entries); 1281 1.2.2.2 skrll if (q->q_cq_dmamem == NULL) 1282 1.2.2.2 skrll goto free_sq; 1283 1.2.2.2 skrll 1284 1.2.2.2 skrll memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1285 1.2.2.2 skrll memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1286 1.2.2.2 skrll 1287 1.2.2.2 skrll mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1288 1.2.2.2 skrll mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1289 1.2.2.2 skrll q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1290 1.2.2.2 skrll q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1291 1.2.2.2 skrll q->q_id = id; 1292 1.2.2.2 skrll q->q_entries = entries; 1293 1.2.2.2 skrll q->q_sq_tail = 0; 1294 1.2.2.2 skrll q->q_cq_head = 0; 1295 1.2.2.2 skrll q->q_cq_phase = NVME_CQE_PHASE; 1296 1.2.2.2 skrll 1297 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1298 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1299 1.2.2.2 skrll 1300 1.2.2.2 skrll if (nvme_ccbs_alloc(q, entries) != 0) { 1301 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1302 1.2.2.2 skrll goto free_cq; 1303 1.2.2.2 skrll } 1304 1.2.2.2 skrll 1305 1.2.2.2 skrll return q; 1306 1.2.2.2 skrll 1307 1.2.2.2 skrll free_cq: 1308 1.2.2.2 skrll nvme_dmamem_free(sc, q->q_cq_dmamem); 1309 1.2.2.2 skrll free_sq: 1310 1.2.2.2 skrll nvme_dmamem_free(sc, q->q_sq_dmamem); 1311 1.2.2.2 skrll free: 1312 1.2.2.2 skrll kmem_free(q, sizeof(*q)); 1313 1.2.2.2 skrll 1314 1.2.2.2 skrll return NULL; 1315 1.2.2.2 skrll } 1316 1.2.2.2 skrll 1317 1.2.2.2 skrll static void 1318 1.2.2.2 skrll nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1319 1.2.2.2 skrll { 1320 1.2.2.2 skrll nvme_ccbs_free(q); 1321 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1322 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1323 1.2.2.2 skrll nvme_dmamem_free(sc, q->q_cq_dmamem); 1324 1.2.2.2 skrll nvme_dmamem_free(sc, q->q_sq_dmamem); 1325 1.2.2.2 skrll kmem_free(q, sizeof(*q)); 1326 1.2.2.2 skrll } 1327 1.2.2.2 skrll 1328 1.2.2.2 skrll int 1329 1.2.2.2 skrll nvme_intr(void *xsc) 1330 1.2.2.2 skrll { 1331 1.2.2.2 skrll struct nvme_softc *sc = xsc; 1332 1.2.2.2 skrll int rv = 0; 1333 1.2.2.2 skrll 1334 1.2.2.2 skrll nvme_write4(sc, NVME_INTMS, 1); 1335 1.2.2.2 skrll 1336 1.2.2.2 skrll if (nvme_q_complete(sc, sc->sc_admin_q)) 1337 1.2.2.2 skrll rv = 1; 1338 1.2.2.2 skrll if (sc->sc_q != NULL) 1339 1.2.2.2 skrll if (nvme_q_complete(sc, sc->sc_q[0])) 1340 1.2.2.2 skrll rv = 1; 1341 1.2.2.2 skrll 1342 1.2.2.2 skrll nvme_write4(sc, NVME_INTMC, 1); 1343 1.2.2.2 skrll 1344 1.2.2.2 skrll return rv; 1345 1.2.2.2 skrll } 1346 1.2.2.2 skrll 1347 1.2.2.2 skrll int 1348 1.2.2.2 skrll nvme_mq_msi_intr(void *xq) 1349 1.2.2.2 skrll { 1350 1.2.2.2 skrll struct nvme_queue *q = xq; 1351 1.2.2.2 skrll struct nvme_softc *sc = q->q_sc; 1352 1.2.2.2 skrll int rv = 0; 1353 1.2.2.2 skrll 1354 1.2.2.2 skrll nvme_write4(sc, NVME_INTMS, 1U << q->q_id); 1355 1.2.2.2 skrll 1356 1.2.2.2 skrll if (nvme_q_complete(sc, q)) 1357 1.2.2.2 skrll rv = 1; 1358 1.2.2.2 skrll 1359 1.2.2.2 skrll nvme_write4(sc, NVME_INTMC, 1U << q->q_id); 1360 1.2.2.2 skrll 1361 1.2.2.2 skrll return rv; 1362 1.2.2.2 skrll } 1363 1.2.2.2 skrll 1364 1.2.2.2 skrll int 1365 1.2.2.2 skrll nvme_mq_msix_intr(void *xq) 1366 1.2.2.2 skrll { 1367 1.2.2.2 skrll struct nvme_queue *q = xq; 1368 1.2.2.2 skrll int rv = 0; 1369 1.2.2.2 skrll 1370 1.2.2.2 skrll if (nvme_q_complete(q->q_sc, q)) 1371 1.2.2.2 skrll rv = 1; 1372 1.2.2.2 skrll 1373 1.2.2.2 skrll return rv; 1374 1.2.2.2 skrll } 1375 1.2.2.2 skrll 1376 1.2.2.2 skrll static struct nvme_dmamem * 1377 1.2.2.2 skrll nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1378 1.2.2.2 skrll { 1379 1.2.2.2 skrll struct nvme_dmamem *ndm; 1380 1.2.2.2 skrll int nsegs; 1381 1.2.2.2 skrll 1382 1.2.2.2 skrll ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1383 1.2.2.2 skrll if (ndm == NULL) 1384 1.2.2.2 skrll return NULL; 1385 1.2.2.2 skrll 1386 1.2.2.2 skrll ndm->ndm_size = size; 1387 1.2.2.2 skrll 1388 1.2.2.2 skrll if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1389 1.2.2.2 skrll BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1390 1.2.2.2 skrll goto ndmfree; 1391 1.2.2.2 skrll 1392 1.2.2.2 skrll if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1393 1.2.2.2 skrll 1, &nsegs, BUS_DMA_WAITOK) != 0) 1394 1.2.2.2 skrll goto destroy; 1395 1.2.2.2 skrll 1396 1.2.2.2 skrll if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1397 1.2.2.2 skrll &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1398 1.2.2.2 skrll goto free; 1399 1.2.2.2 skrll memset(ndm->ndm_kva, 0, size); 1400 1.2.2.2 skrll 1401 1.2.2.2 skrll if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1402 1.2.2.2 skrll NULL, BUS_DMA_WAITOK) != 0) 1403 1.2.2.2 skrll goto unmap; 1404 1.2.2.2 skrll 1405 1.2.2.2 skrll return ndm; 1406 1.2.2.2 skrll 1407 1.2.2.2 skrll unmap: 1408 1.2.2.2 skrll bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1409 1.2.2.2 skrll free: 1410 1.2.2.2 skrll bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1411 1.2.2.2 skrll destroy: 1412 1.2.2.2 skrll bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1413 1.2.2.2 skrll ndmfree: 1414 1.2.2.2 skrll kmem_free(ndm, sizeof(*ndm)); 1415 1.2.2.2 skrll return NULL; 1416 1.2.2.2 skrll } 1417 1.2.2.2 skrll 1418 1.2.2.2 skrll static void 1419 1.2.2.2 skrll nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1420 1.2.2.2 skrll { 1421 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1422 1.2.2.2 skrll 0, NVME_DMA_LEN(mem), ops); 1423 1.2.2.2 skrll } 1424 1.2.2.2 skrll 1425 1.2.2.2 skrll void 1426 1.2.2.2 skrll nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1427 1.2.2.2 skrll { 1428 1.2.2.2 skrll bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1429 1.2.2.2 skrll bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1430 1.2.2.2 skrll bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1431 1.2.2.2 skrll bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1432 1.2.2.2 skrll kmem_free(ndm, sizeof(*ndm)); 1433 1.2.2.2 skrll } 1434 1.2.2.3 skrll 1435 1.2.2.3 skrll /* 1436 1.2.2.3 skrll * ioctl 1437 1.2.2.3 skrll */ 1438 1.2.2.3 skrll 1439 1.2.2.3 skrll /* nvme */ 1440 1.2.2.3 skrll dev_type_open(nvmeopen); 1441 1.2.2.3 skrll dev_type_close(nvmeclose); 1442 1.2.2.3 skrll dev_type_ioctl(nvmeioctl); 1443 1.2.2.3 skrll 1444 1.2.2.3 skrll const struct cdevsw nvme_cdevsw = { 1445 1.2.2.3 skrll .d_open = nvmeopen, 1446 1.2.2.3 skrll .d_close = nvmeclose, 1447 1.2.2.3 skrll .d_read = noread, 1448 1.2.2.3 skrll .d_write = nowrite, 1449 1.2.2.3 skrll .d_ioctl = nvmeioctl, 1450 1.2.2.3 skrll .d_stop = nostop, 1451 1.2.2.3 skrll .d_tty = notty, 1452 1.2.2.3 skrll .d_poll = nopoll, 1453 1.2.2.3 skrll .d_mmap = nommap, 1454 1.2.2.3 skrll .d_kqfilter = nokqfilter, 1455 1.2.2.3 skrll .d_discard = nodiscard, 1456 1.2.2.3 skrll .d_flag = D_OTHER, 1457 1.2.2.3 skrll }; 1458 1.2.2.3 skrll 1459 1.2.2.3 skrll extern struct cfdriver nvme_cd; 1460 1.2.2.3 skrll 1461 1.2.2.3 skrll /* 1462 1.2.2.3 skrll * Accept an open operation on the control device. 1463 1.2.2.3 skrll */ 1464 1.2.2.3 skrll int 1465 1.2.2.3 skrll nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1466 1.2.2.3 skrll { 1467 1.2.2.3 skrll struct nvme_softc *sc; 1468 1.2.2.3 skrll int unit = minor(dev); 1469 1.2.2.3 skrll 1470 1.2.2.3 skrll if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1471 1.2.2.3 skrll return ENXIO; 1472 1.2.2.3 skrll if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1473 1.2.2.3 skrll return ENXIO; 1474 1.2.2.3 skrll if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1475 1.2.2.3 skrll return EBUSY; 1476 1.2.2.3 skrll 1477 1.2.2.3 skrll SET(sc->sc_flags, NVME_F_OPEN); 1478 1.2.2.3 skrll return 0; 1479 1.2.2.3 skrll } 1480 1.2.2.3 skrll 1481 1.2.2.3 skrll /* 1482 1.2.2.3 skrll * Accept the last close on the control device. 1483 1.2.2.3 skrll */ 1484 1.2.2.3 skrll int 1485 1.2.2.3 skrll nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1486 1.2.2.3 skrll { 1487 1.2.2.3 skrll struct nvme_softc *sc; 1488 1.2.2.3 skrll int unit = minor(dev); 1489 1.2.2.3 skrll 1490 1.2.2.3 skrll sc = device_lookup_private(&nvme_cd, unit); 1491 1.2.2.3 skrll if (sc == NULL) 1492 1.2.2.3 skrll return ENXIO; 1493 1.2.2.3 skrll 1494 1.2.2.3 skrll CLR(sc->sc_flags, NVME_F_OPEN); 1495 1.2.2.3 skrll return 0; 1496 1.2.2.3 skrll } 1497 1.2.2.3 skrll 1498 1.2.2.3 skrll /* 1499 1.2.2.3 skrll * Handle control operations. 1500 1.2.2.3 skrll */ 1501 1.2.2.3 skrll int 1502 1.2.2.3 skrll nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1503 1.2.2.3 skrll { 1504 1.2.2.3 skrll struct nvme_softc *sc; 1505 1.2.2.3 skrll struct nvme_pt_command *pt; 1506 1.2.2.3 skrll int unit = minor(dev); 1507 1.2.2.3 skrll 1508 1.2.2.3 skrll sc = device_lookup_private(&nvme_cd, unit); 1509 1.2.2.3 skrll if (sc == NULL) 1510 1.2.2.3 skrll return ENXIO; 1511 1.2.2.3 skrll 1512 1.2.2.3 skrll switch (cmd) { 1513 1.2.2.3 skrll case NVME_PASSTHROUGH_CMD: 1514 1.2.2.3 skrll pt = (struct nvme_pt_command *)data; 1515 1.2.2.3 skrll return nvme_command_passthrough(sc, pt, pt->cmd.nsid, l, true); 1516 1.2.2.3 skrll } 1517 1.2.2.3 skrll 1518 1.2.2.3 skrll return ENOTTY; 1519 1.2.2.3 skrll } 1520 1.2.2.3 skrll 1521 1.2.2.3 skrll /* nvmens */ 1522 1.2.2.3 skrll dev_type_open(nvmensopen); 1523 1.2.2.3 skrll dev_type_close(nvmensclose); 1524 1.2.2.3 skrll dev_type_ioctl(nvmensioctl); 1525 1.2.2.3 skrll 1526 1.2.2.3 skrll const struct cdevsw nvmens_cdevsw = { 1527 1.2.2.3 skrll .d_open = nvmensopen, 1528 1.2.2.3 skrll .d_close = nvmensclose, 1529 1.2.2.3 skrll .d_read = noread, 1530 1.2.2.3 skrll .d_write = nowrite, 1531 1.2.2.3 skrll .d_ioctl = nvmensioctl, 1532 1.2.2.3 skrll .d_stop = nostop, 1533 1.2.2.3 skrll .d_tty = notty, 1534 1.2.2.3 skrll .d_poll = nopoll, 1535 1.2.2.3 skrll .d_mmap = nommap, 1536 1.2.2.3 skrll .d_kqfilter = nokqfilter, 1537 1.2.2.3 skrll .d_discard = nodiscard, 1538 1.2.2.3 skrll .d_flag = D_OTHER, 1539 1.2.2.3 skrll }; 1540 1.2.2.3 skrll 1541 1.2.2.3 skrll extern struct cfdriver nvmens_cd; 1542 1.2.2.3 skrll 1543 1.2.2.3 skrll /* 1544 1.2.2.3 skrll * Accept an open operation on the control device. 1545 1.2.2.3 skrll */ 1546 1.2.2.3 skrll int 1547 1.2.2.3 skrll nvmensopen(dev_t dev, int flag, int mode, struct lwp *l) 1548 1.2.2.3 skrll { 1549 1.2.2.3 skrll struct nvme_softc *sc; 1550 1.2.2.3 skrll int unit = minor(dev) / 0x10000; 1551 1.2.2.3 skrll int nsid = minor(dev) & 0xffff; 1552 1.2.2.3 skrll int nsidx; 1553 1.2.2.3 skrll 1554 1.2.2.3 skrll if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1555 1.2.2.3 skrll return ENXIO; 1556 1.2.2.3 skrll if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1557 1.2.2.3 skrll return ENXIO; 1558 1.2.2.3 skrll if (nsid == 0) 1559 1.2.2.3 skrll return ENXIO; 1560 1.2.2.3 skrll 1561 1.2.2.3 skrll nsidx = nsid - 1; 1562 1.2.2.3 skrll if (nsidx > sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1563 1.2.2.3 skrll return ENXIO; 1564 1.2.2.3 skrll if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1565 1.2.2.3 skrll return EBUSY; 1566 1.2.2.3 skrll 1567 1.2.2.3 skrll SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1568 1.2.2.3 skrll return 0; 1569 1.2.2.3 skrll } 1570 1.2.2.3 skrll 1571 1.2.2.3 skrll /* 1572 1.2.2.3 skrll * Accept the last close on the control device. 1573 1.2.2.3 skrll */ 1574 1.2.2.3 skrll int 1575 1.2.2.3 skrll nvmensclose(dev_t dev, int flag, int mode, struct lwp *l) 1576 1.2.2.3 skrll { 1577 1.2.2.3 skrll struct nvme_softc *sc; 1578 1.2.2.3 skrll int unit = minor(dev) / 0x10000; 1579 1.2.2.3 skrll int nsid = minor(dev) & 0xffff; 1580 1.2.2.3 skrll int nsidx; 1581 1.2.2.3 skrll 1582 1.2.2.3 skrll sc = device_lookup_private(&nvme_cd, unit); 1583 1.2.2.3 skrll if (sc == NULL) 1584 1.2.2.3 skrll return ENXIO; 1585 1.2.2.3 skrll if (nsid == 0) 1586 1.2.2.3 skrll return ENXIO; 1587 1.2.2.3 skrll 1588 1.2.2.3 skrll nsidx = nsid - 1; 1589 1.2.2.3 skrll if (nsidx > sc->sc_nn) 1590 1.2.2.3 skrll return ENXIO; 1591 1.2.2.3 skrll 1592 1.2.2.3 skrll CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1593 1.2.2.3 skrll return 0; 1594 1.2.2.3 skrll } 1595 1.2.2.3 skrll 1596 1.2.2.3 skrll /* 1597 1.2.2.3 skrll * Handle control operations. 1598 1.2.2.3 skrll */ 1599 1.2.2.3 skrll int 1600 1.2.2.3 skrll nvmensioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1601 1.2.2.3 skrll { 1602 1.2.2.3 skrll struct nvme_softc *sc; 1603 1.2.2.3 skrll int unit = minor(dev) / 0x10000; 1604 1.2.2.3 skrll int nsid = minor(dev) & 0xffff; 1605 1.2.2.3 skrll 1606 1.2.2.3 skrll sc = device_lookup_private(&nvme_cd, unit); 1607 1.2.2.3 skrll if (sc == NULL) 1608 1.2.2.3 skrll return ENXIO; 1609 1.2.2.3 skrll if (nsid == 0) 1610 1.2.2.3 skrll return ENXIO; 1611 1.2.2.3 skrll 1612 1.2.2.3 skrll switch (cmd) { 1613 1.2.2.3 skrll case NVME_PASSTHROUGH_CMD: 1614 1.2.2.3 skrll return nvme_command_passthrough(sc, data, nsid, l, false); 1615 1.2.2.3 skrll } 1616 1.2.2.3 skrll 1617 1.2.2.3 skrll return ENOTTY; 1618 1.2.2.3 skrll } 1619
Indexes created Mon Oct 27 17:09:47 GMT 2025