nvme.c revision 1.2.2.2
1 1.2.2.2 skrll /* $NetBSD: nvme.c,v 1.2.2.2 2016/05/29 08:44:21 skrll Exp $ */ 2 1.2.2.2 skrll /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 1.2.2.2 skrll 4 1.2.2.2 skrll /* 5 1.2.2.2 skrll * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 1.2.2.2 skrll * 7 1.2.2.2 skrll * Permission to use, copy, modify, and distribute this software for any 8 1.2.2.2 skrll * purpose with or without fee is hereby granted, provided that the above 9 1.2.2.2 skrll * copyright notice and this permission notice appear in all copies. 10 1.2.2.2 skrll * 11 1.2.2.2 skrll * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 1.2.2.2 skrll * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 1.2.2.2 skrll * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 1.2.2.2 skrll * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 1.2.2.2 skrll * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 1.2.2.2 skrll * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 1.2.2.2 skrll * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 1.2.2.2 skrll */ 19 1.2.2.2 skrll 20 1.2.2.2 skrll #include <sys/cdefs.h> 21 1.2.2.2 skrll __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.2.2.2 2016/05/29 08:44:21 skrll Exp $"); 22 1.2.2.2 skrll 23 1.2.2.2 skrll #include <sys/param.h> 24 1.2.2.2 skrll #include <sys/systm.h> 25 1.2.2.2 skrll #include <sys/kernel.h> 26 1.2.2.2 skrll #include <sys/atomic.h> 27 1.2.2.2 skrll #include <sys/bus.h> 28 1.2.2.2 skrll #include <sys/buf.h> 29 1.2.2.2 skrll #include <sys/device.h> 30 1.2.2.2 skrll #include <sys/kmem.h> 31 1.2.2.2 skrll #include <sys/once.h> 32 1.2.2.2 skrll #include <sys/queue.h> 33 1.2.2.2 skrll #include <sys/mutex.h> 34 1.2.2.2 skrll 35 1.2.2.2 skrll #include <dev/ic/nvmereg.h> 36 1.2.2.2 skrll #include <dev/ic/nvmevar.h> 37 1.2.2.2 skrll 38 1.2.2.2 skrll int nvme_adminq_size = 128; 39 1.2.2.2 skrll int nvme_ioq_size = 128; 40 1.2.2.2 skrll 41 1.2.2.2 skrll static int nvme_print(void *, const char *); 42 1.2.2.2 skrll 43 1.2.2.2 skrll static int nvme_ready(struct nvme_softc *, uint32_t); 44 1.2.2.2 skrll static int nvme_enable(struct nvme_softc *, u_int); 45 1.2.2.2 skrll static int nvme_disable(struct nvme_softc *); 46 1.2.2.2 skrll static int nvme_shutdown(struct nvme_softc *); 47 1.2.2.2 skrll 48 1.2.2.2 skrll static void nvme_version(struct nvme_softc *, uint32_t); 49 1.2.2.2 skrll #ifdef NVME_DEBUG 50 1.2.2.2 skrll static void nvme_dumpregs(struct nvme_softc *); 51 1.2.2.2 skrll #endif 52 1.2.2.2 skrll static int nvme_identify(struct nvme_softc *, u_int); 53 1.2.2.2 skrll static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 54 1.2.2.2 skrll void *); 55 1.2.2.2 skrll 56 1.2.2.2 skrll static int nvme_ccbs_alloc(struct nvme_queue *, u_int); 57 1.2.2.2 skrll static void nvme_ccbs_free(struct nvme_queue *); 58 1.2.2.2 skrll 59 1.2.2.2 skrll static struct nvme_ccb * 60 1.2.2.2 skrll nvme_ccb_get(struct nvme_queue *); 61 1.2.2.2 skrll static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 62 1.2.2.2 skrll 63 1.2.2.2 skrll static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 64 1.2.2.2 skrll struct nvme_ccb *, void (*)(struct nvme_queue *, 65 1.2.2.2 skrll struct nvme_ccb *, void *)); 66 1.2.2.2 skrll static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 67 1.2.2.2 skrll static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 68 1.2.2.2 skrll struct nvme_cqe *); 69 1.2.2.2 skrll static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 70 1.2.2.2 skrll static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 71 1.2.2.2 skrll struct nvme_cqe *); 72 1.2.2.2 skrll 73 1.2.2.2 skrll static struct nvme_queue * 74 1.2.2.2 skrll nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 75 1.2.2.2 skrll static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 76 1.2.2.2 skrll static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 77 1.2.2.2 skrll static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 78 1.2.2.2 skrll struct nvme_ccb *, void (*)(struct nvme_queue *, 79 1.2.2.2 skrll struct nvme_ccb *, void *)); 80 1.2.2.2 skrll static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 81 1.2.2.2 skrll static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 82 1.2.2.2 skrll 83 1.2.2.2 skrll static struct nvme_dmamem * 84 1.2.2.2 skrll nvme_dmamem_alloc(struct nvme_softc *, size_t); 85 1.2.2.2 skrll static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 86 1.2.2.2 skrll static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 87 1.2.2.2 skrll int); 88 1.2.2.2 skrll 89 1.2.2.2 skrll static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 90 1.2.2.2 skrll void *); 91 1.2.2.2 skrll static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 92 1.2.2.2 skrll struct nvme_cqe *); 93 1.2.2.2 skrll static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 94 1.2.2.2 skrll void *); 95 1.2.2.2 skrll static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 96 1.2.2.2 skrll struct nvme_cqe *); 97 1.2.2.2 skrll 98 1.2.2.2 skrll #define nvme_read4(_s, _r) \ 99 1.2.2.2 skrll bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 100 1.2.2.2 skrll #define nvme_write4(_s, _r, _v) \ 101 1.2.2.2 skrll bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 102 1.2.2.2 skrll #ifdef __LP64__ 103 1.2.2.2 skrll #define nvme_read8(_s, _r) \ 104 1.2.2.2 skrll bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r)) 105 1.2.2.2 skrll #define nvme_write8(_s, _r, _v) \ 106 1.2.2.2 skrll bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 107 1.2.2.2 skrll #else /* __LP64__ */ 108 1.2.2.2 skrll static inline uint64_t 109 1.2.2.2 skrll nvme_read8(struct nvme_softc *sc, bus_size_t r) 110 1.2.2.2 skrll { 111 1.2.2.2 skrll uint64_t v; 112 1.2.2.2 skrll uint32_t *a = (uint32_t *)&v; 113 1.2.2.2 skrll 114 1.2.2.2 skrll #if _BYTE_ORDER == _LITTLE_ENDIAN 115 1.2.2.2 skrll a[0] = nvme_read4(sc, r); 116 1.2.2.2 skrll a[1] = nvme_read4(sc, r + 4); 117 1.2.2.2 skrll #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 118 1.2.2.2 skrll a[1] = nvme_read4(sc, r); 119 1.2.2.2 skrll a[0] = nvme_read4(sc, r + 4); 120 1.2.2.2 skrll #endif 121 1.2.2.2 skrll 122 1.2.2.2 skrll return v; 123 1.2.2.2 skrll } 124 1.2.2.2 skrll 125 1.2.2.2 skrll static inline void 126 1.2.2.2 skrll nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 127 1.2.2.2 skrll { 128 1.2.2.2 skrll uint32_t *a = (uint32_t *)&v; 129 1.2.2.2 skrll 130 1.2.2.2 skrll #if _BYTE_ORDER == _LITTLE_ENDIAN 131 1.2.2.2 skrll nvme_write4(sc, r, a[0]); 132 1.2.2.2 skrll nvme_write4(sc, r + 4, a[1]); 133 1.2.2.2 skrll #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 134 1.2.2.2 skrll nvme_write4(sc, r, a[1]); 135 1.2.2.2 skrll nvme_write4(sc, r + 4, a[0]); 136 1.2.2.2 skrll #endif 137 1.2.2.2 skrll } 138 1.2.2.2 skrll #endif /* __LP64__ */ 139 1.2.2.2 skrll #define nvme_barrier(_s, _r, _l, _f) \ 140 1.2.2.2 skrll bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 141 1.2.2.2 skrll 142 1.2.2.2 skrll pool_cache_t nvme_ns_ctx_cache; 143 1.2.2.2 skrll ONCE_DECL(nvme_init_once); 144 1.2.2.2 skrll 145 1.2.2.2 skrll static int 146 1.2.2.2 skrll nvme_init(void) 147 1.2.2.2 skrll { 148 1.2.2.2 skrll nvme_ns_ctx_cache = pool_cache_init(sizeof(struct nvme_ns_context), 149 1.2.2.2 skrll 0, 0, 0, "nvme_ns_ctx", NULL, IPL_BIO, NULL, NULL, NULL); 150 1.2.2.2 skrll KASSERT(nvme_ns_ctx_cache != NULL); 151 1.2.2.2 skrll return 0; 152 1.2.2.2 skrll } 153 1.2.2.2 skrll 154 1.2.2.2 skrll static void 155 1.2.2.2 skrll nvme_version(struct nvme_softc *sc, uint32_t ver) 156 1.2.2.2 skrll { 157 1.2.2.2 skrll const char *v = NULL; 158 1.2.2.2 skrll 159 1.2.2.2 skrll switch (ver) { 160 1.2.2.2 skrll case NVME_VS_1_0: 161 1.2.2.2 skrll v = "1.0"; 162 1.2.2.2 skrll break; 163 1.2.2.2 skrll case NVME_VS_1_1: 164 1.2.2.2 skrll v = "1.1"; 165 1.2.2.2 skrll break; 166 1.2.2.2 skrll case NVME_VS_1_2: 167 1.2.2.2 skrll v = "1.2"; 168 1.2.2.2 skrll break; 169 1.2.2.2 skrll default: 170 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unknown version 0x%08x\n", ver); 171 1.2.2.2 skrll return; 172 1.2.2.2 skrll } 173 1.2.2.2 skrll 174 1.2.2.2 skrll aprint_normal_dev(sc->sc_dev, "NVMe %s\n", v); 175 1.2.2.2 skrll } 176 1.2.2.2 skrll 177 1.2.2.2 skrll #ifdef NVME_DEBUG 178 1.2.2.2 skrll static void 179 1.2.2.2 skrll nvme_dumpregs(struct nvme_softc *sc) 180 1.2.2.2 skrll { 181 1.2.2.2 skrll uint64_t r8; 182 1.2.2.2 skrll uint32_t r4; 183 1.2.2.2 skrll 184 1.2.2.2 skrll #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 185 1.2.2.2 skrll r8 = nvme_read8(sc, NVME_CAP); 186 1.2.2.2 skrll printf("%s: cap 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 187 1.2.2.2 skrll printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 188 1.2.2.2 skrll (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 189 1.2.2.2 skrll printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 190 1.2.2.2 skrll (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 191 1.2.2.2 skrll printf("%s: css %llu\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 192 1.2.2.2 skrll printf("%s: nssrs %llu\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 193 1.2.2.2 skrll printf("%s: dstrd %u\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 194 1.2.2.2 skrll printf("%s: to %llu msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 195 1.2.2.2 skrll printf("%s: ams %llu\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 196 1.2.2.2 skrll printf("%s: cqr %llu\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 197 1.2.2.2 skrll printf("%s: mqes %llu\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 198 1.2.2.2 skrll 199 1.2.2.2 skrll printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 200 1.2.2.2 skrll 201 1.2.2.2 skrll r4 = nvme_read4(sc, NVME_CC); 202 1.2.2.2 skrll printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 203 1.2.2.2 skrll printf("%s: iocqes %u\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4)); 204 1.2.2.2 skrll printf("%s: iosqes %u\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4)); 205 1.2.2.2 skrll printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 206 1.2.2.2 skrll printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 207 1.2.2.2 skrll printf("%s: mps %u\n", DEVNAME(sc), NVME_CC_MPS_R(r4)); 208 1.2.2.2 skrll printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 209 1.2.2.2 skrll printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN)); 210 1.2.2.2 skrll 211 1.2.2.2 skrll printf("%s: csts 0x%08x\n", DEVNAME(sc), nvme_read4(sc, NVME_CSTS)); 212 1.2.2.2 skrll printf("%s: aqa 0x%08x\n", DEVNAME(sc), nvme_read4(sc, NVME_AQA)); 213 1.2.2.2 skrll printf("%s: asq 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 214 1.2.2.2 skrll printf("%s: acq 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 215 1.2.2.2 skrll #undef DEVNAME 216 1.2.2.2 skrll } 217 1.2.2.2 skrll #endif /* NVME_DEBUG */ 218 1.2.2.2 skrll 219 1.2.2.2 skrll static int 220 1.2.2.2 skrll nvme_ready(struct nvme_softc *sc, uint32_t rdy) 221 1.2.2.2 skrll { 222 1.2.2.2 skrll u_int i = 0; 223 1.2.2.2 skrll 224 1.2.2.2 skrll while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 225 1.2.2.2 skrll if (i++ > sc->sc_rdy_to) 226 1.2.2.2 skrll return 1; 227 1.2.2.2 skrll 228 1.2.2.2 skrll delay(1000); 229 1.2.2.2 skrll nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 230 1.2.2.2 skrll } 231 1.2.2.2 skrll 232 1.2.2.2 skrll return 0; 233 1.2.2.2 skrll } 234 1.2.2.2 skrll 235 1.2.2.2 skrll static int 236 1.2.2.2 skrll nvme_enable(struct nvme_softc *sc, u_int mps) 237 1.2.2.2 skrll { 238 1.2.2.2 skrll uint32_t cc; 239 1.2.2.2 skrll 240 1.2.2.2 skrll cc = nvme_read4(sc, NVME_CC); 241 1.2.2.2 skrll if (ISSET(cc, NVME_CC_EN)) 242 1.2.2.2 skrll return nvme_ready(sc, NVME_CSTS_RDY); 243 1.2.2.2 skrll 244 1.2.2.2 skrll nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 245 1.2.2.2 skrll NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 246 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 247 1.2.2.2 skrll 248 1.2.2.2 skrll nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 249 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 250 1.2.2.2 skrll nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 251 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 252 1.2.2.2 skrll 253 1.2.2.2 skrll CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 254 1.2.2.2 skrll NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 255 1.2.2.2 skrll SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 256 1.2.2.2 skrll SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 257 1.2.2.2 skrll SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 258 1.2.2.2 skrll SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 259 1.2.2.2 skrll SET(cc, NVME_CC_MPS(mps)); 260 1.2.2.2 skrll SET(cc, NVME_CC_EN); 261 1.2.2.2 skrll 262 1.2.2.2 skrll nvme_write4(sc, NVME_CC, cc); 263 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, 264 1.2.2.2 skrll BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 265 1.2.2.2 skrll 266 1.2.2.2 skrll return nvme_ready(sc, NVME_CSTS_RDY); 267 1.2.2.2 skrll } 268 1.2.2.2 skrll 269 1.2.2.2 skrll static int 270 1.2.2.2 skrll nvme_disable(struct nvme_softc *sc) 271 1.2.2.2 skrll { 272 1.2.2.2 skrll uint32_t cc, csts; 273 1.2.2.2 skrll 274 1.2.2.2 skrll cc = nvme_read4(sc, NVME_CC); 275 1.2.2.2 skrll if (ISSET(cc, NVME_CC_EN)) { 276 1.2.2.2 skrll csts = nvme_read4(sc, NVME_CSTS); 277 1.2.2.2 skrll if (!ISSET(csts, NVME_CSTS_CFS) && 278 1.2.2.2 skrll nvme_ready(sc, NVME_CSTS_RDY) != 0) 279 1.2.2.2 skrll return 1; 280 1.2.2.2 skrll } 281 1.2.2.2 skrll 282 1.2.2.2 skrll CLR(cc, NVME_CC_EN); 283 1.2.2.2 skrll 284 1.2.2.2 skrll nvme_write4(sc, NVME_CC, cc); 285 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, 286 1.2.2.2 skrll BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 287 1.2.2.2 skrll 288 1.2.2.2 skrll return nvme_ready(sc, 0); 289 1.2.2.2 skrll } 290 1.2.2.2 skrll 291 1.2.2.2 skrll int 292 1.2.2.2 skrll nvme_attach(struct nvme_softc *sc) 293 1.2.2.2 skrll { 294 1.2.2.2 skrll struct nvme_attach_args naa; 295 1.2.2.2 skrll uint64_t cap; 296 1.2.2.2 skrll uint32_t reg; 297 1.2.2.2 skrll u_int dstrd; 298 1.2.2.2 skrll u_int mps = PAGE_SHIFT; 299 1.2.2.2 skrll int adminq_entries = nvme_adminq_size; 300 1.2.2.2 skrll int ioq_entries = nvme_ioq_size; 301 1.2.2.2 skrll int i; 302 1.2.2.2 skrll 303 1.2.2.2 skrll RUN_ONCE(&nvme_init_once, nvme_init); 304 1.2.2.2 skrll 305 1.2.2.2 skrll reg = nvme_read4(sc, NVME_VS); 306 1.2.2.2 skrll if (reg == 0xffffffff) { 307 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 308 1.2.2.2 skrll return 1; 309 1.2.2.2 skrll } 310 1.2.2.2 skrll 311 1.2.2.2 skrll nvme_version(sc, reg); 312 1.2.2.2 skrll 313 1.2.2.2 skrll cap = nvme_read8(sc, NVME_CAP); 314 1.2.2.2 skrll dstrd = NVME_CAP_DSTRD(cap); 315 1.2.2.2 skrll if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 316 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 317 1.2.2.2 skrll "is greater than CPU page size %u\n", 318 1.2.2.2 skrll 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 319 1.2.2.2 skrll return 1; 320 1.2.2.2 skrll } 321 1.2.2.2 skrll if (NVME_CAP_MPSMAX(cap) < mps) 322 1.2.2.2 skrll mps = NVME_CAP_MPSMAX(cap); 323 1.2.2.2 skrll 324 1.2.2.2 skrll sc->sc_rdy_to = NVME_CAP_TO(cap); 325 1.2.2.2 skrll sc->sc_mps = 1 << mps; 326 1.2.2.2 skrll sc->sc_mdts = MAXPHYS; 327 1.2.2.2 skrll sc->sc_max_sgl = 2; 328 1.2.2.2 skrll 329 1.2.2.2 skrll if (nvme_disable(sc) != 0) { 330 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 331 1.2.2.2 skrll return 1; 332 1.2.2.2 skrll } 333 1.2.2.2 skrll 334 1.2.2.2 skrll sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 335 1.2.2.2 skrll if (sc->sc_admin_q == NULL) { 336 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, 337 1.2.2.2 skrll "unable to allocate admin queue\n"); 338 1.2.2.2 skrll return 1; 339 1.2.2.2 skrll } 340 1.2.2.2 skrll if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 341 1.2.2.2 skrll goto free_admin_q; 342 1.2.2.2 skrll 343 1.2.2.2 skrll if (nvme_enable(sc, mps) != 0) { 344 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 345 1.2.2.2 skrll goto disestablish_admin_q; 346 1.2.2.2 skrll } 347 1.2.2.2 skrll 348 1.2.2.2 skrll if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 349 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 350 1.2.2.2 skrll goto disable; 351 1.2.2.2 skrll } 352 1.2.2.2 skrll 353 1.2.2.2 skrll /* we know how big things are now */ 354 1.2.2.2 skrll sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 355 1.2.2.2 skrll 356 1.2.2.2 skrll /* reallocate ccbs of admin queue with new max sgl. */ 357 1.2.2.2 skrll nvme_ccbs_free(sc->sc_admin_q); 358 1.2.2.2 skrll nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 359 1.2.2.2 skrll 360 1.2.2.2 skrll sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 361 1.2.2.2 skrll if (sc->sc_q == NULL) { 362 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n"); 363 1.2.2.2 skrll goto disable; 364 1.2.2.2 skrll } 365 1.2.2.2 skrll for (i = 0; i < sc->sc_nq; i++) { 366 1.2.2.2 skrll sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 367 1.2.2.2 skrll if (sc->sc_q[i] == NULL) { 368 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, 369 1.2.2.2 skrll "unable to allocate io queue\n"); 370 1.2.2.2 skrll goto free_q; 371 1.2.2.2 skrll } 372 1.2.2.2 skrll if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 373 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, 374 1.2.2.2 skrll "unable to create io queue\n"); 375 1.2.2.2 skrll nvme_q_free(sc, sc->sc_q[i]); 376 1.2.2.2 skrll goto free_q; 377 1.2.2.2 skrll } 378 1.2.2.2 skrll } 379 1.2.2.2 skrll 380 1.2.2.2 skrll if (!sc->sc_use_mq) 381 1.2.2.2 skrll nvme_write4(sc, NVME_INTMC, 1); 382 1.2.2.2 skrll 383 1.2.2.2 skrll sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 384 1.2.2.2 skrll KM_SLEEP); 385 1.2.2.2 skrll for (i = 0; i < sc->sc_nn; i++) { 386 1.2.2.2 skrll memset(&naa, 0, sizeof(naa)); 387 1.2.2.2 skrll naa.naa_nsid = i + 1; 388 1.2.2.2 skrll naa.naa_qentries = ioq_entries; 389 1.2.2.2 skrll sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 390 1.2.2.2 skrll nvme_print); 391 1.2.2.2 skrll } 392 1.2.2.2 skrll 393 1.2.2.2 skrll return 0; 394 1.2.2.2 skrll 395 1.2.2.2 skrll free_q: 396 1.2.2.2 skrll while (--i >= 0) { 397 1.2.2.2 skrll nvme_q_delete(sc, sc->sc_q[i]); 398 1.2.2.2 skrll nvme_q_free(sc, sc->sc_q[i]); 399 1.2.2.2 skrll } 400 1.2.2.2 skrll disable: 401 1.2.2.2 skrll nvme_disable(sc); 402 1.2.2.2 skrll disestablish_admin_q: 403 1.2.2.2 skrll sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 404 1.2.2.2 skrll free_admin_q: 405 1.2.2.2 skrll nvme_q_free(sc, sc->sc_admin_q); 406 1.2.2.2 skrll 407 1.2.2.2 skrll return 1; 408 1.2.2.2 skrll } 409 1.2.2.2 skrll 410 1.2.2.2 skrll static int 411 1.2.2.2 skrll nvme_print(void *aux, const char *pnp) 412 1.2.2.2 skrll { 413 1.2.2.2 skrll struct nvme_attach_args *naa = aux; 414 1.2.2.2 skrll 415 1.2.2.2 skrll if (pnp) 416 1.2.2.2 skrll aprint_normal("at %s", pnp); 417 1.2.2.2 skrll 418 1.2.2.2 skrll if (naa->naa_nsid > 0) 419 1.2.2.2 skrll aprint_normal(" nsid %d", naa->naa_nsid); 420 1.2.2.2 skrll 421 1.2.2.2 skrll return UNCONF; 422 1.2.2.2 skrll } 423 1.2.2.2 skrll 424 1.2.2.2 skrll int 425 1.2.2.2 skrll nvme_detach(struct nvme_softc *sc, int flags) 426 1.2.2.2 skrll { 427 1.2.2.2 skrll int i, error; 428 1.2.2.2 skrll 429 1.2.2.2 skrll error = config_detach_children(sc->sc_dev, flags); 430 1.2.2.2 skrll if (error) 431 1.2.2.2 skrll return error; 432 1.2.2.2 skrll 433 1.2.2.2 skrll error = nvme_shutdown(sc); 434 1.2.2.2 skrll if (error) 435 1.2.2.2 skrll return error; 436 1.2.2.2 skrll 437 1.2.2.2 skrll for (i = 0; i < sc->sc_nq; i++) 438 1.2.2.2 skrll nvme_q_free(sc, sc->sc_q[i]); 439 1.2.2.2 skrll kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 440 1.2.2.2 skrll nvme_q_free(sc, sc->sc_admin_q); 441 1.2.2.2 skrll 442 1.2.2.2 skrll return 0; 443 1.2.2.2 skrll } 444 1.2.2.2 skrll 445 1.2.2.2 skrll static int 446 1.2.2.2 skrll nvme_shutdown(struct nvme_softc *sc) 447 1.2.2.2 skrll { 448 1.2.2.2 skrll uint32_t cc, csts; 449 1.2.2.2 skrll bool disabled = false; 450 1.2.2.2 skrll int i; 451 1.2.2.2 skrll 452 1.2.2.2 skrll if (!sc->sc_use_mq) 453 1.2.2.2 skrll nvme_write4(sc, NVME_INTMS, 1); 454 1.2.2.2 skrll 455 1.2.2.2 skrll for (i = 0; i < sc->sc_nq; i++) { 456 1.2.2.2 skrll if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 457 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, 458 1.2.2.2 skrll "unable to delete io queue %d, disabling\n", i + 1); 459 1.2.2.2 skrll disabled = true; 460 1.2.2.2 skrll } 461 1.2.2.2 skrll } 462 1.2.2.2 skrll sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 463 1.2.2.2 skrll if (disabled) 464 1.2.2.2 skrll goto disable; 465 1.2.2.2 skrll 466 1.2.2.2 skrll cc = nvme_read4(sc, NVME_CC); 467 1.2.2.2 skrll CLR(cc, NVME_CC_SHN_MASK); 468 1.2.2.2 skrll SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 469 1.2.2.2 skrll nvme_write4(sc, NVME_CC, cc); 470 1.2.2.2 skrll 471 1.2.2.2 skrll for (i = 0; i < 4000; i++) { 472 1.2.2.2 skrll nvme_barrier(sc, 0, sc->sc_ios, 473 1.2.2.2 skrll BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 474 1.2.2.2 skrll csts = nvme_read4(sc, NVME_CSTS); 475 1.2.2.2 skrll if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 476 1.2.2.2 skrll return 0; 477 1.2.2.2 skrll 478 1.2.2.2 skrll delay(1000); 479 1.2.2.2 skrll } 480 1.2.2.2 skrll 481 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 482 1.2.2.2 skrll 483 1.2.2.2 skrll disable: 484 1.2.2.2 skrll nvme_disable(sc); 485 1.2.2.2 skrll return 0; 486 1.2.2.2 skrll } 487 1.2.2.2 skrll 488 1.2.2.2 skrll void 489 1.2.2.2 skrll nvme_childdet(device_t self, device_t child) 490 1.2.2.2 skrll { 491 1.2.2.2 skrll struct nvme_softc *sc = device_private(self); 492 1.2.2.2 skrll int i; 493 1.2.2.2 skrll 494 1.2.2.2 skrll for (i = 0; i < sc->sc_nn; i++) { 495 1.2.2.2 skrll if (sc->sc_namespaces[i].dev == child) { 496 1.2.2.2 skrll /* Already freed ns->ident. */ 497 1.2.2.2 skrll sc->sc_namespaces[i].dev = NULL; 498 1.2.2.2 skrll break; 499 1.2.2.2 skrll } 500 1.2.2.2 skrll } 501 1.2.2.2 skrll } 502 1.2.2.2 skrll 503 1.2.2.2 skrll int 504 1.2.2.2 skrll nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 505 1.2.2.2 skrll { 506 1.2.2.2 skrll struct nvme_sqe sqe; 507 1.2.2.2 skrll struct nvm_identify_namespace *identify; 508 1.2.2.2 skrll struct nvme_dmamem *mem; 509 1.2.2.2 skrll struct nvme_ccb *ccb; 510 1.2.2.2 skrll struct nvme_namespace *ns; 511 1.2.2.2 skrll int rv; 512 1.2.2.2 skrll 513 1.2.2.2 skrll KASSERT(nsid > 0); 514 1.2.2.2 skrll 515 1.2.2.2 skrll ccb = nvme_ccb_get(sc->sc_admin_q); 516 1.2.2.2 skrll KASSERT(ccb != NULL); 517 1.2.2.2 skrll 518 1.2.2.2 skrll mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 519 1.2.2.2 skrll if (mem == NULL) 520 1.2.2.2 skrll return ENOMEM; 521 1.2.2.2 skrll 522 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 523 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_IDENTIFY; 524 1.2.2.2 skrll htolem32(&sqe.nsid, nsid); 525 1.2.2.2 skrll htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 526 1.2.2.2 skrll htolem32(&sqe.cdw10, 0); 527 1.2.2.2 skrll 528 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 529 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 530 1.2.2.2 skrll 531 1.2.2.2 skrll nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 532 1.2.2.2 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 533 1.2.2.2 skrll nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 534 1.2.2.2 skrll 535 1.2.2.2 skrll nvme_ccb_put(sc->sc_admin_q, ccb); 536 1.2.2.2 skrll 537 1.2.2.2 skrll if (rv != 0) { 538 1.2.2.2 skrll rv = EIO; 539 1.2.2.2 skrll goto done; 540 1.2.2.2 skrll } 541 1.2.2.2 skrll 542 1.2.2.2 skrll /* commit */ 543 1.2.2.2 skrll 544 1.2.2.2 skrll identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 545 1.2.2.2 skrll memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 546 1.2.2.2 skrll 547 1.2.2.2 skrll ns = nvme_ns_get(sc, nsid); 548 1.2.2.2 skrll KASSERT(ns); 549 1.2.2.2 skrll ns->ident = identify; 550 1.2.2.2 skrll 551 1.2.2.2 skrll done: 552 1.2.2.2 skrll nvme_dmamem_free(sc, mem); 553 1.2.2.2 skrll 554 1.2.2.2 skrll return rv; 555 1.2.2.2 skrll } 556 1.2.2.2 skrll 557 1.2.2.2 skrll int 558 1.2.2.2 skrll nvme_ns_dobio(struct nvme_softc *sc, struct nvme_ns_context *ctx) 559 1.2.2.2 skrll { 560 1.2.2.2 skrll struct nvme_queue *q = nvme_get_q(sc); 561 1.2.2.2 skrll struct nvme_ccb *ccb; 562 1.2.2.2 skrll bus_dmamap_t dmap; 563 1.2.2.2 skrll int i, error; 564 1.2.2.2 skrll 565 1.2.2.2 skrll ccb = nvme_ccb_get(q); 566 1.2.2.2 skrll if (ccb == NULL) 567 1.2.2.2 skrll return EAGAIN; 568 1.2.2.2 skrll 569 1.2.2.2 skrll ccb->ccb_done = nvme_ns_io_done; 570 1.2.2.2 skrll ccb->ccb_cookie = ctx; 571 1.2.2.2 skrll 572 1.2.2.2 skrll dmap = ccb->ccb_dmamap; 573 1.2.2.2 skrll error = bus_dmamap_load(sc->sc_dmat, dmap, ctx->nnc_data, 574 1.2.2.2 skrll ctx->nnc_datasize, NULL, 575 1.2.2.2 skrll (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL) ? 576 1.2.2.2 skrll BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 577 1.2.2.2 skrll (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 578 1.2.2.2 skrll BUS_DMA_READ : BUS_DMA_WRITE)); 579 1.2.2.2 skrll if (error) { 580 1.2.2.2 skrll nvme_ccb_put(q, ccb); 581 1.2.2.2 skrll return error; 582 1.2.2.2 skrll } 583 1.2.2.2 skrll 584 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 585 1.2.2.2 skrll ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 586 1.2.2.2 skrll BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 587 1.2.2.2 skrll 588 1.2.2.2 skrll if (dmap->dm_nsegs > 2) { 589 1.2.2.2 skrll for (i = 1; i < dmap->dm_nsegs; i++) { 590 1.2.2.2 skrll htolem64(&ccb->ccb_prpl[i - 1], 591 1.2.2.2 skrll dmap->dm_segs[i].ds_addr); 592 1.2.2.2 skrll } 593 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, 594 1.2.2.2 skrll NVME_DMA_MAP(q->q_ccb_prpls), 595 1.2.2.2 skrll ccb->ccb_prpl_off, 596 1.2.2.2 skrll sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 597 1.2.2.2 skrll BUS_DMASYNC_PREWRITE); 598 1.2.2.2 skrll } 599 1.2.2.2 skrll 600 1.2.2.2 skrll if (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL)) { 601 1.2.2.2 skrll if (nvme_poll(sc, q, ccb, nvme_ns_io_fill) != 0) 602 1.2.2.2 skrll return EIO; 603 1.2.2.2 skrll return 0; 604 1.2.2.2 skrll } 605 1.2.2.2 skrll 606 1.2.2.2 skrll nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 607 1.2.2.2 skrll return 0; 608 1.2.2.2 skrll } 609 1.2.2.2 skrll 610 1.2.2.2 skrll static void 611 1.2.2.2 skrll nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 612 1.2.2.2 skrll { 613 1.2.2.2 skrll struct nvme_sqe_io *sqe = slot; 614 1.2.2.2 skrll struct nvme_ns_context *ctx = ccb->ccb_cookie; 615 1.2.2.2 skrll bus_dmamap_t dmap = ccb->ccb_dmamap; 616 1.2.2.2 skrll 617 1.2.2.2 skrll sqe->opcode = ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 618 1.2.2.2 skrll NVM_CMD_READ : NVM_CMD_WRITE; 619 1.2.2.2 skrll htolem32(&sqe->nsid, ctx->nnc_nsid); 620 1.2.2.2 skrll 621 1.2.2.2 skrll htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 622 1.2.2.2 skrll switch (dmap->dm_nsegs) { 623 1.2.2.2 skrll case 1: 624 1.2.2.2 skrll break; 625 1.2.2.2 skrll case 2: 626 1.2.2.2 skrll htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 627 1.2.2.2 skrll break; 628 1.2.2.2 skrll default: 629 1.2.2.2 skrll /* the prp list is already set up and synced */ 630 1.2.2.2 skrll htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 631 1.2.2.2 skrll break; 632 1.2.2.2 skrll } 633 1.2.2.2 skrll 634 1.2.2.2 skrll htolem64(&sqe->slba, ctx->nnc_blkno); 635 1.2.2.2 skrll htolem16(&sqe->nlb, (ctx->nnc_datasize / ctx->nnc_secsize) - 1); 636 1.2.2.2 skrll } 637 1.2.2.2 skrll 638 1.2.2.2 skrll static void 639 1.2.2.2 skrll nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 640 1.2.2.2 skrll struct nvme_cqe *cqe) 641 1.2.2.2 skrll { 642 1.2.2.2 skrll struct nvme_softc *sc = q->q_sc; 643 1.2.2.2 skrll struct nvme_ns_context *ctx = ccb->ccb_cookie; 644 1.2.2.2 skrll bus_dmamap_t dmap = ccb->ccb_dmamap; 645 1.2.2.2 skrll uint16_t flags; 646 1.2.2.2 skrll 647 1.2.2.2 skrll if (dmap->dm_nsegs > 2) { 648 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, 649 1.2.2.2 skrll NVME_DMA_MAP(q->q_ccb_prpls), 650 1.2.2.2 skrll ccb->ccb_prpl_off, 651 1.2.2.2 skrll sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 652 1.2.2.2 skrll BUS_DMASYNC_POSTWRITE); 653 1.2.2.2 skrll } 654 1.2.2.2 skrll 655 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 656 1.2.2.2 skrll ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 657 1.2.2.2 skrll BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 658 1.2.2.2 skrll 659 1.2.2.2 skrll bus_dmamap_unload(sc->sc_dmat, dmap); 660 1.2.2.2 skrll nvme_ccb_put(q, ccb); 661 1.2.2.2 skrll 662 1.2.2.2 skrll flags = lemtoh16(&cqe->flags); 663 1.2.2.2 skrll 664 1.2.2.2 skrll ctx->nnc_status = flags; 665 1.2.2.2 skrll (*ctx->nnc_done)(ctx); 666 1.2.2.2 skrll } 667 1.2.2.2 skrll 668 1.2.2.2 skrll int 669 1.2.2.2 skrll nvme_ns_sync(struct nvme_softc *sc, struct nvme_ns_context *ctx) 670 1.2.2.2 skrll { 671 1.2.2.2 skrll struct nvme_queue *q = nvme_get_q(sc); 672 1.2.2.2 skrll struct nvme_ccb *ccb; 673 1.2.2.2 skrll 674 1.2.2.2 skrll ccb = nvme_ccb_get(q); 675 1.2.2.2 skrll if (ccb == NULL) 676 1.2.2.2 skrll return EAGAIN; 677 1.2.2.2 skrll 678 1.2.2.2 skrll ccb->ccb_done = nvme_ns_sync_done; 679 1.2.2.2 skrll ccb->ccb_cookie = ctx; 680 1.2.2.2 skrll 681 1.2.2.2 skrll if (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL)) { 682 1.2.2.2 skrll if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill) != 0) 683 1.2.2.2 skrll return EIO; 684 1.2.2.2 skrll return 0; 685 1.2.2.2 skrll } 686 1.2.2.2 skrll 687 1.2.2.2 skrll nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 688 1.2.2.2 skrll return 0; 689 1.2.2.2 skrll } 690 1.2.2.2 skrll 691 1.2.2.2 skrll static void 692 1.2.2.2 skrll nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 693 1.2.2.2 skrll { 694 1.2.2.2 skrll struct nvme_sqe *sqe = slot; 695 1.2.2.2 skrll struct nvme_ns_context *ctx = ccb->ccb_cookie; 696 1.2.2.2 skrll 697 1.2.2.2 skrll sqe->opcode = NVM_CMD_FLUSH; 698 1.2.2.2 skrll htolem32(&sqe->nsid, ctx->nnc_nsid); 699 1.2.2.2 skrll } 700 1.2.2.2 skrll 701 1.2.2.2 skrll static void 702 1.2.2.2 skrll nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 703 1.2.2.2 skrll struct nvme_cqe *cqe) 704 1.2.2.2 skrll { 705 1.2.2.2 skrll struct nvme_ns_context *ctx = ccb->ccb_cookie; 706 1.2.2.2 skrll uint16_t flags; 707 1.2.2.2 skrll 708 1.2.2.2 skrll nvme_ccb_put(q, ccb); 709 1.2.2.2 skrll 710 1.2.2.2 skrll flags = lemtoh16(&cqe->flags); 711 1.2.2.2 skrll 712 1.2.2.2 skrll ctx->nnc_status = flags; 713 1.2.2.2 skrll (*ctx->nnc_done)(ctx); 714 1.2.2.2 skrll } 715 1.2.2.2 skrll 716 1.2.2.2 skrll void 717 1.2.2.2 skrll nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 718 1.2.2.2 skrll { 719 1.2.2.2 skrll struct nvme_namespace *ns; 720 1.2.2.2 skrll struct nvm_identify_namespace *identify; 721 1.2.2.2 skrll 722 1.2.2.2 skrll ns = nvme_ns_get(sc, nsid); 723 1.2.2.2 skrll KASSERT(ns); 724 1.2.2.2 skrll 725 1.2.2.2 skrll identify = ns->ident; 726 1.2.2.2 skrll ns->ident = NULL; 727 1.2.2.2 skrll if (identify != NULL) 728 1.2.2.2 skrll kmem_free(identify, sizeof(*identify)); 729 1.2.2.2 skrll } 730 1.2.2.2 skrll 731 1.2.2.2 skrll static void 732 1.2.2.2 skrll nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 733 1.2.2.2 skrll void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 734 1.2.2.2 skrll { 735 1.2.2.2 skrll struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 736 1.2.2.2 skrll uint32_t tail; 737 1.2.2.2 skrll 738 1.2.2.2 skrll mutex_enter(&q->q_sq_mtx); 739 1.2.2.2 skrll tail = q->q_sq_tail; 740 1.2.2.2 skrll if (++q->q_sq_tail >= q->q_entries) 741 1.2.2.2 skrll q->q_sq_tail = 0; 742 1.2.2.2 skrll 743 1.2.2.2 skrll sqe += tail; 744 1.2.2.2 skrll 745 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 746 1.2.2.2 skrll sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 747 1.2.2.2 skrll memset(sqe, 0, sizeof(*sqe)); 748 1.2.2.2 skrll (*fill)(q, ccb, sqe); 749 1.2.2.2 skrll sqe->cid = ccb->ccb_id; 750 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 751 1.2.2.2 skrll sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 752 1.2.2.2 skrll 753 1.2.2.2 skrll nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 754 1.2.2.2 skrll mutex_exit(&q->q_sq_mtx); 755 1.2.2.2 skrll } 756 1.2.2.2 skrll 757 1.2.2.2 skrll struct nvme_poll_state { 758 1.2.2.2 skrll struct nvme_sqe s; 759 1.2.2.2 skrll struct nvme_cqe c; 760 1.2.2.2 skrll }; 761 1.2.2.2 skrll 762 1.2.2.2 skrll static int 763 1.2.2.2 skrll nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 764 1.2.2.2 skrll void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 765 1.2.2.2 skrll { 766 1.2.2.2 skrll struct nvme_poll_state state; 767 1.2.2.2 skrll void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 768 1.2.2.2 skrll void *cookie; 769 1.2.2.2 skrll uint16_t flags; 770 1.2.2.2 skrll 771 1.2.2.2 skrll memset(&state, 0, sizeof(state)); 772 1.2.2.2 skrll (*fill)(q, ccb, &state.s); 773 1.2.2.2 skrll 774 1.2.2.2 skrll done = ccb->ccb_done; 775 1.2.2.2 skrll cookie = ccb->ccb_cookie; 776 1.2.2.2 skrll 777 1.2.2.2 skrll ccb->ccb_done = nvme_poll_done; 778 1.2.2.2 skrll ccb->ccb_cookie = &state; 779 1.2.2.2 skrll 780 1.2.2.2 skrll nvme_q_submit(sc, q, ccb, nvme_poll_fill); 781 1.2.2.2 skrll while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 782 1.2.2.2 skrll if (nvme_q_complete(sc, q) == 0) 783 1.2.2.2 skrll delay(10); 784 1.2.2.2 skrll 785 1.2.2.2 skrll /* XXX no timeout? */ 786 1.2.2.2 skrll } 787 1.2.2.2 skrll 788 1.2.2.2 skrll ccb->ccb_cookie = cookie; 789 1.2.2.2 skrll done(q, ccb, &state.c); 790 1.2.2.2 skrll 791 1.2.2.2 skrll flags = lemtoh16(&state.c.flags); 792 1.2.2.2 skrll 793 1.2.2.2 skrll return flags & ~NVME_CQE_PHASE; 794 1.2.2.2 skrll } 795 1.2.2.2 skrll 796 1.2.2.2 skrll static void 797 1.2.2.2 skrll nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 798 1.2.2.2 skrll { 799 1.2.2.2 skrll struct nvme_sqe *sqe = slot; 800 1.2.2.2 skrll struct nvme_poll_state *state = ccb->ccb_cookie; 801 1.2.2.2 skrll 802 1.2.2.2 skrll *sqe = state->s; 803 1.2.2.2 skrll } 804 1.2.2.2 skrll 805 1.2.2.2 skrll static void 806 1.2.2.2 skrll nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 807 1.2.2.2 skrll struct nvme_cqe *cqe) 808 1.2.2.2 skrll { 809 1.2.2.2 skrll struct nvme_poll_state *state = ccb->ccb_cookie; 810 1.2.2.2 skrll 811 1.2.2.2 skrll SET(cqe->flags, htole16(NVME_CQE_PHASE)); 812 1.2.2.2 skrll state->c = *cqe; 813 1.2.2.2 skrll } 814 1.2.2.2 skrll 815 1.2.2.2 skrll static void 816 1.2.2.2 skrll nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 817 1.2.2.2 skrll { 818 1.2.2.2 skrll struct nvme_sqe *src = ccb->ccb_cookie; 819 1.2.2.2 skrll struct nvme_sqe *dst = slot; 820 1.2.2.2 skrll 821 1.2.2.2 skrll *dst = *src; 822 1.2.2.2 skrll } 823 1.2.2.2 skrll 824 1.2.2.2 skrll static void 825 1.2.2.2 skrll nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 826 1.2.2.2 skrll struct nvme_cqe *cqe) 827 1.2.2.2 skrll { 828 1.2.2.2 skrll } 829 1.2.2.2 skrll 830 1.2.2.2 skrll static int 831 1.2.2.2 skrll nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 832 1.2.2.2 skrll { 833 1.2.2.2 skrll struct nvme_ccb *ccb; 834 1.2.2.2 skrll struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 835 1.2.2.2 skrll uint32_t head; 836 1.2.2.2 skrll uint16_t flags; 837 1.2.2.2 skrll int rv = 0; 838 1.2.2.2 skrll 839 1.2.2.2 skrll if (!mutex_tryenter(&q->q_cq_mtx)) 840 1.2.2.2 skrll return -1; 841 1.2.2.2 skrll 842 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 843 1.2.2.2 skrll head = q->q_cq_head; 844 1.2.2.2 skrll for (;;) { 845 1.2.2.2 skrll cqe = &ring[head]; 846 1.2.2.2 skrll flags = lemtoh16(&cqe->flags); 847 1.2.2.2 skrll if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 848 1.2.2.2 skrll break; 849 1.2.2.2 skrll 850 1.2.2.2 skrll ccb = &q->q_ccbs[cqe->cid]; 851 1.2.2.2 skrll ccb->ccb_done(q, ccb, cqe); 852 1.2.2.2 skrll 853 1.2.2.2 skrll if (++head >= q->q_entries) { 854 1.2.2.2 skrll head = 0; 855 1.2.2.2 skrll q->q_cq_phase ^= NVME_CQE_PHASE; 856 1.2.2.2 skrll } 857 1.2.2.2 skrll 858 1.2.2.2 skrll rv = 1; 859 1.2.2.2 skrll } 860 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 861 1.2.2.2 skrll 862 1.2.2.2 skrll if (rv) 863 1.2.2.2 skrll nvme_write4(sc, q->q_cqhdbl, q->q_cq_head = head); 864 1.2.2.2 skrll mutex_exit(&q->q_cq_mtx); 865 1.2.2.2 skrll 866 1.2.2.2 skrll return rv; 867 1.2.2.2 skrll } 868 1.2.2.2 skrll 869 1.2.2.2 skrll static int 870 1.2.2.2 skrll nvme_identify(struct nvme_softc *sc, u_int mps) 871 1.2.2.2 skrll { 872 1.2.2.2 skrll char sn[41], mn[81], fr[17]; 873 1.2.2.2 skrll struct nvm_identify_controller *identify; 874 1.2.2.2 skrll struct nvme_dmamem *mem; 875 1.2.2.2 skrll struct nvme_ccb *ccb; 876 1.2.2.2 skrll u_int mdts; 877 1.2.2.2 skrll int rv = 1; 878 1.2.2.2 skrll 879 1.2.2.2 skrll ccb = nvme_ccb_get(sc->sc_admin_q); 880 1.2.2.2 skrll if (ccb == NULL) 881 1.2.2.2 skrll panic("%s: nvme_ccb_get returned NULL", __func__); 882 1.2.2.2 skrll 883 1.2.2.2 skrll mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 884 1.2.2.2 skrll if (mem == NULL) 885 1.2.2.2 skrll return 1; 886 1.2.2.2 skrll 887 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 888 1.2.2.2 skrll ccb->ccb_cookie = mem; 889 1.2.2.2 skrll 890 1.2.2.2 skrll nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 891 1.2.2.2 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify); 892 1.2.2.2 skrll nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 893 1.2.2.2 skrll 894 1.2.2.2 skrll nvme_ccb_put(sc->sc_admin_q, ccb); 895 1.2.2.2 skrll 896 1.2.2.2 skrll if (rv != 0) 897 1.2.2.2 skrll goto done; 898 1.2.2.2 skrll 899 1.2.2.2 skrll identify = NVME_DMA_KVA(mem); 900 1.2.2.2 skrll 901 1.2.2.2 skrll strnvisx(sn, sizeof(sn), (const char *)identify->sn, 902 1.2.2.2 skrll sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 903 1.2.2.2 skrll strnvisx(mn, sizeof(mn), (const char *)identify->mn, 904 1.2.2.2 skrll sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 905 1.2.2.2 skrll strnvisx(fr, sizeof(fr), (const char *)identify->fr, 906 1.2.2.2 skrll sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 907 1.2.2.2 skrll aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 908 1.2.2.2 skrll sn); 909 1.2.2.2 skrll 910 1.2.2.2 skrll if (identify->mdts > 0) { 911 1.2.2.2 skrll mdts = (1 << identify->mdts) * (1 << mps); 912 1.2.2.2 skrll if (mdts < sc->sc_mdts) 913 1.2.2.2 skrll sc->sc_mdts = mdts; 914 1.2.2.2 skrll } 915 1.2.2.2 skrll 916 1.2.2.2 skrll sc->sc_nn = lemtoh32(&identify->nn); 917 1.2.2.2 skrll 918 1.2.2.2 skrll memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 919 1.2.2.2 skrll 920 1.2.2.2 skrll done: 921 1.2.2.2 skrll nvme_dmamem_free(sc, mem); 922 1.2.2.2 skrll 923 1.2.2.2 skrll return rv; 924 1.2.2.2 skrll } 925 1.2.2.2 skrll 926 1.2.2.2 skrll static int 927 1.2.2.2 skrll nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 928 1.2.2.2 skrll { 929 1.2.2.2 skrll struct nvme_sqe_q sqe; 930 1.2.2.2 skrll struct nvme_ccb *ccb; 931 1.2.2.2 skrll int rv; 932 1.2.2.2 skrll 933 1.2.2.2 skrll if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q)) 934 1.2.2.2 skrll return 1; 935 1.2.2.2 skrll 936 1.2.2.2 skrll ccb = nvme_ccb_get(sc->sc_admin_q); 937 1.2.2.2 skrll KASSERT(ccb != NULL); 938 1.2.2.2 skrll 939 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 940 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 941 1.2.2.2 skrll 942 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 943 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_ADD_IOCQ; 944 1.2.2.2 skrll htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 945 1.2.2.2 skrll htolem16(&sqe.qsize, q->q_entries - 1); 946 1.2.2.2 skrll htolem16(&sqe.qid, q->q_id); 947 1.2.2.2 skrll sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 948 1.2.2.2 skrll if (sc->sc_use_mq) 949 1.2.2.2 skrll htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 950 1.2.2.2 skrll 951 1.2.2.2 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 952 1.2.2.2 skrll if (rv != 0) 953 1.2.2.2 skrll goto fail; 954 1.2.2.2 skrll 955 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 956 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 957 1.2.2.2 skrll 958 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 959 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_ADD_IOSQ; 960 1.2.2.2 skrll htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 961 1.2.2.2 skrll htolem16(&sqe.qsize, q->q_entries - 1); 962 1.2.2.2 skrll htolem16(&sqe.qid, q->q_id); 963 1.2.2.2 skrll htolem16(&sqe.cqid, q->q_id); 964 1.2.2.2 skrll sqe.qflags = NVM_SQE_Q_PC; 965 1.2.2.2 skrll 966 1.2.2.2 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 967 1.2.2.2 skrll if (rv != 0) 968 1.2.2.2 skrll goto fail; 969 1.2.2.2 skrll 970 1.2.2.2 skrll fail: 971 1.2.2.2 skrll nvme_ccb_put(sc->sc_admin_q, ccb); 972 1.2.2.2 skrll return rv; 973 1.2.2.2 skrll } 974 1.2.2.2 skrll 975 1.2.2.2 skrll static int 976 1.2.2.2 skrll nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 977 1.2.2.2 skrll { 978 1.2.2.2 skrll struct nvme_sqe_q sqe; 979 1.2.2.2 skrll struct nvme_ccb *ccb; 980 1.2.2.2 skrll int rv; 981 1.2.2.2 skrll 982 1.2.2.2 skrll ccb = nvme_ccb_get(sc->sc_admin_q); 983 1.2.2.2 skrll KASSERT(ccb != NULL); 984 1.2.2.2 skrll 985 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 986 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 987 1.2.2.2 skrll 988 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 989 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_DEL_IOSQ; 990 1.2.2.2 skrll htolem16(&sqe.qid, q->q_id); 991 1.2.2.2 skrll 992 1.2.2.2 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 993 1.2.2.2 skrll if (rv != 0) 994 1.2.2.2 skrll goto fail; 995 1.2.2.2 skrll 996 1.2.2.2 skrll ccb->ccb_done = nvme_empty_done; 997 1.2.2.2 skrll ccb->ccb_cookie = &sqe; 998 1.2.2.2 skrll 999 1.2.2.2 skrll memset(&sqe, 0, sizeof(sqe)); 1000 1.2.2.2 skrll sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1001 1.2.2.2 skrll htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1002 1.2.2.2 skrll htolem16(&sqe.qid, q->q_id); 1003 1.2.2.2 skrll 1004 1.2.2.2 skrll rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1005 1.2.2.2 skrll if (rv != 0) 1006 1.2.2.2 skrll goto fail; 1007 1.2.2.2 skrll 1008 1.2.2.2 skrll fail: 1009 1.2.2.2 skrll nvme_ccb_put(sc->sc_admin_q, ccb); 1010 1.2.2.2 skrll 1011 1.2.2.2 skrll if (rv == 0 && sc->sc_use_mq) { 1012 1.2.2.2 skrll if (sc->sc_intr_disestablish(sc, q->q_id)) 1013 1.2.2.2 skrll rv = 1; 1014 1.2.2.2 skrll } 1015 1.2.2.2 skrll 1016 1.2.2.2 skrll return rv; 1017 1.2.2.2 skrll } 1018 1.2.2.2 skrll 1019 1.2.2.2 skrll static void 1020 1.2.2.2 skrll nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1021 1.2.2.2 skrll { 1022 1.2.2.2 skrll struct nvme_sqe *sqe = slot; 1023 1.2.2.2 skrll struct nvme_dmamem *mem = ccb->ccb_cookie; 1024 1.2.2.2 skrll 1025 1.2.2.2 skrll sqe->opcode = NVM_ADMIN_IDENTIFY; 1026 1.2.2.2 skrll htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1027 1.2.2.2 skrll htolem32(&sqe->cdw10, 1); 1028 1.2.2.2 skrll } 1029 1.2.2.2 skrll 1030 1.2.2.2 skrll static int 1031 1.2.2.2 skrll nvme_ccbs_alloc(struct nvme_queue *q, u_int nccbs) 1032 1.2.2.2 skrll { 1033 1.2.2.2 skrll struct nvme_softc *sc = q->q_sc; 1034 1.2.2.2 skrll struct nvme_ccb *ccb; 1035 1.2.2.2 skrll bus_addr_t off; 1036 1.2.2.2 skrll uint64_t *prpl; 1037 1.2.2.2 skrll u_int i; 1038 1.2.2.2 skrll 1039 1.2.2.2 skrll mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1040 1.2.2.2 skrll SIMPLEQ_INIT(&q->q_ccb_list); 1041 1.2.2.2 skrll 1042 1.2.2.2 skrll q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1043 1.2.2.2 skrll if (q->q_ccbs == NULL) 1044 1.2.2.2 skrll return 1; 1045 1.2.2.2 skrll 1046 1.2.2.2 skrll q->q_nccbs = nccbs; 1047 1.2.2.2 skrll q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1048 1.2.2.2 skrll sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1049 1.2.2.2 skrll 1050 1.2.2.2 skrll prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1051 1.2.2.2 skrll off = 0; 1052 1.2.2.2 skrll 1053 1.2.2.2 skrll for (i = 0; i < nccbs; i++) { 1054 1.2.2.2 skrll ccb = &q->q_ccbs[i]; 1055 1.2.2.2 skrll 1056 1.2.2.2 skrll if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1057 1.2.2.2 skrll sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1058 1.2.2.2 skrll sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1059 1.2.2.2 skrll &ccb->ccb_dmamap) != 0) 1060 1.2.2.2 skrll goto free_maps; 1061 1.2.2.2 skrll 1062 1.2.2.2 skrll ccb->ccb_id = i; 1063 1.2.2.2 skrll ccb->ccb_prpl = prpl; 1064 1.2.2.2 skrll ccb->ccb_prpl_off = off; 1065 1.2.2.2 skrll ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1066 1.2.2.2 skrll 1067 1.2.2.2 skrll SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1068 1.2.2.2 skrll 1069 1.2.2.2 skrll prpl += sc->sc_max_sgl; 1070 1.2.2.2 skrll off += sizeof(*prpl) * sc->sc_max_sgl; 1071 1.2.2.2 skrll } 1072 1.2.2.2 skrll 1073 1.2.2.2 skrll return 0; 1074 1.2.2.2 skrll 1075 1.2.2.2 skrll free_maps: 1076 1.2.2.2 skrll nvme_ccbs_free(q); 1077 1.2.2.2 skrll return 1; 1078 1.2.2.2 skrll } 1079 1.2.2.2 skrll 1080 1.2.2.2 skrll static struct nvme_ccb * 1081 1.2.2.2 skrll nvme_ccb_get(struct nvme_queue *q) 1082 1.2.2.2 skrll { 1083 1.2.2.2 skrll struct nvme_ccb *ccb; 1084 1.2.2.2 skrll 1085 1.2.2.2 skrll mutex_enter(&q->q_ccb_mtx); 1086 1.2.2.2 skrll ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1087 1.2.2.2 skrll if (ccb != NULL) 1088 1.2.2.2 skrll SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1089 1.2.2.2 skrll mutex_exit(&q->q_ccb_mtx); 1090 1.2.2.2 skrll 1091 1.2.2.2 skrll return ccb; 1092 1.2.2.2 skrll } 1093 1.2.2.2 skrll 1094 1.2.2.2 skrll static void 1095 1.2.2.2 skrll nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1096 1.2.2.2 skrll { 1097 1.2.2.2 skrll 1098 1.2.2.2 skrll mutex_enter(&q->q_ccb_mtx); 1099 1.2.2.2 skrll SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1100 1.2.2.2 skrll mutex_exit(&q->q_ccb_mtx); 1101 1.2.2.2 skrll } 1102 1.2.2.2 skrll 1103 1.2.2.2 skrll static void 1104 1.2.2.2 skrll nvme_ccbs_free(struct nvme_queue *q) 1105 1.2.2.2 skrll { 1106 1.2.2.2 skrll struct nvme_softc *sc = q->q_sc; 1107 1.2.2.2 skrll struct nvme_ccb *ccb; 1108 1.2.2.2 skrll 1109 1.2.2.2 skrll mutex_enter(&q->q_ccb_mtx); 1110 1.2.2.2 skrll while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1111 1.2.2.2 skrll SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1112 1.2.2.2 skrll bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1113 1.2.2.2 skrll } 1114 1.2.2.2 skrll mutex_exit(&q->q_ccb_mtx); 1115 1.2.2.2 skrll 1116 1.2.2.2 skrll nvme_dmamem_free(sc, q->q_ccb_prpls); 1117 1.2.2.2 skrll kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1118 1.2.2.2 skrll q->q_ccbs = NULL; 1119 1.2.2.2 skrll mutex_destroy(&q->q_ccb_mtx); 1120 1.2.2.2 skrll } 1121 1.2.2.2 skrll 1122 1.2.2.2 skrll static struct nvme_queue * 1123 1.2.2.2 skrll nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1124 1.2.2.2 skrll { 1125 1.2.2.2 skrll struct nvme_queue *q; 1126 1.2.2.2 skrll 1127 1.2.2.2 skrll q = kmem_alloc(sizeof(*q), KM_SLEEP); 1128 1.2.2.2 skrll if (q == NULL) 1129 1.2.2.2 skrll return NULL; 1130 1.2.2.2 skrll 1131 1.2.2.2 skrll q->q_sc = sc; 1132 1.2.2.2 skrll q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1133 1.2.2.2 skrll sizeof(struct nvme_sqe) * entries); 1134 1.2.2.2 skrll if (q->q_sq_dmamem == NULL) 1135 1.2.2.2 skrll goto free; 1136 1.2.2.2 skrll 1137 1.2.2.2 skrll q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1138 1.2.2.2 skrll sizeof(struct nvme_cqe) * entries); 1139 1.2.2.2 skrll if (q->q_cq_dmamem == NULL) 1140 1.2.2.2 skrll goto free_sq; 1141 1.2.2.2 skrll 1142 1.2.2.2 skrll memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1143 1.2.2.2 skrll memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1144 1.2.2.2 skrll 1145 1.2.2.2 skrll mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1146 1.2.2.2 skrll mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1147 1.2.2.2 skrll q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1148 1.2.2.2 skrll q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1149 1.2.2.2 skrll q->q_id = id; 1150 1.2.2.2 skrll q->q_entries = entries; 1151 1.2.2.2 skrll q->q_sq_tail = 0; 1152 1.2.2.2 skrll q->q_cq_head = 0; 1153 1.2.2.2 skrll q->q_cq_phase = NVME_CQE_PHASE; 1154 1.2.2.2 skrll 1155 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1156 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1157 1.2.2.2 skrll 1158 1.2.2.2 skrll if (nvme_ccbs_alloc(q, entries) != 0) { 1159 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1160 1.2.2.2 skrll goto free_cq; 1161 1.2.2.2 skrll } 1162 1.2.2.2 skrll 1163 1.2.2.2 skrll return q; 1164 1.2.2.2 skrll 1165 1.2.2.2 skrll free_cq: 1166 1.2.2.2 skrll nvme_dmamem_free(sc, q->q_cq_dmamem); 1167 1.2.2.2 skrll free_sq: 1168 1.2.2.2 skrll nvme_dmamem_free(sc, q->q_sq_dmamem); 1169 1.2.2.2 skrll free: 1170 1.2.2.2 skrll kmem_free(q, sizeof(*q)); 1171 1.2.2.2 skrll 1172 1.2.2.2 skrll return NULL; 1173 1.2.2.2 skrll } 1174 1.2.2.2 skrll 1175 1.2.2.2 skrll static void 1176 1.2.2.2 skrll nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1177 1.2.2.2 skrll { 1178 1.2.2.2 skrll nvme_ccbs_free(q); 1179 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1180 1.2.2.2 skrll nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1181 1.2.2.2 skrll nvme_dmamem_free(sc, q->q_cq_dmamem); 1182 1.2.2.2 skrll nvme_dmamem_free(sc, q->q_sq_dmamem); 1183 1.2.2.2 skrll kmem_free(q, sizeof(*q)); 1184 1.2.2.2 skrll } 1185 1.2.2.2 skrll 1186 1.2.2.2 skrll int 1187 1.2.2.2 skrll nvme_intr(void *xsc) 1188 1.2.2.2 skrll { 1189 1.2.2.2 skrll struct nvme_softc *sc = xsc; 1190 1.2.2.2 skrll int rv = 0; 1191 1.2.2.2 skrll 1192 1.2.2.2 skrll nvme_write4(sc, NVME_INTMS, 1); 1193 1.2.2.2 skrll 1194 1.2.2.2 skrll if (nvme_q_complete(sc, sc->sc_admin_q)) 1195 1.2.2.2 skrll rv = 1; 1196 1.2.2.2 skrll if (sc->sc_q != NULL) 1197 1.2.2.2 skrll if (nvme_q_complete(sc, sc->sc_q[0])) 1198 1.2.2.2 skrll rv = 1; 1199 1.2.2.2 skrll 1200 1.2.2.2 skrll nvme_write4(sc, NVME_INTMC, 1); 1201 1.2.2.2 skrll 1202 1.2.2.2 skrll return rv; 1203 1.2.2.2 skrll } 1204 1.2.2.2 skrll 1205 1.2.2.2 skrll int 1206 1.2.2.2 skrll nvme_mq_msi_intr(void *xq) 1207 1.2.2.2 skrll { 1208 1.2.2.2 skrll struct nvme_queue *q = xq; 1209 1.2.2.2 skrll struct nvme_softc *sc = q->q_sc; 1210 1.2.2.2 skrll int rv = 0; 1211 1.2.2.2 skrll 1212 1.2.2.2 skrll nvme_write4(sc, NVME_INTMS, 1U << q->q_id); 1213 1.2.2.2 skrll 1214 1.2.2.2 skrll if (nvme_q_complete(sc, q)) 1215 1.2.2.2 skrll rv = 1; 1216 1.2.2.2 skrll 1217 1.2.2.2 skrll nvme_write4(sc, NVME_INTMC, 1U << q->q_id); 1218 1.2.2.2 skrll 1219 1.2.2.2 skrll return rv; 1220 1.2.2.2 skrll } 1221 1.2.2.2 skrll 1222 1.2.2.2 skrll int 1223 1.2.2.2 skrll nvme_mq_msix_intr(void *xq) 1224 1.2.2.2 skrll { 1225 1.2.2.2 skrll struct nvme_queue *q = xq; 1226 1.2.2.2 skrll int rv = 0; 1227 1.2.2.2 skrll 1228 1.2.2.2 skrll if (nvme_q_complete(q->q_sc, q)) 1229 1.2.2.2 skrll rv = 1; 1230 1.2.2.2 skrll 1231 1.2.2.2 skrll return rv; 1232 1.2.2.2 skrll } 1233 1.2.2.2 skrll 1234 1.2.2.2 skrll static struct nvme_dmamem * 1235 1.2.2.2 skrll nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1236 1.2.2.2 skrll { 1237 1.2.2.2 skrll struct nvme_dmamem *ndm; 1238 1.2.2.2 skrll int nsegs; 1239 1.2.2.2 skrll 1240 1.2.2.2 skrll ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1241 1.2.2.2 skrll if (ndm == NULL) 1242 1.2.2.2 skrll return NULL; 1243 1.2.2.2 skrll 1244 1.2.2.2 skrll ndm->ndm_size = size; 1245 1.2.2.2 skrll 1246 1.2.2.2 skrll if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1247 1.2.2.2 skrll BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1248 1.2.2.2 skrll goto ndmfree; 1249 1.2.2.2 skrll 1250 1.2.2.2 skrll if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1251 1.2.2.2 skrll 1, &nsegs, BUS_DMA_WAITOK) != 0) 1252 1.2.2.2 skrll goto destroy; 1253 1.2.2.2 skrll 1254 1.2.2.2 skrll if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1255 1.2.2.2 skrll &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1256 1.2.2.2 skrll goto free; 1257 1.2.2.2 skrll memset(ndm->ndm_kva, 0, size); 1258 1.2.2.2 skrll 1259 1.2.2.2 skrll if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1260 1.2.2.2 skrll NULL, BUS_DMA_WAITOK) != 0) 1261 1.2.2.2 skrll goto unmap; 1262 1.2.2.2 skrll 1263 1.2.2.2 skrll return ndm; 1264 1.2.2.2 skrll 1265 1.2.2.2 skrll unmap: 1266 1.2.2.2 skrll bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1267 1.2.2.2 skrll free: 1268 1.2.2.2 skrll bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1269 1.2.2.2 skrll destroy: 1270 1.2.2.2 skrll bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1271 1.2.2.2 skrll ndmfree: 1272 1.2.2.2 skrll kmem_free(ndm, sizeof(*ndm)); 1273 1.2.2.2 skrll return NULL; 1274 1.2.2.2 skrll } 1275 1.2.2.2 skrll 1276 1.2.2.2 skrll static void 1277 1.2.2.2 skrll nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1278 1.2.2.2 skrll { 1279 1.2.2.2 skrll bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1280 1.2.2.2 skrll 0, NVME_DMA_LEN(mem), ops); 1281 1.2.2.2 skrll } 1282 1.2.2.2 skrll 1283 1.2.2.2 skrll void 1284 1.2.2.2 skrll nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1285 1.2.2.2 skrll { 1286 1.2.2.2 skrll bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1287 1.2.2.2 skrll bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1288 1.2.2.2 skrll bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1289 1.2.2.2 skrll bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1290 1.2.2.2 skrll kmem_free(ndm, sizeof(*ndm)); 1291 1.2.2.2 skrll } 1292
Indexes created Mon Oct 20 11:09:49 GMT 2025