nvme.c revision 1.42
1 /* $NetBSD: nvme.c,v 1.42 2019/04/24 23:39:23 mlelstv Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.42 2019/04/24 23:39:23 mlelstv Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 #include "ioconf.h" 44 45 #define B4_CHK_RDY_DELAY_MS 2300 /* workaround controller bug */ 46 47 int nvme_adminq_size = 32; 48 int nvme_ioq_size = 1024; 49 50 static int nvme_print(void *, const char *); 51 52 static int nvme_ready(struct nvme_softc *, uint32_t); 53 static int nvme_enable(struct nvme_softc *, u_int); 54 static int nvme_disable(struct nvme_softc *); 55 static int nvme_shutdown(struct nvme_softc *); 56 57 #ifdef NVME_DEBUG 58 static void nvme_dumpregs(struct nvme_softc *); 59 #endif 60 static int nvme_identify(struct nvme_softc *, u_int); 61 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 62 void *); 63 64 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 65 static void nvme_ccbs_free(struct nvme_queue *); 66 67 static struct nvme_ccb * 68 nvme_ccb_get(struct nvme_queue *, bool); 69 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 70 71 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 72 struct nvme_ccb *, void (*)(struct nvme_queue *, 73 struct nvme_ccb *, void *), int); 74 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 76 struct nvme_cqe *); 77 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 78 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 79 struct nvme_cqe *); 80 81 static struct nvme_queue * 82 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 83 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 84 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 85 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 86 struct nvme_ccb *, void (*)(struct nvme_queue *, 87 struct nvme_ccb *, void *)); 88 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 89 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 90 static void nvme_q_wait_complete(struct nvme_softc *, struct nvme_queue *, 91 bool (*)(void *), void *); 92 93 static struct nvme_dmamem * 94 nvme_dmamem_alloc(struct nvme_softc *, size_t); 95 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 96 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 97 int); 98 99 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 100 void *); 101 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 102 struct nvme_cqe *); 103 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 104 void *); 105 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 106 struct nvme_cqe *); 107 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *, 108 void *); 109 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *, 110 struct nvme_cqe *); 111 112 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 113 void *); 114 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 115 struct nvme_cqe *); 116 static int nvme_command_passthrough(struct nvme_softc *, 117 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 118 119 static int nvme_get_number_of_queues(struct nvme_softc *, u_int *); 120 121 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 122 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 123 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 124 #define NVME_TIMO_SY 60 /* sync cache timeout */ 125 126 #define nvme_read4(_s, _r) \ 127 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 128 #define nvme_write4(_s, _r, _v) \ 129 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 130 /* 131 * Some controllers, at least Apple NVMe, always require split 132 * transfers, so don't use bus_space_{read,write}_8() on LP64. 133 */ 134 static inline uint64_t 135 nvme_read8(struct nvme_softc *sc, bus_size_t r) 136 { 137 uint64_t v; 138 uint32_t *a = (uint32_t *)&v; 139 140 #if _BYTE_ORDER == _LITTLE_ENDIAN 141 a[0] = nvme_read4(sc, r); 142 a[1] = nvme_read4(sc, r + 4); 143 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 144 a[1] = nvme_read4(sc, r); 145 a[0] = nvme_read4(sc, r + 4); 146 #endif 147 148 return v; 149 } 150 151 static inline void 152 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 153 { 154 uint32_t *a = (uint32_t *)&v; 155 156 #if _BYTE_ORDER == _LITTLE_ENDIAN 157 nvme_write4(sc, r, a[0]); 158 nvme_write4(sc, r + 4, a[1]); 159 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 160 nvme_write4(sc, r, a[1]); 161 nvme_write4(sc, r + 4, a[0]); 162 #endif 163 } 164 #define nvme_barrier(_s, _r, _l, _f) \ 165 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 166 167 #ifdef NVME_DEBUG 168 static __used void 169 nvme_dumpregs(struct nvme_softc *sc) 170 { 171 uint64_t r8; 172 uint32_t r4; 173 174 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 175 r8 = nvme_read8(sc, NVME_CAP); 176 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 177 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 178 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 179 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 180 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 181 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 182 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 183 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 184 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 185 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 186 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 187 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 188 189 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 190 191 r4 = nvme_read4(sc, NVME_CC); 192 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 193 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 194 (1 << NVME_CC_IOCQES_R(r4))); 195 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 196 (1 << NVME_CC_IOSQES_R(r4))); 197 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 198 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 199 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 200 (1 << NVME_CC_MPS_R(r4))); 201 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 202 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 203 204 r4 = nvme_read4(sc, NVME_CSTS); 205 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 206 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 207 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 208 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 209 210 r4 = nvme_read4(sc, NVME_AQA); 211 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 212 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 213 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 214 215 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 216 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 217 #undef DEVNAME 218 } 219 #endif /* NVME_DEBUG */ 220 221 static int 222 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 223 { 224 u_int i = 0; 225 226 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 227 if (i++ > sc->sc_rdy_to) 228 return ENXIO; 229 230 delay(1000); 231 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 232 } 233 234 return 0; 235 } 236 237 static int 238 nvme_enable(struct nvme_softc *sc, u_int mps) 239 { 240 uint32_t cc, csts; 241 int error; 242 243 cc = nvme_read4(sc, NVME_CC); 244 csts = nvme_read4(sc, NVME_CSTS); 245 246 /* 247 * See note in nvme_disable. Short circuit if we're already enabled. 248 */ 249 if (ISSET(cc, NVME_CC_EN)) { 250 if (ISSET(csts, NVME_CSTS_RDY)) 251 return 0; 252 253 goto waitready; 254 } else { 255 /* EN == 0 already wait for RDY == 0 or fail */ 256 error = nvme_ready(sc, 0); 257 if (error) 258 return error; 259 } 260 261 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 262 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 263 delay(5000); 264 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 265 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 266 delay(5000); 267 268 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 269 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 270 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 271 delay(5000); 272 273 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 274 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 275 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 276 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 277 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 278 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 279 SET(cc, NVME_CC_MPS(mps)); 280 SET(cc, NVME_CC_EN); 281 282 nvme_write4(sc, NVME_CC, cc); 283 nvme_barrier(sc, 0, sc->sc_ios, 284 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 285 286 waitready: 287 return nvme_ready(sc, NVME_CSTS_RDY); 288 } 289 290 static int 291 nvme_disable(struct nvme_softc *sc) 292 { 293 uint32_t cc, csts; 294 int error; 295 296 cc = nvme_read4(sc, NVME_CC); 297 csts = nvme_read4(sc, NVME_CSTS); 298 299 /* 300 * Per 3.1.5 in NVME 1.3 spec, transitioning CC.EN from 0 to 1 301 * when CSTS.RDY is 1 or transitioning CC.EN from 1 to 0 when 302 * CSTS.RDY is 0 "has undefined results" So make sure that CSTS.RDY 303 * isn't the desired value. Short circuit if we're already disabled. 304 */ 305 if (ISSET(cc, NVME_CC_EN)) { 306 if (!ISSET(csts, NVME_CSTS_RDY)) { 307 /* EN == 1, wait for RDY == 1 or fail */ 308 error = nvme_ready(sc, NVME_CSTS_RDY); 309 if (error) 310 return error; 311 } 312 } else { 313 /* EN == 0 already wait for RDY == 0 */ 314 if (!ISSET(csts, NVME_CSTS_RDY)) 315 return 0; 316 317 goto waitready; 318 } 319 320 CLR(cc, NVME_CC_EN); 321 nvme_write4(sc, NVME_CC, cc); 322 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 323 324 /* 325 * Some drives have issues with accessing the mmio after we disable, 326 * so delay for a bit after we write the bit to cope with these issues. 327 */ 328 if (ISSET(sc->sc_quirks, NVME_QUIRK_DELAY_B4_CHK_RDY)) 329 delay(B4_CHK_RDY_DELAY_MS); 330 331 waitready: 332 return nvme_ready(sc, 0); 333 } 334 335 int 336 nvme_attach(struct nvme_softc *sc) 337 { 338 uint64_t cap; 339 uint32_t reg; 340 u_int dstrd; 341 u_int mps = PAGE_SHIFT; 342 u_int ioq_allocated; 343 uint16_t adminq_entries = nvme_adminq_size; 344 uint16_t ioq_entries = nvme_ioq_size; 345 int i; 346 347 reg = nvme_read4(sc, NVME_VS); 348 if (reg == 0xffffffff) { 349 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 350 return 1; 351 } 352 353 if (NVME_VS_TER(reg) == 0) 354 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg), 355 NVME_VS_MNR(reg)); 356 else 357 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg), 358 NVME_VS_MNR(reg), NVME_VS_TER(reg)); 359 360 cap = nvme_read8(sc, NVME_CAP); 361 dstrd = NVME_CAP_DSTRD(cap); 362 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 363 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 364 "is greater than CPU page size %u\n", 365 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 366 return 1; 367 } 368 if (NVME_CAP_MPSMAX(cap) < mps) 369 mps = NVME_CAP_MPSMAX(cap); 370 if (ioq_entries > NVME_CAP_MQES(cap)) 371 ioq_entries = NVME_CAP_MQES(cap); 372 373 /* set initial values to be used for admin queue during probe */ 374 sc->sc_rdy_to = NVME_CAP_TO(cap); 375 sc->sc_mps = 1 << mps; 376 sc->sc_mdts = MAXPHYS; 377 sc->sc_max_sgl = 2; 378 379 if (nvme_disable(sc) != 0) { 380 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 381 return 1; 382 } 383 384 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 385 if (sc->sc_admin_q == NULL) { 386 aprint_error_dev(sc->sc_dev, 387 "unable to allocate admin queue\n"); 388 return 1; 389 } 390 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 391 goto free_admin_q; 392 393 if (nvme_enable(sc, mps) != 0) { 394 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 395 goto disestablish_admin_q; 396 } 397 398 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 399 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 400 goto disable; 401 } 402 403 /* we know how big things are now */ 404 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 405 406 /* reallocate ccbs of admin queue with new max sgl. */ 407 nvme_ccbs_free(sc->sc_admin_q); 408 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 409 410 if (sc->sc_use_mq) { 411 /* Limit the number of queues to the number allocated in HW */ 412 if (nvme_get_number_of_queues(sc, &ioq_allocated) != 0) { 413 aprint_error_dev(sc->sc_dev, 414 "unable to get number of queues\n"); 415 goto disable; 416 } 417 if (sc->sc_nq > ioq_allocated) 418 sc->sc_nq = ioq_allocated; 419 } 420 421 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 422 for (i = 0; i < sc->sc_nq; i++) { 423 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 424 if (sc->sc_q[i] == NULL) { 425 aprint_error_dev(sc->sc_dev, 426 "unable to allocate io queue\n"); 427 goto free_q; 428 } 429 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 430 aprint_error_dev(sc->sc_dev, 431 "unable to create io queue\n"); 432 nvme_q_free(sc, sc->sc_q[i]); 433 goto free_q; 434 } 435 } 436 437 if (!sc->sc_use_mq) 438 nvme_write4(sc, NVME_INTMC, 1); 439 440 /* probe subdevices */ 441 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 442 KM_SLEEP); 443 nvme_rescan(sc->sc_dev, "nvme", &i); 444 445 return 0; 446 447 free_q: 448 while (--i >= 0) { 449 nvme_q_delete(sc, sc->sc_q[i]); 450 nvme_q_free(sc, sc->sc_q[i]); 451 } 452 disable: 453 nvme_disable(sc); 454 disestablish_admin_q: 455 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 456 free_admin_q: 457 nvme_q_free(sc, sc->sc_admin_q); 458 459 return 1; 460 } 461 462 int 463 nvme_rescan(device_t self, const char *attr, const int *flags) 464 { 465 struct nvme_softc *sc = device_private(self); 466 struct nvme_attach_args naa; 467 uint64_t cap; 468 int ioq_entries = nvme_ioq_size; 469 int i; 470 471 cap = nvme_read8(sc, NVME_CAP); 472 if (ioq_entries > NVME_CAP_MQES(cap)) 473 ioq_entries = NVME_CAP_MQES(cap); 474 475 for (i = 0; i < sc->sc_nn; i++) { 476 if (sc->sc_namespaces[i].dev) 477 continue; 478 memset(&naa, 0, sizeof(naa)); 479 naa.naa_nsid = i + 1; 480 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 481 naa.naa_maxphys = sc->sc_mdts; 482 naa.naa_typename = sc->sc_modelname; 483 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 484 nvme_print); 485 } 486 return 0; 487 } 488 489 static int 490 nvme_print(void *aux, const char *pnp) 491 { 492 struct nvme_attach_args *naa = aux; 493 494 if (pnp) 495 aprint_normal("at %s", pnp); 496 497 if (naa->naa_nsid > 0) 498 aprint_normal(" nsid %d", naa->naa_nsid); 499 500 return UNCONF; 501 } 502 503 int 504 nvme_detach(struct nvme_softc *sc, int flags) 505 { 506 int i, error; 507 508 error = config_detach_children(sc->sc_dev, flags); 509 if (error) 510 return error; 511 512 error = nvme_shutdown(sc); 513 if (error) 514 return error; 515 516 /* from now on we are committed to detach, following will never fail */ 517 for (i = 0; i < sc->sc_nq; i++) 518 nvme_q_free(sc, sc->sc_q[i]); 519 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 520 nvme_q_free(sc, sc->sc_admin_q); 521 522 return 0; 523 } 524 525 static int 526 nvme_shutdown(struct nvme_softc *sc) 527 { 528 uint32_t cc, csts; 529 bool disabled = false; 530 int i; 531 532 if (!sc->sc_use_mq) 533 nvme_write4(sc, NVME_INTMS, 1); 534 535 for (i = 0; i < sc->sc_nq; i++) { 536 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 537 aprint_error_dev(sc->sc_dev, 538 "unable to delete io queue %d, disabling\n", i + 1); 539 disabled = true; 540 } 541 } 542 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 543 if (disabled) 544 goto disable; 545 546 cc = nvme_read4(sc, NVME_CC); 547 CLR(cc, NVME_CC_SHN_MASK); 548 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 549 nvme_write4(sc, NVME_CC, cc); 550 551 for (i = 0; i < 4000; i++) { 552 nvme_barrier(sc, 0, sc->sc_ios, 553 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 554 csts = nvme_read4(sc, NVME_CSTS); 555 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 556 return 0; 557 558 delay(1000); 559 } 560 561 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 562 563 disable: 564 nvme_disable(sc); 565 return 0; 566 } 567 568 void 569 nvme_childdet(device_t self, device_t child) 570 { 571 struct nvme_softc *sc = device_private(self); 572 int i; 573 574 for (i = 0; i < sc->sc_nn; i++) { 575 if (sc->sc_namespaces[i].dev == child) { 576 /* Already freed ns->ident. */ 577 sc->sc_namespaces[i].dev = NULL; 578 break; 579 } 580 } 581 } 582 583 int 584 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 585 { 586 struct nvme_sqe sqe; 587 struct nvm_identify_namespace *identify; 588 struct nvme_dmamem *mem; 589 struct nvme_ccb *ccb; 590 struct nvme_namespace *ns; 591 int rv; 592 593 KASSERT(nsid > 0); 594 595 ccb = nvme_ccb_get(sc->sc_admin_q, false); 596 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 597 598 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 599 if (mem == NULL) { 600 nvme_ccb_put(sc->sc_admin_q, ccb); 601 return ENOMEM; 602 } 603 604 memset(&sqe, 0, sizeof(sqe)); 605 sqe.opcode = NVM_ADMIN_IDENTIFY; 606 htolem32(&sqe.nsid, nsid); 607 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 608 htolem32(&sqe.cdw10, 0); 609 610 ccb->ccb_done = nvme_empty_done; 611 ccb->ccb_cookie = &sqe; 612 613 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 614 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 615 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 616 617 nvme_ccb_put(sc->sc_admin_q, ccb); 618 619 if (rv != 0) { 620 rv = EIO; 621 goto done; 622 } 623 624 /* commit */ 625 626 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 627 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 628 629 /* Convert data to host endian */ 630 nvme_identify_namespace_swapbytes(identify); 631 632 ns = nvme_ns_get(sc, nsid); 633 KASSERT(ns); 634 KASSERT(ns->ident == NULL); 635 ns->ident = identify; 636 637 done: 638 nvme_dmamem_free(sc, mem); 639 640 return rv; 641 } 642 643 int 644 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 645 struct buf *bp, void *data, size_t datasize, 646 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 647 { 648 struct nvme_queue *q = nvme_get_q(sc); 649 struct nvme_ccb *ccb; 650 bus_dmamap_t dmap; 651 int i, error; 652 653 ccb = nvme_ccb_get(q, false); 654 if (ccb == NULL) 655 return EAGAIN; 656 657 ccb->ccb_done = nvme_ns_io_done; 658 ccb->ccb_cookie = cookie; 659 660 /* namespace context */ 661 ccb->nnc_nsid = nsid; 662 ccb->nnc_flags = flags; 663 ccb->nnc_buf = bp; 664 ccb->nnc_datasize = datasize; 665 ccb->nnc_secsize = secsize; 666 ccb->nnc_blkno = blkno; 667 ccb->nnc_done = nnc_done; 668 669 dmap = ccb->ccb_dmamap; 670 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 671 datasize, NULL, 672 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 673 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 674 (ISSET(flags, NVME_NS_CTX_F_READ) ? 675 BUS_DMA_READ : BUS_DMA_WRITE)); 676 if (error) { 677 nvme_ccb_put(q, ccb); 678 return error; 679 } 680 681 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 682 ISSET(flags, NVME_NS_CTX_F_READ) ? 683 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 684 685 if (dmap->dm_nsegs > 2) { 686 for (i = 1; i < dmap->dm_nsegs; i++) { 687 htolem64(&ccb->ccb_prpl[i - 1], 688 dmap->dm_segs[i].ds_addr); 689 } 690 bus_dmamap_sync(sc->sc_dmat, 691 NVME_DMA_MAP(q->q_ccb_prpls), 692 ccb->ccb_prpl_off, 693 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 694 BUS_DMASYNC_PREWRITE); 695 } 696 697 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 698 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 699 return EIO; 700 return 0; 701 } 702 703 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 704 return 0; 705 } 706 707 static void 708 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 709 { 710 struct nvme_sqe_io *sqe = slot; 711 bus_dmamap_t dmap = ccb->ccb_dmamap; 712 713 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 714 NVM_CMD_READ : NVM_CMD_WRITE; 715 htolem32(&sqe->nsid, ccb->nnc_nsid); 716 717 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 718 switch (dmap->dm_nsegs) { 719 case 1: 720 break; 721 case 2: 722 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 723 break; 724 default: 725 /* the prp list is already set up and synced */ 726 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 727 break; 728 } 729 730 htolem64(&sqe->slba, ccb->nnc_blkno); 731 732 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) 733 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA); 734 735 /* guaranteed by upper layers, but check just in case */ 736 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 737 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 738 } 739 740 static void 741 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 742 struct nvme_cqe *cqe) 743 { 744 struct nvme_softc *sc = q->q_sc; 745 bus_dmamap_t dmap = ccb->ccb_dmamap; 746 void *nnc_cookie = ccb->ccb_cookie; 747 nvme_nnc_done nnc_done = ccb->nnc_done; 748 struct buf *bp = ccb->nnc_buf; 749 750 if (dmap->dm_nsegs > 2) { 751 bus_dmamap_sync(sc->sc_dmat, 752 NVME_DMA_MAP(q->q_ccb_prpls), 753 ccb->ccb_prpl_off, 754 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 755 BUS_DMASYNC_POSTWRITE); 756 } 757 758 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 759 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 760 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 761 762 bus_dmamap_unload(sc->sc_dmat, dmap); 763 nvme_ccb_put(q, ccb); 764 765 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 766 } 767 768 /* 769 * If there is no volatile write cache, it makes no sense to issue 770 * flush commands or query for the status. 771 */ 772 static bool 773 nvme_has_volatile_write_cache(struct nvme_softc *sc) 774 { 775 /* sc_identify is filled during attachment */ 776 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0); 777 } 778 779 static bool 780 nvme_ns_sync_finished(void *cookie) 781 { 782 int *result = cookie; 783 784 return (*result != 0); 785 } 786 787 int 788 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, int flags) 789 { 790 struct nvme_queue *q = nvme_get_q(sc); 791 struct nvme_ccb *ccb; 792 int result = 0; 793 794 if (!nvme_has_volatile_write_cache(sc)) { 795 /* cache not present, no value in trying to flush it */ 796 return 0; 797 } 798 799 ccb = nvme_ccb_get(q, true); 800 if (ccb == NULL) 801 return EAGAIN; 802 803 ccb->ccb_done = nvme_ns_sync_done; 804 ccb->ccb_cookie = &result; 805 806 /* namespace context */ 807 ccb->nnc_nsid = nsid; 808 ccb->nnc_flags = flags; 809 ccb->nnc_done = NULL; 810 811 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 812 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 813 return EIO; 814 return 0; 815 } 816 817 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 818 819 /* wait for completion */ 820 nvme_q_wait_complete(sc, q, nvme_ns_sync_finished, &result); 821 KASSERT(result != 0); 822 823 return (result > 0) ? 0 : EIO; 824 } 825 826 static void 827 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 828 { 829 struct nvme_sqe *sqe = slot; 830 831 sqe->opcode = NVM_CMD_FLUSH; 832 htolem32(&sqe->nsid, ccb->nnc_nsid); 833 } 834 835 static void 836 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 837 struct nvme_cqe *cqe) 838 { 839 int *result = ccb->ccb_cookie; 840 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 841 842 if (status == NVME_CQE_SC_SUCCESS) 843 *result = 1; 844 else 845 *result = -1; 846 847 nvme_ccb_put(q, ccb); 848 } 849 850 static bool 851 nvme_getcache_finished(void *xc) 852 { 853 int *addr = xc; 854 855 return (*addr != 0); 856 } 857 858 /* 859 * Get status of volatile write cache. Always asynchronous. 860 */ 861 int 862 nvme_admin_getcache(struct nvme_softc *sc, int *addr) 863 { 864 struct nvme_ccb *ccb; 865 struct nvme_queue *q = sc->sc_admin_q; 866 int result = 0, error; 867 868 if (!nvme_has_volatile_write_cache(sc)) { 869 /* cache simply not present */ 870 *addr = 0; 871 return 0; 872 } 873 874 ccb = nvme_ccb_get(q, true); 875 KASSERT(ccb != NULL); 876 877 ccb->ccb_done = nvme_getcache_done; 878 ccb->ccb_cookie = &result; 879 880 /* namespace context */ 881 ccb->nnc_flags = 0; 882 ccb->nnc_done = NULL; 883 884 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); 885 886 /* wait for completion */ 887 nvme_q_wait_complete(sc, q, nvme_getcache_finished, &result); 888 KASSERT(result != 0); 889 890 if (result > 0) { 891 *addr = result; 892 error = 0; 893 } else 894 error = EINVAL; 895 896 return error; 897 } 898 899 static void 900 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 901 { 902 struct nvme_sqe *sqe = slot; 903 904 sqe->opcode = NVM_ADMIN_GET_FEATURES; 905 htolem32(&sqe->cdw10, NVM_FEATURE_VOLATILE_WRITE_CACHE); 906 htolem32(&sqe->cdw11, NVM_VOLATILE_WRITE_CACHE_WCE); 907 } 908 909 static void 910 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 911 struct nvme_cqe *cqe) 912 { 913 int *addr = ccb->ccb_cookie; 914 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 915 uint32_t cdw0 = lemtoh32(&cqe->cdw0); 916 int result; 917 918 if (status == NVME_CQE_SC_SUCCESS) { 919 result = 0; 920 921 /* 922 * DPO not supported, Dataset Management (DSM) field doesn't 923 * specify the same semantics. FUA is always supported. 924 */ 925 result = DKCACHE_FUA; 926 927 if (cdw0 & NVM_VOLATILE_WRITE_CACHE_WCE) 928 result |= DKCACHE_WRITE; 929 930 /* 931 * If volatile write cache is present, the flag shall also be 932 * settable. 933 */ 934 result |= DKCACHE_WCHANGE; 935 936 /* 937 * ONCS field indicates whether the optional SAVE is also 938 * supported for Set Features. According to spec v1.3, 939 * Volatile Write Cache however doesn't support persistency 940 * across power cycle/reset. 941 */ 942 943 } else { 944 result = -1; 945 } 946 947 *addr = result; 948 949 nvme_ccb_put(q, ccb); 950 } 951 952 struct nvme_setcache_state { 953 int dkcache; 954 int result; 955 }; 956 957 static bool 958 nvme_setcache_finished(void *xc) 959 { 960 struct nvme_setcache_state *st = xc; 961 962 return (st->result != 0); 963 } 964 965 static void 966 nvme_setcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 967 { 968 struct nvme_sqe *sqe = slot; 969 struct nvme_setcache_state *st = ccb->ccb_cookie; 970 971 sqe->opcode = NVM_ADMIN_SET_FEATURES; 972 htolem32(&sqe->cdw10, NVM_FEATURE_VOLATILE_WRITE_CACHE); 973 if (st->dkcache & DKCACHE_WRITE) 974 htolem32(&sqe->cdw11, NVM_VOLATILE_WRITE_CACHE_WCE); 975 } 976 977 static void 978 nvme_setcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 979 struct nvme_cqe *cqe) 980 { 981 struct nvme_setcache_state *st = ccb->ccb_cookie; 982 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 983 984 if (status == NVME_CQE_SC_SUCCESS) { 985 st->result = 1; 986 } else { 987 st->result = -1; 988 } 989 990 nvme_ccb_put(q, ccb); 991 } 992 993 /* 994 * Set status of volatile write cache. Always asynchronous. 995 */ 996 int 997 nvme_admin_setcache(struct nvme_softc *sc, int dkcache) 998 { 999 struct nvme_ccb *ccb; 1000 struct nvme_queue *q = sc->sc_admin_q; 1001 int error; 1002 struct nvme_setcache_state st; 1003 1004 if (!nvme_has_volatile_write_cache(sc)) { 1005 /* cache simply not present */ 1006 return EOPNOTSUPP; 1007 } 1008 1009 if (dkcache & ~(DKCACHE_WRITE)) { 1010 /* unsupported parameters */ 1011 return EOPNOTSUPP; 1012 } 1013 1014 ccb = nvme_ccb_get(q, true); 1015 KASSERT(ccb != NULL); 1016 1017 memset(&st, 0, sizeof(st)); 1018 st.dkcache = dkcache; 1019 1020 ccb->ccb_done = nvme_setcache_done; 1021 ccb->ccb_cookie = &st; 1022 1023 /* namespace context */ 1024 ccb->nnc_flags = 0; 1025 ccb->nnc_done = NULL; 1026 1027 nvme_q_submit(sc, q, ccb, nvme_setcache_fill); 1028 1029 /* wait for completion */ 1030 nvme_q_wait_complete(sc, q, nvme_setcache_finished, &st); 1031 KASSERT(st.result != 0); 1032 1033 if (st.result > 0) 1034 error = 0; 1035 else 1036 error = EINVAL; 1037 1038 return error; 1039 } 1040 1041 void 1042 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 1043 { 1044 struct nvme_namespace *ns; 1045 struct nvm_identify_namespace *identify; 1046 1047 ns = nvme_ns_get(sc, nsid); 1048 KASSERT(ns); 1049 1050 identify = ns->ident; 1051 ns->ident = NULL; 1052 if (identify != NULL) 1053 kmem_free(identify, sizeof(*identify)); 1054 } 1055 1056 struct nvme_pt_state { 1057 struct nvme_pt_command *pt; 1058 bool finished; 1059 }; 1060 1061 static void 1062 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1063 { 1064 struct nvme_softc *sc = q->q_sc; 1065 struct nvme_sqe *sqe = slot; 1066 struct nvme_pt_state *state = ccb->ccb_cookie; 1067 struct nvme_pt_command *pt = state->pt; 1068 bus_dmamap_t dmap = ccb->ccb_dmamap; 1069 int i; 1070 1071 sqe->opcode = pt->cmd.opcode; 1072 htolem32(&sqe->nsid, pt->cmd.nsid); 1073 1074 if (pt->buf != NULL && pt->len > 0) { 1075 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 1076 switch (dmap->dm_nsegs) { 1077 case 1: 1078 break; 1079 case 2: 1080 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 1081 break; 1082 default: 1083 for (i = 1; i < dmap->dm_nsegs; i++) { 1084 htolem64(&ccb->ccb_prpl[i - 1], 1085 dmap->dm_segs[i].ds_addr); 1086 } 1087 bus_dmamap_sync(sc->sc_dmat, 1088 NVME_DMA_MAP(q->q_ccb_prpls), 1089 ccb->ccb_prpl_off, 1090 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 1091 BUS_DMASYNC_PREWRITE); 1092 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 1093 break; 1094 } 1095 } 1096 1097 htolem32(&sqe->cdw10, pt->cmd.cdw10); 1098 htolem32(&sqe->cdw11, pt->cmd.cdw11); 1099 htolem32(&sqe->cdw12, pt->cmd.cdw12); 1100 htolem32(&sqe->cdw13, pt->cmd.cdw13); 1101 htolem32(&sqe->cdw14, pt->cmd.cdw14); 1102 htolem32(&sqe->cdw15, pt->cmd.cdw15); 1103 } 1104 1105 static void 1106 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 1107 { 1108 struct nvme_softc *sc = q->q_sc; 1109 struct nvme_pt_state *state = ccb->ccb_cookie; 1110 struct nvme_pt_command *pt = state->pt; 1111 bus_dmamap_t dmap = ccb->ccb_dmamap; 1112 1113 if (pt->buf != NULL && pt->len > 0) { 1114 if (dmap->dm_nsegs > 2) { 1115 bus_dmamap_sync(sc->sc_dmat, 1116 NVME_DMA_MAP(q->q_ccb_prpls), 1117 ccb->ccb_prpl_off, 1118 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 1119 BUS_DMASYNC_POSTWRITE); 1120 } 1121 1122 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1123 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 1124 bus_dmamap_unload(sc->sc_dmat, dmap); 1125 } 1126 1127 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 1128 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 1129 1130 state->finished = true; 1131 1132 nvme_ccb_put(q, ccb); 1133 } 1134 1135 static bool 1136 nvme_pt_finished(void *cookie) 1137 { 1138 struct nvme_pt_state *state = cookie; 1139 1140 return state->finished; 1141 } 1142 1143 static int 1144 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 1145 uint16_t nsid, struct lwp *l, bool is_adminq) 1146 { 1147 struct nvme_queue *q; 1148 struct nvme_ccb *ccb; 1149 void *buf = NULL; 1150 struct nvme_pt_state state; 1151 int error; 1152 1153 /* limit command size to maximum data transfer size */ 1154 if ((pt->buf == NULL && pt->len > 0) || 1155 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 1156 return EINVAL; 1157 1158 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 1159 ccb = nvme_ccb_get(q, true); 1160 KASSERT(ccb != NULL); 1161 1162 if (pt->buf != NULL) { 1163 KASSERT(pt->len > 0); 1164 buf = kmem_alloc(pt->len, KM_SLEEP); 1165 if (!pt->is_read) { 1166 error = copyin(pt->buf, buf, pt->len); 1167 if (error) 1168 goto kmem_free; 1169 } 1170 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 1171 pt->len, NULL, 1172 BUS_DMA_WAITOK | 1173 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 1174 if (error) 1175 goto kmem_free; 1176 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 1177 0, ccb->ccb_dmamap->dm_mapsize, 1178 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 1179 } 1180 1181 memset(&state, 0, sizeof(state)); 1182 state.pt = pt; 1183 state.finished = false; 1184 1185 ccb->ccb_done = nvme_pt_done; 1186 ccb->ccb_cookie = &state; 1187 1188 pt->cmd.nsid = nsid; 1189 1190 nvme_q_submit(sc, q, ccb, nvme_pt_fill); 1191 1192 /* wait for completion */ 1193 nvme_q_wait_complete(sc, q, nvme_pt_finished, &state); 1194 KASSERT(state.finished); 1195 1196 error = 0; 1197 1198 if (buf != NULL) { 1199 if (error == 0 && pt->is_read) 1200 error = copyout(buf, pt->buf, pt->len); 1201 kmem_free: 1202 kmem_free(buf, pt->len); 1203 } 1204 1205 return error; 1206 } 1207 1208 static void 1209 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1210 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 1211 { 1212 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 1213 uint32_t tail; 1214 1215 mutex_enter(&q->q_sq_mtx); 1216 tail = q->q_sq_tail; 1217 if (++q->q_sq_tail >= q->q_entries) 1218 q->q_sq_tail = 0; 1219 1220 sqe += tail; 1221 1222 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1223 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 1224 memset(sqe, 0, sizeof(*sqe)); 1225 (*fill)(q, ccb, sqe); 1226 htolem16(&sqe->cid, ccb->ccb_id); 1227 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1228 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 1229 1230 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 1231 mutex_exit(&q->q_sq_mtx); 1232 } 1233 1234 struct nvme_poll_state { 1235 struct nvme_sqe s; 1236 struct nvme_cqe c; 1237 void *cookie; 1238 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 1239 }; 1240 1241 static int 1242 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1243 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 1244 { 1245 struct nvme_poll_state state; 1246 uint16_t flags; 1247 int step = 10; 1248 int maxloop = timo_sec * 1000000 / step; 1249 int error = 0; 1250 1251 memset(&state, 0, sizeof(state)); 1252 (*fill)(q, ccb, &state.s); 1253 1254 state.done = ccb->ccb_done; 1255 state.cookie = ccb->ccb_cookie; 1256 1257 ccb->ccb_done = nvme_poll_done; 1258 ccb->ccb_cookie = &state; 1259 1260 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1261 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1262 if (nvme_q_complete(sc, q) == 0) 1263 delay(step); 1264 1265 if (timo_sec >= 0 && --maxloop <= 0) { 1266 error = ETIMEDOUT; 1267 break; 1268 } 1269 } 1270 1271 if (error == 0) { 1272 flags = lemtoh16(&state.c.flags); 1273 return flags & ~NVME_CQE_PHASE; 1274 } else { 1275 /* 1276 * If it succeds later, it would hit ccb which will have been 1277 * already reused for something else. Not good. Cross 1278 * fingers and hope for best. XXX do controller reset? 1279 */ 1280 aprint_error_dev(sc->sc_dev, "polled command timed out\n"); 1281 1282 /* Invoke the callback to clean state anyway */ 1283 struct nvme_cqe cqe; 1284 memset(&cqe, 0, sizeof(cqe)); 1285 ccb->ccb_done(q, ccb, &cqe); 1286 1287 return 1; 1288 } 1289 } 1290 1291 static void 1292 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1293 { 1294 struct nvme_sqe *sqe = slot; 1295 struct nvme_poll_state *state = ccb->ccb_cookie; 1296 1297 *sqe = state->s; 1298 } 1299 1300 static void 1301 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1302 struct nvme_cqe *cqe) 1303 { 1304 struct nvme_poll_state *state = ccb->ccb_cookie; 1305 1306 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1307 state->c = *cqe; 1308 1309 ccb->ccb_cookie = state->cookie; 1310 state->done(q, ccb, &state->c); 1311 } 1312 1313 static void 1314 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1315 { 1316 struct nvme_sqe *src = ccb->ccb_cookie; 1317 struct nvme_sqe *dst = slot; 1318 1319 *dst = *src; 1320 } 1321 1322 static void 1323 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1324 struct nvme_cqe *cqe) 1325 { 1326 } 1327 1328 static int 1329 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1330 { 1331 struct nvme_ccb *ccb; 1332 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1333 uint16_t flags; 1334 int rv = 0; 1335 1336 mutex_enter(&q->q_cq_mtx); 1337 1338 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1339 for (;;) { 1340 cqe = &ring[q->q_cq_head]; 1341 flags = lemtoh16(&cqe->flags); 1342 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1343 break; 1344 1345 ccb = &q->q_ccbs[cqe->cid]; 1346 1347 if (++q->q_cq_head >= q->q_entries) { 1348 q->q_cq_head = 0; 1349 q->q_cq_phase ^= NVME_CQE_PHASE; 1350 } 1351 1352 #ifdef DEBUG 1353 /* 1354 * If we get spurious completion notification, something 1355 * is seriously hosed up. Very likely DMA to some random 1356 * memory place happened, so just bail out. 1357 */ 1358 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1359 panic("%s: invalid ccb detected", 1360 device_xname(sc->sc_dev)); 1361 /* NOTREACHED */ 1362 } 1363 #endif 1364 1365 rv++; 1366 1367 /* 1368 * Unlock the mutex before calling the ccb_done callback 1369 * and re-lock afterwards. The callback triggers lddone() 1370 * which schedules another i/o, and also calls nvme_ccb_put(). 1371 * Unlock/relock avoids possibility of deadlock. 1372 */ 1373 mutex_exit(&q->q_cq_mtx); 1374 ccb->ccb_done(q, ccb, cqe); 1375 mutex_enter(&q->q_cq_mtx); 1376 } 1377 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1378 1379 if (rv) 1380 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1381 1382 mutex_exit(&q->q_cq_mtx); 1383 1384 return rv; 1385 } 1386 1387 static void 1388 nvme_q_wait_complete(struct nvme_softc *sc, 1389 struct nvme_queue *q, bool (*finished)(void *), void *cookie) 1390 { 1391 mutex_enter(&q->q_ccb_mtx); 1392 if (finished(cookie)) 1393 goto out; 1394 1395 for(;;) { 1396 q->q_ccb_waiting = true; 1397 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1398 1399 if (finished(cookie)) 1400 break; 1401 } 1402 1403 out: 1404 mutex_exit(&q->q_ccb_mtx); 1405 } 1406 1407 static int 1408 nvme_identify(struct nvme_softc *sc, u_int mps) 1409 { 1410 char sn[41], mn[81], fr[17]; 1411 struct nvm_identify_controller *identify; 1412 struct nvme_dmamem *mem; 1413 struct nvme_ccb *ccb; 1414 u_int mdts; 1415 int rv = 1; 1416 1417 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1418 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1419 1420 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1421 if (mem == NULL) 1422 return 1; 1423 1424 ccb->ccb_done = nvme_empty_done; 1425 ccb->ccb_cookie = mem; 1426 1427 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1428 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1429 NVME_TIMO_IDENT); 1430 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1431 1432 nvme_ccb_put(sc->sc_admin_q, ccb); 1433 1434 if (rv != 0) 1435 goto done; 1436 1437 identify = NVME_DMA_KVA(mem); 1438 sc->sc_identify = *identify; 1439 identify = NULL; 1440 1441 /* Convert data to host endian */ 1442 nvme_identify_controller_swapbytes(&sc->sc_identify); 1443 1444 strnvisx(sn, sizeof(sn), (const char *)sc->sc_identify.sn, 1445 sizeof(sc->sc_identify.sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1446 strnvisx(mn, sizeof(mn), (const char *)sc->sc_identify.mn, 1447 sizeof(sc->sc_identify.mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1448 strnvisx(fr, sizeof(fr), (const char *)sc->sc_identify.fr, 1449 sizeof(sc->sc_identify.fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1450 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1451 sn); 1452 1453 strlcpy(sc->sc_modelname, mn, sizeof(sc->sc_modelname)); 1454 1455 if (sc->sc_identify.mdts > 0) { 1456 mdts = (1 << sc->sc_identify.mdts) * (1 << mps); 1457 if (mdts < sc->sc_mdts) 1458 sc->sc_mdts = mdts; 1459 } 1460 1461 sc->sc_nn = sc->sc_identify.nn; 1462 1463 done: 1464 nvme_dmamem_free(sc, mem); 1465 1466 return rv; 1467 } 1468 1469 static int 1470 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1471 { 1472 struct nvme_sqe_q sqe; 1473 struct nvme_ccb *ccb; 1474 int rv; 1475 1476 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1477 return 1; 1478 1479 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1480 KASSERT(ccb != NULL); 1481 1482 ccb->ccb_done = nvme_empty_done; 1483 ccb->ccb_cookie = &sqe; 1484 1485 memset(&sqe, 0, sizeof(sqe)); 1486 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1487 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1488 htolem16(&sqe.qsize, q->q_entries - 1); 1489 htolem16(&sqe.qid, q->q_id); 1490 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1491 if (sc->sc_use_mq) 1492 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1493 1494 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1495 if (rv != 0) 1496 goto fail; 1497 1498 ccb->ccb_done = nvme_empty_done; 1499 ccb->ccb_cookie = &sqe; 1500 1501 memset(&sqe, 0, sizeof(sqe)); 1502 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1503 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1504 htolem16(&sqe.qsize, q->q_entries - 1); 1505 htolem16(&sqe.qid, q->q_id); 1506 htolem16(&sqe.cqid, q->q_id); 1507 sqe.qflags = NVM_SQE_Q_PC; 1508 1509 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1510 if (rv != 0) 1511 goto fail; 1512 1513 nvme_ccb_put(sc->sc_admin_q, ccb); 1514 return 0; 1515 1516 fail: 1517 if (sc->sc_use_mq) 1518 sc->sc_intr_disestablish(sc, q->q_id); 1519 1520 nvme_ccb_put(sc->sc_admin_q, ccb); 1521 return rv; 1522 } 1523 1524 static int 1525 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1526 { 1527 struct nvme_sqe_q sqe; 1528 struct nvme_ccb *ccb; 1529 int rv; 1530 1531 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1532 KASSERT(ccb != NULL); 1533 1534 ccb->ccb_done = nvme_empty_done; 1535 ccb->ccb_cookie = &sqe; 1536 1537 memset(&sqe, 0, sizeof(sqe)); 1538 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1539 htolem16(&sqe.qid, q->q_id); 1540 1541 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1542 if (rv != 0) 1543 goto fail; 1544 1545 ccb->ccb_done = nvme_empty_done; 1546 ccb->ccb_cookie = &sqe; 1547 1548 memset(&sqe, 0, sizeof(sqe)); 1549 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1550 htolem16(&sqe.qid, q->q_id); 1551 1552 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1553 if (rv != 0) 1554 goto fail; 1555 1556 fail: 1557 nvme_ccb_put(sc->sc_admin_q, ccb); 1558 1559 if (rv == 0 && sc->sc_use_mq) { 1560 if (sc->sc_intr_disestablish(sc, q->q_id)) 1561 rv = 1; 1562 } 1563 1564 return rv; 1565 } 1566 1567 static void 1568 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1569 { 1570 struct nvme_sqe *sqe = slot; 1571 struct nvme_dmamem *mem = ccb->ccb_cookie; 1572 1573 sqe->opcode = NVM_ADMIN_IDENTIFY; 1574 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1575 htolem32(&sqe->cdw10, 1); 1576 } 1577 1578 static int 1579 nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap) 1580 { 1581 struct nvme_pt_state state; 1582 struct nvme_pt_command pt; 1583 struct nvme_ccb *ccb; 1584 uint16_t ncqa, nsqa; 1585 int rv; 1586 1587 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1588 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1589 1590 memset(&pt, 0, sizeof(pt)); 1591 pt.cmd.opcode = NVM_ADMIN_GET_FEATURES; 1592 pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES; 1593 1594 memset(&state, 0, sizeof(state)); 1595 state.pt = &pt; 1596 state.finished = false; 1597 1598 ccb->ccb_done = nvme_pt_done; 1599 ccb->ccb_cookie = &state; 1600 1601 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1602 1603 if (rv != 0) { 1604 *nqap = 0; 1605 return EIO; 1606 } 1607 1608 ncqa = pt.cpl.cdw0 >> 16; 1609 nsqa = pt.cpl.cdw0 & 0xffff; 1610 *nqap = MIN(ncqa, nsqa) + 1; 1611 1612 return 0; 1613 } 1614 1615 static int 1616 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1617 { 1618 struct nvme_softc *sc = q->q_sc; 1619 struct nvme_ccb *ccb; 1620 bus_addr_t off; 1621 uint64_t *prpl; 1622 u_int i; 1623 1624 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1625 cv_init(&q->q_ccb_wait, "nvmeqw"); 1626 q->q_ccb_waiting = false; 1627 SIMPLEQ_INIT(&q->q_ccb_list); 1628 1629 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1630 1631 q->q_nccbs = nccbs; 1632 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1633 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1634 1635 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1636 off = 0; 1637 1638 for (i = 0; i < nccbs; i++) { 1639 ccb = &q->q_ccbs[i]; 1640 1641 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1642 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1643 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1644 &ccb->ccb_dmamap) != 0) 1645 goto free_maps; 1646 1647 ccb->ccb_id = i; 1648 ccb->ccb_prpl = prpl; 1649 ccb->ccb_prpl_off = off; 1650 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1651 1652 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1653 1654 prpl += sc->sc_max_sgl; 1655 off += sizeof(*prpl) * sc->sc_max_sgl; 1656 } 1657 1658 return 0; 1659 1660 free_maps: 1661 nvme_ccbs_free(q); 1662 return 1; 1663 } 1664 1665 static struct nvme_ccb * 1666 nvme_ccb_get(struct nvme_queue *q, bool wait) 1667 { 1668 struct nvme_ccb *ccb = NULL; 1669 1670 mutex_enter(&q->q_ccb_mtx); 1671 again: 1672 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1673 if (ccb != NULL) { 1674 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1675 #ifdef DEBUG 1676 ccb->ccb_cookie = NULL; 1677 #endif 1678 } else { 1679 if (__predict_false(wait)) { 1680 q->q_ccb_waiting = true; 1681 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1682 goto again; 1683 } 1684 } 1685 mutex_exit(&q->q_ccb_mtx); 1686 1687 return ccb; 1688 } 1689 1690 static void 1691 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1692 { 1693 1694 mutex_enter(&q->q_ccb_mtx); 1695 #ifdef DEBUG 1696 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1697 #endif 1698 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1699 1700 /* It's unlikely there are any waiters, it's not used for regular I/O */ 1701 if (__predict_false(q->q_ccb_waiting)) { 1702 q->q_ccb_waiting = false; 1703 cv_broadcast(&q->q_ccb_wait); 1704 } 1705 1706 mutex_exit(&q->q_ccb_mtx); 1707 } 1708 1709 static void 1710 nvme_ccbs_free(struct nvme_queue *q) 1711 { 1712 struct nvme_softc *sc = q->q_sc; 1713 struct nvme_ccb *ccb; 1714 1715 mutex_enter(&q->q_ccb_mtx); 1716 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1717 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1718 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1719 } 1720 mutex_exit(&q->q_ccb_mtx); 1721 1722 nvme_dmamem_free(sc, q->q_ccb_prpls); 1723 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1724 q->q_ccbs = NULL; 1725 cv_destroy(&q->q_ccb_wait); 1726 mutex_destroy(&q->q_ccb_mtx); 1727 } 1728 1729 static struct nvme_queue * 1730 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1731 { 1732 struct nvme_queue *q; 1733 1734 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1735 q->q_sc = sc; 1736 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1737 sizeof(struct nvme_sqe) * entries); 1738 if (q->q_sq_dmamem == NULL) 1739 goto free; 1740 1741 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1742 sizeof(struct nvme_cqe) * entries); 1743 if (q->q_cq_dmamem == NULL) 1744 goto free_sq; 1745 1746 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1747 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1748 1749 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1750 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1751 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1752 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1753 q->q_id = id; 1754 q->q_entries = entries; 1755 q->q_sq_tail = 0; 1756 q->q_cq_head = 0; 1757 q->q_cq_phase = NVME_CQE_PHASE; 1758 1759 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1760 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1761 1762 /* 1763 * Due to definition of full and empty queue (queue is empty 1764 * when head == tail, full when tail is one less then head), 1765 * we can actually only have (entries - 1) in-flight commands. 1766 */ 1767 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1768 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1769 goto free_cq; 1770 } 1771 1772 return q; 1773 1774 free_cq: 1775 nvme_dmamem_free(sc, q->q_cq_dmamem); 1776 free_sq: 1777 nvme_dmamem_free(sc, q->q_sq_dmamem); 1778 free: 1779 kmem_free(q, sizeof(*q)); 1780 1781 return NULL; 1782 } 1783 1784 static void 1785 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1786 { 1787 nvme_ccbs_free(q); 1788 mutex_destroy(&q->q_sq_mtx); 1789 mutex_destroy(&q->q_cq_mtx); 1790 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1791 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1792 nvme_dmamem_free(sc, q->q_cq_dmamem); 1793 nvme_dmamem_free(sc, q->q_sq_dmamem); 1794 kmem_free(q, sizeof(*q)); 1795 } 1796 1797 int 1798 nvme_intr(void *xsc) 1799 { 1800 struct nvme_softc *sc = xsc; 1801 1802 /* 1803 * INTx is level triggered, controller deasserts the interrupt only 1804 * when we advance command queue head via write to the doorbell. 1805 * Tell the controller to block the interrupts while we process 1806 * the queue(s). 1807 */ 1808 nvme_write4(sc, NVME_INTMS, 1); 1809 1810 softint_schedule(sc->sc_softih[0]); 1811 1812 /* don't know, might not have been for us */ 1813 return 1; 1814 } 1815 1816 void 1817 nvme_softintr_intx(void *xq) 1818 { 1819 struct nvme_queue *q = xq; 1820 struct nvme_softc *sc = q->q_sc; 1821 1822 nvme_q_complete(sc, sc->sc_admin_q); 1823 if (sc->sc_q != NULL) 1824 nvme_q_complete(sc, sc->sc_q[0]); 1825 1826 /* 1827 * Processing done, tell controller to issue interrupts again. There 1828 * is no race, as NVMe spec requires the controller to maintain state, 1829 * and assert the interrupt whenever there are unacknowledged 1830 * completion queue entries. 1831 */ 1832 nvme_write4(sc, NVME_INTMC, 1); 1833 } 1834 1835 int 1836 nvme_intr_msi(void *xq) 1837 { 1838 struct nvme_queue *q = xq; 1839 1840 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1841 && q->q_sc->sc_softih[q->q_id]); 1842 1843 /* 1844 * MSI/MSI-X are edge triggered, so can handover processing to softint 1845 * without masking the interrupt. 1846 */ 1847 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1848 1849 return 1; 1850 } 1851 1852 void 1853 nvme_softintr_msi(void *xq) 1854 { 1855 struct nvme_queue *q = xq; 1856 struct nvme_softc *sc = q->q_sc; 1857 1858 nvme_q_complete(sc, q); 1859 } 1860 1861 static struct nvme_dmamem * 1862 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1863 { 1864 struct nvme_dmamem *ndm; 1865 int nsegs; 1866 1867 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1868 if (ndm == NULL) 1869 return NULL; 1870 1871 ndm->ndm_size = size; 1872 1873 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1874 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1875 goto ndmfree; 1876 1877 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1878 1, &nsegs, BUS_DMA_WAITOK) != 0) 1879 goto destroy; 1880 1881 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1882 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1883 goto free; 1884 memset(ndm->ndm_kva, 0, size); 1885 1886 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1887 NULL, BUS_DMA_WAITOK) != 0) 1888 goto unmap; 1889 1890 return ndm; 1891 1892 unmap: 1893 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1894 free: 1895 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1896 destroy: 1897 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1898 ndmfree: 1899 kmem_free(ndm, sizeof(*ndm)); 1900 return NULL; 1901 } 1902 1903 static void 1904 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1905 { 1906 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1907 0, NVME_DMA_LEN(mem), ops); 1908 } 1909 1910 void 1911 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1912 { 1913 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1914 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1915 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1916 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1917 kmem_free(ndm, sizeof(*ndm)); 1918 } 1919 1920 /* 1921 * ioctl 1922 */ 1923 1924 dev_type_open(nvmeopen); 1925 dev_type_close(nvmeclose); 1926 dev_type_ioctl(nvmeioctl); 1927 1928 const struct cdevsw nvme_cdevsw = { 1929 .d_open = nvmeopen, 1930 .d_close = nvmeclose, 1931 .d_read = noread, 1932 .d_write = nowrite, 1933 .d_ioctl = nvmeioctl, 1934 .d_stop = nostop, 1935 .d_tty = notty, 1936 .d_poll = nopoll, 1937 .d_mmap = nommap, 1938 .d_kqfilter = nokqfilter, 1939 .d_discard = nodiscard, 1940 .d_flag = D_OTHER, 1941 }; 1942 1943 /* 1944 * Accept an open operation on the control device. 1945 */ 1946 int 1947 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1948 { 1949 struct nvme_softc *sc; 1950 int unit = minor(dev) / 0x10000; 1951 int nsid = minor(dev) & 0xffff; 1952 int nsidx; 1953 1954 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1955 return ENXIO; 1956 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1957 return ENXIO; 1958 1959 if (nsid == 0) { 1960 /* controller */ 1961 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1962 return EBUSY; 1963 SET(sc->sc_flags, NVME_F_OPEN); 1964 } else { 1965 /* namespace */ 1966 nsidx = nsid - 1; 1967 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1968 return ENXIO; 1969 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1970 return EBUSY; 1971 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1972 } 1973 return 0; 1974 } 1975 1976 /* 1977 * Accept the last close on the control device. 1978 */ 1979 int 1980 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1981 { 1982 struct nvme_softc *sc; 1983 int unit = minor(dev) / 0x10000; 1984 int nsid = minor(dev) & 0xffff; 1985 int nsidx; 1986 1987 sc = device_lookup_private(&nvme_cd, unit); 1988 if (sc == NULL) 1989 return ENXIO; 1990 1991 if (nsid == 0) { 1992 /* controller */ 1993 CLR(sc->sc_flags, NVME_F_OPEN); 1994 } else { 1995 /* namespace */ 1996 nsidx = nsid - 1; 1997 if (nsidx >= sc->sc_nn) 1998 return ENXIO; 1999 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 2000 } 2001 2002 return 0; 2003 } 2004 2005 /* 2006 * Handle control operations. 2007 */ 2008 int 2009 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 2010 { 2011 struct nvme_softc *sc; 2012 int unit = minor(dev) / 0x10000; 2013 int nsid = minor(dev) & 0xffff; 2014 struct nvme_pt_command *pt; 2015 2016 sc = device_lookup_private(&nvme_cd, unit); 2017 if (sc == NULL) 2018 return ENXIO; 2019 2020 switch (cmd) { 2021 case NVME_PASSTHROUGH_CMD: 2022 pt = data; 2023 return nvme_command_passthrough(sc, data, 2024 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 2025 } 2026 2027 return ENOTTY; 2028 } 2029
Indexes created Mon Sep 29 21:09:56 GMT 2025