nvme.c revision 1.9
1 /* $NetBSD: nvme.c,v 1.9 2016/09/18 21:19:39 jdolecek Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.9 2016/09/18 21:19:39 jdolecek Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 int nvme_adminq_size = 128; 44 int nvme_ioq_size = 1024; 45 46 static int nvme_print(void *, const char *); 47 48 static int nvme_ready(struct nvme_softc *, uint32_t); 49 static int nvme_enable(struct nvme_softc *, u_int); 50 static int nvme_disable(struct nvme_softc *); 51 static int nvme_shutdown(struct nvme_softc *); 52 53 static void nvme_version(struct nvme_softc *, uint32_t); 54 #ifdef NVME_DEBUG 55 static void nvme_dumpregs(struct nvme_softc *); 56 #endif 57 static int nvme_identify(struct nvme_softc *, u_int); 58 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 59 void *); 60 61 static int nvme_ccbs_alloc(struct nvme_queue *, u_int); 62 static void nvme_ccbs_free(struct nvme_queue *); 63 64 static struct nvme_ccb * 65 nvme_ccb_get(struct nvme_queue *); 66 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 67 68 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 69 struct nvme_ccb *, void (*)(struct nvme_queue *, 70 struct nvme_ccb *, void *), int); 71 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 72 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 73 struct nvme_cqe *); 74 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 76 struct nvme_cqe *); 77 78 static struct nvme_queue * 79 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 80 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 81 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 82 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 83 struct nvme_ccb *, void (*)(struct nvme_queue *, 84 struct nvme_ccb *, void *)); 85 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 86 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 87 88 static struct nvme_dmamem * 89 nvme_dmamem_alloc(struct nvme_softc *, size_t); 90 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 91 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 92 int); 93 94 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 95 void *); 96 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 97 struct nvme_cqe *); 98 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 99 void *); 100 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 101 struct nvme_cqe *); 102 103 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 104 void *); 105 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 106 struct nvme_cqe *); 107 static int nvme_command_passthrough(struct nvme_softc *, 108 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 109 110 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 111 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 112 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 113 #define NVME_TIMO_SY -1 /* sync cache timeout */ 114 115 #define nvme_read4(_s, _r) \ 116 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 117 #define nvme_write4(_s, _r, _v) \ 118 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 119 #ifdef __LP64__ 120 #define nvme_read8(_s, _r) \ 121 bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r)) 122 #define nvme_write8(_s, _r, _v) \ 123 bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 124 #else /* __LP64__ */ 125 static inline uint64_t 126 nvme_read8(struct nvme_softc *sc, bus_size_t r) 127 { 128 uint64_t v; 129 uint32_t *a = (uint32_t *)&v; 130 131 #if _BYTE_ORDER == _LITTLE_ENDIAN 132 a[0] = nvme_read4(sc, r); 133 a[1] = nvme_read4(sc, r + 4); 134 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 135 a[1] = nvme_read4(sc, r); 136 a[0] = nvme_read4(sc, r + 4); 137 #endif 138 139 return v; 140 } 141 142 static inline void 143 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 144 { 145 uint32_t *a = (uint32_t *)&v; 146 147 #if _BYTE_ORDER == _LITTLE_ENDIAN 148 nvme_write4(sc, r, a[0]); 149 nvme_write4(sc, r + 4, a[1]); 150 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 151 nvme_write4(sc, r, a[1]); 152 nvme_write4(sc, r + 4, a[0]); 153 #endif 154 } 155 #endif /* __LP64__ */ 156 #define nvme_barrier(_s, _r, _l, _f) \ 157 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 158 159 static void 160 nvme_version(struct nvme_softc *sc, uint32_t ver) 161 { 162 const char *v = NULL; 163 164 switch (ver) { 165 case NVME_VS_1_0: 166 v = "1.0"; 167 break; 168 case NVME_VS_1_1: 169 v = "1.1"; 170 break; 171 case NVME_VS_1_2: 172 v = "1.2"; 173 break; 174 default: 175 aprint_error_dev(sc->sc_dev, "unknown version 0x%08x\n", ver); 176 return; 177 } 178 179 aprint_normal_dev(sc->sc_dev, "NVMe %s\n", v); 180 } 181 182 #ifdef NVME_DEBUG 183 static __used void 184 nvme_dumpregs(struct nvme_softc *sc) 185 { 186 uint64_t r8; 187 uint32_t r4; 188 189 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 190 r8 = nvme_read8(sc, NVME_CAP); 191 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 192 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 193 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 194 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 195 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 196 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 197 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 198 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 199 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 200 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 201 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 202 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 203 204 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 205 206 r4 = nvme_read4(sc, NVME_CC); 207 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 208 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 209 (1 << NVME_CC_IOCQES_R(r4))); 210 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 211 (1 << NVME_CC_IOSQES_R(r4))); 212 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 213 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 214 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 215 (1 << NVME_CC_MPS_R(r4))); 216 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 217 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 218 219 r4 = nvme_read4(sc, NVME_CSTS); 220 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 221 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 222 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 223 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 224 225 r4 = nvme_read4(sc, NVME_AQA); 226 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 227 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 228 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 229 230 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 231 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 232 #undef DEVNAME 233 } 234 #endif /* NVME_DEBUG */ 235 236 static int 237 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 238 { 239 u_int i = 0; 240 uint32_t cc; 241 242 cc = nvme_read4(sc, NVME_CC); 243 if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) { 244 aprint_error_dev(sc->sc_dev, 245 "controller enabled status expected %d, found to be %d\n", 246 (rdy != 0), ((cc & NVME_CC_EN) != 0)); 247 return ENXIO; 248 } 249 250 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 251 if (i++ > sc->sc_rdy_to) 252 return ENXIO; 253 254 delay(1000); 255 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 256 } 257 258 return 0; 259 } 260 261 static int 262 nvme_enable(struct nvme_softc *sc, u_int mps) 263 { 264 uint32_t cc, csts; 265 266 cc = nvme_read4(sc, NVME_CC); 267 csts = nvme_read4(sc, NVME_CSTS); 268 269 if (ISSET(cc, NVME_CC_EN)) { 270 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 271 272 if (ISSET(csts, NVME_CSTS_RDY)) 273 return 1; 274 275 goto waitready; 276 } 277 278 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 279 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 280 delay(5000); 281 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 282 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 283 delay(5000); 284 285 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 286 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 287 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 288 delay(5000); 289 290 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 291 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 292 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 293 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 294 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 295 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 296 SET(cc, NVME_CC_MPS(mps)); 297 SET(cc, NVME_CC_EN); 298 299 nvme_write4(sc, NVME_CC, cc); 300 nvme_barrier(sc, 0, sc->sc_ios, 301 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 302 delay(5000); 303 304 waitready: 305 return nvme_ready(sc, NVME_CSTS_RDY); 306 } 307 308 static int 309 nvme_disable(struct nvme_softc *sc) 310 { 311 uint32_t cc, csts; 312 313 cc = nvme_read4(sc, NVME_CC); 314 csts = nvme_read4(sc, NVME_CSTS); 315 316 if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY)) 317 nvme_ready(sc, NVME_CSTS_RDY); 318 319 CLR(cc, NVME_CC_EN); 320 321 nvme_write4(sc, NVME_CC, cc); 322 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 323 324 delay(5000); 325 326 return nvme_ready(sc, 0); 327 } 328 329 int 330 nvme_attach(struct nvme_softc *sc) 331 { 332 struct nvme_attach_args naa; 333 uint64_t cap; 334 uint32_t reg; 335 u_int dstrd; 336 u_int mps = PAGE_SHIFT; 337 int adminq_entries = nvme_adminq_size; 338 int ioq_entries = nvme_ioq_size; 339 int i; 340 341 reg = nvme_read4(sc, NVME_VS); 342 if (reg == 0xffffffff) { 343 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 344 return 1; 345 } 346 347 nvme_version(sc, reg); 348 349 cap = nvme_read8(sc, NVME_CAP); 350 dstrd = NVME_CAP_DSTRD(cap); 351 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 352 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 353 "is greater than CPU page size %u\n", 354 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 355 return 1; 356 } 357 if (NVME_CAP_MPSMAX(cap) < mps) 358 mps = NVME_CAP_MPSMAX(cap); 359 360 /* set initial values to be used for admin queue during probe */ 361 sc->sc_rdy_to = NVME_CAP_TO(cap); 362 sc->sc_mps = 1 << mps; 363 sc->sc_mdts = MAXPHYS; 364 sc->sc_max_sgl = 2; 365 366 if (nvme_disable(sc) != 0) { 367 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 368 return 1; 369 } 370 371 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 372 if (sc->sc_admin_q == NULL) { 373 aprint_error_dev(sc->sc_dev, 374 "unable to allocate admin queue\n"); 375 return 1; 376 } 377 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 378 goto free_admin_q; 379 380 if (nvme_enable(sc, mps) != 0) { 381 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 382 goto disestablish_admin_q; 383 } 384 385 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 386 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 387 goto disable; 388 } 389 390 /* we know how big things are now */ 391 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 392 393 /* reallocate ccbs of admin queue with new max sgl. */ 394 nvme_ccbs_free(sc->sc_admin_q); 395 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 396 397 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 398 if (sc->sc_q == NULL) { 399 aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n"); 400 goto disable; 401 } 402 for (i = 0; i < sc->sc_nq; i++) { 403 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 404 if (sc->sc_q[i] == NULL) { 405 aprint_error_dev(sc->sc_dev, 406 "unable to allocate io queue\n"); 407 goto free_q; 408 } 409 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 410 aprint_error_dev(sc->sc_dev, 411 "unable to create io queue\n"); 412 nvme_q_free(sc, sc->sc_q[i]); 413 goto free_q; 414 } 415 } 416 417 if (!sc->sc_use_mq) 418 nvme_write4(sc, NVME_INTMC, 1); 419 420 snprintf(sc->sc_ctxpoolname, sizeof(sc->sc_ctxpoolname), 421 "%s_ns_ctx", device_xname(sc->sc_dev)); 422 sc->sc_ctxpool = pool_cache_init(sizeof(struct nvme_ns_context), 423 0, 0, 0, sc->sc_ctxpoolname, NULL, IPL_BIO, NULL, NULL, NULL); 424 if (sc->sc_ctxpool == NULL) { 425 aprint_error_dev(sc->sc_dev, "unable to create ctx pool\n"); 426 goto free_q; 427 } 428 429 /* probe subdevices */ 430 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 431 KM_SLEEP); 432 if (sc->sc_namespaces == NULL) 433 goto free_q; 434 for (i = 0; i < sc->sc_nn; i++) { 435 memset(&naa, 0, sizeof(naa)); 436 naa.naa_nsid = i + 1; 437 naa.naa_qentries = ioq_entries; 438 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 439 nvme_print); 440 } 441 442 return 0; 443 444 free_q: 445 while (--i >= 0) { 446 nvme_q_delete(sc, sc->sc_q[i]); 447 nvme_q_free(sc, sc->sc_q[i]); 448 } 449 disable: 450 nvme_disable(sc); 451 disestablish_admin_q: 452 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 453 free_admin_q: 454 nvme_q_free(sc, sc->sc_admin_q); 455 456 return 1; 457 } 458 459 static int 460 nvme_print(void *aux, const char *pnp) 461 { 462 struct nvme_attach_args *naa = aux; 463 464 if (pnp) 465 aprint_normal("at %s", pnp); 466 467 if (naa->naa_nsid > 0) 468 aprint_normal(" nsid %d", naa->naa_nsid); 469 470 return UNCONF; 471 } 472 473 int 474 nvme_detach(struct nvme_softc *sc, int flags) 475 { 476 int i, error; 477 478 error = config_detach_children(sc->sc_dev, flags); 479 if (error) 480 return error; 481 482 error = nvme_shutdown(sc); 483 if (error) 484 return error; 485 486 /* from now on we are committed to detach, following will never fail */ 487 pool_cache_destroy(sc->sc_ctxpool); 488 489 for (i = 0; i < sc->sc_nq; i++) 490 nvme_q_free(sc, sc->sc_q[i]); 491 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 492 nvme_q_free(sc, sc->sc_admin_q); 493 494 return 0; 495 } 496 497 static int 498 nvme_shutdown(struct nvme_softc *sc) 499 { 500 uint32_t cc, csts; 501 bool disabled = false; 502 int i; 503 504 if (!sc->sc_use_mq) 505 nvme_write4(sc, NVME_INTMS, 1); 506 507 for (i = 0; i < sc->sc_nq; i++) { 508 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 509 aprint_error_dev(sc->sc_dev, 510 "unable to delete io queue %d, disabling\n", i + 1); 511 disabled = true; 512 } 513 } 514 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 515 if (disabled) 516 goto disable; 517 518 cc = nvme_read4(sc, NVME_CC); 519 CLR(cc, NVME_CC_SHN_MASK); 520 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 521 nvme_write4(sc, NVME_CC, cc); 522 523 for (i = 0; i < 4000; i++) { 524 nvme_barrier(sc, 0, sc->sc_ios, 525 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 526 csts = nvme_read4(sc, NVME_CSTS); 527 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 528 return 0; 529 530 delay(1000); 531 } 532 533 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 534 535 disable: 536 nvme_disable(sc); 537 return 0; 538 } 539 540 void 541 nvme_childdet(device_t self, device_t child) 542 { 543 struct nvme_softc *sc = device_private(self); 544 int i; 545 546 for (i = 0; i < sc->sc_nn; i++) { 547 if (sc->sc_namespaces[i].dev == child) { 548 /* Already freed ns->ident. */ 549 sc->sc_namespaces[i].dev = NULL; 550 break; 551 } 552 } 553 } 554 555 int 556 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 557 { 558 struct nvme_sqe sqe; 559 struct nvm_identify_namespace *identify; 560 struct nvme_dmamem *mem; 561 struct nvme_ccb *ccb; 562 struct nvme_namespace *ns; 563 int rv; 564 565 KASSERT(nsid > 0); 566 567 ccb = nvme_ccb_get(sc->sc_admin_q); 568 if (ccb == NULL) 569 return EAGAIN; 570 571 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 572 if (mem == NULL) 573 return ENOMEM; 574 575 memset(&sqe, 0, sizeof(sqe)); 576 sqe.opcode = NVM_ADMIN_IDENTIFY; 577 htolem32(&sqe.nsid, nsid); 578 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 579 htolem32(&sqe.cdw10, 0); 580 581 ccb->ccb_done = nvme_empty_done; 582 ccb->ccb_cookie = &sqe; 583 584 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 585 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 586 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 587 588 nvme_ccb_put(sc->sc_admin_q, ccb); 589 590 if (rv != 0) { 591 rv = EIO; 592 goto done; 593 } 594 595 /* commit */ 596 597 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 598 memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 599 600 ns = nvme_ns_get(sc, nsid); 601 KASSERT(ns); 602 ns->ident = identify; 603 604 done: 605 nvme_dmamem_free(sc, mem); 606 607 return rv; 608 } 609 610 int 611 nvme_ns_dobio(struct nvme_softc *sc, struct nvme_ns_context *ctx) 612 { 613 struct nvme_queue *q = nvme_get_q(sc); 614 struct nvme_ccb *ccb; 615 bus_dmamap_t dmap; 616 int i, error; 617 618 ccb = nvme_ccb_get(q); 619 if (ccb == NULL) 620 return EAGAIN; 621 622 ccb->ccb_done = nvme_ns_io_done; 623 ccb->ccb_cookie = ctx; 624 625 dmap = ccb->ccb_dmamap; 626 error = bus_dmamap_load(sc->sc_dmat, dmap, ctx->nnc_data, 627 ctx->nnc_datasize, NULL, 628 (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL) ? 629 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 630 (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 631 BUS_DMA_READ : BUS_DMA_WRITE)); 632 if (error) { 633 nvme_ccb_put(q, ccb); 634 return error; 635 } 636 637 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 638 ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 639 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 640 641 if (dmap->dm_nsegs > 2) { 642 for (i = 1; i < dmap->dm_nsegs; i++) { 643 htolem64(&ccb->ccb_prpl[i - 1], 644 dmap->dm_segs[i].ds_addr); 645 } 646 bus_dmamap_sync(sc->sc_dmat, 647 NVME_DMA_MAP(q->q_ccb_prpls), 648 ccb->ccb_prpl_off, 649 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 650 BUS_DMASYNC_PREWRITE); 651 } 652 653 if (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL)) { 654 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 655 return EIO; 656 return 0; 657 } 658 659 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 660 return 0; 661 } 662 663 static void 664 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 665 { 666 struct nvme_sqe_io *sqe = slot; 667 struct nvme_ns_context *ctx = ccb->ccb_cookie; 668 bus_dmamap_t dmap = ccb->ccb_dmamap; 669 670 sqe->opcode = ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 671 NVM_CMD_READ : NVM_CMD_WRITE; 672 htolem32(&sqe->nsid, ctx->nnc_nsid); 673 674 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 675 switch (dmap->dm_nsegs) { 676 case 1: 677 break; 678 case 2: 679 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 680 break; 681 default: 682 /* the prp list is already set up and synced */ 683 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 684 break; 685 } 686 687 htolem64(&sqe->slba, ctx->nnc_blkno); 688 htolem16(&sqe->nlb, (ctx->nnc_datasize / ctx->nnc_secsize) - 1); 689 } 690 691 static void 692 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 693 struct nvme_cqe *cqe) 694 { 695 struct nvme_softc *sc = q->q_sc; 696 struct nvme_ns_context *ctx = ccb->ccb_cookie; 697 bus_dmamap_t dmap = ccb->ccb_dmamap; 698 uint16_t flags; 699 700 if (dmap->dm_nsegs > 2) { 701 bus_dmamap_sync(sc->sc_dmat, 702 NVME_DMA_MAP(q->q_ccb_prpls), 703 ccb->ccb_prpl_off, 704 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 705 BUS_DMASYNC_POSTWRITE); 706 } 707 708 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 709 ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 710 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 711 712 bus_dmamap_unload(sc->sc_dmat, dmap); 713 nvme_ccb_put(q, ccb); 714 715 flags = lemtoh16(&cqe->flags); 716 717 ctx->nnc_status = flags; 718 (*ctx->nnc_done)(ctx); 719 } 720 721 int 722 nvme_ns_sync(struct nvme_softc *sc, struct nvme_ns_context *ctx) 723 { 724 struct nvme_queue *q = nvme_get_q(sc); 725 struct nvme_ccb *ccb; 726 727 ccb = nvme_ccb_get(q); 728 if (ccb == NULL) 729 return EAGAIN; 730 731 ccb->ccb_done = nvme_ns_sync_done; 732 ccb->ccb_cookie = ctx; 733 734 if (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL)) { 735 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 736 return EIO; 737 return 0; 738 } 739 740 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 741 return 0; 742 } 743 744 static void 745 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 746 { 747 struct nvme_sqe *sqe = slot; 748 struct nvme_ns_context *ctx = ccb->ccb_cookie; 749 750 sqe->opcode = NVM_CMD_FLUSH; 751 htolem32(&sqe->nsid, ctx->nnc_nsid); 752 } 753 754 static void 755 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 756 struct nvme_cqe *cqe) 757 { 758 struct nvme_ns_context *ctx = ccb->ccb_cookie; 759 uint16_t flags; 760 761 nvme_ccb_put(q, ccb); 762 763 flags = lemtoh16(&cqe->flags); 764 765 ctx->nnc_status = flags; 766 (*ctx->nnc_done)(ctx); 767 } 768 769 void 770 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 771 { 772 struct nvme_namespace *ns; 773 struct nvm_identify_namespace *identify; 774 775 ns = nvme_ns_get(sc, nsid); 776 KASSERT(ns); 777 778 identify = ns->ident; 779 ns->ident = NULL; 780 if (identify != NULL) 781 kmem_free(identify, sizeof(*identify)); 782 } 783 784 static void 785 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 786 { 787 struct nvme_softc *sc = q->q_sc; 788 struct nvme_sqe *sqe = slot; 789 struct nvme_pt_command *pt = ccb->ccb_cookie; 790 bus_dmamap_t dmap = ccb->ccb_dmamap; 791 int i; 792 793 sqe->opcode = pt->cmd.opcode; 794 htolem32(&sqe->nsid, pt->cmd.nsid); 795 796 if (pt->buf != NULL && pt->len > 0) { 797 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 798 switch (dmap->dm_nsegs) { 799 case 1: 800 break; 801 case 2: 802 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 803 break; 804 default: 805 for (i = 1; i < dmap->dm_nsegs; i++) { 806 htolem64(&ccb->ccb_prpl[i - 1], 807 dmap->dm_segs[i].ds_addr); 808 } 809 bus_dmamap_sync(sc->sc_dmat, 810 NVME_DMA_MAP(q->q_ccb_prpls), 811 ccb->ccb_prpl_off, 812 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 813 BUS_DMASYNC_PREWRITE); 814 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 815 break; 816 } 817 } 818 819 htolem32(&sqe->cdw10, pt->cmd.cdw10); 820 htolem32(&sqe->cdw11, pt->cmd.cdw11); 821 htolem32(&sqe->cdw12, pt->cmd.cdw12); 822 htolem32(&sqe->cdw13, pt->cmd.cdw13); 823 htolem32(&sqe->cdw14, pt->cmd.cdw14); 824 htolem32(&sqe->cdw15, pt->cmd.cdw15); 825 } 826 827 static void 828 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 829 { 830 struct nvme_softc *sc = q->q_sc; 831 struct nvme_pt_command *pt = ccb->ccb_cookie; 832 bus_dmamap_t dmap = ccb->ccb_dmamap; 833 834 if (pt->buf != NULL && pt->len > 0) { 835 if (dmap->dm_nsegs > 2) { 836 bus_dmamap_sync(sc->sc_dmat, 837 NVME_DMA_MAP(q->q_ccb_prpls), 838 ccb->ccb_prpl_off, 839 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 840 BUS_DMASYNC_POSTWRITE); 841 } 842 843 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 844 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 845 bus_dmamap_unload(sc->sc_dmat, dmap); 846 } 847 848 pt->cpl.cdw0 = cqe->cdw0; 849 pt->cpl.flags = cqe->flags & ~NVME_CQE_PHASE; 850 } 851 852 static int 853 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 854 uint16_t nsid, struct lwp *l, bool is_adminq) 855 { 856 struct nvme_queue *q; 857 struct nvme_ccb *ccb; 858 void *buf = NULL; 859 int error; 860 861 /* limit command size to maximum data transfer size */ 862 if ((pt->buf == NULL && pt->len > 0) || 863 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 864 return EINVAL; 865 866 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 867 ccb = nvme_ccb_get(q); 868 if (ccb == NULL) 869 return EBUSY; 870 871 if (pt->buf != NULL) { 872 KASSERT(pt->len > 0); 873 buf = kmem_alloc(pt->len, KM_SLEEP); 874 if (buf == NULL) { 875 error = ENOMEM; 876 goto ccb_put; 877 } 878 if (!pt->is_read) { 879 error = copyin(pt->buf, buf, pt->len); 880 if (error) 881 goto kmem_free; 882 } 883 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 884 pt->len, NULL, 885 BUS_DMA_WAITOK | 886 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 887 if (error) 888 goto kmem_free; 889 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 890 0, ccb->ccb_dmamap->dm_mapsize, 891 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 892 } 893 894 ccb->ccb_done = nvme_pt_done; 895 ccb->ccb_cookie = pt; 896 897 pt->cmd.nsid = nsid; 898 if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) { 899 error = EIO; 900 goto out; 901 } 902 903 error = 0; 904 out: 905 if (buf != NULL) { 906 if (error == 0 && pt->is_read) 907 error = copyout(buf, pt->buf, pt->len); 908 kmem_free: 909 kmem_free(buf, pt->len); 910 } 911 ccb_put: 912 nvme_ccb_put(q, ccb); 913 return error; 914 } 915 916 static void 917 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 918 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 919 { 920 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 921 uint32_t tail; 922 923 mutex_enter(&q->q_sq_mtx); 924 tail = q->q_sq_tail; 925 if (++q->q_sq_tail >= q->q_entries) 926 q->q_sq_tail = 0; 927 928 sqe += tail; 929 930 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 931 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 932 memset(sqe, 0, sizeof(*sqe)); 933 (*fill)(q, ccb, sqe); 934 sqe->cid = ccb->ccb_id; 935 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 936 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 937 938 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 939 mutex_exit(&q->q_sq_mtx); 940 } 941 942 struct nvme_poll_state { 943 struct nvme_sqe s; 944 struct nvme_cqe c; 945 }; 946 947 static int 948 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 949 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 950 { 951 struct nvme_poll_state state; 952 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 953 void *cookie; 954 uint16_t flags; 955 int step = 10; 956 int maxloop = timo_sec * 1000000 / step; 957 int error = 0; 958 959 memset(&state, 0, sizeof(state)); 960 (*fill)(q, ccb, &state.s); 961 962 done = ccb->ccb_done; 963 cookie = ccb->ccb_cookie; 964 965 ccb->ccb_done = nvme_poll_done; 966 ccb->ccb_cookie = &state; 967 968 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 969 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 970 if (nvme_q_complete(sc, q) == 0) 971 delay(step); 972 973 if (timo_sec >= 0 && --maxloop <= 0) { 974 error = ETIMEDOUT; 975 break; 976 } 977 } 978 979 ccb->ccb_cookie = cookie; 980 done(q, ccb, &state.c); 981 982 if (error == 0) { 983 flags = lemtoh16(&state.c.flags); 984 return flags & ~NVME_CQE_PHASE; 985 } else { 986 return 1; 987 } 988 } 989 990 static void 991 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 992 { 993 struct nvme_sqe *sqe = slot; 994 struct nvme_poll_state *state = ccb->ccb_cookie; 995 996 *sqe = state->s; 997 } 998 999 static void 1000 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1001 struct nvme_cqe *cqe) 1002 { 1003 struct nvme_poll_state *state = ccb->ccb_cookie; 1004 1005 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1006 state->c = *cqe; 1007 } 1008 1009 static void 1010 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1011 { 1012 struct nvme_sqe *src = ccb->ccb_cookie; 1013 struct nvme_sqe *dst = slot; 1014 1015 *dst = *src; 1016 } 1017 1018 static void 1019 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1020 struct nvme_cqe *cqe) 1021 { 1022 } 1023 1024 static int 1025 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1026 { 1027 struct nvme_ccb *ccb; 1028 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1029 uint16_t flags; 1030 int rv = 0; 1031 1032 mutex_enter(&q->q_cq_mtx); 1033 1034 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1035 for (;;) { 1036 cqe = &ring[q->q_cq_head]; 1037 flags = lemtoh16(&cqe->flags); 1038 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1039 break; 1040 1041 ccb = &q->q_ccbs[cqe->cid]; 1042 1043 if (++q->q_cq_head >= q->q_entries) { 1044 q->q_cq_head = 0; 1045 q->q_cq_phase ^= NVME_CQE_PHASE; 1046 } 1047 1048 rv = 1; 1049 1050 /* 1051 * Unlock the mutext before calling the ccb_done callback 1052 * and re-lock afterwards. The callback triggers lddone() 1053 * which schedules another i/o, and also calls nvme_ccb_put(). 1054 * Unlock/relock avoids possibility of deadlock. 1055 */ 1056 mutex_exit(&q->q_cq_mtx); 1057 ccb->ccb_done(q, ccb, cqe); 1058 mutex_enter(&q->q_cq_mtx); 1059 } 1060 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1061 1062 if (rv) 1063 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1064 1065 mutex_exit(&q->q_cq_mtx); 1066 1067 return rv; 1068 } 1069 1070 static int 1071 nvme_identify(struct nvme_softc *sc, u_int mps) 1072 { 1073 char sn[41], mn[81], fr[17]; 1074 struct nvm_identify_controller *identify; 1075 struct nvme_dmamem *mem; 1076 struct nvme_ccb *ccb; 1077 u_int mdts; 1078 int rv = 1; 1079 1080 ccb = nvme_ccb_get(sc->sc_admin_q); 1081 if (ccb == NULL) 1082 panic("%s: nvme_ccb_get returned NULL", __func__); 1083 1084 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1085 if (mem == NULL) 1086 return 1; 1087 1088 ccb->ccb_done = nvme_empty_done; 1089 ccb->ccb_cookie = mem; 1090 1091 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1092 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1093 NVME_TIMO_IDENT); 1094 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1095 1096 nvme_ccb_put(sc->sc_admin_q, ccb); 1097 1098 if (rv != 0) 1099 goto done; 1100 1101 identify = NVME_DMA_KVA(mem); 1102 1103 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1104 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1105 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1106 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1107 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1108 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1109 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1110 sn); 1111 1112 if (identify->mdts > 0) { 1113 mdts = (1 << identify->mdts) * (1 << mps); 1114 if (mdts < sc->sc_mdts) 1115 sc->sc_mdts = mdts; 1116 } 1117 1118 sc->sc_nn = lemtoh32(&identify->nn); 1119 1120 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1121 1122 done: 1123 nvme_dmamem_free(sc, mem); 1124 1125 return rv; 1126 } 1127 1128 static int 1129 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1130 { 1131 struct nvme_sqe_q sqe; 1132 struct nvme_ccb *ccb; 1133 int rv; 1134 1135 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1136 return 1; 1137 1138 ccb = nvme_ccb_get(sc->sc_admin_q); 1139 KASSERT(ccb != NULL); 1140 1141 ccb->ccb_done = nvme_empty_done; 1142 ccb->ccb_cookie = &sqe; 1143 1144 memset(&sqe, 0, sizeof(sqe)); 1145 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1146 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1147 htolem16(&sqe.qsize, q->q_entries - 1); 1148 htolem16(&sqe.qid, q->q_id); 1149 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1150 if (sc->sc_use_mq) 1151 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1152 1153 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1154 if (rv != 0) 1155 goto fail; 1156 1157 ccb->ccb_done = nvme_empty_done; 1158 ccb->ccb_cookie = &sqe; 1159 1160 memset(&sqe, 0, sizeof(sqe)); 1161 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1162 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1163 htolem16(&sqe.qsize, q->q_entries - 1); 1164 htolem16(&sqe.qid, q->q_id); 1165 htolem16(&sqe.cqid, q->q_id); 1166 sqe.qflags = NVM_SQE_Q_PC; 1167 1168 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1169 if (rv != 0) 1170 goto fail; 1171 1172 fail: 1173 nvme_ccb_put(sc->sc_admin_q, ccb); 1174 return rv; 1175 } 1176 1177 static int 1178 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1179 { 1180 struct nvme_sqe_q sqe; 1181 struct nvme_ccb *ccb; 1182 int rv; 1183 1184 ccb = nvme_ccb_get(sc->sc_admin_q); 1185 KASSERT(ccb != NULL); 1186 1187 ccb->ccb_done = nvme_empty_done; 1188 ccb->ccb_cookie = &sqe; 1189 1190 memset(&sqe, 0, sizeof(sqe)); 1191 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1192 htolem16(&sqe.qid, q->q_id); 1193 1194 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1195 if (rv != 0) 1196 goto fail; 1197 1198 ccb->ccb_done = nvme_empty_done; 1199 ccb->ccb_cookie = &sqe; 1200 1201 memset(&sqe, 0, sizeof(sqe)); 1202 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1203 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1204 htolem16(&sqe.qid, q->q_id); 1205 1206 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1207 if (rv != 0) 1208 goto fail; 1209 1210 fail: 1211 nvme_ccb_put(sc->sc_admin_q, ccb); 1212 1213 if (rv == 0 && sc->sc_use_mq) { 1214 if (sc->sc_intr_disestablish(sc, q->q_id)) 1215 rv = 1; 1216 } 1217 1218 return rv; 1219 } 1220 1221 static void 1222 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1223 { 1224 struct nvme_sqe *sqe = slot; 1225 struct nvme_dmamem *mem = ccb->ccb_cookie; 1226 1227 sqe->opcode = NVM_ADMIN_IDENTIFY; 1228 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1229 htolem32(&sqe->cdw10, 1); 1230 } 1231 1232 static int 1233 nvme_ccbs_alloc(struct nvme_queue *q, u_int nccbs) 1234 { 1235 struct nvme_softc *sc = q->q_sc; 1236 struct nvme_ccb *ccb; 1237 bus_addr_t off; 1238 uint64_t *prpl; 1239 u_int i; 1240 1241 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1242 SIMPLEQ_INIT(&q->q_ccb_list); 1243 1244 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1245 if (q->q_ccbs == NULL) 1246 return 1; 1247 1248 q->q_nccbs = nccbs; 1249 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1250 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1251 1252 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1253 off = 0; 1254 1255 for (i = 0; i < nccbs; i++) { 1256 ccb = &q->q_ccbs[i]; 1257 1258 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1259 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1260 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1261 &ccb->ccb_dmamap) != 0) 1262 goto free_maps; 1263 1264 ccb->ccb_id = i; 1265 ccb->ccb_prpl = prpl; 1266 ccb->ccb_prpl_off = off; 1267 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1268 1269 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1270 1271 prpl += sc->sc_max_sgl; 1272 off += sizeof(*prpl) * sc->sc_max_sgl; 1273 } 1274 1275 return 0; 1276 1277 free_maps: 1278 nvme_ccbs_free(q); 1279 return 1; 1280 } 1281 1282 static struct nvme_ccb * 1283 nvme_ccb_get(struct nvme_queue *q) 1284 { 1285 struct nvme_ccb *ccb; 1286 1287 mutex_enter(&q->q_ccb_mtx); 1288 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1289 if (ccb != NULL) 1290 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1291 mutex_exit(&q->q_ccb_mtx); 1292 1293 return ccb; 1294 } 1295 1296 static void 1297 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1298 { 1299 1300 mutex_enter(&q->q_ccb_mtx); 1301 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1302 mutex_exit(&q->q_ccb_mtx); 1303 } 1304 1305 static void 1306 nvme_ccbs_free(struct nvme_queue *q) 1307 { 1308 struct nvme_softc *sc = q->q_sc; 1309 struct nvme_ccb *ccb; 1310 1311 mutex_enter(&q->q_ccb_mtx); 1312 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1313 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1314 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1315 } 1316 mutex_exit(&q->q_ccb_mtx); 1317 1318 nvme_dmamem_free(sc, q->q_ccb_prpls); 1319 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1320 q->q_ccbs = NULL; 1321 mutex_destroy(&q->q_ccb_mtx); 1322 } 1323 1324 static struct nvme_queue * 1325 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1326 { 1327 struct nvme_queue *q; 1328 1329 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1330 if (q == NULL) 1331 return NULL; 1332 1333 q->q_sc = sc; 1334 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1335 sizeof(struct nvme_sqe) * entries); 1336 if (q->q_sq_dmamem == NULL) 1337 goto free; 1338 1339 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1340 sizeof(struct nvme_cqe) * entries); 1341 if (q->q_cq_dmamem == NULL) 1342 goto free_sq; 1343 1344 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1345 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1346 1347 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1348 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1349 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1350 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1351 q->q_id = id; 1352 q->q_entries = entries; 1353 q->q_sq_tail = 0; 1354 q->q_cq_head = 0; 1355 q->q_cq_phase = NVME_CQE_PHASE; 1356 1357 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1358 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1359 1360 if (nvme_ccbs_alloc(q, entries) != 0) { 1361 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1362 goto free_cq; 1363 } 1364 1365 return q; 1366 1367 free_cq: 1368 nvme_dmamem_free(sc, q->q_cq_dmamem); 1369 free_sq: 1370 nvme_dmamem_free(sc, q->q_sq_dmamem); 1371 free: 1372 kmem_free(q, sizeof(*q)); 1373 1374 return NULL; 1375 } 1376 1377 static void 1378 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1379 { 1380 nvme_ccbs_free(q); 1381 mutex_destroy(&q->q_sq_mtx); 1382 mutex_destroy(&q->q_cq_mtx); 1383 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1384 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1385 nvme_dmamem_free(sc, q->q_cq_dmamem); 1386 nvme_dmamem_free(sc, q->q_sq_dmamem); 1387 kmem_free(q, sizeof(*q)); 1388 } 1389 1390 int 1391 nvme_intr(void *xsc) 1392 { 1393 struct nvme_softc *sc = xsc; 1394 int rv = 0; 1395 1396 /* INTx is level triggered, controller deasserts the interrupt only 1397 * when we advance command queue head via write to the doorbell */ 1398 if (nvme_q_complete(sc, sc->sc_admin_q)) 1399 rv = 1; 1400 if (sc->sc_q != NULL) 1401 if (nvme_q_complete(sc, sc->sc_q[0])) 1402 rv = 1; 1403 1404 return rv; 1405 } 1406 1407 int 1408 nvme_intr_msi(void *xq) 1409 { 1410 struct nvme_queue *q = xq; 1411 1412 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1413 && q->q_sc->sc_softih[q->q_id]); 1414 1415 /* MSI are edge triggered, so can handover processing to softint */ 1416 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1417 1418 return 1; 1419 } 1420 1421 void 1422 nvme_softintr_msi(void *xq) 1423 { 1424 struct nvme_queue *q = xq; 1425 struct nvme_softc *sc = q->q_sc; 1426 1427 nvme_q_complete(sc, q); 1428 } 1429 1430 static struct nvme_dmamem * 1431 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1432 { 1433 struct nvme_dmamem *ndm; 1434 int nsegs; 1435 1436 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1437 if (ndm == NULL) 1438 return NULL; 1439 1440 ndm->ndm_size = size; 1441 1442 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1443 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1444 goto ndmfree; 1445 1446 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1447 1, &nsegs, BUS_DMA_WAITOK) != 0) 1448 goto destroy; 1449 1450 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1451 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1452 goto free; 1453 memset(ndm->ndm_kva, 0, size); 1454 1455 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1456 NULL, BUS_DMA_WAITOK) != 0) 1457 goto unmap; 1458 1459 return ndm; 1460 1461 unmap: 1462 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1463 free: 1464 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1465 destroy: 1466 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1467 ndmfree: 1468 kmem_free(ndm, sizeof(*ndm)); 1469 return NULL; 1470 } 1471 1472 static void 1473 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1474 { 1475 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1476 0, NVME_DMA_LEN(mem), ops); 1477 } 1478 1479 void 1480 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1481 { 1482 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1483 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1484 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1485 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1486 kmem_free(ndm, sizeof(*ndm)); 1487 } 1488 1489 /* 1490 * ioctl 1491 */ 1492 1493 dev_type_open(nvmeopen); 1494 dev_type_close(nvmeclose); 1495 dev_type_ioctl(nvmeioctl); 1496 1497 const struct cdevsw nvme_cdevsw = { 1498 .d_open = nvmeopen, 1499 .d_close = nvmeclose, 1500 .d_read = noread, 1501 .d_write = nowrite, 1502 .d_ioctl = nvmeioctl, 1503 .d_stop = nostop, 1504 .d_tty = notty, 1505 .d_poll = nopoll, 1506 .d_mmap = nommap, 1507 .d_kqfilter = nokqfilter, 1508 .d_discard = nodiscard, 1509 .d_flag = D_OTHER, 1510 }; 1511 1512 extern struct cfdriver nvme_cd; 1513 1514 /* 1515 * Accept an open operation on the control device. 1516 */ 1517 int 1518 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1519 { 1520 struct nvme_softc *sc; 1521 int unit = minor(dev) / 0x10000; 1522 int nsid = minor(dev) & 0xffff; 1523 int nsidx; 1524 1525 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1526 return ENXIO; 1527 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1528 return ENXIO; 1529 1530 if (nsid == 0) { 1531 /* controller */ 1532 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1533 return EBUSY; 1534 SET(sc->sc_flags, NVME_F_OPEN); 1535 } else { 1536 /* namespace */ 1537 nsidx = nsid - 1; 1538 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1539 return ENXIO; 1540 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1541 return EBUSY; 1542 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1543 } 1544 return 0; 1545 } 1546 1547 /* 1548 * Accept the last close on the control device. 1549 */ 1550 int 1551 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1552 { 1553 struct nvme_softc *sc; 1554 int unit = minor(dev) / 0x10000; 1555 int nsid = minor(dev) & 0xffff; 1556 int nsidx; 1557 1558 sc = device_lookup_private(&nvme_cd, unit); 1559 if (sc == NULL) 1560 return ENXIO; 1561 1562 if (nsid == 0) { 1563 /* controller */ 1564 CLR(sc->sc_flags, NVME_F_OPEN); 1565 } else { 1566 /* namespace */ 1567 nsidx = nsid - 1; 1568 if (nsidx >= sc->sc_nn) 1569 return ENXIO; 1570 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1571 } 1572 1573 return 0; 1574 } 1575 1576 /* 1577 * Handle control operations. 1578 */ 1579 int 1580 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1581 { 1582 struct nvme_softc *sc; 1583 int unit = minor(dev) / 0x10000; 1584 int nsid = minor(dev) & 0xffff; 1585 struct nvme_pt_command *pt; 1586 1587 sc = device_lookup_private(&nvme_cd, unit); 1588 if (sc == NULL) 1589 return ENXIO; 1590 1591 switch (cmd) { 1592 case NVME_PASSTHROUGH_CMD: 1593 pt = data; 1594 return nvme_command_passthrough(sc, data, 1595 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1596 } 1597 1598 return ENOTTY; 1599 } 1600
Indexes created Wed Sep 24 22:09:59 GMT 2025