nvme.c revision 1.30.2.1
1 /* $NetBSD: nvme.c,v 1.30.2.1 2018/03/17 08:11:18 martin Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.30.2.1 2018/03/17 08:11:18 martin Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 int nvme_adminq_size = 32; 44 int nvme_ioq_size = 1024; 45 46 static int nvme_print(void *, const char *); 47 48 static int nvme_ready(struct nvme_softc *, uint32_t); 49 static int nvme_enable(struct nvme_softc *, u_int); 50 static int nvme_disable(struct nvme_softc *); 51 static int nvme_shutdown(struct nvme_softc *); 52 53 #ifdef NVME_DEBUG 54 static void nvme_dumpregs(struct nvme_softc *); 55 #endif 56 static int nvme_identify(struct nvme_softc *, u_int); 57 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 58 void *); 59 60 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 61 static void nvme_ccbs_free(struct nvme_queue *); 62 63 static struct nvme_ccb * 64 nvme_ccb_get(struct nvme_queue *); 65 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 66 67 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 68 struct nvme_ccb *, void (*)(struct nvme_queue *, 69 struct nvme_ccb *, void *), int); 70 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 71 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 72 struct nvme_cqe *); 73 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 74 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 75 struct nvme_cqe *); 76 77 static struct nvme_queue * 78 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 79 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 80 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 81 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 82 struct nvme_ccb *, void (*)(struct nvme_queue *, 83 struct nvme_ccb *, void *)); 84 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 85 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 86 87 static struct nvme_dmamem * 88 nvme_dmamem_alloc(struct nvme_softc *, size_t); 89 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 90 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 91 int); 92 93 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 94 void *); 95 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 96 struct nvme_cqe *); 97 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 98 void *); 99 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 100 struct nvme_cqe *); 101 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *, 102 void *); 103 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *, 104 struct nvme_cqe *); 105 106 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 107 void *); 108 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 109 struct nvme_cqe *); 110 static int nvme_command_passthrough(struct nvme_softc *, 111 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 112 113 static int nvme_get_number_of_queues(struct nvme_softc *, u_int *); 114 115 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 116 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 117 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 118 #define NVME_TIMO_SY 60 /* sync cache timeout */ 119 120 #define nvme_read4(_s, _r) \ 121 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 122 #define nvme_write4(_s, _r, _v) \ 123 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 124 /* 125 * Some controllers, at least Apple NVMe, always require split 126 * transfers, so don't use bus_space_{read,write}_8() on LP64. 127 */ 128 static inline uint64_t 129 nvme_read8(struct nvme_softc *sc, bus_size_t r) 130 { 131 uint64_t v; 132 uint32_t *a = (uint32_t *)&v; 133 134 #if _BYTE_ORDER == _LITTLE_ENDIAN 135 a[0] = nvme_read4(sc, r); 136 a[1] = nvme_read4(sc, r + 4); 137 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 138 a[1] = nvme_read4(sc, r); 139 a[0] = nvme_read4(sc, r + 4); 140 #endif 141 142 return v; 143 } 144 145 static inline void 146 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 147 { 148 uint32_t *a = (uint32_t *)&v; 149 150 #if _BYTE_ORDER == _LITTLE_ENDIAN 151 nvme_write4(sc, r, a[0]); 152 nvme_write4(sc, r + 4, a[1]); 153 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 154 nvme_write4(sc, r, a[1]); 155 nvme_write4(sc, r + 4, a[0]); 156 #endif 157 } 158 #define nvme_barrier(_s, _r, _l, _f) \ 159 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 160 161 #ifdef NVME_DEBUG 162 static __used void 163 nvme_dumpregs(struct nvme_softc *sc) 164 { 165 uint64_t r8; 166 uint32_t r4; 167 168 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 169 r8 = nvme_read8(sc, NVME_CAP); 170 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 171 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 172 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 173 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 174 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 175 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 176 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 177 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 178 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 179 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 180 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 181 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 182 183 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 184 185 r4 = nvme_read4(sc, NVME_CC); 186 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 187 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 188 (1 << NVME_CC_IOCQES_R(r4))); 189 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 190 (1 << NVME_CC_IOSQES_R(r4))); 191 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 192 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 193 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 194 (1 << NVME_CC_MPS_R(r4))); 195 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 196 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 197 198 r4 = nvme_read4(sc, NVME_CSTS); 199 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 200 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 201 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 202 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 203 204 r4 = nvme_read4(sc, NVME_AQA); 205 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 206 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 207 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 208 209 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 210 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 211 #undef DEVNAME 212 } 213 #endif /* NVME_DEBUG */ 214 215 static int 216 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 217 { 218 u_int i = 0; 219 uint32_t cc; 220 221 cc = nvme_read4(sc, NVME_CC); 222 if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) { 223 aprint_error_dev(sc->sc_dev, 224 "controller enabled status expected %d, found to be %d\n", 225 (rdy != 0), ((cc & NVME_CC_EN) != 0)); 226 return ENXIO; 227 } 228 229 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 230 if (i++ > sc->sc_rdy_to) 231 return ENXIO; 232 233 delay(1000); 234 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 235 } 236 237 return 0; 238 } 239 240 static int 241 nvme_enable(struct nvme_softc *sc, u_int mps) 242 { 243 uint32_t cc, csts; 244 245 cc = nvme_read4(sc, NVME_CC); 246 csts = nvme_read4(sc, NVME_CSTS); 247 248 if (ISSET(cc, NVME_CC_EN)) { 249 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 250 251 if (ISSET(csts, NVME_CSTS_RDY)) 252 return 1; 253 254 goto waitready; 255 } 256 257 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 258 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 259 delay(5000); 260 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 261 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 262 delay(5000); 263 264 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 265 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 266 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 267 delay(5000); 268 269 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 270 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 271 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 272 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 273 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 274 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 275 SET(cc, NVME_CC_MPS(mps)); 276 SET(cc, NVME_CC_EN); 277 278 nvme_write4(sc, NVME_CC, cc); 279 nvme_barrier(sc, 0, sc->sc_ios, 280 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 281 delay(5000); 282 283 waitready: 284 return nvme_ready(sc, NVME_CSTS_RDY); 285 } 286 287 static int 288 nvme_disable(struct nvme_softc *sc) 289 { 290 uint32_t cc, csts; 291 292 cc = nvme_read4(sc, NVME_CC); 293 csts = nvme_read4(sc, NVME_CSTS); 294 295 if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY)) 296 nvme_ready(sc, NVME_CSTS_RDY); 297 298 CLR(cc, NVME_CC_EN); 299 300 nvme_write4(sc, NVME_CC, cc); 301 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 302 303 delay(5000); 304 305 return nvme_ready(sc, 0); 306 } 307 308 int 309 nvme_attach(struct nvme_softc *sc) 310 { 311 uint64_t cap; 312 uint32_t reg; 313 u_int dstrd; 314 u_int mps = PAGE_SHIFT; 315 u_int ioq_allocated; 316 uint16_t adminq_entries = nvme_adminq_size; 317 uint16_t ioq_entries = nvme_ioq_size; 318 int i; 319 320 reg = nvme_read4(sc, NVME_VS); 321 if (reg == 0xffffffff) { 322 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 323 return 1; 324 } 325 326 if (NVME_VS_TER(reg) == 0) 327 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg), 328 NVME_VS_MNR(reg)); 329 else 330 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg), 331 NVME_VS_MNR(reg), NVME_VS_TER(reg)); 332 333 cap = nvme_read8(sc, NVME_CAP); 334 dstrd = NVME_CAP_DSTRD(cap); 335 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 336 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 337 "is greater than CPU page size %u\n", 338 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 339 return 1; 340 } 341 if (NVME_CAP_MPSMAX(cap) < mps) 342 mps = NVME_CAP_MPSMAX(cap); 343 if (ioq_entries > NVME_CAP_MQES(cap)) 344 ioq_entries = NVME_CAP_MQES(cap); 345 346 /* set initial values to be used for admin queue during probe */ 347 sc->sc_rdy_to = NVME_CAP_TO(cap); 348 sc->sc_mps = 1 << mps; 349 sc->sc_mdts = MAXPHYS; 350 sc->sc_max_sgl = 2; 351 352 if (nvme_disable(sc) != 0) { 353 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 354 return 1; 355 } 356 357 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 358 if (sc->sc_admin_q == NULL) { 359 aprint_error_dev(sc->sc_dev, 360 "unable to allocate admin queue\n"); 361 return 1; 362 } 363 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 364 goto free_admin_q; 365 366 if (nvme_enable(sc, mps) != 0) { 367 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 368 goto disestablish_admin_q; 369 } 370 371 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 372 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 373 goto disable; 374 } 375 376 /* we know how big things are now */ 377 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 378 379 /* reallocate ccbs of admin queue with new max sgl. */ 380 nvme_ccbs_free(sc->sc_admin_q); 381 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 382 383 if (sc->sc_use_mq) { 384 /* Limit the number of queues to the number allocated in HW */ 385 if (nvme_get_number_of_queues(sc, &ioq_allocated) != 0) { 386 aprint_error_dev(sc->sc_dev, 387 "unable to get number of queues\n"); 388 goto disable; 389 } 390 if (sc->sc_nq > ioq_allocated) 391 sc->sc_nq = ioq_allocated; 392 } 393 394 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 395 for (i = 0; i < sc->sc_nq; i++) { 396 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 397 if (sc->sc_q[i] == NULL) { 398 aprint_error_dev(sc->sc_dev, 399 "unable to allocate io queue\n"); 400 goto free_q; 401 } 402 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 403 aprint_error_dev(sc->sc_dev, 404 "unable to create io queue\n"); 405 nvme_q_free(sc, sc->sc_q[i]); 406 goto free_q; 407 } 408 } 409 410 if (!sc->sc_use_mq) 411 nvme_write4(sc, NVME_INTMC, 1); 412 413 /* probe subdevices */ 414 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 415 KM_SLEEP); 416 nvme_rescan(sc->sc_dev, "nvme", &i); 417 418 return 0; 419 420 free_q: 421 while (--i >= 0) { 422 nvme_q_delete(sc, sc->sc_q[i]); 423 nvme_q_free(sc, sc->sc_q[i]); 424 } 425 disable: 426 nvme_disable(sc); 427 disestablish_admin_q: 428 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 429 free_admin_q: 430 nvme_q_free(sc, sc->sc_admin_q); 431 432 return 1; 433 } 434 435 int 436 nvme_rescan(device_t self, const char *attr, const int *flags) 437 { 438 struct nvme_softc *sc = device_private(self); 439 struct nvme_attach_args naa; 440 uint64_t cap; 441 int ioq_entries = nvme_ioq_size; 442 int i; 443 444 cap = nvme_read8(sc, NVME_CAP); 445 if (ioq_entries > NVME_CAP_MQES(cap)) 446 ioq_entries = NVME_CAP_MQES(cap); 447 448 for (i = 0; i < sc->sc_nn; i++) { 449 if (sc->sc_namespaces[i].dev) 450 continue; 451 memset(&naa, 0, sizeof(naa)); 452 naa.naa_nsid = i + 1; 453 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 454 naa.naa_maxphys = sc->sc_mdts; 455 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 456 nvme_print); 457 } 458 return 0; 459 } 460 461 static int 462 nvme_print(void *aux, const char *pnp) 463 { 464 struct nvme_attach_args *naa = aux; 465 466 if (pnp) 467 aprint_normal("at %s", pnp); 468 469 if (naa->naa_nsid > 0) 470 aprint_normal(" nsid %d", naa->naa_nsid); 471 472 return UNCONF; 473 } 474 475 int 476 nvme_detach(struct nvme_softc *sc, int flags) 477 { 478 int i, error; 479 480 error = config_detach_children(sc->sc_dev, flags); 481 if (error) 482 return error; 483 484 error = nvme_shutdown(sc); 485 if (error) 486 return error; 487 488 /* from now on we are committed to detach, following will never fail */ 489 for (i = 0; i < sc->sc_nq; i++) 490 nvme_q_free(sc, sc->sc_q[i]); 491 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 492 nvme_q_free(sc, sc->sc_admin_q); 493 494 return 0; 495 } 496 497 static int 498 nvme_shutdown(struct nvme_softc *sc) 499 { 500 uint32_t cc, csts; 501 bool disabled = false; 502 int i; 503 504 if (!sc->sc_use_mq) 505 nvme_write4(sc, NVME_INTMS, 1); 506 507 for (i = 0; i < sc->sc_nq; i++) { 508 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 509 aprint_error_dev(sc->sc_dev, 510 "unable to delete io queue %d, disabling\n", i + 1); 511 disabled = true; 512 } 513 } 514 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 515 if (disabled) 516 goto disable; 517 518 cc = nvme_read4(sc, NVME_CC); 519 CLR(cc, NVME_CC_SHN_MASK); 520 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 521 nvme_write4(sc, NVME_CC, cc); 522 523 for (i = 0; i < 4000; i++) { 524 nvme_barrier(sc, 0, sc->sc_ios, 525 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 526 csts = nvme_read4(sc, NVME_CSTS); 527 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 528 return 0; 529 530 delay(1000); 531 } 532 533 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 534 535 disable: 536 nvme_disable(sc); 537 return 0; 538 } 539 540 void 541 nvme_childdet(device_t self, device_t child) 542 { 543 struct nvme_softc *sc = device_private(self); 544 int i; 545 546 for (i = 0; i < sc->sc_nn; i++) { 547 if (sc->sc_namespaces[i].dev == child) { 548 /* Already freed ns->ident. */ 549 sc->sc_namespaces[i].dev = NULL; 550 break; 551 } 552 } 553 } 554 555 int 556 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 557 { 558 struct nvme_sqe sqe; 559 struct nvm_identify_namespace *identify; 560 struct nvme_dmamem *mem; 561 struct nvme_ccb *ccb; 562 struct nvme_namespace *ns; 563 int rv; 564 565 KASSERT(nsid > 0); 566 567 ccb = nvme_ccb_get(sc->sc_admin_q); 568 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 569 570 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 571 if (mem == NULL) 572 return ENOMEM; 573 574 memset(&sqe, 0, sizeof(sqe)); 575 sqe.opcode = NVM_ADMIN_IDENTIFY; 576 htolem32(&sqe.nsid, nsid); 577 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 578 htolem32(&sqe.cdw10, 0); 579 580 ccb->ccb_done = nvme_empty_done; 581 ccb->ccb_cookie = &sqe; 582 583 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 584 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 585 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 586 587 nvme_ccb_put(sc->sc_admin_q, ccb); 588 589 if (rv != 0) { 590 rv = EIO; 591 goto done; 592 } 593 594 /* commit */ 595 596 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 597 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 598 //memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 599 600 ns = nvme_ns_get(sc, nsid); 601 KASSERT(ns); 602 ns->ident = identify; 603 604 done: 605 nvme_dmamem_free(sc, mem); 606 607 return rv; 608 } 609 610 int 611 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 612 struct buf *bp, void *data, size_t datasize, 613 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 614 { 615 struct nvme_queue *q = nvme_get_q(sc); 616 struct nvme_ccb *ccb; 617 bus_dmamap_t dmap; 618 int i, error; 619 620 ccb = nvme_ccb_get(q); 621 if (ccb == NULL) 622 return EAGAIN; 623 624 ccb->ccb_done = nvme_ns_io_done; 625 ccb->ccb_cookie = cookie; 626 627 /* namespace context */ 628 ccb->nnc_nsid = nsid; 629 ccb->nnc_flags = flags; 630 ccb->nnc_buf = bp; 631 ccb->nnc_datasize = datasize; 632 ccb->nnc_secsize = secsize; 633 ccb->nnc_blkno = blkno; 634 ccb->nnc_done = nnc_done; 635 636 dmap = ccb->ccb_dmamap; 637 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 638 datasize, NULL, 639 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 640 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 641 (ISSET(flags, NVME_NS_CTX_F_READ) ? 642 BUS_DMA_READ : BUS_DMA_WRITE)); 643 if (error) { 644 nvme_ccb_put(q, ccb); 645 return error; 646 } 647 648 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 649 ISSET(flags, NVME_NS_CTX_F_READ) ? 650 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 651 652 if (dmap->dm_nsegs > 2) { 653 for (i = 1; i < dmap->dm_nsegs; i++) { 654 htolem64(&ccb->ccb_prpl[i - 1], 655 dmap->dm_segs[i].ds_addr); 656 } 657 bus_dmamap_sync(sc->sc_dmat, 658 NVME_DMA_MAP(q->q_ccb_prpls), 659 ccb->ccb_prpl_off, 660 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 661 BUS_DMASYNC_PREWRITE); 662 } 663 664 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 665 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 666 return EIO; 667 return 0; 668 } 669 670 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 671 return 0; 672 } 673 674 static void 675 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 676 { 677 struct nvme_sqe_io *sqe = slot; 678 bus_dmamap_t dmap = ccb->ccb_dmamap; 679 680 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 681 NVM_CMD_READ : NVM_CMD_WRITE; 682 htolem32(&sqe->nsid, ccb->nnc_nsid); 683 684 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 685 switch (dmap->dm_nsegs) { 686 case 1: 687 break; 688 case 2: 689 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 690 break; 691 default: 692 /* the prp list is already set up and synced */ 693 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 694 break; 695 } 696 697 htolem64(&sqe->slba, ccb->nnc_blkno); 698 699 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) 700 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA); 701 702 /* guaranteed by upper layers, but check just in case */ 703 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 704 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 705 } 706 707 static void 708 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 709 struct nvme_cqe *cqe) 710 { 711 struct nvme_softc *sc = q->q_sc; 712 bus_dmamap_t dmap = ccb->ccb_dmamap; 713 void *nnc_cookie = ccb->ccb_cookie; 714 nvme_nnc_done nnc_done = ccb->nnc_done; 715 struct buf *bp = ccb->nnc_buf; 716 717 if (dmap->dm_nsegs > 2) { 718 bus_dmamap_sync(sc->sc_dmat, 719 NVME_DMA_MAP(q->q_ccb_prpls), 720 ccb->ccb_prpl_off, 721 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 722 BUS_DMASYNC_POSTWRITE); 723 } 724 725 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 726 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 727 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 728 729 bus_dmamap_unload(sc->sc_dmat, dmap); 730 nvme_ccb_put(q, ccb); 731 732 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 733 } 734 735 /* 736 * If there is no volatile write cache, it makes no sense to issue 737 * flush commands or query for the status. 738 */ 739 bool 740 nvme_has_volatile_write_cache(struct nvme_softc *sc) 741 { 742 /* sc_identify is filled during attachment */ 743 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0); 744 } 745 746 int 747 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie, 748 int flags, nvme_nnc_done nnc_done) 749 { 750 struct nvme_queue *q = nvme_get_q(sc); 751 struct nvme_ccb *ccb; 752 753 ccb = nvme_ccb_get(q); 754 if (ccb == NULL) 755 return EAGAIN; 756 757 ccb->ccb_done = nvme_ns_sync_done; 758 ccb->ccb_cookie = cookie; 759 760 /* namespace context */ 761 ccb->nnc_nsid = nsid; 762 ccb->nnc_flags = flags; 763 ccb->nnc_done = nnc_done; 764 765 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 766 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 767 return EIO; 768 return 0; 769 } 770 771 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 772 return 0; 773 } 774 775 static void 776 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 777 { 778 struct nvme_sqe *sqe = slot; 779 780 sqe->opcode = NVM_CMD_FLUSH; 781 htolem32(&sqe->nsid, ccb->nnc_nsid); 782 } 783 784 static void 785 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 786 struct nvme_cqe *cqe) 787 { 788 void *cookie = ccb->ccb_cookie; 789 nvme_nnc_done nnc_done = ccb->nnc_done; 790 791 nvme_ccb_put(q, ccb); 792 793 nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 794 } 795 796 /* 797 * Get status of volatile write cache. Always asynchronous. 798 */ 799 int 800 nvme_admin_getcache(struct nvme_softc *sc, void *cookie, nvme_nnc_done nnc_done) 801 { 802 struct nvme_ccb *ccb; 803 struct nvme_queue *q = sc->sc_admin_q; 804 805 ccb = nvme_ccb_get(q); 806 if (ccb == NULL) 807 return EAGAIN; 808 809 ccb->ccb_done = nvme_getcache_done; 810 ccb->ccb_cookie = cookie; 811 812 /* namespace context */ 813 ccb->nnc_flags = 0; 814 ccb->nnc_done = nnc_done; 815 816 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); 817 return 0; 818 } 819 820 static void 821 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 822 { 823 struct nvme_sqe *sqe = slot; 824 825 sqe->opcode = NVM_ADMIN_GET_FEATURES; 826 sqe->cdw10 = NVM_FEATURE_VOLATILE_WRITE_CACHE; 827 } 828 829 static void 830 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 831 struct nvme_cqe *cqe) 832 { 833 void *cookie = ccb->ccb_cookie; 834 nvme_nnc_done nnc_done = ccb->nnc_done; 835 836 nvme_ccb_put(q, ccb); 837 838 nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 839 } 840 841 void 842 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 843 { 844 struct nvme_namespace *ns; 845 struct nvm_identify_namespace *identify; 846 847 ns = nvme_ns_get(sc, nsid); 848 KASSERT(ns); 849 850 identify = ns->ident; 851 ns->ident = NULL; 852 if (identify != NULL) 853 kmem_free(identify, sizeof(*identify)); 854 } 855 856 static void 857 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 858 { 859 struct nvme_softc *sc = q->q_sc; 860 struct nvme_sqe *sqe = slot; 861 struct nvme_pt_command *pt = ccb->ccb_cookie; 862 bus_dmamap_t dmap = ccb->ccb_dmamap; 863 int i; 864 865 sqe->opcode = pt->cmd.opcode; 866 htolem32(&sqe->nsid, pt->cmd.nsid); 867 868 if (pt->buf != NULL && pt->len > 0) { 869 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 870 switch (dmap->dm_nsegs) { 871 case 1: 872 break; 873 case 2: 874 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 875 break; 876 default: 877 for (i = 1; i < dmap->dm_nsegs; i++) { 878 htolem64(&ccb->ccb_prpl[i - 1], 879 dmap->dm_segs[i].ds_addr); 880 } 881 bus_dmamap_sync(sc->sc_dmat, 882 NVME_DMA_MAP(q->q_ccb_prpls), 883 ccb->ccb_prpl_off, 884 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 885 BUS_DMASYNC_PREWRITE); 886 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 887 break; 888 } 889 } 890 891 htolem32(&sqe->cdw10, pt->cmd.cdw10); 892 htolem32(&sqe->cdw11, pt->cmd.cdw11); 893 htolem32(&sqe->cdw12, pt->cmd.cdw12); 894 htolem32(&sqe->cdw13, pt->cmd.cdw13); 895 htolem32(&sqe->cdw14, pt->cmd.cdw14); 896 htolem32(&sqe->cdw15, pt->cmd.cdw15); 897 } 898 899 static void 900 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 901 { 902 struct nvme_softc *sc = q->q_sc; 903 struct nvme_pt_command *pt = ccb->ccb_cookie; 904 bus_dmamap_t dmap = ccb->ccb_dmamap; 905 906 if (pt->buf != NULL && pt->len > 0) { 907 if (dmap->dm_nsegs > 2) { 908 bus_dmamap_sync(sc->sc_dmat, 909 NVME_DMA_MAP(q->q_ccb_prpls), 910 ccb->ccb_prpl_off, 911 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 912 BUS_DMASYNC_POSTWRITE); 913 } 914 915 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 916 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 917 bus_dmamap_unload(sc->sc_dmat, dmap); 918 } 919 920 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 921 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 922 } 923 924 static int 925 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 926 uint16_t nsid, struct lwp *l, bool is_adminq) 927 { 928 struct nvme_queue *q; 929 struct nvme_ccb *ccb; 930 void *buf = NULL; 931 int error; 932 933 /* limit command size to maximum data transfer size */ 934 if ((pt->buf == NULL && pt->len > 0) || 935 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 936 return EINVAL; 937 938 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 939 ccb = nvme_ccb_get(q); 940 if (ccb == NULL) 941 return EBUSY; 942 943 if (pt->buf != NULL) { 944 KASSERT(pt->len > 0); 945 buf = kmem_alloc(pt->len, KM_SLEEP); 946 if (!pt->is_read) { 947 error = copyin(pt->buf, buf, pt->len); 948 if (error) 949 goto kmem_free; 950 } 951 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 952 pt->len, NULL, 953 BUS_DMA_WAITOK | 954 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 955 if (error) 956 goto kmem_free; 957 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 958 0, ccb->ccb_dmamap->dm_mapsize, 959 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 960 } 961 962 ccb->ccb_done = nvme_pt_done; 963 ccb->ccb_cookie = pt; 964 965 pt->cmd.nsid = nsid; 966 if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) { 967 error = EIO; 968 goto out; 969 } 970 971 error = 0; 972 out: 973 if (buf != NULL) { 974 if (error == 0 && pt->is_read) 975 error = copyout(buf, pt->buf, pt->len); 976 kmem_free: 977 kmem_free(buf, pt->len); 978 } 979 nvme_ccb_put(q, ccb); 980 return error; 981 } 982 983 static void 984 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 985 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 986 { 987 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 988 uint32_t tail; 989 990 mutex_enter(&q->q_sq_mtx); 991 tail = q->q_sq_tail; 992 if (++q->q_sq_tail >= q->q_entries) 993 q->q_sq_tail = 0; 994 995 sqe += tail; 996 997 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 998 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 999 memset(sqe, 0, sizeof(*sqe)); 1000 (*fill)(q, ccb, sqe); 1001 sqe->cid = ccb->ccb_id; 1002 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1003 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 1004 1005 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 1006 mutex_exit(&q->q_sq_mtx); 1007 } 1008 1009 struct nvme_poll_state { 1010 struct nvme_sqe s; 1011 struct nvme_cqe c; 1012 }; 1013 1014 static int 1015 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1016 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 1017 { 1018 struct nvme_poll_state state; 1019 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 1020 void *cookie; 1021 uint16_t flags; 1022 int step = 10; 1023 int maxloop = timo_sec * 1000000 / step; 1024 int error = 0; 1025 1026 memset(&state, 0, sizeof(state)); 1027 (*fill)(q, ccb, &state.s); 1028 1029 done = ccb->ccb_done; 1030 cookie = ccb->ccb_cookie; 1031 1032 ccb->ccb_done = nvme_poll_done; 1033 ccb->ccb_cookie = &state; 1034 1035 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1036 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1037 if (nvme_q_complete(sc, q) == 0) 1038 delay(step); 1039 1040 if (timo_sec >= 0 && --maxloop <= 0) { 1041 error = ETIMEDOUT; 1042 break; 1043 } 1044 } 1045 1046 ccb->ccb_cookie = cookie; 1047 done(q, ccb, &state.c); 1048 1049 if (error == 0) { 1050 flags = lemtoh16(&state.c.flags); 1051 return flags & ~NVME_CQE_PHASE; 1052 } else { 1053 return 1; 1054 } 1055 } 1056 1057 static void 1058 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1059 { 1060 struct nvme_sqe *sqe = slot; 1061 struct nvme_poll_state *state = ccb->ccb_cookie; 1062 1063 *sqe = state->s; 1064 } 1065 1066 static void 1067 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1068 struct nvme_cqe *cqe) 1069 { 1070 struct nvme_poll_state *state = ccb->ccb_cookie; 1071 1072 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1073 state->c = *cqe; 1074 } 1075 1076 static void 1077 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1078 { 1079 struct nvme_sqe *src = ccb->ccb_cookie; 1080 struct nvme_sqe *dst = slot; 1081 1082 *dst = *src; 1083 } 1084 1085 static void 1086 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1087 struct nvme_cqe *cqe) 1088 { 1089 } 1090 1091 static int 1092 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1093 { 1094 struct nvme_ccb *ccb; 1095 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1096 uint16_t flags; 1097 int rv = 0; 1098 1099 mutex_enter(&q->q_cq_mtx); 1100 1101 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1102 for (;;) { 1103 cqe = &ring[q->q_cq_head]; 1104 flags = lemtoh16(&cqe->flags); 1105 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1106 break; 1107 1108 ccb = &q->q_ccbs[cqe->cid]; 1109 1110 if (++q->q_cq_head >= q->q_entries) { 1111 q->q_cq_head = 0; 1112 q->q_cq_phase ^= NVME_CQE_PHASE; 1113 } 1114 1115 #ifdef DEBUG 1116 /* 1117 * If we get spurious completion notification, something 1118 * is seriously hosed up. Very likely DMA to some random 1119 * memory place happened, so just bail out. 1120 */ 1121 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1122 panic("%s: invalid ccb detected", 1123 device_xname(sc->sc_dev)); 1124 /* NOTREACHED */ 1125 } 1126 #endif 1127 1128 rv++; 1129 1130 /* 1131 * Unlock the mutex before calling the ccb_done callback 1132 * and re-lock afterwards. The callback triggers lddone() 1133 * which schedules another i/o, and also calls nvme_ccb_put(). 1134 * Unlock/relock avoids possibility of deadlock. 1135 */ 1136 mutex_exit(&q->q_cq_mtx); 1137 ccb->ccb_done(q, ccb, cqe); 1138 mutex_enter(&q->q_cq_mtx); 1139 } 1140 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1141 1142 if (rv) 1143 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1144 1145 mutex_exit(&q->q_cq_mtx); 1146 1147 return rv; 1148 } 1149 1150 static int 1151 nvme_identify(struct nvme_softc *sc, u_int mps) 1152 { 1153 char sn[41], mn[81], fr[17]; 1154 struct nvm_identify_controller *identify; 1155 struct nvme_dmamem *mem; 1156 struct nvme_ccb *ccb; 1157 u_int mdts; 1158 int rv = 1; 1159 1160 ccb = nvme_ccb_get(sc->sc_admin_q); 1161 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1162 1163 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1164 if (mem == NULL) 1165 return 1; 1166 1167 ccb->ccb_done = nvme_empty_done; 1168 ccb->ccb_cookie = mem; 1169 1170 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1171 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1172 NVME_TIMO_IDENT); 1173 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1174 1175 nvme_ccb_put(sc->sc_admin_q, ccb); 1176 1177 if (rv != 0) 1178 goto done; 1179 1180 identify = NVME_DMA_KVA(mem); 1181 1182 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1183 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1184 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1185 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1186 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1187 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1188 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1189 sn); 1190 1191 if (identify->mdts > 0) { 1192 mdts = (1 << identify->mdts) * (1 << mps); 1193 if (mdts < sc->sc_mdts) 1194 sc->sc_mdts = mdts; 1195 } 1196 1197 sc->sc_nn = lemtoh32(&identify->nn); 1198 1199 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1200 1201 done: 1202 nvme_dmamem_free(sc, mem); 1203 1204 return rv; 1205 } 1206 1207 static int 1208 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1209 { 1210 struct nvme_sqe_q sqe; 1211 struct nvme_ccb *ccb; 1212 int rv; 1213 1214 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1215 return 1; 1216 1217 ccb = nvme_ccb_get(sc->sc_admin_q); 1218 KASSERT(ccb != NULL); 1219 1220 ccb->ccb_done = nvme_empty_done; 1221 ccb->ccb_cookie = &sqe; 1222 1223 memset(&sqe, 0, sizeof(sqe)); 1224 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1225 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1226 htolem16(&sqe.qsize, q->q_entries - 1); 1227 htolem16(&sqe.qid, q->q_id); 1228 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1229 if (sc->sc_use_mq) 1230 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1231 1232 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1233 if (rv != 0) 1234 goto fail; 1235 1236 ccb->ccb_done = nvme_empty_done; 1237 ccb->ccb_cookie = &sqe; 1238 1239 memset(&sqe, 0, sizeof(sqe)); 1240 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1241 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1242 htolem16(&sqe.qsize, q->q_entries - 1); 1243 htolem16(&sqe.qid, q->q_id); 1244 htolem16(&sqe.cqid, q->q_id); 1245 sqe.qflags = NVM_SQE_Q_PC; 1246 1247 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1248 if (rv != 0) 1249 goto fail; 1250 1251 fail: 1252 nvme_ccb_put(sc->sc_admin_q, ccb); 1253 return rv; 1254 } 1255 1256 static int 1257 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1258 { 1259 struct nvme_sqe_q sqe; 1260 struct nvme_ccb *ccb; 1261 int rv; 1262 1263 ccb = nvme_ccb_get(sc->sc_admin_q); 1264 KASSERT(ccb != NULL); 1265 1266 ccb->ccb_done = nvme_empty_done; 1267 ccb->ccb_cookie = &sqe; 1268 1269 memset(&sqe, 0, sizeof(sqe)); 1270 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1271 htolem16(&sqe.qid, q->q_id); 1272 1273 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1274 if (rv != 0) 1275 goto fail; 1276 1277 ccb->ccb_done = nvme_empty_done; 1278 ccb->ccb_cookie = &sqe; 1279 1280 memset(&sqe, 0, sizeof(sqe)); 1281 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1282 htolem16(&sqe.qid, q->q_id); 1283 1284 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1285 if (rv != 0) 1286 goto fail; 1287 1288 fail: 1289 nvme_ccb_put(sc->sc_admin_q, ccb); 1290 1291 if (rv == 0 && sc->sc_use_mq) { 1292 if (sc->sc_intr_disestablish(sc, q->q_id)) 1293 rv = 1; 1294 } 1295 1296 return rv; 1297 } 1298 1299 static void 1300 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1301 { 1302 struct nvme_sqe *sqe = slot; 1303 struct nvme_dmamem *mem = ccb->ccb_cookie; 1304 1305 sqe->opcode = NVM_ADMIN_IDENTIFY; 1306 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1307 htolem32(&sqe->cdw10, 1); 1308 } 1309 1310 static int 1311 nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap) 1312 { 1313 struct nvme_pt_command pt; 1314 struct nvme_ccb *ccb; 1315 uint16_t ncqa, nsqa; 1316 int rv; 1317 1318 ccb = nvme_ccb_get(sc->sc_admin_q); 1319 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1320 1321 memset(&pt, 0, sizeof(pt)); 1322 pt.cmd.opcode = NVM_ADMIN_GET_FEATURES; 1323 pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES; 1324 1325 ccb->ccb_done = nvme_pt_done; 1326 ccb->ccb_cookie = &pt; 1327 1328 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1329 1330 nvme_ccb_put(sc->sc_admin_q, ccb); 1331 1332 if (rv != 0) { 1333 *nqap = 0; 1334 return EIO; 1335 } 1336 1337 ncqa = pt.cpl.cdw0 >> 16; 1338 nsqa = pt.cpl.cdw0 & 0xffff; 1339 *nqap = MIN(ncqa, nsqa) + 1; 1340 1341 return 0; 1342 } 1343 1344 static int 1345 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1346 { 1347 struct nvme_softc *sc = q->q_sc; 1348 struct nvme_ccb *ccb; 1349 bus_addr_t off; 1350 uint64_t *prpl; 1351 u_int i; 1352 1353 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1354 SIMPLEQ_INIT(&q->q_ccb_list); 1355 1356 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1357 1358 q->q_nccbs = nccbs; 1359 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1360 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1361 1362 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1363 off = 0; 1364 1365 for (i = 0; i < nccbs; i++) { 1366 ccb = &q->q_ccbs[i]; 1367 1368 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1369 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1370 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1371 &ccb->ccb_dmamap) != 0) 1372 goto free_maps; 1373 1374 ccb->ccb_id = i; 1375 ccb->ccb_prpl = prpl; 1376 ccb->ccb_prpl_off = off; 1377 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1378 1379 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1380 1381 prpl += sc->sc_max_sgl; 1382 off += sizeof(*prpl) * sc->sc_max_sgl; 1383 } 1384 1385 return 0; 1386 1387 free_maps: 1388 nvme_ccbs_free(q); 1389 return 1; 1390 } 1391 1392 static struct nvme_ccb * 1393 nvme_ccb_get(struct nvme_queue *q) 1394 { 1395 struct nvme_ccb *ccb = NULL; 1396 1397 mutex_enter(&q->q_ccb_mtx); 1398 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1399 if (ccb != NULL) { 1400 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1401 #ifdef DEBUG 1402 ccb->ccb_cookie = NULL; 1403 #endif 1404 } 1405 mutex_exit(&q->q_ccb_mtx); 1406 1407 return ccb; 1408 } 1409 1410 static void 1411 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1412 { 1413 1414 mutex_enter(&q->q_ccb_mtx); 1415 #ifdef DEBUG 1416 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1417 #endif 1418 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1419 mutex_exit(&q->q_ccb_mtx); 1420 } 1421 1422 static void 1423 nvme_ccbs_free(struct nvme_queue *q) 1424 { 1425 struct nvme_softc *sc = q->q_sc; 1426 struct nvme_ccb *ccb; 1427 1428 mutex_enter(&q->q_ccb_mtx); 1429 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1430 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1431 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1432 } 1433 mutex_exit(&q->q_ccb_mtx); 1434 1435 nvme_dmamem_free(sc, q->q_ccb_prpls); 1436 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1437 q->q_ccbs = NULL; 1438 mutex_destroy(&q->q_ccb_mtx); 1439 } 1440 1441 static struct nvme_queue * 1442 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1443 { 1444 struct nvme_queue *q; 1445 1446 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1447 q->q_sc = sc; 1448 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1449 sizeof(struct nvme_sqe) * entries); 1450 if (q->q_sq_dmamem == NULL) 1451 goto free; 1452 1453 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1454 sizeof(struct nvme_cqe) * entries); 1455 if (q->q_cq_dmamem == NULL) 1456 goto free_sq; 1457 1458 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1459 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1460 1461 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1462 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1463 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1464 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1465 q->q_id = id; 1466 q->q_entries = entries; 1467 q->q_sq_tail = 0; 1468 q->q_cq_head = 0; 1469 q->q_cq_phase = NVME_CQE_PHASE; 1470 1471 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1472 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1473 1474 /* 1475 * Due to definition of full and empty queue (queue is empty 1476 * when head == tail, full when tail is one less then head), 1477 * we can actually only have (entries - 1) in-flight commands. 1478 */ 1479 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1480 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1481 goto free_cq; 1482 } 1483 1484 return q; 1485 1486 free_cq: 1487 nvme_dmamem_free(sc, q->q_cq_dmamem); 1488 free_sq: 1489 nvme_dmamem_free(sc, q->q_sq_dmamem); 1490 free: 1491 kmem_free(q, sizeof(*q)); 1492 1493 return NULL; 1494 } 1495 1496 static void 1497 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1498 { 1499 nvme_ccbs_free(q); 1500 mutex_destroy(&q->q_sq_mtx); 1501 mutex_destroy(&q->q_cq_mtx); 1502 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1503 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1504 nvme_dmamem_free(sc, q->q_cq_dmamem); 1505 nvme_dmamem_free(sc, q->q_sq_dmamem); 1506 kmem_free(q, sizeof(*q)); 1507 } 1508 1509 int 1510 nvme_intr(void *xsc) 1511 { 1512 struct nvme_softc *sc = xsc; 1513 1514 /* 1515 * INTx is level triggered, controller deasserts the interrupt only 1516 * when we advance command queue head via write to the doorbell. 1517 * Tell the controller to block the interrupts while we process 1518 * the queue(s). 1519 */ 1520 nvme_write4(sc, NVME_INTMS, 1); 1521 1522 softint_schedule(sc->sc_softih[0]); 1523 1524 /* don't know, might not have been for us */ 1525 return 1; 1526 } 1527 1528 void 1529 nvme_softintr_intx(void *xq) 1530 { 1531 struct nvme_queue *q = xq; 1532 struct nvme_softc *sc = q->q_sc; 1533 1534 nvme_q_complete(sc, sc->sc_admin_q); 1535 if (sc->sc_q != NULL) 1536 nvme_q_complete(sc, sc->sc_q[0]); 1537 1538 /* 1539 * Processing done, tell controller to issue interrupts again. There 1540 * is no race, as NVMe spec requires the controller to maintain state, 1541 * and assert the interrupt whenever there are unacknowledged 1542 * completion queue entries. 1543 */ 1544 nvme_write4(sc, NVME_INTMC, 1); 1545 } 1546 1547 int 1548 nvme_intr_msi(void *xq) 1549 { 1550 struct nvme_queue *q = xq; 1551 1552 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1553 && q->q_sc->sc_softih[q->q_id]); 1554 1555 /* 1556 * MSI/MSI-X are edge triggered, so can handover processing to softint 1557 * without masking the interrupt. 1558 */ 1559 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1560 1561 return 1; 1562 } 1563 1564 void 1565 nvme_softintr_msi(void *xq) 1566 { 1567 struct nvme_queue *q = xq; 1568 struct nvme_softc *sc = q->q_sc; 1569 1570 nvme_q_complete(sc, q); 1571 } 1572 1573 static struct nvme_dmamem * 1574 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1575 { 1576 struct nvme_dmamem *ndm; 1577 int nsegs; 1578 1579 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1580 if (ndm == NULL) 1581 return NULL; 1582 1583 ndm->ndm_size = size; 1584 1585 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1586 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1587 goto ndmfree; 1588 1589 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1590 1, &nsegs, BUS_DMA_WAITOK) != 0) 1591 goto destroy; 1592 1593 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1594 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1595 goto free; 1596 memset(ndm->ndm_kva, 0, size); 1597 1598 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1599 NULL, BUS_DMA_WAITOK) != 0) 1600 goto unmap; 1601 1602 return ndm; 1603 1604 unmap: 1605 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1606 free: 1607 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1608 destroy: 1609 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1610 ndmfree: 1611 kmem_free(ndm, sizeof(*ndm)); 1612 return NULL; 1613 } 1614 1615 static void 1616 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1617 { 1618 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1619 0, NVME_DMA_LEN(mem), ops); 1620 } 1621 1622 void 1623 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1624 { 1625 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1626 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1627 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1628 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1629 kmem_free(ndm, sizeof(*ndm)); 1630 } 1631 1632 /* 1633 * ioctl 1634 */ 1635 1636 dev_type_open(nvmeopen); 1637 dev_type_close(nvmeclose); 1638 dev_type_ioctl(nvmeioctl); 1639 1640 const struct cdevsw nvme_cdevsw = { 1641 .d_open = nvmeopen, 1642 .d_close = nvmeclose, 1643 .d_read = noread, 1644 .d_write = nowrite, 1645 .d_ioctl = nvmeioctl, 1646 .d_stop = nostop, 1647 .d_tty = notty, 1648 .d_poll = nopoll, 1649 .d_mmap = nommap, 1650 .d_kqfilter = nokqfilter, 1651 .d_discard = nodiscard, 1652 .d_flag = D_OTHER, 1653 }; 1654 1655 extern struct cfdriver nvme_cd; 1656 1657 /* 1658 * Accept an open operation on the control device. 1659 */ 1660 int 1661 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1662 { 1663 struct nvme_softc *sc; 1664 int unit = minor(dev) / 0x10000; 1665 int nsid = minor(dev) & 0xffff; 1666 int nsidx; 1667 1668 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1669 return ENXIO; 1670 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1671 return ENXIO; 1672 1673 if (nsid == 0) { 1674 /* controller */ 1675 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1676 return EBUSY; 1677 SET(sc->sc_flags, NVME_F_OPEN); 1678 } else { 1679 /* namespace */ 1680 nsidx = nsid - 1; 1681 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1682 return ENXIO; 1683 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1684 return EBUSY; 1685 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1686 } 1687 return 0; 1688 } 1689 1690 /* 1691 * Accept the last close on the control device. 1692 */ 1693 int 1694 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1695 { 1696 struct nvme_softc *sc; 1697 int unit = minor(dev) / 0x10000; 1698 int nsid = minor(dev) & 0xffff; 1699 int nsidx; 1700 1701 sc = device_lookup_private(&nvme_cd, unit); 1702 if (sc == NULL) 1703 return ENXIO; 1704 1705 if (nsid == 0) { 1706 /* controller */ 1707 CLR(sc->sc_flags, NVME_F_OPEN); 1708 } else { 1709 /* namespace */ 1710 nsidx = nsid - 1; 1711 if (nsidx >= sc->sc_nn) 1712 return ENXIO; 1713 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1714 } 1715 1716 return 0; 1717 } 1718 1719 /* 1720 * Handle control operations. 1721 */ 1722 int 1723 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1724 { 1725 struct nvme_softc *sc; 1726 int unit = minor(dev) / 0x10000; 1727 int nsid = minor(dev) & 0xffff; 1728 struct nvme_pt_command *pt; 1729 1730 sc = device_lookup_private(&nvme_cd, unit); 1731 if (sc == NULL) 1732 return ENXIO; 1733 1734 switch (cmd) { 1735 case NVME_PASSTHROUGH_CMD: 1736 pt = data; 1737 return nvme_command_passthrough(sc, data, 1738 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1739 } 1740 1741 return ENOTTY; 1742 } 1743
Indexes created Wed Oct 22 13:09:56 GMT 2025