nvme.c revision 1.58
1 /* $NetBSD: nvme.c,v 1.58 2021/08/07 16:19:12 thorpej Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.58 2021/08/07 16:19:12 thorpej Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 #include "ioconf.h" 44 #include "locators.h" 45 46 #define B4_CHK_RDY_DELAY_MS 2300 /* workaround controller bug */ 47 48 int nvme_adminq_size = 32; 49 int nvme_ioq_size = 1024; 50 51 static int nvme_print(void *, const char *); 52 53 static int nvme_ready(struct nvme_softc *, uint32_t); 54 static int nvme_enable(struct nvme_softc *, u_int); 55 static int nvme_disable(struct nvme_softc *); 56 static int nvme_shutdown(struct nvme_softc *); 57 58 #ifdef NVME_DEBUG 59 static void nvme_dumpregs(struct nvme_softc *); 60 #endif 61 static int nvme_identify(struct nvme_softc *, u_int); 62 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 63 void *); 64 65 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 66 static void nvme_ccbs_free(struct nvme_queue *); 67 68 static struct nvme_ccb * 69 nvme_ccb_get(struct nvme_queue *, bool); 70 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 71 72 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 73 struct nvme_ccb *, void (*)(struct nvme_queue *, 74 struct nvme_ccb *, void *), int); 75 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 76 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 77 struct nvme_cqe *); 78 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 79 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 80 struct nvme_cqe *); 81 82 static struct nvme_queue * 83 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 84 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 85 static void nvme_q_reset(struct nvme_softc *, struct nvme_queue *); 86 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 87 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 88 struct nvme_ccb *, void (*)(struct nvme_queue *, 89 struct nvme_ccb *, void *)); 90 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 91 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 92 static void nvme_q_wait_complete(struct nvme_softc *, struct nvme_queue *, 93 bool (*)(void *), void *); 94 95 static struct nvme_dmamem * 96 nvme_dmamem_alloc(struct nvme_softc *, size_t); 97 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 98 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 99 int); 100 101 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 102 void *); 103 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 104 struct nvme_cqe *); 105 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 106 void *); 107 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 108 struct nvme_cqe *); 109 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *, 110 void *); 111 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *, 112 struct nvme_cqe *); 113 114 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 115 void *); 116 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 117 struct nvme_cqe *); 118 static int nvme_command_passthrough(struct nvme_softc *, 119 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 120 121 static int nvme_set_number_of_queues(struct nvme_softc *, u_int, u_int *, 122 u_int *); 123 124 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 125 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 126 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 127 #define NVME_TIMO_SY 60 /* sync cache timeout */ 128 129 #define nvme_read4(_s, _r) \ 130 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 131 #define nvme_write4(_s, _r, _v) \ 132 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 133 /* 134 * Some controllers, at least Apple NVMe, always require split 135 * transfers, so don't use bus_space_{read,write}_8() on LP64. 136 */ 137 static inline uint64_t 138 nvme_read8(struct nvme_softc *sc, bus_size_t r) 139 { 140 uint64_t v; 141 uint32_t *a = (uint32_t *)&v; 142 143 #if _BYTE_ORDER == _LITTLE_ENDIAN 144 a[0] = nvme_read4(sc, r); 145 a[1] = nvme_read4(sc, r + 4); 146 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 147 a[1] = nvme_read4(sc, r); 148 a[0] = nvme_read4(sc, r + 4); 149 #endif 150 151 return v; 152 } 153 154 static inline void 155 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 156 { 157 uint32_t *a = (uint32_t *)&v; 158 159 #if _BYTE_ORDER == _LITTLE_ENDIAN 160 nvme_write4(sc, r, a[0]); 161 nvme_write4(sc, r + 4, a[1]); 162 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 163 nvme_write4(sc, r, a[1]); 164 nvme_write4(sc, r + 4, a[0]); 165 #endif 166 } 167 #define nvme_barrier(_s, _r, _l, _f) \ 168 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 169 170 #ifdef NVME_DEBUG 171 static __used void 172 nvme_dumpregs(struct nvme_softc *sc) 173 { 174 uint64_t r8; 175 uint32_t r4; 176 177 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 178 r8 = nvme_read8(sc, NVME_CAP); 179 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 180 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 181 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 182 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 183 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 184 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 185 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 186 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 187 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 188 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 189 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 190 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 191 192 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 193 194 r4 = nvme_read4(sc, NVME_CC); 195 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 196 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 197 (1 << NVME_CC_IOCQES_R(r4))); 198 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 199 (1 << NVME_CC_IOSQES_R(r4))); 200 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 201 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 202 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 203 (1 << NVME_CC_MPS_R(r4))); 204 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 205 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 206 207 r4 = nvme_read4(sc, NVME_CSTS); 208 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 209 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 210 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 211 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 212 213 r4 = nvme_read4(sc, NVME_AQA); 214 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 215 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 216 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 217 218 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 219 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 220 #undef DEVNAME 221 } 222 #endif /* NVME_DEBUG */ 223 224 static int 225 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 226 { 227 u_int i = 0; 228 229 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 230 if (i++ > sc->sc_rdy_to) 231 return ENXIO; 232 233 delay(1000); 234 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 235 } 236 237 return 0; 238 } 239 240 static int 241 nvme_enable(struct nvme_softc *sc, u_int mps) 242 { 243 uint32_t cc, csts; 244 int error; 245 246 cc = nvme_read4(sc, NVME_CC); 247 csts = nvme_read4(sc, NVME_CSTS); 248 249 /* 250 * See note in nvme_disable. Short circuit if we're already enabled. 251 */ 252 if (ISSET(cc, NVME_CC_EN)) { 253 if (ISSET(csts, NVME_CSTS_RDY)) 254 return 0; 255 256 goto waitready; 257 } else { 258 /* EN == 0 already wait for RDY == 0 or fail */ 259 error = nvme_ready(sc, 0); 260 if (error) 261 return error; 262 } 263 264 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 265 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 266 delay(5000); 267 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 268 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 269 delay(5000); 270 271 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 272 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 273 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 274 delay(5000); 275 276 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 277 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 278 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 279 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 280 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 281 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 282 SET(cc, NVME_CC_MPS(mps)); 283 SET(cc, NVME_CC_EN); 284 285 nvme_write4(sc, NVME_CC, cc); 286 nvme_barrier(sc, 0, sc->sc_ios, 287 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 288 289 waitready: 290 return nvme_ready(sc, NVME_CSTS_RDY); 291 } 292 293 static int 294 nvme_disable(struct nvme_softc *sc) 295 { 296 uint32_t cc, csts; 297 int error; 298 299 cc = nvme_read4(sc, NVME_CC); 300 csts = nvme_read4(sc, NVME_CSTS); 301 302 /* 303 * Per 3.1.5 in NVME 1.3 spec, transitioning CC.EN from 0 to 1 304 * when CSTS.RDY is 1 or transitioning CC.EN from 1 to 0 when 305 * CSTS.RDY is 0 "has undefined results" So make sure that CSTS.RDY 306 * isn't the desired value. Short circuit if we're already disabled. 307 */ 308 if (ISSET(cc, NVME_CC_EN)) { 309 if (!ISSET(csts, NVME_CSTS_RDY)) { 310 /* EN == 1, wait for RDY == 1 or fail */ 311 error = nvme_ready(sc, NVME_CSTS_RDY); 312 if (error) 313 return error; 314 } 315 } else { 316 /* EN == 0 already wait for RDY == 0 */ 317 if (!ISSET(csts, NVME_CSTS_RDY)) 318 return 0; 319 320 goto waitready; 321 } 322 323 CLR(cc, NVME_CC_EN); 324 nvme_write4(sc, NVME_CC, cc); 325 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 326 327 /* 328 * Some drives have issues with accessing the mmio after we disable, 329 * so delay for a bit after we write the bit to cope with these issues. 330 */ 331 if (ISSET(sc->sc_quirks, NVME_QUIRK_DELAY_B4_CHK_RDY)) 332 delay(B4_CHK_RDY_DELAY_MS); 333 334 waitready: 335 return nvme_ready(sc, 0); 336 } 337 338 int 339 nvme_attach(struct nvme_softc *sc) 340 { 341 uint64_t cap; 342 uint32_t reg; 343 u_int mps = PAGE_SHIFT; 344 u_int ncq, nsq; 345 uint16_t adminq_entries = nvme_adminq_size; 346 uint16_t ioq_entries = nvme_ioq_size; 347 int i; 348 349 reg = nvme_read4(sc, NVME_VS); 350 if (reg == 0xffffffff) { 351 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 352 return 1; 353 } 354 355 if (NVME_VS_TER(reg) == 0) 356 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg), 357 NVME_VS_MNR(reg)); 358 else 359 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg), 360 NVME_VS_MNR(reg), NVME_VS_TER(reg)); 361 362 cap = nvme_read8(sc, NVME_CAP); 363 sc->sc_dstrd = NVME_CAP_DSTRD(cap); 364 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 365 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 366 "is greater than CPU page size %u\n", 367 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 368 return 1; 369 } 370 if (NVME_CAP_MPSMAX(cap) < mps) 371 mps = NVME_CAP_MPSMAX(cap); 372 if (ioq_entries > NVME_CAP_MQES(cap)) 373 ioq_entries = NVME_CAP_MQES(cap); 374 375 /* set initial values to be used for admin queue during probe */ 376 sc->sc_rdy_to = NVME_CAP_TO(cap); 377 sc->sc_mps = 1 << mps; 378 sc->sc_mdts = MAXPHYS; 379 sc->sc_max_sgl = btoc(round_page(sc->sc_mdts)); 380 381 if (nvme_disable(sc) != 0) { 382 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 383 return 1; 384 } 385 386 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, 387 sc->sc_dstrd); 388 if (sc->sc_admin_q == NULL) { 389 aprint_error_dev(sc->sc_dev, 390 "unable to allocate admin queue\n"); 391 return 1; 392 } 393 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 394 goto free_admin_q; 395 396 if (nvme_enable(sc, mps) != 0) { 397 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 398 goto disestablish_admin_q; 399 } 400 401 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 402 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 403 goto disable; 404 } 405 if (sc->sc_nn == 0) { 406 aprint_error_dev(sc->sc_dev, "namespace not found\n"); 407 goto disable; 408 } 409 410 /* we know how big things are now */ 411 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 412 413 /* reallocate ccbs of admin queue with new max sgl. */ 414 nvme_ccbs_free(sc->sc_admin_q); 415 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 416 417 if (sc->sc_use_mq) { 418 /* Limit the number of queues to the number allocated in HW */ 419 if (nvme_set_number_of_queues(sc, sc->sc_nq, &ncq, &nsq) != 0) { 420 aprint_error_dev(sc->sc_dev, 421 "unable to get number of queues\n"); 422 goto disable; 423 } 424 if (sc->sc_nq > ncq) 425 sc->sc_nq = ncq; 426 if (sc->sc_nq > nsq) 427 sc->sc_nq = nsq; 428 } 429 430 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 431 for (i = 0; i < sc->sc_nq; i++) { 432 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, 433 sc->sc_dstrd); 434 if (sc->sc_q[i] == NULL) { 435 aprint_error_dev(sc->sc_dev, 436 "unable to allocate io queue\n"); 437 goto free_q; 438 } 439 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 440 aprint_error_dev(sc->sc_dev, 441 "unable to create io queue\n"); 442 nvme_q_free(sc, sc->sc_q[i]); 443 goto free_q; 444 } 445 } 446 447 if (!sc->sc_use_mq) 448 nvme_write4(sc, NVME_INTMC, 1); 449 450 /* probe subdevices */ 451 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 452 KM_SLEEP); 453 nvme_rescan(sc->sc_dev, NULL, NULL); 454 455 return 0; 456 457 free_q: 458 while (--i >= 0) { 459 nvme_q_delete(sc, sc->sc_q[i]); 460 nvme_q_free(sc, sc->sc_q[i]); 461 } 462 disable: 463 nvme_disable(sc); 464 disestablish_admin_q: 465 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 466 free_admin_q: 467 nvme_q_free(sc, sc->sc_admin_q); 468 469 return 1; 470 } 471 472 int 473 nvme_rescan(device_t self, const char *ifattr, const int *locs) 474 { 475 struct nvme_softc *sc = device_private(self); 476 struct nvme_attach_args naa; 477 struct nvm_namespace_format *f; 478 struct nvme_namespace *ns; 479 uint64_t cap; 480 int ioq_entries = nvme_ioq_size; 481 int i, mlocs[NVMECF_NLOCS]; 482 int error; 483 484 cap = nvme_read8(sc, NVME_CAP); 485 if (ioq_entries > NVME_CAP_MQES(cap)) 486 ioq_entries = NVME_CAP_MQES(cap); 487 488 for (i = 1; i <= sc->sc_nn; i++) { 489 if (sc->sc_namespaces[i - 1].dev) 490 continue; 491 492 /* identify to check for availability */ 493 error = nvme_ns_identify(sc, i); 494 if (error) { 495 aprint_error_dev(self, "couldn't identify namespace #%d\n", i); 496 continue; 497 } 498 499 ns = nvme_ns_get(sc, i); 500 KASSERT(ns); 501 502 f = &ns->ident->lbaf[NVME_ID_NS_FLBAS(ns->ident->flbas)]; 503 504 /* 505 * NVME1.0e 6.11 Identify command 506 * 507 * LBADS values smaller than 9 are not supported, a value 508 * of zero means that the format is not used. 509 */ 510 if (f->lbads < 9) { 511 if (f->lbads > 0) 512 aprint_error_dev(self, 513 "unsupported logical data size %u\n", f->lbads); 514 continue; 515 } 516 517 mlocs[NVMECF_NSID] = i; 518 519 memset(&naa, 0, sizeof(naa)); 520 naa.naa_nsid = i; 521 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 522 naa.naa_maxphys = sc->sc_mdts; 523 naa.naa_typename = sc->sc_modelname; 524 sc->sc_namespaces[i - 1].dev = 525 config_found(sc->sc_dev, &naa, nvme_print, 526 CFARGS(.submatch = config_stdsubmatch, 527 .locators = mlocs)); 528 } 529 return 0; 530 } 531 532 static int 533 nvme_print(void *aux, const char *pnp) 534 { 535 struct nvme_attach_args *naa = aux; 536 537 if (pnp) 538 aprint_normal("ld at %s", pnp); 539 540 if (naa->naa_nsid > 0) 541 aprint_normal(" nsid %d", naa->naa_nsid); 542 543 return UNCONF; 544 } 545 546 int 547 nvme_detach(struct nvme_softc *sc, int flags) 548 { 549 int i, error; 550 551 error = config_detach_children(sc->sc_dev, flags); 552 if (error) 553 return error; 554 555 error = nvme_shutdown(sc); 556 if (error) 557 return error; 558 559 /* from now on we are committed to detach, following will never fail */ 560 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 561 for (i = 0; i < sc->sc_nq; i++) 562 nvme_q_free(sc, sc->sc_q[i]); 563 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 564 nvme_q_free(sc, sc->sc_admin_q); 565 566 return 0; 567 } 568 569 int 570 nvme_suspend(struct nvme_softc *sc) 571 { 572 573 return nvme_shutdown(sc); 574 } 575 576 int 577 nvme_resume(struct nvme_softc *sc) 578 { 579 int ioq_entries = nvme_ioq_size; 580 uint64_t cap; 581 int i, error; 582 583 error = nvme_disable(sc); 584 if (error) { 585 device_printf(sc->sc_dev, "unable to disable controller\n"); 586 return error; 587 } 588 589 nvme_q_reset(sc, sc->sc_admin_q); 590 591 error = nvme_enable(sc, ffs(sc->sc_mps) - 1); 592 if (error) { 593 device_printf(sc->sc_dev, "unable to enable controller\n"); 594 return error; 595 } 596 597 for (i = 0; i < sc->sc_nq; i++) { 598 cap = nvme_read8(sc, NVME_CAP); 599 if (ioq_entries > NVME_CAP_MQES(cap)) 600 ioq_entries = NVME_CAP_MQES(cap); 601 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, 602 sc->sc_dstrd); 603 if (sc->sc_q[i] == NULL) { 604 error = ENOMEM; 605 device_printf(sc->sc_dev, "unable to allocate io q %d" 606 "\n", i); 607 goto disable; 608 } 609 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 610 error = EIO; 611 device_printf(sc->sc_dev, "unable to create io q %d" 612 "\n", i); 613 nvme_q_free(sc, sc->sc_q[i]); 614 goto free_q; 615 } 616 } 617 618 nvme_write4(sc, NVME_INTMC, 1); 619 620 return 0; 621 622 free_q: 623 while (i --> 0) 624 nvme_q_free(sc, sc->sc_q[i]); 625 disable: 626 (void)nvme_disable(sc); 627 628 return error; 629 } 630 631 static int 632 nvme_shutdown(struct nvme_softc *sc) 633 { 634 uint32_t cc, csts; 635 bool disabled = false; 636 int i; 637 638 if (!sc->sc_use_mq) 639 nvme_write4(sc, NVME_INTMS, 1); 640 641 for (i = 0; i < sc->sc_nq; i++) { 642 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 643 aprint_error_dev(sc->sc_dev, 644 "unable to delete io queue %d, disabling\n", i + 1); 645 disabled = true; 646 } 647 } 648 if (disabled) 649 goto disable; 650 651 cc = nvme_read4(sc, NVME_CC); 652 CLR(cc, NVME_CC_SHN_MASK); 653 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 654 nvme_write4(sc, NVME_CC, cc); 655 656 for (i = 0; i < 4000; i++) { 657 nvme_barrier(sc, 0, sc->sc_ios, 658 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 659 csts = nvme_read4(sc, NVME_CSTS); 660 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 661 return 0; 662 663 delay(1000); 664 } 665 666 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 667 668 disable: 669 nvme_disable(sc); 670 return 0; 671 } 672 673 void 674 nvme_childdet(device_t self, device_t child) 675 { 676 struct nvme_softc *sc = device_private(self); 677 int i; 678 679 for (i = 0; i < sc->sc_nn; i++) { 680 if (sc->sc_namespaces[i].dev == child) { 681 /* Already freed ns->ident. */ 682 sc->sc_namespaces[i].dev = NULL; 683 break; 684 } 685 } 686 } 687 688 int 689 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 690 { 691 struct nvme_sqe sqe; 692 struct nvm_identify_namespace *identify; 693 struct nvme_dmamem *mem; 694 struct nvme_ccb *ccb; 695 struct nvme_namespace *ns; 696 int rv; 697 698 KASSERT(nsid > 0); 699 700 ns = nvme_ns_get(sc, nsid); 701 KASSERT(ns); 702 703 if (ns->ident != NULL) 704 return 0; 705 706 ccb = nvme_ccb_get(sc->sc_admin_q, false); 707 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 708 709 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 710 if (mem == NULL) { 711 nvme_ccb_put(sc->sc_admin_q, ccb); 712 return ENOMEM; 713 } 714 715 memset(&sqe, 0, sizeof(sqe)); 716 sqe.opcode = NVM_ADMIN_IDENTIFY; 717 htolem32(&sqe.nsid, nsid); 718 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 719 htolem32(&sqe.cdw10, 0); 720 721 ccb->ccb_done = nvme_empty_done; 722 ccb->ccb_cookie = &sqe; 723 724 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 725 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 726 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 727 728 nvme_ccb_put(sc->sc_admin_q, ccb); 729 730 if (rv != 0) { 731 rv = EIO; 732 goto done; 733 } 734 735 /* commit */ 736 737 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 738 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 739 740 /* Convert data to host endian */ 741 nvme_identify_namespace_swapbytes(identify); 742 743 ns->ident = identify; 744 745 done: 746 nvme_dmamem_free(sc, mem); 747 748 return rv; 749 } 750 751 int 752 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 753 struct buf *bp, void *data, size_t datasize, 754 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 755 { 756 struct nvme_queue *q = nvme_get_q(sc, bp, false); 757 struct nvme_ccb *ccb; 758 bus_dmamap_t dmap; 759 int i, error; 760 761 ccb = nvme_ccb_get(q, false); 762 if (ccb == NULL) 763 return EAGAIN; 764 765 ccb->ccb_done = nvme_ns_io_done; 766 ccb->ccb_cookie = cookie; 767 768 /* namespace context */ 769 ccb->nnc_nsid = nsid; 770 ccb->nnc_flags = flags; 771 ccb->nnc_buf = bp; 772 ccb->nnc_datasize = datasize; 773 ccb->nnc_secsize = secsize; 774 ccb->nnc_blkno = blkno; 775 ccb->nnc_done = nnc_done; 776 777 dmap = ccb->ccb_dmamap; 778 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 779 datasize, NULL, 780 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 781 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 782 (ISSET(flags, NVME_NS_CTX_F_READ) ? 783 BUS_DMA_READ : BUS_DMA_WRITE)); 784 if (error) { 785 nvme_ccb_put(q, ccb); 786 return error; 787 } 788 789 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 790 ISSET(flags, NVME_NS_CTX_F_READ) ? 791 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 792 793 if (dmap->dm_nsegs > 2) { 794 for (i = 1; i < dmap->dm_nsegs; i++) { 795 htolem64(&ccb->ccb_prpl[i - 1], 796 dmap->dm_segs[i].ds_addr); 797 } 798 bus_dmamap_sync(sc->sc_dmat, 799 NVME_DMA_MAP(q->q_ccb_prpls), 800 ccb->ccb_prpl_off, 801 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 802 BUS_DMASYNC_PREWRITE); 803 } 804 805 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 806 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 807 return EIO; 808 return 0; 809 } 810 811 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 812 return 0; 813 } 814 815 static void 816 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 817 { 818 struct nvme_sqe_io *sqe = slot; 819 bus_dmamap_t dmap = ccb->ccb_dmamap; 820 821 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 822 NVM_CMD_READ : NVM_CMD_WRITE; 823 htolem32(&sqe->nsid, ccb->nnc_nsid); 824 825 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 826 switch (dmap->dm_nsegs) { 827 case 1: 828 break; 829 case 2: 830 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 831 break; 832 default: 833 /* the prp list is already set up and synced */ 834 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 835 break; 836 } 837 838 htolem64(&sqe->slba, ccb->nnc_blkno); 839 840 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) 841 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA); 842 843 /* guaranteed by upper layers, but check just in case */ 844 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 845 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 846 } 847 848 static void 849 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 850 struct nvme_cqe *cqe) 851 { 852 struct nvme_softc *sc = q->q_sc; 853 bus_dmamap_t dmap = ccb->ccb_dmamap; 854 void *nnc_cookie = ccb->ccb_cookie; 855 nvme_nnc_done nnc_done = ccb->nnc_done; 856 struct buf *bp = ccb->nnc_buf; 857 858 if (dmap->dm_nsegs > 2) { 859 bus_dmamap_sync(sc->sc_dmat, 860 NVME_DMA_MAP(q->q_ccb_prpls), 861 ccb->ccb_prpl_off, 862 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 863 BUS_DMASYNC_POSTWRITE); 864 } 865 866 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 867 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 868 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 869 870 bus_dmamap_unload(sc->sc_dmat, dmap); 871 nvme_ccb_put(q, ccb); 872 873 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 874 } 875 876 /* 877 * If there is no volatile write cache, it makes no sense to issue 878 * flush commands or query for the status. 879 */ 880 static bool 881 nvme_has_volatile_write_cache(struct nvme_softc *sc) 882 { 883 /* sc_identify is filled during attachment */ 884 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0); 885 } 886 887 static bool 888 nvme_ns_sync_finished(void *cookie) 889 { 890 int *result = cookie; 891 892 return (*result != 0); 893 } 894 895 int 896 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, int flags) 897 { 898 struct nvme_queue *q = nvme_get_q(sc, NULL, true); 899 struct nvme_ccb *ccb; 900 int result = 0; 901 902 if (!nvme_has_volatile_write_cache(sc)) { 903 /* cache not present, no value in trying to flush it */ 904 return 0; 905 } 906 907 ccb = nvme_ccb_get(q, true); 908 KASSERT(ccb != NULL); 909 910 ccb->ccb_done = nvme_ns_sync_done; 911 ccb->ccb_cookie = &result; 912 913 /* namespace context */ 914 ccb->nnc_nsid = nsid; 915 ccb->nnc_flags = flags; 916 ccb->nnc_done = NULL; 917 918 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 919 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 920 return EIO; 921 return 0; 922 } 923 924 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 925 926 /* wait for completion */ 927 nvme_q_wait_complete(sc, q, nvme_ns_sync_finished, &result); 928 KASSERT(result != 0); 929 930 return (result > 0) ? 0 : EIO; 931 } 932 933 static void 934 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 935 { 936 struct nvme_sqe *sqe = slot; 937 938 sqe->opcode = NVM_CMD_FLUSH; 939 htolem32(&sqe->nsid, ccb->nnc_nsid); 940 } 941 942 static void 943 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 944 struct nvme_cqe *cqe) 945 { 946 int *result = ccb->ccb_cookie; 947 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 948 949 if (status == NVME_CQE_SC_SUCCESS) 950 *result = 1; 951 else 952 *result = -1; 953 954 nvme_ccb_put(q, ccb); 955 } 956 957 static bool 958 nvme_getcache_finished(void *xc) 959 { 960 int *addr = xc; 961 962 return (*addr != 0); 963 } 964 965 /* 966 * Get status of volatile write cache. Always asynchronous. 967 */ 968 int 969 nvme_admin_getcache(struct nvme_softc *sc, int *addr) 970 { 971 struct nvme_ccb *ccb; 972 struct nvme_queue *q = sc->sc_admin_q; 973 int result = 0, error; 974 975 if (!nvme_has_volatile_write_cache(sc)) { 976 /* cache simply not present */ 977 *addr = 0; 978 return 0; 979 } 980 981 ccb = nvme_ccb_get(q, true); 982 KASSERT(ccb != NULL); 983 984 ccb->ccb_done = nvme_getcache_done; 985 ccb->ccb_cookie = &result; 986 987 /* namespace context */ 988 ccb->nnc_flags = 0; 989 ccb->nnc_done = NULL; 990 991 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); 992 993 /* wait for completion */ 994 nvme_q_wait_complete(sc, q, nvme_getcache_finished, &result); 995 KASSERT(result != 0); 996 997 if (result > 0) { 998 *addr = result; 999 error = 0; 1000 } else 1001 error = EINVAL; 1002 1003 return error; 1004 } 1005 1006 static void 1007 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1008 { 1009 struct nvme_sqe *sqe = slot; 1010 1011 sqe->opcode = NVM_ADMIN_GET_FEATURES; 1012 htolem32(&sqe->cdw10, NVM_FEATURE_VOLATILE_WRITE_CACHE); 1013 htolem32(&sqe->cdw11, NVM_VOLATILE_WRITE_CACHE_WCE); 1014 } 1015 1016 static void 1017 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1018 struct nvme_cqe *cqe) 1019 { 1020 int *addr = ccb->ccb_cookie; 1021 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 1022 uint32_t cdw0 = lemtoh32(&cqe->cdw0); 1023 int result; 1024 1025 if (status == NVME_CQE_SC_SUCCESS) { 1026 result = 0; 1027 1028 /* 1029 * DPO not supported, Dataset Management (DSM) field doesn't 1030 * specify the same semantics. FUA is always supported. 1031 */ 1032 result = DKCACHE_FUA; 1033 1034 if (cdw0 & NVM_VOLATILE_WRITE_CACHE_WCE) 1035 result |= DKCACHE_WRITE; 1036 1037 /* 1038 * If volatile write cache is present, the flag shall also be 1039 * settable. 1040 */ 1041 result |= DKCACHE_WCHANGE; 1042 1043 /* 1044 * ONCS field indicates whether the optional SAVE is also 1045 * supported for Set Features. According to spec v1.3, 1046 * Volatile Write Cache however doesn't support persistency 1047 * across power cycle/reset. 1048 */ 1049 1050 } else { 1051 result = -1; 1052 } 1053 1054 *addr = result; 1055 1056 nvme_ccb_put(q, ccb); 1057 } 1058 1059 struct nvme_setcache_state { 1060 int dkcache; 1061 int result; 1062 }; 1063 1064 static bool 1065 nvme_setcache_finished(void *xc) 1066 { 1067 struct nvme_setcache_state *st = xc; 1068 1069 return (st->result != 0); 1070 } 1071 1072 static void 1073 nvme_setcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1074 { 1075 struct nvme_sqe *sqe = slot; 1076 struct nvme_setcache_state *st = ccb->ccb_cookie; 1077 1078 sqe->opcode = NVM_ADMIN_SET_FEATURES; 1079 htolem32(&sqe->cdw10, NVM_FEATURE_VOLATILE_WRITE_CACHE); 1080 if (st->dkcache & DKCACHE_WRITE) 1081 htolem32(&sqe->cdw11, NVM_VOLATILE_WRITE_CACHE_WCE); 1082 } 1083 1084 static void 1085 nvme_setcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1086 struct nvme_cqe *cqe) 1087 { 1088 struct nvme_setcache_state *st = ccb->ccb_cookie; 1089 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 1090 1091 if (status == NVME_CQE_SC_SUCCESS) { 1092 st->result = 1; 1093 } else { 1094 st->result = -1; 1095 } 1096 1097 nvme_ccb_put(q, ccb); 1098 } 1099 1100 /* 1101 * Set status of volatile write cache. Always asynchronous. 1102 */ 1103 int 1104 nvme_admin_setcache(struct nvme_softc *sc, int dkcache) 1105 { 1106 struct nvme_ccb *ccb; 1107 struct nvme_queue *q = sc->sc_admin_q; 1108 int error; 1109 struct nvme_setcache_state st; 1110 1111 if (!nvme_has_volatile_write_cache(sc)) { 1112 /* cache simply not present */ 1113 return EOPNOTSUPP; 1114 } 1115 1116 if (dkcache & ~(DKCACHE_WRITE)) { 1117 /* unsupported parameters */ 1118 return EOPNOTSUPP; 1119 } 1120 1121 ccb = nvme_ccb_get(q, true); 1122 KASSERT(ccb != NULL); 1123 1124 memset(&st, 0, sizeof(st)); 1125 st.dkcache = dkcache; 1126 1127 ccb->ccb_done = nvme_setcache_done; 1128 ccb->ccb_cookie = &st; 1129 1130 /* namespace context */ 1131 ccb->nnc_flags = 0; 1132 ccb->nnc_done = NULL; 1133 1134 nvme_q_submit(sc, q, ccb, nvme_setcache_fill); 1135 1136 /* wait for completion */ 1137 nvme_q_wait_complete(sc, q, nvme_setcache_finished, &st); 1138 KASSERT(st.result != 0); 1139 1140 if (st.result > 0) 1141 error = 0; 1142 else 1143 error = EINVAL; 1144 1145 return error; 1146 } 1147 1148 void 1149 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 1150 { 1151 struct nvme_namespace *ns; 1152 struct nvm_identify_namespace *identify; 1153 1154 ns = nvme_ns_get(sc, nsid); 1155 KASSERT(ns); 1156 1157 identify = ns->ident; 1158 ns->ident = NULL; 1159 if (identify != NULL) 1160 kmem_free(identify, sizeof(*identify)); 1161 } 1162 1163 struct nvme_pt_state { 1164 struct nvme_pt_command *pt; 1165 bool finished; 1166 }; 1167 1168 static void 1169 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1170 { 1171 struct nvme_softc *sc = q->q_sc; 1172 struct nvme_sqe *sqe = slot; 1173 struct nvme_pt_state *state = ccb->ccb_cookie; 1174 struct nvme_pt_command *pt = state->pt; 1175 bus_dmamap_t dmap = ccb->ccb_dmamap; 1176 int i; 1177 1178 sqe->opcode = pt->cmd.opcode; 1179 htolem32(&sqe->nsid, pt->cmd.nsid); 1180 1181 if (pt->buf != NULL && pt->len > 0) { 1182 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 1183 switch (dmap->dm_nsegs) { 1184 case 1: 1185 break; 1186 case 2: 1187 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 1188 break; 1189 default: 1190 for (i = 1; i < dmap->dm_nsegs; i++) { 1191 htolem64(&ccb->ccb_prpl[i - 1], 1192 dmap->dm_segs[i].ds_addr); 1193 } 1194 bus_dmamap_sync(sc->sc_dmat, 1195 NVME_DMA_MAP(q->q_ccb_prpls), 1196 ccb->ccb_prpl_off, 1197 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 1198 BUS_DMASYNC_PREWRITE); 1199 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 1200 break; 1201 } 1202 } 1203 1204 htolem32(&sqe->cdw10, pt->cmd.cdw10); 1205 htolem32(&sqe->cdw11, pt->cmd.cdw11); 1206 htolem32(&sqe->cdw12, pt->cmd.cdw12); 1207 htolem32(&sqe->cdw13, pt->cmd.cdw13); 1208 htolem32(&sqe->cdw14, pt->cmd.cdw14); 1209 htolem32(&sqe->cdw15, pt->cmd.cdw15); 1210 } 1211 1212 static void 1213 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 1214 { 1215 struct nvme_softc *sc = q->q_sc; 1216 struct nvme_pt_state *state = ccb->ccb_cookie; 1217 struct nvme_pt_command *pt = state->pt; 1218 bus_dmamap_t dmap = ccb->ccb_dmamap; 1219 1220 if (pt->buf != NULL && pt->len > 0) { 1221 if (dmap->dm_nsegs > 2) { 1222 bus_dmamap_sync(sc->sc_dmat, 1223 NVME_DMA_MAP(q->q_ccb_prpls), 1224 ccb->ccb_prpl_off, 1225 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 1226 BUS_DMASYNC_POSTWRITE); 1227 } 1228 1229 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1230 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 1231 bus_dmamap_unload(sc->sc_dmat, dmap); 1232 } 1233 1234 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 1235 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 1236 1237 state->finished = true; 1238 1239 nvme_ccb_put(q, ccb); 1240 } 1241 1242 static bool 1243 nvme_pt_finished(void *cookie) 1244 { 1245 struct nvme_pt_state *state = cookie; 1246 1247 return state->finished; 1248 } 1249 1250 static int 1251 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 1252 uint16_t nsid, struct lwp *l, bool is_adminq) 1253 { 1254 struct nvme_queue *q; 1255 struct nvme_ccb *ccb; 1256 void *buf = NULL; 1257 struct nvme_pt_state state; 1258 int error; 1259 1260 /* limit command size to maximum data transfer size */ 1261 if ((pt->buf == NULL && pt->len > 0) || 1262 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 1263 return EINVAL; 1264 1265 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc, NULL, true); 1266 ccb = nvme_ccb_get(q, true); 1267 KASSERT(ccb != NULL); 1268 1269 if (pt->buf != NULL) { 1270 KASSERT(pt->len > 0); 1271 buf = kmem_alloc(pt->len, KM_SLEEP); 1272 if (!pt->is_read) { 1273 error = copyin(pt->buf, buf, pt->len); 1274 if (error) 1275 goto kmem_free; 1276 } 1277 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 1278 pt->len, NULL, 1279 BUS_DMA_WAITOK | 1280 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 1281 if (error) 1282 goto kmem_free; 1283 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 1284 0, ccb->ccb_dmamap->dm_mapsize, 1285 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 1286 } 1287 1288 memset(&state, 0, sizeof(state)); 1289 state.pt = pt; 1290 state.finished = false; 1291 1292 ccb->ccb_done = nvme_pt_done; 1293 ccb->ccb_cookie = &state; 1294 1295 pt->cmd.nsid = nsid; 1296 1297 nvme_q_submit(sc, q, ccb, nvme_pt_fill); 1298 1299 /* wait for completion */ 1300 nvme_q_wait_complete(sc, q, nvme_pt_finished, &state); 1301 KASSERT(state.finished); 1302 1303 error = 0; 1304 1305 if (buf != NULL) { 1306 if (error == 0 && pt->is_read) 1307 error = copyout(buf, pt->buf, pt->len); 1308 kmem_free: 1309 kmem_free(buf, pt->len); 1310 } 1311 1312 return error; 1313 } 1314 1315 static void 1316 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1317 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 1318 { 1319 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 1320 uint32_t tail; 1321 1322 mutex_enter(&q->q_sq_mtx); 1323 tail = q->q_sq_tail; 1324 if (++q->q_sq_tail >= q->q_entries) 1325 q->q_sq_tail = 0; 1326 1327 sqe += tail; 1328 1329 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1330 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 1331 memset(sqe, 0, sizeof(*sqe)); 1332 (*fill)(q, ccb, sqe); 1333 htolem16(&sqe->cid, ccb->ccb_id); 1334 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1335 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 1336 1337 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 1338 mutex_exit(&q->q_sq_mtx); 1339 } 1340 1341 struct nvme_poll_state { 1342 struct nvme_sqe s; 1343 struct nvme_cqe c; 1344 void *cookie; 1345 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 1346 }; 1347 1348 static int 1349 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1350 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 1351 { 1352 struct nvme_poll_state state; 1353 uint16_t flags; 1354 int step = 10; 1355 int maxloop = timo_sec * 1000000 / step; 1356 int error = 0; 1357 1358 memset(&state, 0, sizeof(state)); 1359 (*fill)(q, ccb, &state.s); 1360 1361 state.done = ccb->ccb_done; 1362 state.cookie = ccb->ccb_cookie; 1363 1364 ccb->ccb_done = nvme_poll_done; 1365 ccb->ccb_cookie = &state; 1366 1367 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1368 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1369 if (nvme_q_complete(sc, q) == 0) 1370 delay(step); 1371 1372 if (timo_sec >= 0 && --maxloop <= 0) { 1373 error = ETIMEDOUT; 1374 break; 1375 } 1376 } 1377 1378 if (error == 0) { 1379 flags = lemtoh16(&state.c.flags); 1380 return flags & ~NVME_CQE_PHASE; 1381 } else { 1382 /* 1383 * If it succeds later, it would hit ccb which will have been 1384 * already reused for something else. Not good. Cross 1385 * fingers and hope for best. XXX do controller reset? 1386 */ 1387 aprint_error_dev(sc->sc_dev, "polled command timed out\n"); 1388 1389 /* Invoke the callback to clean state anyway */ 1390 struct nvme_cqe cqe; 1391 memset(&cqe, 0, sizeof(cqe)); 1392 ccb->ccb_done(q, ccb, &cqe); 1393 1394 return 1; 1395 } 1396 } 1397 1398 static void 1399 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1400 { 1401 struct nvme_sqe *sqe = slot; 1402 struct nvme_poll_state *state = ccb->ccb_cookie; 1403 1404 *sqe = state->s; 1405 } 1406 1407 static void 1408 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1409 struct nvme_cqe *cqe) 1410 { 1411 struct nvme_poll_state *state = ccb->ccb_cookie; 1412 1413 state->c = *cqe; 1414 SET(state->c.flags, htole16(NVME_CQE_PHASE)); 1415 1416 ccb->ccb_cookie = state->cookie; 1417 state->done(q, ccb, &state->c); 1418 } 1419 1420 static void 1421 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1422 { 1423 struct nvme_sqe *src = ccb->ccb_cookie; 1424 struct nvme_sqe *dst = slot; 1425 1426 *dst = *src; 1427 } 1428 1429 static void 1430 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1431 struct nvme_cqe *cqe) 1432 { 1433 } 1434 1435 static int 1436 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1437 { 1438 struct nvme_ccb *ccb; 1439 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1440 uint16_t flags; 1441 int rv = 0; 1442 1443 mutex_enter(&q->q_cq_mtx); 1444 1445 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1446 for (;;) { 1447 cqe = &ring[q->q_cq_head]; 1448 flags = lemtoh16(&cqe->flags); 1449 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1450 break; 1451 1452 ccb = &q->q_ccbs[lemtoh16(&cqe->cid)]; 1453 1454 if (++q->q_cq_head >= q->q_entries) { 1455 q->q_cq_head = 0; 1456 q->q_cq_phase ^= NVME_CQE_PHASE; 1457 } 1458 1459 #ifdef DEBUG 1460 /* 1461 * If we get spurious completion notification, something 1462 * is seriously hosed up. Very likely DMA to some random 1463 * memory place happened, so just bail out. 1464 */ 1465 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1466 panic("%s: invalid ccb detected", 1467 device_xname(sc->sc_dev)); 1468 /* NOTREACHED */ 1469 } 1470 #endif 1471 1472 rv++; 1473 1474 /* 1475 * Unlock the mutex before calling the ccb_done callback 1476 * and re-lock afterwards. The callback triggers lddone() 1477 * which schedules another i/o, and also calls nvme_ccb_put(). 1478 * Unlock/relock avoids possibility of deadlock. 1479 */ 1480 mutex_exit(&q->q_cq_mtx); 1481 ccb->ccb_done(q, ccb, cqe); 1482 mutex_enter(&q->q_cq_mtx); 1483 } 1484 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1485 1486 if (rv) 1487 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1488 1489 mutex_exit(&q->q_cq_mtx); 1490 1491 return rv; 1492 } 1493 1494 static void 1495 nvme_q_wait_complete(struct nvme_softc *sc, 1496 struct nvme_queue *q, bool (*finished)(void *), void *cookie) 1497 { 1498 mutex_enter(&q->q_ccb_mtx); 1499 if (finished(cookie)) 1500 goto out; 1501 1502 for(;;) { 1503 q->q_ccb_waiting = true; 1504 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1505 1506 if (finished(cookie)) 1507 break; 1508 } 1509 1510 out: 1511 mutex_exit(&q->q_ccb_mtx); 1512 } 1513 1514 static int 1515 nvme_identify(struct nvme_softc *sc, u_int mps) 1516 { 1517 char sn[41], mn[81], fr[17]; 1518 struct nvm_identify_controller *identify; 1519 struct nvme_dmamem *mem; 1520 struct nvme_ccb *ccb; 1521 u_int mdts; 1522 int rv = 1; 1523 1524 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1525 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1526 1527 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1528 if (mem == NULL) 1529 return 1; 1530 1531 ccb->ccb_done = nvme_empty_done; 1532 ccb->ccb_cookie = mem; 1533 1534 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1535 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1536 NVME_TIMO_IDENT); 1537 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1538 1539 nvme_ccb_put(sc->sc_admin_q, ccb); 1540 1541 if (rv != 0) 1542 goto done; 1543 1544 identify = NVME_DMA_KVA(mem); 1545 sc->sc_identify = *identify; 1546 identify = NULL; 1547 1548 /* Convert data to host endian */ 1549 nvme_identify_controller_swapbytes(&sc->sc_identify); 1550 1551 strnvisx(sn, sizeof(sn), (const char *)sc->sc_identify.sn, 1552 sizeof(sc->sc_identify.sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1553 strnvisx(mn, sizeof(mn), (const char *)sc->sc_identify.mn, 1554 sizeof(sc->sc_identify.mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1555 strnvisx(fr, sizeof(fr), (const char *)sc->sc_identify.fr, 1556 sizeof(sc->sc_identify.fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1557 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1558 sn); 1559 1560 strlcpy(sc->sc_modelname, mn, sizeof(sc->sc_modelname)); 1561 1562 if (sc->sc_identify.mdts > 0) { 1563 mdts = (1 << sc->sc_identify.mdts) * (1 << mps); 1564 if (mdts < sc->sc_mdts) 1565 sc->sc_mdts = mdts; 1566 } 1567 1568 sc->sc_nn = sc->sc_identify.nn; 1569 1570 done: 1571 nvme_dmamem_free(sc, mem); 1572 1573 return rv; 1574 } 1575 1576 static int 1577 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1578 { 1579 struct nvme_sqe_q sqe; 1580 struct nvme_ccb *ccb; 1581 int rv; 1582 1583 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1584 return 1; 1585 1586 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1587 KASSERT(ccb != NULL); 1588 1589 ccb->ccb_done = nvme_empty_done; 1590 ccb->ccb_cookie = &sqe; 1591 1592 memset(&sqe, 0, sizeof(sqe)); 1593 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1594 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1595 htolem16(&sqe.qsize, q->q_entries - 1); 1596 htolem16(&sqe.qid, q->q_id); 1597 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1598 if (sc->sc_use_mq) 1599 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1600 1601 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1602 if (rv != 0) 1603 goto fail; 1604 1605 ccb->ccb_done = nvme_empty_done; 1606 ccb->ccb_cookie = &sqe; 1607 1608 memset(&sqe, 0, sizeof(sqe)); 1609 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1610 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1611 htolem16(&sqe.qsize, q->q_entries - 1); 1612 htolem16(&sqe.qid, q->q_id); 1613 htolem16(&sqe.cqid, q->q_id); 1614 sqe.qflags = NVM_SQE_Q_PC; 1615 1616 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1617 if (rv != 0) 1618 goto fail; 1619 1620 nvme_ccb_put(sc->sc_admin_q, ccb); 1621 return 0; 1622 1623 fail: 1624 if (sc->sc_use_mq) 1625 sc->sc_intr_disestablish(sc, q->q_id); 1626 1627 nvme_ccb_put(sc->sc_admin_q, ccb); 1628 return rv; 1629 } 1630 1631 static int 1632 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1633 { 1634 struct nvme_sqe_q sqe; 1635 struct nvme_ccb *ccb; 1636 int rv; 1637 1638 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1639 KASSERT(ccb != NULL); 1640 1641 ccb->ccb_done = nvme_empty_done; 1642 ccb->ccb_cookie = &sqe; 1643 1644 memset(&sqe, 0, sizeof(sqe)); 1645 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1646 htolem16(&sqe.qid, q->q_id); 1647 1648 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1649 if (rv != 0) 1650 goto fail; 1651 1652 ccb->ccb_done = nvme_empty_done; 1653 ccb->ccb_cookie = &sqe; 1654 1655 memset(&sqe, 0, sizeof(sqe)); 1656 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1657 htolem16(&sqe.qid, q->q_id); 1658 1659 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1660 if (rv != 0) 1661 goto fail; 1662 1663 fail: 1664 nvme_ccb_put(sc->sc_admin_q, ccb); 1665 1666 if (rv == 0 && sc->sc_use_mq) { 1667 if (sc->sc_intr_disestablish(sc, q->q_id)) 1668 rv = 1; 1669 } 1670 1671 return rv; 1672 } 1673 1674 static void 1675 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1676 { 1677 struct nvme_sqe *sqe = slot; 1678 struct nvme_dmamem *mem = ccb->ccb_cookie; 1679 1680 sqe->opcode = NVM_ADMIN_IDENTIFY; 1681 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1682 htolem32(&sqe->cdw10, 1); 1683 } 1684 1685 static int 1686 nvme_set_number_of_queues(struct nvme_softc *sc, u_int nq, u_int *ncqa, 1687 u_int *nsqa) 1688 { 1689 struct nvme_pt_state state; 1690 struct nvme_pt_command pt; 1691 struct nvme_ccb *ccb; 1692 int rv; 1693 1694 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1695 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1696 1697 memset(&pt, 0, sizeof(pt)); 1698 pt.cmd.opcode = NVM_ADMIN_SET_FEATURES; 1699 pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES; 1700 pt.cmd.cdw11 = ((nq - 1) << 16) | (nq - 1); 1701 1702 memset(&state, 0, sizeof(state)); 1703 state.pt = &pt; 1704 state.finished = false; 1705 1706 ccb->ccb_done = nvme_pt_done; 1707 ccb->ccb_cookie = &state; 1708 1709 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1710 1711 if (rv != 0) { 1712 *ncqa = *nsqa = 0; 1713 return EIO; 1714 } 1715 1716 *ncqa = (pt.cpl.cdw0 >> 16) + 1; 1717 *nsqa = (pt.cpl.cdw0 & 0xffff) + 1; 1718 1719 return 0; 1720 } 1721 1722 static int 1723 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1724 { 1725 struct nvme_softc *sc = q->q_sc; 1726 struct nvme_ccb *ccb; 1727 bus_addr_t off; 1728 uint64_t *prpl; 1729 u_int i; 1730 1731 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1732 cv_init(&q->q_ccb_wait, "nvmeqw"); 1733 q->q_ccb_waiting = false; 1734 SIMPLEQ_INIT(&q->q_ccb_list); 1735 1736 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1737 1738 q->q_nccbs = nccbs; 1739 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1740 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1741 1742 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1743 off = 0; 1744 1745 for (i = 0; i < nccbs; i++) { 1746 ccb = &q->q_ccbs[i]; 1747 1748 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1749 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1750 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1751 &ccb->ccb_dmamap) != 0) 1752 goto free_maps; 1753 1754 ccb->ccb_id = i; 1755 ccb->ccb_prpl = prpl; 1756 ccb->ccb_prpl_off = off; 1757 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1758 1759 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1760 1761 prpl += sc->sc_max_sgl; 1762 off += sizeof(*prpl) * sc->sc_max_sgl; 1763 } 1764 1765 return 0; 1766 1767 free_maps: 1768 nvme_ccbs_free(q); 1769 return 1; 1770 } 1771 1772 static struct nvme_ccb * 1773 nvme_ccb_get(struct nvme_queue *q, bool wait) 1774 { 1775 struct nvme_ccb *ccb = NULL; 1776 1777 mutex_enter(&q->q_ccb_mtx); 1778 again: 1779 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1780 if (ccb != NULL) { 1781 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1782 #ifdef DEBUG 1783 ccb->ccb_cookie = NULL; 1784 #endif 1785 } else { 1786 if (__predict_false(wait)) { 1787 q->q_ccb_waiting = true; 1788 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1789 goto again; 1790 } 1791 } 1792 mutex_exit(&q->q_ccb_mtx); 1793 1794 return ccb; 1795 } 1796 1797 static void 1798 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1799 { 1800 1801 mutex_enter(&q->q_ccb_mtx); 1802 #ifdef DEBUG 1803 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1804 #endif 1805 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1806 1807 /* It's unlikely there are any waiters, it's not used for regular I/O */ 1808 if (__predict_false(q->q_ccb_waiting)) { 1809 q->q_ccb_waiting = false; 1810 cv_broadcast(&q->q_ccb_wait); 1811 } 1812 1813 mutex_exit(&q->q_ccb_mtx); 1814 } 1815 1816 static void 1817 nvme_ccbs_free(struct nvme_queue *q) 1818 { 1819 struct nvme_softc *sc = q->q_sc; 1820 struct nvme_ccb *ccb; 1821 1822 mutex_enter(&q->q_ccb_mtx); 1823 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1824 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1825 /* 1826 * bus_dmamap_destroy() may call vm_map_lock() and rw_enter() 1827 * internally. don't hold spin mutex 1828 */ 1829 mutex_exit(&q->q_ccb_mtx); 1830 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1831 mutex_enter(&q->q_ccb_mtx); 1832 } 1833 mutex_exit(&q->q_ccb_mtx); 1834 1835 nvme_dmamem_free(sc, q->q_ccb_prpls); 1836 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1837 q->q_ccbs = NULL; 1838 cv_destroy(&q->q_ccb_wait); 1839 mutex_destroy(&q->q_ccb_mtx); 1840 } 1841 1842 static struct nvme_queue * 1843 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1844 { 1845 struct nvme_queue *q; 1846 1847 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1848 q->q_sc = sc; 1849 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1850 sizeof(struct nvme_sqe) * entries); 1851 if (q->q_sq_dmamem == NULL) 1852 goto free; 1853 1854 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1855 sizeof(struct nvme_cqe) * entries); 1856 if (q->q_cq_dmamem == NULL) 1857 goto free_sq; 1858 1859 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1860 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1861 1862 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1863 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1864 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1865 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1866 q->q_id = id; 1867 q->q_entries = entries; 1868 q->q_sq_tail = 0; 1869 q->q_cq_head = 0; 1870 q->q_cq_phase = NVME_CQE_PHASE; 1871 1872 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1873 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1874 1875 /* 1876 * Due to definition of full and empty queue (queue is empty 1877 * when head == tail, full when tail is one less then head), 1878 * we can actually only have (entries - 1) in-flight commands. 1879 */ 1880 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1881 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1882 goto free_cq; 1883 } 1884 1885 return q; 1886 1887 free_cq: 1888 nvme_dmamem_free(sc, q->q_cq_dmamem); 1889 free_sq: 1890 nvme_dmamem_free(sc, q->q_sq_dmamem); 1891 free: 1892 kmem_free(q, sizeof(*q)); 1893 1894 return NULL; 1895 } 1896 1897 static void 1898 nvme_q_reset(struct nvme_softc *sc, struct nvme_queue *q) 1899 { 1900 1901 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1902 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1903 1904 q->q_sqtdbl = NVME_SQTDBL(q->q_id, sc->sc_dstrd); 1905 q->q_cqhdbl = NVME_CQHDBL(q->q_id, sc->sc_dstrd); 1906 1907 q->q_sq_tail = 0; 1908 q->q_cq_head = 0; 1909 q->q_cq_phase = NVME_CQE_PHASE; 1910 1911 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1912 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1913 } 1914 1915 static void 1916 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1917 { 1918 nvme_ccbs_free(q); 1919 mutex_destroy(&q->q_sq_mtx); 1920 mutex_destroy(&q->q_cq_mtx); 1921 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1922 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1923 nvme_dmamem_free(sc, q->q_cq_dmamem); 1924 nvme_dmamem_free(sc, q->q_sq_dmamem); 1925 kmem_free(q, sizeof(*q)); 1926 } 1927 1928 int 1929 nvme_intr(void *xsc) 1930 { 1931 struct nvme_softc *sc = xsc; 1932 1933 /* 1934 * INTx is level triggered, controller deasserts the interrupt only 1935 * when we advance command queue head via write to the doorbell. 1936 * Tell the controller to block the interrupts while we process 1937 * the queue(s). 1938 */ 1939 nvme_write4(sc, NVME_INTMS, 1); 1940 1941 softint_schedule(sc->sc_softih[0]); 1942 1943 /* don't know, might not have been for us */ 1944 return 1; 1945 } 1946 1947 void 1948 nvme_softintr_intx(void *xq) 1949 { 1950 struct nvme_queue *q = xq; 1951 struct nvme_softc *sc = q->q_sc; 1952 1953 nvme_q_complete(sc, sc->sc_admin_q); 1954 if (sc->sc_q != NULL) 1955 nvme_q_complete(sc, sc->sc_q[0]); 1956 1957 /* 1958 * Processing done, tell controller to issue interrupts again. There 1959 * is no race, as NVMe spec requires the controller to maintain state, 1960 * and assert the interrupt whenever there are unacknowledged 1961 * completion queue entries. 1962 */ 1963 nvme_write4(sc, NVME_INTMC, 1); 1964 } 1965 1966 int 1967 nvme_intr_msi(void *xq) 1968 { 1969 struct nvme_queue *q = xq; 1970 1971 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1972 && q->q_sc->sc_softih[q->q_id]); 1973 1974 /* 1975 * MSI/MSI-X are edge triggered, so can handover processing to softint 1976 * without masking the interrupt. 1977 */ 1978 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1979 1980 return 1; 1981 } 1982 1983 void 1984 nvme_softintr_msi(void *xq) 1985 { 1986 struct nvme_queue *q = xq; 1987 struct nvme_softc *sc = q->q_sc; 1988 1989 nvme_q_complete(sc, q); 1990 } 1991 1992 static struct nvme_dmamem * 1993 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1994 { 1995 struct nvme_dmamem *ndm; 1996 int nsegs; 1997 1998 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1999 if (ndm == NULL) 2000 return NULL; 2001 2002 ndm->ndm_size = size; 2003 2004 if (bus_dmamap_create(sc->sc_dmat, size, btoc(round_page(size)), size, 0, 2005 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 2006 goto ndmfree; 2007 2008 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 2009 1, &nsegs, BUS_DMA_WAITOK) != 0) 2010 goto destroy; 2011 2012 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 2013 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 2014 goto free; 2015 2016 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 2017 NULL, BUS_DMA_WAITOK) != 0) 2018 goto unmap; 2019 2020 memset(ndm->ndm_kva, 0, size); 2021 bus_dmamap_sync(sc->sc_dmat, ndm->ndm_map, 0, size, BUS_DMASYNC_PREREAD); 2022 2023 return ndm; 2024 2025 unmap: 2026 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 2027 free: 2028 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 2029 destroy: 2030 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 2031 ndmfree: 2032 kmem_free(ndm, sizeof(*ndm)); 2033 return NULL; 2034 } 2035 2036 static void 2037 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 2038 { 2039 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 2040 0, NVME_DMA_LEN(mem), ops); 2041 } 2042 2043 void 2044 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 2045 { 2046 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 2047 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 2048 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 2049 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 2050 kmem_free(ndm, sizeof(*ndm)); 2051 } 2052 2053 /* 2054 * ioctl 2055 */ 2056 2057 dev_type_open(nvmeopen); 2058 dev_type_close(nvmeclose); 2059 dev_type_ioctl(nvmeioctl); 2060 2061 const struct cdevsw nvme_cdevsw = { 2062 .d_open = nvmeopen, 2063 .d_close = nvmeclose, 2064 .d_read = noread, 2065 .d_write = nowrite, 2066 .d_ioctl = nvmeioctl, 2067 .d_stop = nostop, 2068 .d_tty = notty, 2069 .d_poll = nopoll, 2070 .d_mmap = nommap, 2071 .d_kqfilter = nokqfilter, 2072 .d_discard = nodiscard, 2073 .d_flag = D_OTHER, 2074 }; 2075 2076 /* 2077 * Accept an open operation on the control device. 2078 */ 2079 int 2080 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 2081 { 2082 struct nvme_softc *sc; 2083 int unit = minor(dev) / 0x10000; 2084 int nsid = minor(dev) & 0xffff; 2085 int nsidx; 2086 2087 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 2088 return ENXIO; 2089 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 2090 return ENXIO; 2091 2092 if (nsid == 0) { 2093 /* controller */ 2094 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 2095 return EBUSY; 2096 SET(sc->sc_flags, NVME_F_OPEN); 2097 } else { 2098 /* namespace */ 2099 nsidx = nsid - 1; 2100 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 2101 return ENXIO; 2102 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 2103 return EBUSY; 2104 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 2105 } 2106 return 0; 2107 } 2108 2109 /* 2110 * Accept the last close on the control device. 2111 */ 2112 int 2113 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 2114 { 2115 struct nvme_softc *sc; 2116 int unit = minor(dev) / 0x10000; 2117 int nsid = minor(dev) & 0xffff; 2118 int nsidx; 2119 2120 sc = device_lookup_private(&nvme_cd, unit); 2121 if (sc == NULL) 2122 return ENXIO; 2123 2124 if (nsid == 0) { 2125 /* controller */ 2126 CLR(sc->sc_flags, NVME_F_OPEN); 2127 } else { 2128 /* namespace */ 2129 nsidx = nsid - 1; 2130 if (nsidx >= sc->sc_nn) 2131 return ENXIO; 2132 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 2133 } 2134 2135 return 0; 2136 } 2137 2138 /* 2139 * Handle control operations. 2140 */ 2141 int 2142 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 2143 { 2144 struct nvme_softc *sc; 2145 int unit = minor(dev) / 0x10000; 2146 int nsid = minor(dev) & 0xffff; 2147 struct nvme_pt_command *pt; 2148 2149 sc = device_lookup_private(&nvme_cd, unit); 2150 if (sc == NULL) 2151 return ENXIO; 2152 2153 switch (cmd) { 2154 case NVME_PASSTHROUGH_CMD: 2155 pt = data; 2156 return nvme_command_passthrough(sc, data, 2157 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 2158 } 2159 2160 return ENOTTY; 2161 } 2162
Indexes created Sun Sep 28 12:09:53 GMT 2025