nvme.c revision 1.44.2.2
1 /* $NetBSD: nvme.c,v 1.44.2.2 2019/09/26 19:13:14 martin Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.44.2.2 2019/09/26 19:13:14 martin Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 #include "ioconf.h" 44 45 #define B4_CHK_RDY_DELAY_MS 2300 /* workaround controller bug */ 46 47 int nvme_adminq_size = 32; 48 int nvme_ioq_size = 1024; 49 50 static int nvme_print(void *, const char *); 51 52 static int nvme_ready(struct nvme_softc *, uint32_t); 53 static int nvme_enable(struct nvme_softc *, u_int); 54 static int nvme_disable(struct nvme_softc *); 55 static int nvme_shutdown(struct nvme_softc *); 56 57 #ifdef NVME_DEBUG 58 static void nvme_dumpregs(struct nvme_softc *); 59 #endif 60 static int nvme_identify(struct nvme_softc *, u_int); 61 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 62 void *); 63 64 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 65 static void nvme_ccbs_free(struct nvme_queue *); 66 67 static struct nvme_ccb * 68 nvme_ccb_get(struct nvme_queue *, bool); 69 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 70 71 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 72 struct nvme_ccb *, void (*)(struct nvme_queue *, 73 struct nvme_ccb *, void *), int); 74 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 76 struct nvme_cqe *); 77 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 78 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 79 struct nvme_cqe *); 80 81 static struct nvme_queue * 82 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 83 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 84 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 85 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 86 struct nvme_ccb *, void (*)(struct nvme_queue *, 87 struct nvme_ccb *, void *)); 88 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 89 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 90 static void nvme_q_wait_complete(struct nvme_softc *, struct nvme_queue *, 91 bool (*)(void *), void *); 92 93 static struct nvme_dmamem * 94 nvme_dmamem_alloc(struct nvme_softc *, size_t); 95 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 96 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 97 int); 98 99 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 100 void *); 101 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 102 struct nvme_cqe *); 103 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 104 void *); 105 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 106 struct nvme_cqe *); 107 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *, 108 void *); 109 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *, 110 struct nvme_cqe *); 111 112 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 113 void *); 114 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 115 struct nvme_cqe *); 116 static int nvme_command_passthrough(struct nvme_softc *, 117 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 118 119 static int nvme_get_number_of_queues(struct nvme_softc *, u_int *); 120 121 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 122 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 123 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 124 #define NVME_TIMO_SY 60 /* sync cache timeout */ 125 126 #define nvme_read4(_s, _r) \ 127 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 128 #define nvme_write4(_s, _r, _v) \ 129 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 130 /* 131 * Some controllers, at least Apple NVMe, always require split 132 * transfers, so don't use bus_space_{read,write}_8() on LP64. 133 */ 134 static inline uint64_t 135 nvme_read8(struct nvme_softc *sc, bus_size_t r) 136 { 137 uint64_t v; 138 uint32_t *a = (uint32_t *)&v; 139 140 #if _BYTE_ORDER == _LITTLE_ENDIAN 141 a[0] = nvme_read4(sc, r); 142 a[1] = nvme_read4(sc, r + 4); 143 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 144 a[1] = nvme_read4(sc, r); 145 a[0] = nvme_read4(sc, r + 4); 146 #endif 147 148 return v; 149 } 150 151 static inline void 152 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 153 { 154 uint32_t *a = (uint32_t *)&v; 155 156 #if _BYTE_ORDER == _LITTLE_ENDIAN 157 nvme_write4(sc, r, a[0]); 158 nvme_write4(sc, r + 4, a[1]); 159 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 160 nvme_write4(sc, r, a[1]); 161 nvme_write4(sc, r + 4, a[0]); 162 #endif 163 } 164 #define nvme_barrier(_s, _r, _l, _f) \ 165 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 166 167 #ifdef NVME_DEBUG 168 static __used void 169 nvme_dumpregs(struct nvme_softc *sc) 170 { 171 uint64_t r8; 172 uint32_t r4; 173 174 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 175 r8 = nvme_read8(sc, NVME_CAP); 176 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 177 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 178 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 179 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 180 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 181 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 182 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 183 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 184 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 185 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 186 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 187 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 188 189 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 190 191 r4 = nvme_read4(sc, NVME_CC); 192 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 193 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 194 (1 << NVME_CC_IOCQES_R(r4))); 195 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 196 (1 << NVME_CC_IOSQES_R(r4))); 197 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 198 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 199 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 200 (1 << NVME_CC_MPS_R(r4))); 201 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 202 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 203 204 r4 = nvme_read4(sc, NVME_CSTS); 205 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 206 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 207 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 208 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 209 210 r4 = nvme_read4(sc, NVME_AQA); 211 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 212 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 213 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 214 215 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 216 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 217 #undef DEVNAME 218 } 219 #endif /* NVME_DEBUG */ 220 221 static int 222 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 223 { 224 u_int i = 0; 225 226 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 227 if (i++ > sc->sc_rdy_to) 228 return ENXIO; 229 230 delay(1000); 231 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 232 } 233 234 return 0; 235 } 236 237 static int 238 nvme_enable(struct nvme_softc *sc, u_int mps) 239 { 240 uint32_t cc, csts; 241 int error; 242 243 cc = nvme_read4(sc, NVME_CC); 244 csts = nvme_read4(sc, NVME_CSTS); 245 246 /* 247 * See note in nvme_disable. Short circuit if we're already enabled. 248 */ 249 if (ISSET(cc, NVME_CC_EN)) { 250 if (ISSET(csts, NVME_CSTS_RDY)) 251 return 0; 252 253 goto waitready; 254 } else { 255 /* EN == 0 already wait for RDY == 0 or fail */ 256 error = nvme_ready(sc, 0); 257 if (error) 258 return error; 259 } 260 261 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 262 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 263 delay(5000); 264 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 265 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 266 delay(5000); 267 268 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 269 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 270 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 271 delay(5000); 272 273 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 274 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 275 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 276 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 277 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 278 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 279 SET(cc, NVME_CC_MPS(mps)); 280 SET(cc, NVME_CC_EN); 281 282 nvme_write4(sc, NVME_CC, cc); 283 nvme_barrier(sc, 0, sc->sc_ios, 284 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 285 286 waitready: 287 return nvme_ready(sc, NVME_CSTS_RDY); 288 } 289 290 static int 291 nvme_disable(struct nvme_softc *sc) 292 { 293 uint32_t cc, csts; 294 int error; 295 296 cc = nvme_read4(sc, NVME_CC); 297 csts = nvme_read4(sc, NVME_CSTS); 298 299 /* 300 * Per 3.1.5 in NVME 1.3 spec, transitioning CC.EN from 0 to 1 301 * when CSTS.RDY is 1 or transitioning CC.EN from 1 to 0 when 302 * CSTS.RDY is 0 "has undefined results" So make sure that CSTS.RDY 303 * isn't the desired value. Short circuit if we're already disabled. 304 */ 305 if (ISSET(cc, NVME_CC_EN)) { 306 if (!ISSET(csts, NVME_CSTS_RDY)) { 307 /* EN == 1, wait for RDY == 1 or fail */ 308 error = nvme_ready(sc, NVME_CSTS_RDY); 309 if (error) 310 return error; 311 } 312 } else { 313 /* EN == 0 already wait for RDY == 0 */ 314 if (!ISSET(csts, NVME_CSTS_RDY)) 315 return 0; 316 317 goto waitready; 318 } 319 320 CLR(cc, NVME_CC_EN); 321 nvme_write4(sc, NVME_CC, cc); 322 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 323 324 /* 325 * Some drives have issues with accessing the mmio after we disable, 326 * so delay for a bit after we write the bit to cope with these issues. 327 */ 328 if (ISSET(sc->sc_quirks, NVME_QUIRK_DELAY_B4_CHK_RDY)) 329 delay(B4_CHK_RDY_DELAY_MS); 330 331 waitready: 332 return nvme_ready(sc, 0); 333 } 334 335 int 336 nvme_attach(struct nvme_softc *sc) 337 { 338 uint64_t cap; 339 uint32_t reg; 340 u_int dstrd; 341 u_int mps = PAGE_SHIFT; 342 u_int ioq_allocated; 343 uint16_t adminq_entries = nvme_adminq_size; 344 uint16_t ioq_entries = nvme_ioq_size; 345 int i; 346 347 reg = nvme_read4(sc, NVME_VS); 348 if (reg == 0xffffffff) { 349 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 350 return 1; 351 } 352 353 if (NVME_VS_TER(reg) == 0) 354 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg), 355 NVME_VS_MNR(reg)); 356 else 357 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg), 358 NVME_VS_MNR(reg), NVME_VS_TER(reg)); 359 360 cap = nvme_read8(sc, NVME_CAP); 361 dstrd = NVME_CAP_DSTRD(cap); 362 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 363 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 364 "is greater than CPU page size %u\n", 365 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 366 return 1; 367 } 368 if (NVME_CAP_MPSMAX(cap) < mps) 369 mps = NVME_CAP_MPSMAX(cap); 370 if (ioq_entries > NVME_CAP_MQES(cap)) 371 ioq_entries = NVME_CAP_MQES(cap); 372 373 /* set initial values to be used for admin queue during probe */ 374 sc->sc_rdy_to = NVME_CAP_TO(cap); 375 sc->sc_mps = 1 << mps; 376 sc->sc_mdts = MAXPHYS; 377 sc->sc_max_sgl = btoc(round_page(sc->sc_mdts)); 378 379 if (nvme_disable(sc) != 0) { 380 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 381 return 1; 382 } 383 384 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 385 if (sc->sc_admin_q == NULL) { 386 aprint_error_dev(sc->sc_dev, 387 "unable to allocate admin queue\n"); 388 return 1; 389 } 390 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 391 goto free_admin_q; 392 393 if (nvme_enable(sc, mps) != 0) { 394 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 395 goto disestablish_admin_q; 396 } 397 398 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 399 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 400 goto disable; 401 } 402 if (sc->sc_nn == 0) { 403 aprint_error_dev(sc->sc_dev, "namespace not found\n"); 404 goto disable; 405 } 406 407 /* we know how big things are now */ 408 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 409 410 /* reallocate ccbs of admin queue with new max sgl. */ 411 nvme_ccbs_free(sc->sc_admin_q); 412 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 413 414 if (sc->sc_use_mq) { 415 /* Limit the number of queues to the number allocated in HW */ 416 if (nvme_get_number_of_queues(sc, &ioq_allocated) != 0) { 417 aprint_error_dev(sc->sc_dev, 418 "unable to get number of queues\n"); 419 goto disable; 420 } 421 if (sc->sc_nq > ioq_allocated) 422 sc->sc_nq = ioq_allocated; 423 } 424 425 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 426 for (i = 0; i < sc->sc_nq; i++) { 427 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 428 if (sc->sc_q[i] == NULL) { 429 aprint_error_dev(sc->sc_dev, 430 "unable to allocate io queue\n"); 431 goto free_q; 432 } 433 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 434 aprint_error_dev(sc->sc_dev, 435 "unable to create io queue\n"); 436 nvme_q_free(sc, sc->sc_q[i]); 437 goto free_q; 438 } 439 } 440 441 if (!sc->sc_use_mq) 442 nvme_write4(sc, NVME_INTMC, 1); 443 444 /* probe subdevices */ 445 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 446 KM_SLEEP); 447 nvme_rescan(sc->sc_dev, "nvme", &i); 448 449 return 0; 450 451 free_q: 452 while (--i >= 0) { 453 nvme_q_delete(sc, sc->sc_q[i]); 454 nvme_q_free(sc, sc->sc_q[i]); 455 } 456 disable: 457 nvme_disable(sc); 458 disestablish_admin_q: 459 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 460 free_admin_q: 461 nvme_q_free(sc, sc->sc_admin_q); 462 463 return 1; 464 } 465 466 int 467 nvme_rescan(device_t self, const char *attr, const int *flags) 468 { 469 struct nvme_softc *sc = device_private(self); 470 struct nvme_attach_args naa; 471 uint64_t cap; 472 int ioq_entries = nvme_ioq_size; 473 int i; 474 475 cap = nvme_read8(sc, NVME_CAP); 476 if (ioq_entries > NVME_CAP_MQES(cap)) 477 ioq_entries = NVME_CAP_MQES(cap); 478 479 for (i = 0; i < sc->sc_nn; i++) { 480 if (sc->sc_namespaces[i].dev) 481 continue; 482 memset(&naa, 0, sizeof(naa)); 483 naa.naa_nsid = i + 1; 484 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 485 naa.naa_maxphys = sc->sc_mdts; 486 naa.naa_typename = sc->sc_modelname; 487 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 488 nvme_print); 489 } 490 return 0; 491 } 492 493 static int 494 nvme_print(void *aux, const char *pnp) 495 { 496 struct nvme_attach_args *naa = aux; 497 498 if (pnp) 499 aprint_normal("at %s", pnp); 500 501 if (naa->naa_nsid > 0) 502 aprint_normal(" nsid %d", naa->naa_nsid); 503 504 return UNCONF; 505 } 506 507 int 508 nvme_detach(struct nvme_softc *sc, int flags) 509 { 510 int i, error; 511 512 error = config_detach_children(sc->sc_dev, flags); 513 if (error) 514 return error; 515 516 error = nvme_shutdown(sc); 517 if (error) 518 return error; 519 520 /* from now on we are committed to detach, following will never fail */ 521 for (i = 0; i < sc->sc_nq; i++) 522 nvme_q_free(sc, sc->sc_q[i]); 523 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 524 nvme_q_free(sc, sc->sc_admin_q); 525 526 return 0; 527 } 528 529 static int 530 nvme_shutdown(struct nvme_softc *sc) 531 { 532 uint32_t cc, csts; 533 bool disabled = false; 534 int i; 535 536 if (!sc->sc_use_mq) 537 nvme_write4(sc, NVME_INTMS, 1); 538 539 for (i = 0; i < sc->sc_nq; i++) { 540 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 541 aprint_error_dev(sc->sc_dev, 542 "unable to delete io queue %d, disabling\n", i + 1); 543 disabled = true; 544 } 545 } 546 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 547 if (disabled) 548 goto disable; 549 550 cc = nvme_read4(sc, NVME_CC); 551 CLR(cc, NVME_CC_SHN_MASK); 552 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 553 nvme_write4(sc, NVME_CC, cc); 554 555 for (i = 0; i < 4000; i++) { 556 nvme_barrier(sc, 0, sc->sc_ios, 557 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 558 csts = nvme_read4(sc, NVME_CSTS); 559 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 560 return 0; 561 562 delay(1000); 563 } 564 565 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 566 567 disable: 568 nvme_disable(sc); 569 return 0; 570 } 571 572 void 573 nvme_childdet(device_t self, device_t child) 574 { 575 struct nvme_softc *sc = device_private(self); 576 int i; 577 578 for (i = 0; i < sc->sc_nn; i++) { 579 if (sc->sc_namespaces[i].dev == child) { 580 /* Already freed ns->ident. */ 581 sc->sc_namespaces[i].dev = NULL; 582 break; 583 } 584 } 585 } 586 587 int 588 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 589 { 590 struct nvme_sqe sqe; 591 struct nvm_identify_namespace *identify; 592 struct nvme_dmamem *mem; 593 struct nvme_ccb *ccb; 594 struct nvme_namespace *ns; 595 int rv; 596 597 KASSERT(nsid > 0); 598 599 ccb = nvme_ccb_get(sc->sc_admin_q, false); 600 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 601 602 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 603 if (mem == NULL) { 604 nvme_ccb_put(sc->sc_admin_q, ccb); 605 return ENOMEM; 606 } 607 608 memset(&sqe, 0, sizeof(sqe)); 609 sqe.opcode = NVM_ADMIN_IDENTIFY; 610 htolem32(&sqe.nsid, nsid); 611 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 612 htolem32(&sqe.cdw10, 0); 613 614 ccb->ccb_done = nvme_empty_done; 615 ccb->ccb_cookie = &sqe; 616 617 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 618 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 619 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 620 621 nvme_ccb_put(sc->sc_admin_q, ccb); 622 623 if (rv != 0) { 624 rv = EIO; 625 goto done; 626 } 627 628 /* commit */ 629 630 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 631 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 632 633 /* Convert data to host endian */ 634 nvme_identify_namespace_swapbytes(identify); 635 636 ns = nvme_ns_get(sc, nsid); 637 KASSERT(ns); 638 KASSERT(ns->ident == NULL); 639 ns->ident = identify; 640 641 done: 642 nvme_dmamem_free(sc, mem); 643 644 return rv; 645 } 646 647 int 648 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 649 struct buf *bp, void *data, size_t datasize, 650 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 651 { 652 struct nvme_queue *q = nvme_get_q(sc, bp, false); 653 struct nvme_ccb *ccb; 654 bus_dmamap_t dmap; 655 int i, error; 656 657 ccb = nvme_ccb_get(q, false); 658 if (ccb == NULL) 659 return EAGAIN; 660 661 ccb->ccb_done = nvme_ns_io_done; 662 ccb->ccb_cookie = cookie; 663 664 /* namespace context */ 665 ccb->nnc_nsid = nsid; 666 ccb->nnc_flags = flags; 667 ccb->nnc_buf = bp; 668 ccb->nnc_datasize = datasize; 669 ccb->nnc_secsize = secsize; 670 ccb->nnc_blkno = blkno; 671 ccb->nnc_done = nnc_done; 672 673 dmap = ccb->ccb_dmamap; 674 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 675 datasize, NULL, 676 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 677 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 678 (ISSET(flags, NVME_NS_CTX_F_READ) ? 679 BUS_DMA_READ : BUS_DMA_WRITE)); 680 if (error) { 681 nvme_ccb_put(q, ccb); 682 return error; 683 } 684 685 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 686 ISSET(flags, NVME_NS_CTX_F_READ) ? 687 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 688 689 if (dmap->dm_nsegs > 2) { 690 for (i = 1; i < dmap->dm_nsegs; i++) { 691 htolem64(&ccb->ccb_prpl[i - 1], 692 dmap->dm_segs[i].ds_addr); 693 } 694 bus_dmamap_sync(sc->sc_dmat, 695 NVME_DMA_MAP(q->q_ccb_prpls), 696 ccb->ccb_prpl_off, 697 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 698 BUS_DMASYNC_PREWRITE); 699 } 700 701 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 702 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 703 return EIO; 704 return 0; 705 } 706 707 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 708 return 0; 709 } 710 711 static void 712 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 713 { 714 struct nvme_sqe_io *sqe = slot; 715 bus_dmamap_t dmap = ccb->ccb_dmamap; 716 717 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 718 NVM_CMD_READ : NVM_CMD_WRITE; 719 htolem32(&sqe->nsid, ccb->nnc_nsid); 720 721 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 722 switch (dmap->dm_nsegs) { 723 case 1: 724 break; 725 case 2: 726 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 727 break; 728 default: 729 /* the prp list is already set up and synced */ 730 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 731 break; 732 } 733 734 htolem64(&sqe->slba, ccb->nnc_blkno); 735 736 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) 737 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA); 738 739 /* guaranteed by upper layers, but check just in case */ 740 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 741 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 742 } 743 744 static void 745 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 746 struct nvme_cqe *cqe) 747 { 748 struct nvme_softc *sc = q->q_sc; 749 bus_dmamap_t dmap = ccb->ccb_dmamap; 750 void *nnc_cookie = ccb->ccb_cookie; 751 nvme_nnc_done nnc_done = ccb->nnc_done; 752 struct buf *bp = ccb->nnc_buf; 753 754 if (dmap->dm_nsegs > 2) { 755 bus_dmamap_sync(sc->sc_dmat, 756 NVME_DMA_MAP(q->q_ccb_prpls), 757 ccb->ccb_prpl_off, 758 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 759 BUS_DMASYNC_POSTWRITE); 760 } 761 762 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 763 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 764 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 765 766 bus_dmamap_unload(sc->sc_dmat, dmap); 767 nvme_ccb_put(q, ccb); 768 769 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 770 } 771 772 /* 773 * If there is no volatile write cache, it makes no sense to issue 774 * flush commands or query for the status. 775 */ 776 static bool 777 nvme_has_volatile_write_cache(struct nvme_softc *sc) 778 { 779 /* sc_identify is filled during attachment */ 780 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0); 781 } 782 783 static bool 784 nvme_ns_sync_finished(void *cookie) 785 { 786 int *result = cookie; 787 788 return (*result != 0); 789 } 790 791 int 792 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, int flags) 793 { 794 struct nvme_queue *q = nvme_get_q(sc, NULL, true); 795 struct nvme_ccb *ccb; 796 int result = 0; 797 798 if (!nvme_has_volatile_write_cache(sc)) { 799 /* cache not present, no value in trying to flush it */ 800 return 0; 801 } 802 803 ccb = nvme_ccb_get(q, true); 804 KASSERT(ccb != NULL); 805 806 ccb->ccb_done = nvme_ns_sync_done; 807 ccb->ccb_cookie = &result; 808 809 /* namespace context */ 810 ccb->nnc_nsid = nsid; 811 ccb->nnc_flags = flags; 812 ccb->nnc_done = NULL; 813 814 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 815 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 816 return EIO; 817 return 0; 818 } 819 820 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 821 822 /* wait for completion */ 823 nvme_q_wait_complete(sc, q, nvme_ns_sync_finished, &result); 824 KASSERT(result != 0); 825 826 return (result > 0) ? 0 : EIO; 827 } 828 829 static void 830 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 831 { 832 struct nvme_sqe *sqe = slot; 833 834 sqe->opcode = NVM_CMD_FLUSH; 835 htolem32(&sqe->nsid, ccb->nnc_nsid); 836 } 837 838 static void 839 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 840 struct nvme_cqe *cqe) 841 { 842 int *result = ccb->ccb_cookie; 843 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 844 845 if (status == NVME_CQE_SC_SUCCESS) 846 *result = 1; 847 else 848 *result = -1; 849 850 nvme_ccb_put(q, ccb); 851 } 852 853 static bool 854 nvme_getcache_finished(void *xc) 855 { 856 int *addr = xc; 857 858 return (*addr != 0); 859 } 860 861 /* 862 * Get status of volatile write cache. Always asynchronous. 863 */ 864 int 865 nvme_admin_getcache(struct nvme_softc *sc, int *addr) 866 { 867 struct nvme_ccb *ccb; 868 struct nvme_queue *q = sc->sc_admin_q; 869 int result = 0, error; 870 871 if (!nvme_has_volatile_write_cache(sc)) { 872 /* cache simply not present */ 873 *addr = 0; 874 return 0; 875 } 876 877 ccb = nvme_ccb_get(q, true); 878 KASSERT(ccb != NULL); 879 880 ccb->ccb_done = nvme_getcache_done; 881 ccb->ccb_cookie = &result; 882 883 /* namespace context */ 884 ccb->nnc_flags = 0; 885 ccb->nnc_done = NULL; 886 887 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); 888 889 /* wait for completion */ 890 nvme_q_wait_complete(sc, q, nvme_getcache_finished, &result); 891 KASSERT(result != 0); 892 893 if (result > 0) { 894 *addr = result; 895 error = 0; 896 } else 897 error = EINVAL; 898 899 return error; 900 } 901 902 static void 903 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 904 { 905 struct nvme_sqe *sqe = slot; 906 907 sqe->opcode = NVM_ADMIN_GET_FEATURES; 908 htolem32(&sqe->cdw10, NVM_FEATURE_VOLATILE_WRITE_CACHE); 909 htolem32(&sqe->cdw11, NVM_VOLATILE_WRITE_CACHE_WCE); 910 } 911 912 static void 913 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 914 struct nvme_cqe *cqe) 915 { 916 int *addr = ccb->ccb_cookie; 917 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 918 uint32_t cdw0 = lemtoh32(&cqe->cdw0); 919 int result; 920 921 if (status == NVME_CQE_SC_SUCCESS) { 922 result = 0; 923 924 /* 925 * DPO not supported, Dataset Management (DSM) field doesn't 926 * specify the same semantics. FUA is always supported. 927 */ 928 result = DKCACHE_FUA; 929 930 if (cdw0 & NVM_VOLATILE_WRITE_CACHE_WCE) 931 result |= DKCACHE_WRITE; 932 933 /* 934 * If volatile write cache is present, the flag shall also be 935 * settable. 936 */ 937 result |= DKCACHE_WCHANGE; 938 939 /* 940 * ONCS field indicates whether the optional SAVE is also 941 * supported for Set Features. According to spec v1.3, 942 * Volatile Write Cache however doesn't support persistency 943 * across power cycle/reset. 944 */ 945 946 } else { 947 result = -1; 948 } 949 950 *addr = result; 951 952 nvme_ccb_put(q, ccb); 953 } 954 955 struct nvme_setcache_state { 956 int dkcache; 957 int result; 958 }; 959 960 static bool 961 nvme_setcache_finished(void *xc) 962 { 963 struct nvme_setcache_state *st = xc; 964 965 return (st->result != 0); 966 } 967 968 static void 969 nvme_setcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 970 { 971 struct nvme_sqe *sqe = slot; 972 struct nvme_setcache_state *st = ccb->ccb_cookie; 973 974 sqe->opcode = NVM_ADMIN_SET_FEATURES; 975 htolem32(&sqe->cdw10, NVM_FEATURE_VOLATILE_WRITE_CACHE); 976 if (st->dkcache & DKCACHE_WRITE) 977 htolem32(&sqe->cdw11, NVM_VOLATILE_WRITE_CACHE_WCE); 978 } 979 980 static void 981 nvme_setcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 982 struct nvme_cqe *cqe) 983 { 984 struct nvme_setcache_state *st = ccb->ccb_cookie; 985 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 986 987 if (status == NVME_CQE_SC_SUCCESS) { 988 st->result = 1; 989 } else { 990 st->result = -1; 991 } 992 993 nvme_ccb_put(q, ccb); 994 } 995 996 /* 997 * Set status of volatile write cache. Always asynchronous. 998 */ 999 int 1000 nvme_admin_setcache(struct nvme_softc *sc, int dkcache) 1001 { 1002 struct nvme_ccb *ccb; 1003 struct nvme_queue *q = sc->sc_admin_q; 1004 int error; 1005 struct nvme_setcache_state st; 1006 1007 if (!nvme_has_volatile_write_cache(sc)) { 1008 /* cache simply not present */ 1009 return EOPNOTSUPP; 1010 } 1011 1012 if (dkcache & ~(DKCACHE_WRITE)) { 1013 /* unsupported parameters */ 1014 return EOPNOTSUPP; 1015 } 1016 1017 ccb = nvme_ccb_get(q, true); 1018 KASSERT(ccb != NULL); 1019 1020 memset(&st, 0, sizeof(st)); 1021 st.dkcache = dkcache; 1022 1023 ccb->ccb_done = nvme_setcache_done; 1024 ccb->ccb_cookie = &st; 1025 1026 /* namespace context */ 1027 ccb->nnc_flags = 0; 1028 ccb->nnc_done = NULL; 1029 1030 nvme_q_submit(sc, q, ccb, nvme_setcache_fill); 1031 1032 /* wait for completion */ 1033 nvme_q_wait_complete(sc, q, nvme_setcache_finished, &st); 1034 KASSERT(st.result != 0); 1035 1036 if (st.result > 0) 1037 error = 0; 1038 else 1039 error = EINVAL; 1040 1041 return error; 1042 } 1043 1044 void 1045 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 1046 { 1047 struct nvme_namespace *ns; 1048 struct nvm_identify_namespace *identify; 1049 1050 ns = nvme_ns_get(sc, nsid); 1051 KASSERT(ns); 1052 1053 identify = ns->ident; 1054 ns->ident = NULL; 1055 if (identify != NULL) 1056 kmem_free(identify, sizeof(*identify)); 1057 } 1058 1059 struct nvme_pt_state { 1060 struct nvme_pt_command *pt; 1061 bool finished; 1062 }; 1063 1064 static void 1065 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1066 { 1067 struct nvme_softc *sc = q->q_sc; 1068 struct nvme_sqe *sqe = slot; 1069 struct nvme_pt_state *state = ccb->ccb_cookie; 1070 struct nvme_pt_command *pt = state->pt; 1071 bus_dmamap_t dmap = ccb->ccb_dmamap; 1072 int i; 1073 1074 sqe->opcode = pt->cmd.opcode; 1075 htolem32(&sqe->nsid, pt->cmd.nsid); 1076 1077 if (pt->buf != NULL && pt->len > 0) { 1078 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 1079 switch (dmap->dm_nsegs) { 1080 case 1: 1081 break; 1082 case 2: 1083 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 1084 break; 1085 default: 1086 for (i = 1; i < dmap->dm_nsegs; i++) { 1087 htolem64(&ccb->ccb_prpl[i - 1], 1088 dmap->dm_segs[i].ds_addr); 1089 } 1090 bus_dmamap_sync(sc->sc_dmat, 1091 NVME_DMA_MAP(q->q_ccb_prpls), 1092 ccb->ccb_prpl_off, 1093 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 1094 BUS_DMASYNC_PREWRITE); 1095 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 1096 break; 1097 } 1098 } 1099 1100 htolem32(&sqe->cdw10, pt->cmd.cdw10); 1101 htolem32(&sqe->cdw11, pt->cmd.cdw11); 1102 htolem32(&sqe->cdw12, pt->cmd.cdw12); 1103 htolem32(&sqe->cdw13, pt->cmd.cdw13); 1104 htolem32(&sqe->cdw14, pt->cmd.cdw14); 1105 htolem32(&sqe->cdw15, pt->cmd.cdw15); 1106 } 1107 1108 static void 1109 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 1110 { 1111 struct nvme_softc *sc = q->q_sc; 1112 struct nvme_pt_state *state = ccb->ccb_cookie; 1113 struct nvme_pt_command *pt = state->pt; 1114 bus_dmamap_t dmap = ccb->ccb_dmamap; 1115 1116 if (pt->buf != NULL && pt->len > 0) { 1117 if (dmap->dm_nsegs > 2) { 1118 bus_dmamap_sync(sc->sc_dmat, 1119 NVME_DMA_MAP(q->q_ccb_prpls), 1120 ccb->ccb_prpl_off, 1121 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 1122 BUS_DMASYNC_POSTWRITE); 1123 } 1124 1125 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1126 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 1127 bus_dmamap_unload(sc->sc_dmat, dmap); 1128 } 1129 1130 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 1131 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 1132 1133 state->finished = true; 1134 1135 nvme_ccb_put(q, ccb); 1136 } 1137 1138 static bool 1139 nvme_pt_finished(void *cookie) 1140 { 1141 struct nvme_pt_state *state = cookie; 1142 1143 return state->finished; 1144 } 1145 1146 static int 1147 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 1148 uint16_t nsid, struct lwp *l, bool is_adminq) 1149 { 1150 struct nvme_queue *q; 1151 struct nvme_ccb *ccb; 1152 void *buf = NULL; 1153 struct nvme_pt_state state; 1154 int error; 1155 1156 /* limit command size to maximum data transfer size */ 1157 if ((pt->buf == NULL && pt->len > 0) || 1158 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 1159 return EINVAL; 1160 1161 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc, NULL, true); 1162 ccb = nvme_ccb_get(q, true); 1163 KASSERT(ccb != NULL); 1164 1165 if (pt->buf != NULL) { 1166 KASSERT(pt->len > 0); 1167 buf = kmem_alloc(pt->len, KM_SLEEP); 1168 if (!pt->is_read) { 1169 error = copyin(pt->buf, buf, pt->len); 1170 if (error) 1171 goto kmem_free; 1172 } 1173 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 1174 pt->len, NULL, 1175 BUS_DMA_WAITOK | 1176 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 1177 if (error) 1178 goto kmem_free; 1179 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 1180 0, ccb->ccb_dmamap->dm_mapsize, 1181 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 1182 } 1183 1184 memset(&state, 0, sizeof(state)); 1185 state.pt = pt; 1186 state.finished = false; 1187 1188 ccb->ccb_done = nvme_pt_done; 1189 ccb->ccb_cookie = &state; 1190 1191 pt->cmd.nsid = nsid; 1192 1193 nvme_q_submit(sc, q, ccb, nvme_pt_fill); 1194 1195 /* wait for completion */ 1196 nvme_q_wait_complete(sc, q, nvme_pt_finished, &state); 1197 KASSERT(state.finished); 1198 1199 error = 0; 1200 1201 if (buf != NULL) { 1202 if (error == 0 && pt->is_read) 1203 error = copyout(buf, pt->buf, pt->len); 1204 kmem_free: 1205 kmem_free(buf, pt->len); 1206 } 1207 1208 return error; 1209 } 1210 1211 static void 1212 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1213 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 1214 { 1215 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 1216 uint32_t tail; 1217 1218 mutex_enter(&q->q_sq_mtx); 1219 tail = q->q_sq_tail; 1220 if (++q->q_sq_tail >= q->q_entries) 1221 q->q_sq_tail = 0; 1222 1223 sqe += tail; 1224 1225 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1226 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 1227 memset(sqe, 0, sizeof(*sqe)); 1228 (*fill)(q, ccb, sqe); 1229 htolem16(&sqe->cid, ccb->ccb_id); 1230 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1231 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 1232 1233 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 1234 mutex_exit(&q->q_sq_mtx); 1235 } 1236 1237 struct nvme_poll_state { 1238 struct nvme_sqe s; 1239 struct nvme_cqe c; 1240 void *cookie; 1241 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 1242 }; 1243 1244 static int 1245 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1246 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 1247 { 1248 struct nvme_poll_state state; 1249 uint16_t flags; 1250 int step = 10; 1251 int maxloop = timo_sec * 1000000 / step; 1252 int error = 0; 1253 1254 memset(&state, 0, sizeof(state)); 1255 (*fill)(q, ccb, &state.s); 1256 1257 state.done = ccb->ccb_done; 1258 state.cookie = ccb->ccb_cookie; 1259 1260 ccb->ccb_done = nvme_poll_done; 1261 ccb->ccb_cookie = &state; 1262 1263 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1264 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1265 if (nvme_q_complete(sc, q) == 0) 1266 delay(step); 1267 1268 if (timo_sec >= 0 && --maxloop <= 0) { 1269 error = ETIMEDOUT; 1270 break; 1271 } 1272 } 1273 1274 if (error == 0) { 1275 flags = lemtoh16(&state.c.flags); 1276 return flags & ~NVME_CQE_PHASE; 1277 } else { 1278 /* 1279 * If it succeds later, it would hit ccb which will have been 1280 * already reused for something else. Not good. Cross 1281 * fingers and hope for best. XXX do controller reset? 1282 */ 1283 aprint_error_dev(sc->sc_dev, "polled command timed out\n"); 1284 1285 /* Invoke the callback to clean state anyway */ 1286 struct nvme_cqe cqe; 1287 memset(&cqe, 0, sizeof(cqe)); 1288 ccb->ccb_done(q, ccb, &cqe); 1289 1290 return 1; 1291 } 1292 } 1293 1294 static void 1295 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1296 { 1297 struct nvme_sqe *sqe = slot; 1298 struct nvme_poll_state *state = ccb->ccb_cookie; 1299 1300 *sqe = state->s; 1301 } 1302 1303 static void 1304 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1305 struct nvme_cqe *cqe) 1306 { 1307 struct nvme_poll_state *state = ccb->ccb_cookie; 1308 1309 state->c = *cqe; 1310 SET(state->c.flags, htole16(NVME_CQE_PHASE)); 1311 1312 ccb->ccb_cookie = state->cookie; 1313 state->done(q, ccb, &state->c); 1314 } 1315 1316 static void 1317 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1318 { 1319 struct nvme_sqe *src = ccb->ccb_cookie; 1320 struct nvme_sqe *dst = slot; 1321 1322 *dst = *src; 1323 } 1324 1325 static void 1326 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1327 struct nvme_cqe *cqe) 1328 { 1329 } 1330 1331 static int 1332 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1333 { 1334 struct nvme_ccb *ccb; 1335 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1336 uint16_t flags; 1337 int rv = 0; 1338 1339 mutex_enter(&q->q_cq_mtx); 1340 1341 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1342 for (;;) { 1343 cqe = &ring[q->q_cq_head]; 1344 flags = lemtoh16(&cqe->flags); 1345 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1346 break; 1347 1348 ccb = &q->q_ccbs[cqe->cid]; 1349 1350 if (++q->q_cq_head >= q->q_entries) { 1351 q->q_cq_head = 0; 1352 q->q_cq_phase ^= NVME_CQE_PHASE; 1353 } 1354 1355 #ifdef DEBUG 1356 /* 1357 * If we get spurious completion notification, something 1358 * is seriously hosed up. Very likely DMA to some random 1359 * memory place happened, so just bail out. 1360 */ 1361 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1362 panic("%s: invalid ccb detected", 1363 device_xname(sc->sc_dev)); 1364 /* NOTREACHED */ 1365 } 1366 #endif 1367 1368 rv++; 1369 1370 /* 1371 * Unlock the mutex before calling the ccb_done callback 1372 * and re-lock afterwards. The callback triggers lddone() 1373 * which schedules another i/o, and also calls nvme_ccb_put(). 1374 * Unlock/relock avoids possibility of deadlock. 1375 */ 1376 mutex_exit(&q->q_cq_mtx); 1377 ccb->ccb_done(q, ccb, cqe); 1378 mutex_enter(&q->q_cq_mtx); 1379 } 1380 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1381 1382 if (rv) 1383 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1384 1385 mutex_exit(&q->q_cq_mtx); 1386 1387 return rv; 1388 } 1389 1390 static void 1391 nvme_q_wait_complete(struct nvme_softc *sc, 1392 struct nvme_queue *q, bool (*finished)(void *), void *cookie) 1393 { 1394 mutex_enter(&q->q_ccb_mtx); 1395 if (finished(cookie)) 1396 goto out; 1397 1398 for(;;) { 1399 q->q_ccb_waiting = true; 1400 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1401 1402 if (finished(cookie)) 1403 break; 1404 } 1405 1406 out: 1407 mutex_exit(&q->q_ccb_mtx); 1408 } 1409 1410 static int 1411 nvme_identify(struct nvme_softc *sc, u_int mps) 1412 { 1413 char sn[41], mn[81], fr[17]; 1414 struct nvm_identify_controller *identify; 1415 struct nvme_dmamem *mem; 1416 struct nvme_ccb *ccb; 1417 u_int mdts; 1418 int rv = 1; 1419 1420 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1421 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1422 1423 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1424 if (mem == NULL) 1425 return 1; 1426 1427 ccb->ccb_done = nvme_empty_done; 1428 ccb->ccb_cookie = mem; 1429 1430 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1431 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1432 NVME_TIMO_IDENT); 1433 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1434 1435 nvme_ccb_put(sc->sc_admin_q, ccb); 1436 1437 if (rv != 0) 1438 goto done; 1439 1440 identify = NVME_DMA_KVA(mem); 1441 sc->sc_identify = *identify; 1442 identify = NULL; 1443 1444 /* Convert data to host endian */ 1445 nvme_identify_controller_swapbytes(&sc->sc_identify); 1446 1447 strnvisx(sn, sizeof(sn), (const char *)sc->sc_identify.sn, 1448 sizeof(sc->sc_identify.sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1449 strnvisx(mn, sizeof(mn), (const char *)sc->sc_identify.mn, 1450 sizeof(sc->sc_identify.mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1451 strnvisx(fr, sizeof(fr), (const char *)sc->sc_identify.fr, 1452 sizeof(sc->sc_identify.fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1453 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1454 sn); 1455 1456 strlcpy(sc->sc_modelname, mn, sizeof(sc->sc_modelname)); 1457 1458 if (sc->sc_identify.mdts > 0) { 1459 mdts = (1 << sc->sc_identify.mdts) * (1 << mps); 1460 if (mdts < sc->sc_mdts) 1461 sc->sc_mdts = mdts; 1462 } 1463 1464 sc->sc_nn = sc->sc_identify.nn; 1465 1466 done: 1467 nvme_dmamem_free(sc, mem); 1468 1469 return rv; 1470 } 1471 1472 static int 1473 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1474 { 1475 struct nvme_sqe_q sqe; 1476 struct nvme_ccb *ccb; 1477 int rv; 1478 1479 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1480 return 1; 1481 1482 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1483 KASSERT(ccb != NULL); 1484 1485 ccb->ccb_done = nvme_empty_done; 1486 ccb->ccb_cookie = &sqe; 1487 1488 memset(&sqe, 0, sizeof(sqe)); 1489 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1490 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1491 htolem16(&sqe.qsize, q->q_entries - 1); 1492 htolem16(&sqe.qid, q->q_id); 1493 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1494 if (sc->sc_use_mq) 1495 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1496 1497 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1498 if (rv != 0) 1499 goto fail; 1500 1501 ccb->ccb_done = nvme_empty_done; 1502 ccb->ccb_cookie = &sqe; 1503 1504 memset(&sqe, 0, sizeof(sqe)); 1505 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1506 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1507 htolem16(&sqe.qsize, q->q_entries - 1); 1508 htolem16(&sqe.qid, q->q_id); 1509 htolem16(&sqe.cqid, q->q_id); 1510 sqe.qflags = NVM_SQE_Q_PC; 1511 1512 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1513 if (rv != 0) 1514 goto fail; 1515 1516 nvme_ccb_put(sc->sc_admin_q, ccb); 1517 return 0; 1518 1519 fail: 1520 if (sc->sc_use_mq) 1521 sc->sc_intr_disestablish(sc, q->q_id); 1522 1523 nvme_ccb_put(sc->sc_admin_q, ccb); 1524 return rv; 1525 } 1526 1527 static int 1528 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1529 { 1530 struct nvme_sqe_q sqe; 1531 struct nvme_ccb *ccb; 1532 int rv; 1533 1534 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1535 KASSERT(ccb != NULL); 1536 1537 ccb->ccb_done = nvme_empty_done; 1538 ccb->ccb_cookie = &sqe; 1539 1540 memset(&sqe, 0, sizeof(sqe)); 1541 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1542 htolem16(&sqe.qid, q->q_id); 1543 1544 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1545 if (rv != 0) 1546 goto fail; 1547 1548 ccb->ccb_done = nvme_empty_done; 1549 ccb->ccb_cookie = &sqe; 1550 1551 memset(&sqe, 0, sizeof(sqe)); 1552 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1553 htolem16(&sqe.qid, q->q_id); 1554 1555 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1556 if (rv != 0) 1557 goto fail; 1558 1559 fail: 1560 nvme_ccb_put(sc->sc_admin_q, ccb); 1561 1562 if (rv == 0 && sc->sc_use_mq) { 1563 if (sc->sc_intr_disestablish(sc, q->q_id)) 1564 rv = 1; 1565 } 1566 1567 return rv; 1568 } 1569 1570 static void 1571 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1572 { 1573 struct nvme_sqe *sqe = slot; 1574 struct nvme_dmamem *mem = ccb->ccb_cookie; 1575 1576 sqe->opcode = NVM_ADMIN_IDENTIFY; 1577 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1578 htolem32(&sqe->cdw10, 1); 1579 } 1580 1581 static int 1582 nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap) 1583 { 1584 struct nvme_pt_state state; 1585 struct nvme_pt_command pt; 1586 struct nvme_ccb *ccb; 1587 uint16_t ncqa, nsqa; 1588 int rv; 1589 1590 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1591 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1592 1593 memset(&pt, 0, sizeof(pt)); 1594 pt.cmd.opcode = NVM_ADMIN_GET_FEATURES; 1595 pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES; 1596 1597 memset(&state, 0, sizeof(state)); 1598 state.pt = &pt; 1599 state.finished = false; 1600 1601 ccb->ccb_done = nvme_pt_done; 1602 ccb->ccb_cookie = &state; 1603 1604 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1605 1606 if (rv != 0) { 1607 *nqap = 0; 1608 return EIO; 1609 } 1610 1611 ncqa = pt.cpl.cdw0 >> 16; 1612 nsqa = pt.cpl.cdw0 & 0xffff; 1613 *nqap = MIN(ncqa, nsqa) + 1; 1614 1615 return 0; 1616 } 1617 1618 static int 1619 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1620 { 1621 struct nvme_softc *sc = q->q_sc; 1622 struct nvme_ccb *ccb; 1623 bus_addr_t off; 1624 uint64_t *prpl; 1625 u_int i; 1626 1627 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1628 cv_init(&q->q_ccb_wait, "nvmeqw"); 1629 q->q_ccb_waiting = false; 1630 SIMPLEQ_INIT(&q->q_ccb_list); 1631 1632 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1633 1634 q->q_nccbs = nccbs; 1635 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1636 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1637 1638 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1639 off = 0; 1640 1641 for (i = 0; i < nccbs; i++) { 1642 ccb = &q->q_ccbs[i]; 1643 1644 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1645 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1646 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1647 &ccb->ccb_dmamap) != 0) 1648 goto free_maps; 1649 1650 ccb->ccb_id = i; 1651 ccb->ccb_prpl = prpl; 1652 ccb->ccb_prpl_off = off; 1653 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1654 1655 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1656 1657 prpl += sc->sc_max_sgl; 1658 off += sizeof(*prpl) * sc->sc_max_sgl; 1659 } 1660 1661 return 0; 1662 1663 free_maps: 1664 nvme_ccbs_free(q); 1665 return 1; 1666 } 1667 1668 static struct nvme_ccb * 1669 nvme_ccb_get(struct nvme_queue *q, bool wait) 1670 { 1671 struct nvme_ccb *ccb = NULL; 1672 1673 mutex_enter(&q->q_ccb_mtx); 1674 again: 1675 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1676 if (ccb != NULL) { 1677 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1678 #ifdef DEBUG 1679 ccb->ccb_cookie = NULL; 1680 #endif 1681 } else { 1682 if (__predict_false(wait)) { 1683 q->q_ccb_waiting = true; 1684 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1685 goto again; 1686 } 1687 } 1688 mutex_exit(&q->q_ccb_mtx); 1689 1690 return ccb; 1691 } 1692 1693 static void 1694 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1695 { 1696 1697 mutex_enter(&q->q_ccb_mtx); 1698 #ifdef DEBUG 1699 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1700 #endif 1701 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1702 1703 /* It's unlikely there are any waiters, it's not used for regular I/O */ 1704 if (__predict_false(q->q_ccb_waiting)) { 1705 q->q_ccb_waiting = false; 1706 cv_broadcast(&q->q_ccb_wait); 1707 } 1708 1709 mutex_exit(&q->q_ccb_mtx); 1710 } 1711 1712 static void 1713 nvme_ccbs_free(struct nvme_queue *q) 1714 { 1715 struct nvme_softc *sc = q->q_sc; 1716 struct nvme_ccb *ccb; 1717 1718 mutex_enter(&q->q_ccb_mtx); 1719 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1720 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1721 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1722 } 1723 mutex_exit(&q->q_ccb_mtx); 1724 1725 nvme_dmamem_free(sc, q->q_ccb_prpls); 1726 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1727 q->q_ccbs = NULL; 1728 cv_destroy(&q->q_ccb_wait); 1729 mutex_destroy(&q->q_ccb_mtx); 1730 } 1731 1732 static struct nvme_queue * 1733 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1734 { 1735 struct nvme_queue *q; 1736 1737 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1738 q->q_sc = sc; 1739 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1740 sizeof(struct nvme_sqe) * entries); 1741 if (q->q_sq_dmamem == NULL) 1742 goto free; 1743 1744 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1745 sizeof(struct nvme_cqe) * entries); 1746 if (q->q_cq_dmamem == NULL) 1747 goto free_sq; 1748 1749 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1750 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1751 1752 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1753 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1754 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1755 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1756 q->q_id = id; 1757 q->q_entries = entries; 1758 q->q_sq_tail = 0; 1759 q->q_cq_head = 0; 1760 q->q_cq_phase = NVME_CQE_PHASE; 1761 1762 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1763 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1764 1765 /* 1766 * Due to definition of full and empty queue (queue is empty 1767 * when head == tail, full when tail is one less then head), 1768 * we can actually only have (entries - 1) in-flight commands. 1769 */ 1770 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1771 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1772 goto free_cq; 1773 } 1774 1775 return q; 1776 1777 free_cq: 1778 nvme_dmamem_free(sc, q->q_cq_dmamem); 1779 free_sq: 1780 nvme_dmamem_free(sc, q->q_sq_dmamem); 1781 free: 1782 kmem_free(q, sizeof(*q)); 1783 1784 return NULL; 1785 } 1786 1787 static void 1788 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1789 { 1790 nvme_ccbs_free(q); 1791 mutex_destroy(&q->q_sq_mtx); 1792 mutex_destroy(&q->q_cq_mtx); 1793 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1794 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1795 nvme_dmamem_free(sc, q->q_cq_dmamem); 1796 nvme_dmamem_free(sc, q->q_sq_dmamem); 1797 kmem_free(q, sizeof(*q)); 1798 } 1799 1800 int 1801 nvme_intr(void *xsc) 1802 { 1803 struct nvme_softc *sc = xsc; 1804 1805 /* 1806 * INTx is level triggered, controller deasserts the interrupt only 1807 * when we advance command queue head via write to the doorbell. 1808 * Tell the controller to block the interrupts while we process 1809 * the queue(s). 1810 */ 1811 nvme_write4(sc, NVME_INTMS, 1); 1812 1813 softint_schedule(sc->sc_softih[0]); 1814 1815 /* don't know, might not have been for us */ 1816 return 1; 1817 } 1818 1819 void 1820 nvme_softintr_intx(void *xq) 1821 { 1822 struct nvme_queue *q = xq; 1823 struct nvme_softc *sc = q->q_sc; 1824 1825 nvme_q_complete(sc, sc->sc_admin_q); 1826 if (sc->sc_q != NULL) 1827 nvme_q_complete(sc, sc->sc_q[0]); 1828 1829 /* 1830 * Processing done, tell controller to issue interrupts again. There 1831 * is no race, as NVMe spec requires the controller to maintain state, 1832 * and assert the interrupt whenever there are unacknowledged 1833 * completion queue entries. 1834 */ 1835 nvme_write4(sc, NVME_INTMC, 1); 1836 } 1837 1838 int 1839 nvme_intr_msi(void *xq) 1840 { 1841 struct nvme_queue *q = xq; 1842 1843 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1844 && q->q_sc->sc_softih[q->q_id]); 1845 1846 /* 1847 * MSI/MSI-X are edge triggered, so can handover processing to softint 1848 * without masking the interrupt. 1849 */ 1850 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1851 1852 return 1; 1853 } 1854 1855 void 1856 nvme_softintr_msi(void *xq) 1857 { 1858 struct nvme_queue *q = xq; 1859 struct nvme_softc *sc = q->q_sc; 1860 1861 nvme_q_complete(sc, q); 1862 } 1863 1864 static struct nvme_dmamem * 1865 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1866 { 1867 struct nvme_dmamem *ndm; 1868 int nsegs; 1869 1870 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1871 if (ndm == NULL) 1872 return NULL; 1873 1874 ndm->ndm_size = size; 1875 1876 if (bus_dmamap_create(sc->sc_dmat, size, btoc(round_page(size)), size, 0, 1877 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1878 goto ndmfree; 1879 1880 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1881 1, &nsegs, BUS_DMA_WAITOK) != 0) 1882 goto destroy; 1883 1884 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1885 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1886 goto free; 1887 memset(ndm->ndm_kva, 0, size); 1888 1889 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1890 NULL, BUS_DMA_WAITOK) != 0) 1891 goto unmap; 1892 1893 return ndm; 1894 1895 unmap: 1896 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1897 free: 1898 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1899 destroy: 1900 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1901 ndmfree: 1902 kmem_free(ndm, sizeof(*ndm)); 1903 return NULL; 1904 } 1905 1906 static void 1907 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1908 { 1909 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1910 0, NVME_DMA_LEN(mem), ops); 1911 } 1912 1913 void 1914 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1915 { 1916 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1917 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1918 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1919 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1920 kmem_free(ndm, sizeof(*ndm)); 1921 } 1922 1923 /* 1924 * ioctl 1925 */ 1926 1927 dev_type_open(nvmeopen); 1928 dev_type_close(nvmeclose); 1929 dev_type_ioctl(nvmeioctl); 1930 1931 const struct cdevsw nvme_cdevsw = { 1932 .d_open = nvmeopen, 1933 .d_close = nvmeclose, 1934 .d_read = noread, 1935 .d_write = nowrite, 1936 .d_ioctl = nvmeioctl, 1937 .d_stop = nostop, 1938 .d_tty = notty, 1939 .d_poll = nopoll, 1940 .d_mmap = nommap, 1941 .d_kqfilter = nokqfilter, 1942 .d_discard = nodiscard, 1943 .d_flag = D_OTHER, 1944 }; 1945 1946 /* 1947 * Accept an open operation on the control device. 1948 */ 1949 int 1950 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1951 { 1952 struct nvme_softc *sc; 1953 int unit = minor(dev) / 0x10000; 1954 int nsid = minor(dev) & 0xffff; 1955 int nsidx; 1956 1957 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1958 return ENXIO; 1959 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1960 return ENXIO; 1961 1962 if (nsid == 0) { 1963 /* controller */ 1964 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1965 return EBUSY; 1966 SET(sc->sc_flags, NVME_F_OPEN); 1967 } else { 1968 /* namespace */ 1969 nsidx = nsid - 1; 1970 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1971 return ENXIO; 1972 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1973 return EBUSY; 1974 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1975 } 1976 return 0; 1977 } 1978 1979 /* 1980 * Accept the last close on the control device. 1981 */ 1982 int 1983 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1984 { 1985 struct nvme_softc *sc; 1986 int unit = minor(dev) / 0x10000; 1987 int nsid = minor(dev) & 0xffff; 1988 int nsidx; 1989 1990 sc = device_lookup_private(&nvme_cd, unit); 1991 if (sc == NULL) 1992 return ENXIO; 1993 1994 if (nsid == 0) { 1995 /* controller */ 1996 CLR(sc->sc_flags, NVME_F_OPEN); 1997 } else { 1998 /* namespace */ 1999 nsidx = nsid - 1; 2000 if (nsidx >= sc->sc_nn) 2001 return ENXIO; 2002 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 2003 } 2004 2005 return 0; 2006 } 2007 2008 /* 2009 * Handle control operations. 2010 */ 2011 int 2012 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 2013 { 2014 struct nvme_softc *sc; 2015 int unit = minor(dev) / 0x10000; 2016 int nsid = minor(dev) & 0xffff; 2017 struct nvme_pt_command *pt; 2018 2019 sc = device_lookup_private(&nvme_cd, unit); 2020 if (sc == NULL) 2021 return ENXIO; 2022 2023 switch (cmd) { 2024 case NVME_PASSTHROUGH_CMD: 2025 pt = data; 2026 return nvme_command_passthrough(sc, data, 2027 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 2028 } 2029 2030 return ENOTTY; 2031 } 2032
Indexes created Sat Oct 25 16:10:12 GMT 2025