nvme.c revision 1.44.2.6
1 /* $NetBSD: nvme.c,v 1.44.2.6 2021/06/21 17:25:48 martin Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.44.2.6 2021/06/21 17:25:48 martin Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 #include "ioconf.h" 44 45 #define B4_CHK_RDY_DELAY_MS 2300 /* workaround controller bug */ 46 47 int nvme_adminq_size = 32; 48 int nvme_ioq_size = 1024; 49 50 static int nvme_print(void *, const char *); 51 52 static int nvme_ready(struct nvme_softc *, uint32_t); 53 static int nvme_enable(struct nvme_softc *, u_int); 54 static int nvme_disable(struct nvme_softc *); 55 static int nvme_shutdown(struct nvme_softc *); 56 57 #ifdef NVME_DEBUG 58 static void nvme_dumpregs(struct nvme_softc *); 59 #endif 60 static int nvme_identify(struct nvme_softc *, u_int); 61 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 62 void *); 63 64 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 65 static void nvme_ccbs_free(struct nvme_queue *); 66 67 static struct nvme_ccb * 68 nvme_ccb_get(struct nvme_queue *, bool); 69 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 70 71 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 72 struct nvme_ccb *, void (*)(struct nvme_queue *, 73 struct nvme_ccb *, void *), int); 74 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 76 struct nvme_cqe *); 77 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 78 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 79 struct nvme_cqe *); 80 81 static struct nvme_queue * 82 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 83 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 84 static void nvme_q_reset(struct nvme_softc *, struct nvme_queue *); 85 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 86 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 87 struct nvme_ccb *, void (*)(struct nvme_queue *, 88 struct nvme_ccb *, void *)); 89 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 90 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 91 static void nvme_q_wait_complete(struct nvme_softc *, struct nvme_queue *, 92 bool (*)(void *), void *); 93 94 static struct nvme_dmamem * 95 nvme_dmamem_alloc(struct nvme_softc *, size_t); 96 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 97 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 98 int); 99 100 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 101 void *); 102 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 103 struct nvme_cqe *); 104 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 105 void *); 106 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 107 struct nvme_cqe *); 108 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *, 109 void *); 110 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *, 111 struct nvme_cqe *); 112 113 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 114 void *); 115 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 116 struct nvme_cqe *); 117 static int nvme_command_passthrough(struct nvme_softc *, 118 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 119 120 static int nvme_set_number_of_queues(struct nvme_softc *, u_int, u_int *, 121 u_int *); 122 123 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 124 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 125 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 126 #define NVME_TIMO_SY 60 /* sync cache timeout */ 127 128 #define nvme_read4(_s, _r) \ 129 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 130 #define nvme_write4(_s, _r, _v) \ 131 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 132 /* 133 * Some controllers, at least Apple NVMe, always require split 134 * transfers, so don't use bus_space_{read,write}_8() on LP64. 135 */ 136 static inline uint64_t 137 nvme_read8(struct nvme_softc *sc, bus_size_t r) 138 { 139 uint64_t v; 140 uint32_t *a = (uint32_t *)&v; 141 142 #if _BYTE_ORDER == _LITTLE_ENDIAN 143 a[0] = nvme_read4(sc, r); 144 a[1] = nvme_read4(sc, r + 4); 145 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 146 a[1] = nvme_read4(sc, r); 147 a[0] = nvme_read4(sc, r + 4); 148 #endif 149 150 return v; 151 } 152 153 static inline void 154 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 155 { 156 uint32_t *a = (uint32_t *)&v; 157 158 #if _BYTE_ORDER == _LITTLE_ENDIAN 159 nvme_write4(sc, r, a[0]); 160 nvme_write4(sc, r + 4, a[1]); 161 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 162 nvme_write4(sc, r, a[1]); 163 nvme_write4(sc, r + 4, a[0]); 164 #endif 165 } 166 #define nvme_barrier(_s, _r, _l, _f) \ 167 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 168 169 #ifdef NVME_DEBUG 170 static __used void 171 nvme_dumpregs(struct nvme_softc *sc) 172 { 173 uint64_t r8; 174 uint32_t r4; 175 176 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 177 r8 = nvme_read8(sc, NVME_CAP); 178 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 179 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 180 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 181 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 182 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 183 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 184 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 185 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 186 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 187 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 188 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 189 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 190 191 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 192 193 r4 = nvme_read4(sc, NVME_CC); 194 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 195 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 196 (1 << NVME_CC_IOCQES_R(r4))); 197 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 198 (1 << NVME_CC_IOSQES_R(r4))); 199 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 200 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 201 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 202 (1 << NVME_CC_MPS_R(r4))); 203 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 204 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 205 206 r4 = nvme_read4(sc, NVME_CSTS); 207 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 208 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 209 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 210 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 211 212 r4 = nvme_read4(sc, NVME_AQA); 213 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 214 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 215 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 216 217 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 218 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 219 #undef DEVNAME 220 } 221 #endif /* NVME_DEBUG */ 222 223 static int 224 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 225 { 226 u_int i = 0; 227 228 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 229 if (i++ > sc->sc_rdy_to) 230 return ENXIO; 231 232 delay(1000); 233 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 234 } 235 236 return 0; 237 } 238 239 static int 240 nvme_enable(struct nvme_softc *sc, u_int mps) 241 { 242 uint32_t cc, csts; 243 int error; 244 245 cc = nvme_read4(sc, NVME_CC); 246 csts = nvme_read4(sc, NVME_CSTS); 247 248 /* 249 * See note in nvme_disable. Short circuit if we're already enabled. 250 */ 251 if (ISSET(cc, NVME_CC_EN)) { 252 if (ISSET(csts, NVME_CSTS_RDY)) 253 return 0; 254 255 goto waitready; 256 } else { 257 /* EN == 0 already wait for RDY == 0 or fail */ 258 error = nvme_ready(sc, 0); 259 if (error) 260 return error; 261 } 262 263 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 264 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 265 delay(5000); 266 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 267 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 268 delay(5000); 269 270 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 271 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 272 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 273 delay(5000); 274 275 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 276 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 277 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 278 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 279 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 280 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 281 SET(cc, NVME_CC_MPS(mps)); 282 SET(cc, NVME_CC_EN); 283 284 nvme_write4(sc, NVME_CC, cc); 285 nvme_barrier(sc, 0, sc->sc_ios, 286 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 287 288 waitready: 289 return nvme_ready(sc, NVME_CSTS_RDY); 290 } 291 292 static int 293 nvme_disable(struct nvme_softc *sc) 294 { 295 uint32_t cc, csts; 296 int error; 297 298 cc = nvme_read4(sc, NVME_CC); 299 csts = nvme_read4(sc, NVME_CSTS); 300 301 /* 302 * Per 3.1.5 in NVME 1.3 spec, transitioning CC.EN from 0 to 1 303 * when CSTS.RDY is 1 or transitioning CC.EN from 1 to 0 when 304 * CSTS.RDY is 0 "has undefined results" So make sure that CSTS.RDY 305 * isn't the desired value. Short circuit if we're already disabled. 306 */ 307 if (ISSET(cc, NVME_CC_EN)) { 308 if (!ISSET(csts, NVME_CSTS_RDY)) { 309 /* EN == 1, wait for RDY == 1 or fail */ 310 error = nvme_ready(sc, NVME_CSTS_RDY); 311 if (error) 312 return error; 313 } 314 } else { 315 /* EN == 0 already wait for RDY == 0 */ 316 if (!ISSET(csts, NVME_CSTS_RDY)) 317 return 0; 318 319 goto waitready; 320 } 321 322 CLR(cc, NVME_CC_EN); 323 nvme_write4(sc, NVME_CC, cc); 324 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 325 326 /* 327 * Some drives have issues with accessing the mmio after we disable, 328 * so delay for a bit after we write the bit to cope with these issues. 329 */ 330 if (ISSET(sc->sc_quirks, NVME_QUIRK_DELAY_B4_CHK_RDY)) 331 delay(B4_CHK_RDY_DELAY_MS); 332 333 waitready: 334 return nvme_ready(sc, 0); 335 } 336 337 int 338 nvme_attach(struct nvme_softc *sc) 339 { 340 uint64_t cap; 341 uint32_t reg; 342 u_int mps = PAGE_SHIFT; 343 u_int ncq, nsq; 344 uint16_t adminq_entries = nvme_adminq_size; 345 uint16_t ioq_entries = nvme_ioq_size; 346 int i; 347 348 reg = nvme_read4(sc, NVME_VS); 349 if (reg == 0xffffffff) { 350 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 351 return 1; 352 } 353 354 if (NVME_VS_TER(reg) == 0) 355 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg), 356 NVME_VS_MNR(reg)); 357 else 358 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg), 359 NVME_VS_MNR(reg), NVME_VS_TER(reg)); 360 361 cap = nvme_read8(sc, NVME_CAP); 362 sc->sc_dstrd = NVME_CAP_DSTRD(cap); 363 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 364 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 365 "is greater than CPU page size %u\n", 366 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 367 return 1; 368 } 369 if (NVME_CAP_MPSMAX(cap) < mps) 370 mps = NVME_CAP_MPSMAX(cap); 371 if (ioq_entries > NVME_CAP_MQES(cap)) 372 ioq_entries = NVME_CAP_MQES(cap); 373 374 /* set initial values to be used for admin queue during probe */ 375 sc->sc_rdy_to = NVME_CAP_TO(cap); 376 sc->sc_mps = 1 << mps; 377 sc->sc_mdts = MAXPHYS; 378 sc->sc_max_sgl = btoc(round_page(sc->sc_mdts)); 379 380 if (nvme_disable(sc) != 0) { 381 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 382 return 1; 383 } 384 385 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, 386 sc->sc_dstrd); 387 if (sc->sc_admin_q == NULL) { 388 aprint_error_dev(sc->sc_dev, 389 "unable to allocate admin queue\n"); 390 return 1; 391 } 392 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 393 goto free_admin_q; 394 395 if (nvme_enable(sc, mps) != 0) { 396 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 397 goto disestablish_admin_q; 398 } 399 400 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 401 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 402 goto disable; 403 } 404 if (sc->sc_nn == 0) { 405 aprint_error_dev(sc->sc_dev, "namespace not found\n"); 406 goto disable; 407 } 408 409 /* we know how big things are now */ 410 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 411 412 /* reallocate ccbs of admin queue with new max sgl. */ 413 nvme_ccbs_free(sc->sc_admin_q); 414 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 415 416 if (sc->sc_use_mq) { 417 /* Limit the number of queues to the number allocated in HW */ 418 if (nvme_set_number_of_queues(sc, sc->sc_nq, &ncq, &nsq) != 0) { 419 aprint_error_dev(sc->sc_dev, 420 "unable to get number of queues\n"); 421 goto disable; 422 } 423 if (sc->sc_nq > ncq) 424 sc->sc_nq = ncq; 425 if (sc->sc_nq > nsq) 426 sc->sc_nq = nsq; 427 } 428 429 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 430 for (i = 0; i < sc->sc_nq; i++) { 431 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, 432 sc->sc_dstrd); 433 if (sc->sc_q[i] == NULL) { 434 aprint_error_dev(sc->sc_dev, 435 "unable to allocate io queue\n"); 436 goto free_q; 437 } 438 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 439 aprint_error_dev(sc->sc_dev, 440 "unable to create io queue\n"); 441 nvme_q_free(sc, sc->sc_q[i]); 442 goto free_q; 443 } 444 } 445 446 if (!sc->sc_use_mq) 447 nvme_write4(sc, NVME_INTMC, 1); 448 449 /* probe subdevices */ 450 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 451 KM_SLEEP); 452 nvme_rescan(sc->sc_dev, "nvme", &i); 453 454 return 0; 455 456 free_q: 457 while (--i >= 0) { 458 nvme_q_delete(sc, sc->sc_q[i]); 459 nvme_q_free(sc, sc->sc_q[i]); 460 } 461 disable: 462 nvme_disable(sc); 463 disestablish_admin_q: 464 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 465 free_admin_q: 466 nvme_q_free(sc, sc->sc_admin_q); 467 468 return 1; 469 } 470 471 int 472 nvme_rescan(device_t self, const char *attr, const int *flags) 473 { 474 struct nvme_softc *sc = device_private(self); 475 struct nvme_attach_args naa; 476 struct nvm_namespace_format *f; 477 struct nvme_namespace *ns; 478 uint64_t cap; 479 int ioq_entries = nvme_ioq_size; 480 int i; 481 int error; 482 483 cap = nvme_read8(sc, NVME_CAP); 484 if (ioq_entries > NVME_CAP_MQES(cap)) 485 ioq_entries = NVME_CAP_MQES(cap); 486 487 for (i = 1; i <= sc->sc_nn; i++) { 488 if (sc->sc_namespaces[i - 1].dev) 489 continue; 490 491 /* identify to check for availability */ 492 error = nvme_ns_identify(sc, i); 493 if (error) { 494 aprint_error_dev(self, "couldn't identify namespace #%d\n", i); 495 continue; 496 } 497 498 ns = nvme_ns_get(sc, i); 499 KASSERT(ns); 500 501 f = &ns->ident->lbaf[NVME_ID_NS_FLBAS(ns->ident->flbas)]; 502 503 /* 504 * NVME1.0e 6.11 Identify command 505 * 506 * LBADS values smaller than 9 are not supported, a value 507 * of zero means that the format is not used. 508 */ 509 if (f->lbads < 9) { 510 if (f->lbads > 0) 511 aprint_error_dev(self, 512 "unsupported logical data size %u\n", f->lbads); 513 continue; 514 } 515 516 memset(&naa, 0, sizeof(naa)); 517 naa.naa_nsid = i; 518 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 519 naa.naa_maxphys = sc->sc_mdts; 520 naa.naa_typename = sc->sc_modelname; 521 sc->sc_namespaces[i - 1].dev = config_found(sc->sc_dev, &naa, 522 nvme_print); 523 } 524 return 0; 525 } 526 527 static int 528 nvme_print(void *aux, const char *pnp) 529 { 530 struct nvme_attach_args *naa = aux; 531 532 if (pnp) 533 aprint_normal("at %s", pnp); 534 535 if (naa->naa_nsid > 0) 536 aprint_normal(" nsid %d", naa->naa_nsid); 537 538 return UNCONF; 539 } 540 541 int 542 nvme_detach(struct nvme_softc *sc, int flags) 543 { 544 int i, error; 545 546 error = config_detach_children(sc->sc_dev, flags); 547 if (error) 548 return error; 549 550 error = nvme_shutdown(sc); 551 if (error) 552 return error; 553 554 /* from now on we are committed to detach, following will never fail */ 555 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 556 for (i = 0; i < sc->sc_nq; i++) 557 nvme_q_free(sc, sc->sc_q[i]); 558 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 559 nvme_q_free(sc, sc->sc_admin_q); 560 561 return 0; 562 } 563 564 int 565 nvme_suspend(struct nvme_softc *sc) 566 { 567 568 return nvme_shutdown(sc); 569 } 570 571 int 572 nvme_resume(struct nvme_softc *sc) 573 { 574 int ioq_entries = nvme_ioq_size; 575 uint64_t cap; 576 int i, error; 577 578 error = nvme_disable(sc); 579 if (error) { 580 device_printf(sc->sc_dev, "unable to disable controller\n"); 581 return error; 582 } 583 584 nvme_q_reset(sc, sc->sc_admin_q); 585 586 error = nvme_enable(sc, ffs(sc->sc_mps) - 1); 587 if (error) { 588 device_printf(sc->sc_dev, "unable to enable controller\n"); 589 return error; 590 } 591 592 for (i = 0; i < sc->sc_nq; i++) { 593 cap = nvme_read8(sc, NVME_CAP); 594 if (ioq_entries > NVME_CAP_MQES(cap)) 595 ioq_entries = NVME_CAP_MQES(cap); 596 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, 597 sc->sc_dstrd); 598 if (sc->sc_q[i] == NULL) { 599 error = ENOMEM; 600 device_printf(sc->sc_dev, "unable to allocate io q %d" 601 "\n", i); 602 goto disable; 603 } 604 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 605 error = EIO; 606 device_printf(sc->sc_dev, "unable to create io q %d" 607 "\n", i); 608 nvme_q_free(sc, sc->sc_q[i]); 609 goto free_q; 610 } 611 } 612 613 nvme_write4(sc, NVME_INTMC, 1); 614 615 return 0; 616 617 free_q: 618 while (i --> 0) 619 nvme_q_free(sc, sc->sc_q[i]); 620 disable: 621 (void)nvme_disable(sc); 622 623 return error; 624 } 625 626 static int 627 nvme_shutdown(struct nvme_softc *sc) 628 { 629 uint32_t cc, csts; 630 bool disabled = false; 631 int i; 632 633 if (!sc->sc_use_mq) 634 nvme_write4(sc, NVME_INTMS, 1); 635 636 for (i = 0; i < sc->sc_nq; i++) { 637 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 638 aprint_error_dev(sc->sc_dev, 639 "unable to delete io queue %d, disabling\n", i + 1); 640 disabled = true; 641 } 642 } 643 if (disabled) 644 goto disable; 645 646 cc = nvme_read4(sc, NVME_CC); 647 CLR(cc, NVME_CC_SHN_MASK); 648 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 649 nvme_write4(sc, NVME_CC, cc); 650 651 for (i = 0; i < 4000; i++) { 652 nvme_barrier(sc, 0, sc->sc_ios, 653 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 654 csts = nvme_read4(sc, NVME_CSTS); 655 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 656 return 0; 657 658 delay(1000); 659 } 660 661 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 662 663 disable: 664 nvme_disable(sc); 665 return 0; 666 } 667 668 void 669 nvme_childdet(device_t self, device_t child) 670 { 671 struct nvme_softc *sc = device_private(self); 672 int i; 673 674 for (i = 0; i < sc->sc_nn; i++) { 675 if (sc->sc_namespaces[i].dev == child) { 676 /* Already freed ns->ident. */ 677 sc->sc_namespaces[i].dev = NULL; 678 break; 679 } 680 } 681 } 682 683 int 684 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 685 { 686 struct nvme_sqe sqe; 687 struct nvm_identify_namespace *identify; 688 struct nvme_dmamem *mem; 689 struct nvme_ccb *ccb; 690 struct nvme_namespace *ns; 691 int rv; 692 693 KASSERT(nsid > 0); 694 695 ns = nvme_ns_get(sc, nsid); 696 KASSERT(ns); 697 698 if (ns->ident != NULL) 699 return 0; 700 701 ccb = nvme_ccb_get(sc->sc_admin_q, false); 702 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 703 704 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 705 if (mem == NULL) { 706 nvme_ccb_put(sc->sc_admin_q, ccb); 707 return ENOMEM; 708 } 709 710 memset(&sqe, 0, sizeof(sqe)); 711 sqe.opcode = NVM_ADMIN_IDENTIFY; 712 htolem32(&sqe.nsid, nsid); 713 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 714 htolem32(&sqe.cdw10, 0); 715 716 ccb->ccb_done = nvme_empty_done; 717 ccb->ccb_cookie = &sqe; 718 719 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 720 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 721 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 722 723 nvme_ccb_put(sc->sc_admin_q, ccb); 724 725 if (rv != 0) { 726 rv = EIO; 727 goto done; 728 } 729 730 /* commit */ 731 732 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 733 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 734 735 /* Convert data to host endian */ 736 nvme_identify_namespace_swapbytes(identify); 737 738 ns->ident = identify; 739 740 done: 741 nvme_dmamem_free(sc, mem); 742 743 return rv; 744 } 745 746 int 747 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 748 struct buf *bp, void *data, size_t datasize, 749 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 750 { 751 struct nvme_queue *q = nvme_get_q(sc, bp, false); 752 struct nvme_ccb *ccb; 753 bus_dmamap_t dmap; 754 int i, error; 755 756 ccb = nvme_ccb_get(q, false); 757 if (ccb == NULL) 758 return EAGAIN; 759 760 ccb->ccb_done = nvme_ns_io_done; 761 ccb->ccb_cookie = cookie; 762 763 /* namespace context */ 764 ccb->nnc_nsid = nsid; 765 ccb->nnc_flags = flags; 766 ccb->nnc_buf = bp; 767 ccb->nnc_datasize = datasize; 768 ccb->nnc_secsize = secsize; 769 ccb->nnc_blkno = blkno; 770 ccb->nnc_done = nnc_done; 771 772 dmap = ccb->ccb_dmamap; 773 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 774 datasize, NULL, 775 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 776 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 777 (ISSET(flags, NVME_NS_CTX_F_READ) ? 778 BUS_DMA_READ : BUS_DMA_WRITE)); 779 if (error) { 780 nvme_ccb_put(q, ccb); 781 return error; 782 } 783 784 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 785 ISSET(flags, NVME_NS_CTX_F_READ) ? 786 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 787 788 if (dmap->dm_nsegs > 2) { 789 for (i = 1; i < dmap->dm_nsegs; i++) { 790 htolem64(&ccb->ccb_prpl[i - 1], 791 dmap->dm_segs[i].ds_addr); 792 } 793 bus_dmamap_sync(sc->sc_dmat, 794 NVME_DMA_MAP(q->q_ccb_prpls), 795 ccb->ccb_prpl_off, 796 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 797 BUS_DMASYNC_PREWRITE); 798 } 799 800 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 801 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 802 return EIO; 803 return 0; 804 } 805 806 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 807 return 0; 808 } 809 810 static void 811 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 812 { 813 struct nvme_sqe_io *sqe = slot; 814 bus_dmamap_t dmap = ccb->ccb_dmamap; 815 816 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 817 NVM_CMD_READ : NVM_CMD_WRITE; 818 htolem32(&sqe->nsid, ccb->nnc_nsid); 819 820 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 821 switch (dmap->dm_nsegs) { 822 case 1: 823 break; 824 case 2: 825 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 826 break; 827 default: 828 /* the prp list is already set up and synced */ 829 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 830 break; 831 } 832 833 htolem64(&sqe->slba, ccb->nnc_blkno); 834 835 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) 836 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA); 837 838 /* guaranteed by upper layers, but check just in case */ 839 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 840 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 841 } 842 843 static void 844 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 845 struct nvme_cqe *cqe) 846 { 847 struct nvme_softc *sc = q->q_sc; 848 bus_dmamap_t dmap = ccb->ccb_dmamap; 849 void *nnc_cookie = ccb->ccb_cookie; 850 nvme_nnc_done nnc_done = ccb->nnc_done; 851 struct buf *bp = ccb->nnc_buf; 852 853 if (dmap->dm_nsegs > 2) { 854 bus_dmamap_sync(sc->sc_dmat, 855 NVME_DMA_MAP(q->q_ccb_prpls), 856 ccb->ccb_prpl_off, 857 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 858 BUS_DMASYNC_POSTWRITE); 859 } 860 861 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 862 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 863 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 864 865 bus_dmamap_unload(sc->sc_dmat, dmap); 866 nvme_ccb_put(q, ccb); 867 868 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 869 } 870 871 /* 872 * If there is no volatile write cache, it makes no sense to issue 873 * flush commands or query for the status. 874 */ 875 static bool 876 nvme_has_volatile_write_cache(struct nvme_softc *sc) 877 { 878 /* sc_identify is filled during attachment */ 879 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0); 880 } 881 882 static bool 883 nvme_ns_sync_finished(void *cookie) 884 { 885 int *result = cookie; 886 887 return (*result != 0); 888 } 889 890 int 891 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, int flags) 892 { 893 struct nvme_queue *q = nvme_get_q(sc, NULL, true); 894 struct nvme_ccb *ccb; 895 int result = 0; 896 897 if (!nvme_has_volatile_write_cache(sc)) { 898 /* cache not present, no value in trying to flush it */ 899 return 0; 900 } 901 902 ccb = nvme_ccb_get(q, true); 903 KASSERT(ccb != NULL); 904 905 ccb->ccb_done = nvme_ns_sync_done; 906 ccb->ccb_cookie = &result; 907 908 /* namespace context */ 909 ccb->nnc_nsid = nsid; 910 ccb->nnc_flags = flags; 911 ccb->nnc_done = NULL; 912 913 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 914 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 915 return EIO; 916 return 0; 917 } 918 919 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 920 921 /* wait for completion */ 922 nvme_q_wait_complete(sc, q, nvme_ns_sync_finished, &result); 923 KASSERT(result != 0); 924 925 return (result > 0) ? 0 : EIO; 926 } 927 928 static void 929 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 930 { 931 struct nvme_sqe *sqe = slot; 932 933 sqe->opcode = NVM_CMD_FLUSH; 934 htolem32(&sqe->nsid, ccb->nnc_nsid); 935 } 936 937 static void 938 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 939 struct nvme_cqe *cqe) 940 { 941 int *result = ccb->ccb_cookie; 942 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 943 944 if (status == NVME_CQE_SC_SUCCESS) 945 *result = 1; 946 else 947 *result = -1; 948 949 nvme_ccb_put(q, ccb); 950 } 951 952 static bool 953 nvme_getcache_finished(void *xc) 954 { 955 int *addr = xc; 956 957 return (*addr != 0); 958 } 959 960 /* 961 * Get status of volatile write cache. Always asynchronous. 962 */ 963 int 964 nvme_admin_getcache(struct nvme_softc *sc, int *addr) 965 { 966 struct nvme_ccb *ccb; 967 struct nvme_queue *q = sc->sc_admin_q; 968 int result = 0, error; 969 970 if (!nvme_has_volatile_write_cache(sc)) { 971 /* cache simply not present */ 972 *addr = 0; 973 return 0; 974 } 975 976 ccb = nvme_ccb_get(q, true); 977 KASSERT(ccb != NULL); 978 979 ccb->ccb_done = nvme_getcache_done; 980 ccb->ccb_cookie = &result; 981 982 /* namespace context */ 983 ccb->nnc_flags = 0; 984 ccb->nnc_done = NULL; 985 986 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); 987 988 /* wait for completion */ 989 nvme_q_wait_complete(sc, q, nvme_getcache_finished, &result); 990 KASSERT(result != 0); 991 992 if (result > 0) { 993 *addr = result; 994 error = 0; 995 } else 996 error = EINVAL; 997 998 return error; 999 } 1000 1001 static void 1002 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1003 { 1004 struct nvme_sqe *sqe = slot; 1005 1006 sqe->opcode = NVM_ADMIN_GET_FEATURES; 1007 htolem32(&sqe->cdw10, NVM_FEATURE_VOLATILE_WRITE_CACHE); 1008 htolem32(&sqe->cdw11, NVM_VOLATILE_WRITE_CACHE_WCE); 1009 } 1010 1011 static void 1012 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1013 struct nvme_cqe *cqe) 1014 { 1015 int *addr = ccb->ccb_cookie; 1016 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 1017 uint32_t cdw0 = lemtoh32(&cqe->cdw0); 1018 int result; 1019 1020 if (status == NVME_CQE_SC_SUCCESS) { 1021 result = 0; 1022 1023 /* 1024 * DPO not supported, Dataset Management (DSM) field doesn't 1025 * specify the same semantics. FUA is always supported. 1026 */ 1027 result = DKCACHE_FUA; 1028 1029 if (cdw0 & NVM_VOLATILE_WRITE_CACHE_WCE) 1030 result |= DKCACHE_WRITE; 1031 1032 /* 1033 * If volatile write cache is present, the flag shall also be 1034 * settable. 1035 */ 1036 result |= DKCACHE_WCHANGE; 1037 1038 /* 1039 * ONCS field indicates whether the optional SAVE is also 1040 * supported for Set Features. According to spec v1.3, 1041 * Volatile Write Cache however doesn't support persistency 1042 * across power cycle/reset. 1043 */ 1044 1045 } else { 1046 result = -1; 1047 } 1048 1049 *addr = result; 1050 1051 nvme_ccb_put(q, ccb); 1052 } 1053 1054 struct nvme_setcache_state { 1055 int dkcache; 1056 int result; 1057 }; 1058 1059 static bool 1060 nvme_setcache_finished(void *xc) 1061 { 1062 struct nvme_setcache_state *st = xc; 1063 1064 return (st->result != 0); 1065 } 1066 1067 static void 1068 nvme_setcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1069 { 1070 struct nvme_sqe *sqe = slot; 1071 struct nvme_setcache_state *st = ccb->ccb_cookie; 1072 1073 sqe->opcode = NVM_ADMIN_SET_FEATURES; 1074 htolem32(&sqe->cdw10, NVM_FEATURE_VOLATILE_WRITE_CACHE); 1075 if (st->dkcache & DKCACHE_WRITE) 1076 htolem32(&sqe->cdw11, NVM_VOLATILE_WRITE_CACHE_WCE); 1077 } 1078 1079 static void 1080 nvme_setcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1081 struct nvme_cqe *cqe) 1082 { 1083 struct nvme_setcache_state *st = ccb->ccb_cookie; 1084 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 1085 1086 if (status == NVME_CQE_SC_SUCCESS) { 1087 st->result = 1; 1088 } else { 1089 st->result = -1; 1090 } 1091 1092 nvme_ccb_put(q, ccb); 1093 } 1094 1095 /* 1096 * Set status of volatile write cache. Always asynchronous. 1097 */ 1098 int 1099 nvme_admin_setcache(struct nvme_softc *sc, int dkcache) 1100 { 1101 struct nvme_ccb *ccb; 1102 struct nvme_queue *q = sc->sc_admin_q; 1103 int error; 1104 struct nvme_setcache_state st; 1105 1106 if (!nvme_has_volatile_write_cache(sc)) { 1107 /* cache simply not present */ 1108 return EOPNOTSUPP; 1109 } 1110 1111 if (dkcache & ~(DKCACHE_WRITE)) { 1112 /* unsupported parameters */ 1113 return EOPNOTSUPP; 1114 } 1115 1116 ccb = nvme_ccb_get(q, true); 1117 KASSERT(ccb != NULL); 1118 1119 memset(&st, 0, sizeof(st)); 1120 st.dkcache = dkcache; 1121 1122 ccb->ccb_done = nvme_setcache_done; 1123 ccb->ccb_cookie = &st; 1124 1125 /* namespace context */ 1126 ccb->nnc_flags = 0; 1127 ccb->nnc_done = NULL; 1128 1129 nvme_q_submit(sc, q, ccb, nvme_setcache_fill); 1130 1131 /* wait for completion */ 1132 nvme_q_wait_complete(sc, q, nvme_setcache_finished, &st); 1133 KASSERT(st.result != 0); 1134 1135 if (st.result > 0) 1136 error = 0; 1137 else 1138 error = EINVAL; 1139 1140 return error; 1141 } 1142 1143 void 1144 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 1145 { 1146 struct nvme_namespace *ns; 1147 struct nvm_identify_namespace *identify; 1148 1149 ns = nvme_ns_get(sc, nsid); 1150 KASSERT(ns); 1151 1152 identify = ns->ident; 1153 ns->ident = NULL; 1154 if (identify != NULL) 1155 kmem_free(identify, sizeof(*identify)); 1156 } 1157 1158 struct nvme_pt_state { 1159 struct nvme_pt_command *pt; 1160 bool finished; 1161 }; 1162 1163 static void 1164 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1165 { 1166 struct nvme_softc *sc = q->q_sc; 1167 struct nvme_sqe *sqe = slot; 1168 struct nvme_pt_state *state = ccb->ccb_cookie; 1169 struct nvme_pt_command *pt = state->pt; 1170 bus_dmamap_t dmap = ccb->ccb_dmamap; 1171 int i; 1172 1173 sqe->opcode = pt->cmd.opcode; 1174 htolem32(&sqe->nsid, pt->cmd.nsid); 1175 1176 if (pt->buf != NULL && pt->len > 0) { 1177 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 1178 switch (dmap->dm_nsegs) { 1179 case 1: 1180 break; 1181 case 2: 1182 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 1183 break; 1184 default: 1185 for (i = 1; i < dmap->dm_nsegs; i++) { 1186 htolem64(&ccb->ccb_prpl[i - 1], 1187 dmap->dm_segs[i].ds_addr); 1188 } 1189 bus_dmamap_sync(sc->sc_dmat, 1190 NVME_DMA_MAP(q->q_ccb_prpls), 1191 ccb->ccb_prpl_off, 1192 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 1193 BUS_DMASYNC_PREWRITE); 1194 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 1195 break; 1196 } 1197 } 1198 1199 htolem32(&sqe->cdw10, pt->cmd.cdw10); 1200 htolem32(&sqe->cdw11, pt->cmd.cdw11); 1201 htolem32(&sqe->cdw12, pt->cmd.cdw12); 1202 htolem32(&sqe->cdw13, pt->cmd.cdw13); 1203 htolem32(&sqe->cdw14, pt->cmd.cdw14); 1204 htolem32(&sqe->cdw15, pt->cmd.cdw15); 1205 } 1206 1207 static void 1208 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 1209 { 1210 struct nvme_softc *sc = q->q_sc; 1211 struct nvme_pt_state *state = ccb->ccb_cookie; 1212 struct nvme_pt_command *pt = state->pt; 1213 bus_dmamap_t dmap = ccb->ccb_dmamap; 1214 1215 if (pt->buf != NULL && pt->len > 0) { 1216 if (dmap->dm_nsegs > 2) { 1217 bus_dmamap_sync(sc->sc_dmat, 1218 NVME_DMA_MAP(q->q_ccb_prpls), 1219 ccb->ccb_prpl_off, 1220 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 1221 BUS_DMASYNC_POSTWRITE); 1222 } 1223 1224 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1225 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 1226 bus_dmamap_unload(sc->sc_dmat, dmap); 1227 } 1228 1229 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 1230 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 1231 1232 state->finished = true; 1233 1234 nvme_ccb_put(q, ccb); 1235 } 1236 1237 static bool 1238 nvme_pt_finished(void *cookie) 1239 { 1240 struct nvme_pt_state *state = cookie; 1241 1242 return state->finished; 1243 } 1244 1245 static int 1246 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 1247 uint16_t nsid, struct lwp *l, bool is_adminq) 1248 { 1249 struct nvme_queue *q; 1250 struct nvme_ccb *ccb; 1251 void *buf = NULL; 1252 struct nvme_pt_state state; 1253 int error; 1254 1255 /* limit command size to maximum data transfer size */ 1256 if ((pt->buf == NULL && pt->len > 0) || 1257 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 1258 return EINVAL; 1259 1260 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc, NULL, true); 1261 ccb = nvme_ccb_get(q, true); 1262 KASSERT(ccb != NULL); 1263 1264 if (pt->buf != NULL) { 1265 KASSERT(pt->len > 0); 1266 buf = kmem_alloc(pt->len, KM_SLEEP); 1267 if (!pt->is_read) { 1268 error = copyin(pt->buf, buf, pt->len); 1269 if (error) 1270 goto kmem_free; 1271 } 1272 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 1273 pt->len, NULL, 1274 BUS_DMA_WAITOK | 1275 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 1276 if (error) 1277 goto kmem_free; 1278 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 1279 0, ccb->ccb_dmamap->dm_mapsize, 1280 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 1281 } 1282 1283 memset(&state, 0, sizeof(state)); 1284 state.pt = pt; 1285 state.finished = false; 1286 1287 ccb->ccb_done = nvme_pt_done; 1288 ccb->ccb_cookie = &state; 1289 1290 pt->cmd.nsid = nsid; 1291 1292 nvme_q_submit(sc, q, ccb, nvme_pt_fill); 1293 1294 /* wait for completion */ 1295 nvme_q_wait_complete(sc, q, nvme_pt_finished, &state); 1296 KASSERT(state.finished); 1297 1298 error = 0; 1299 1300 if (buf != NULL) { 1301 if (error == 0 && pt->is_read) 1302 error = copyout(buf, pt->buf, pt->len); 1303 kmem_free: 1304 kmem_free(buf, pt->len); 1305 } 1306 1307 return error; 1308 } 1309 1310 static void 1311 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1312 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 1313 { 1314 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 1315 uint32_t tail; 1316 1317 mutex_enter(&q->q_sq_mtx); 1318 tail = q->q_sq_tail; 1319 if (++q->q_sq_tail >= q->q_entries) 1320 q->q_sq_tail = 0; 1321 1322 sqe += tail; 1323 1324 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1325 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 1326 memset(sqe, 0, sizeof(*sqe)); 1327 (*fill)(q, ccb, sqe); 1328 htolem16(&sqe->cid, ccb->ccb_id); 1329 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1330 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 1331 1332 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 1333 mutex_exit(&q->q_sq_mtx); 1334 } 1335 1336 struct nvme_poll_state { 1337 struct nvme_sqe s; 1338 struct nvme_cqe c; 1339 void *cookie; 1340 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 1341 }; 1342 1343 static int 1344 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1345 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 1346 { 1347 struct nvme_poll_state state; 1348 uint16_t flags; 1349 int step = 10; 1350 int maxloop = timo_sec * 1000000 / step; 1351 int error = 0; 1352 1353 memset(&state, 0, sizeof(state)); 1354 (*fill)(q, ccb, &state.s); 1355 1356 state.done = ccb->ccb_done; 1357 state.cookie = ccb->ccb_cookie; 1358 1359 ccb->ccb_done = nvme_poll_done; 1360 ccb->ccb_cookie = &state; 1361 1362 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1363 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1364 if (nvme_q_complete(sc, q) == 0) 1365 delay(step); 1366 1367 if (timo_sec >= 0 && --maxloop <= 0) { 1368 error = ETIMEDOUT; 1369 break; 1370 } 1371 } 1372 1373 if (error == 0) { 1374 flags = lemtoh16(&state.c.flags); 1375 return flags & ~NVME_CQE_PHASE; 1376 } else { 1377 /* 1378 * If it succeds later, it would hit ccb which will have been 1379 * already reused for something else. Not good. Cross 1380 * fingers and hope for best. XXX do controller reset? 1381 */ 1382 aprint_error_dev(sc->sc_dev, "polled command timed out\n"); 1383 1384 /* Invoke the callback to clean state anyway */ 1385 struct nvme_cqe cqe; 1386 memset(&cqe, 0, sizeof(cqe)); 1387 ccb->ccb_done(q, ccb, &cqe); 1388 1389 return 1; 1390 } 1391 } 1392 1393 static void 1394 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1395 { 1396 struct nvme_sqe *sqe = slot; 1397 struct nvme_poll_state *state = ccb->ccb_cookie; 1398 1399 *sqe = state->s; 1400 } 1401 1402 static void 1403 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1404 struct nvme_cqe *cqe) 1405 { 1406 struct nvme_poll_state *state = ccb->ccb_cookie; 1407 1408 state->c = *cqe; 1409 SET(state->c.flags, htole16(NVME_CQE_PHASE)); 1410 1411 ccb->ccb_cookie = state->cookie; 1412 state->done(q, ccb, &state->c); 1413 } 1414 1415 static void 1416 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1417 { 1418 struct nvme_sqe *src = ccb->ccb_cookie; 1419 struct nvme_sqe *dst = slot; 1420 1421 *dst = *src; 1422 } 1423 1424 static void 1425 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1426 struct nvme_cqe *cqe) 1427 { 1428 } 1429 1430 static int 1431 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1432 { 1433 struct nvme_ccb *ccb; 1434 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1435 uint16_t flags; 1436 int rv = 0; 1437 1438 mutex_enter(&q->q_cq_mtx); 1439 1440 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1441 for (;;) { 1442 cqe = &ring[q->q_cq_head]; 1443 flags = lemtoh16(&cqe->flags); 1444 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1445 break; 1446 1447 ccb = &q->q_ccbs[cqe->cid]; 1448 1449 if (++q->q_cq_head >= q->q_entries) { 1450 q->q_cq_head = 0; 1451 q->q_cq_phase ^= NVME_CQE_PHASE; 1452 } 1453 1454 #ifdef DEBUG 1455 /* 1456 * If we get spurious completion notification, something 1457 * is seriously hosed up. Very likely DMA to some random 1458 * memory place happened, so just bail out. 1459 */ 1460 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1461 panic("%s: invalid ccb detected", 1462 device_xname(sc->sc_dev)); 1463 /* NOTREACHED */ 1464 } 1465 #endif 1466 1467 rv++; 1468 1469 /* 1470 * Unlock the mutex before calling the ccb_done callback 1471 * and re-lock afterwards. The callback triggers lddone() 1472 * which schedules another i/o, and also calls nvme_ccb_put(). 1473 * Unlock/relock avoids possibility of deadlock. 1474 */ 1475 mutex_exit(&q->q_cq_mtx); 1476 ccb->ccb_done(q, ccb, cqe); 1477 mutex_enter(&q->q_cq_mtx); 1478 } 1479 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1480 1481 if (rv) 1482 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1483 1484 mutex_exit(&q->q_cq_mtx); 1485 1486 return rv; 1487 } 1488 1489 static void 1490 nvme_q_wait_complete(struct nvme_softc *sc, 1491 struct nvme_queue *q, bool (*finished)(void *), void *cookie) 1492 { 1493 mutex_enter(&q->q_ccb_mtx); 1494 if (finished(cookie)) 1495 goto out; 1496 1497 for(;;) { 1498 q->q_ccb_waiting = true; 1499 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1500 1501 if (finished(cookie)) 1502 break; 1503 } 1504 1505 out: 1506 mutex_exit(&q->q_ccb_mtx); 1507 } 1508 1509 static int 1510 nvme_identify(struct nvme_softc *sc, u_int mps) 1511 { 1512 char sn[41], mn[81], fr[17]; 1513 struct nvm_identify_controller *identify; 1514 struct nvme_dmamem *mem; 1515 struct nvme_ccb *ccb; 1516 u_int mdts; 1517 int rv = 1; 1518 1519 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1520 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1521 1522 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1523 if (mem == NULL) 1524 return 1; 1525 1526 ccb->ccb_done = nvme_empty_done; 1527 ccb->ccb_cookie = mem; 1528 1529 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1530 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1531 NVME_TIMO_IDENT); 1532 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1533 1534 nvme_ccb_put(sc->sc_admin_q, ccb); 1535 1536 if (rv != 0) 1537 goto done; 1538 1539 identify = NVME_DMA_KVA(mem); 1540 sc->sc_identify = *identify; 1541 identify = NULL; 1542 1543 /* Convert data to host endian */ 1544 nvme_identify_controller_swapbytes(&sc->sc_identify); 1545 1546 strnvisx(sn, sizeof(sn), (const char *)sc->sc_identify.sn, 1547 sizeof(sc->sc_identify.sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1548 strnvisx(mn, sizeof(mn), (const char *)sc->sc_identify.mn, 1549 sizeof(sc->sc_identify.mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1550 strnvisx(fr, sizeof(fr), (const char *)sc->sc_identify.fr, 1551 sizeof(sc->sc_identify.fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1552 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1553 sn); 1554 1555 strlcpy(sc->sc_modelname, mn, sizeof(sc->sc_modelname)); 1556 1557 if (sc->sc_identify.mdts > 0) { 1558 mdts = (1 << sc->sc_identify.mdts) * (1 << mps); 1559 if (mdts < sc->sc_mdts) 1560 sc->sc_mdts = mdts; 1561 } 1562 1563 sc->sc_nn = sc->sc_identify.nn; 1564 1565 done: 1566 nvme_dmamem_free(sc, mem); 1567 1568 return rv; 1569 } 1570 1571 static int 1572 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1573 { 1574 struct nvme_sqe_q sqe; 1575 struct nvme_ccb *ccb; 1576 int rv; 1577 1578 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1579 return 1; 1580 1581 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1582 KASSERT(ccb != NULL); 1583 1584 ccb->ccb_done = nvme_empty_done; 1585 ccb->ccb_cookie = &sqe; 1586 1587 memset(&sqe, 0, sizeof(sqe)); 1588 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1589 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1590 htolem16(&sqe.qsize, q->q_entries - 1); 1591 htolem16(&sqe.qid, q->q_id); 1592 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1593 if (sc->sc_use_mq) 1594 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1595 1596 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1597 if (rv != 0) 1598 goto fail; 1599 1600 ccb->ccb_done = nvme_empty_done; 1601 ccb->ccb_cookie = &sqe; 1602 1603 memset(&sqe, 0, sizeof(sqe)); 1604 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1605 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1606 htolem16(&sqe.qsize, q->q_entries - 1); 1607 htolem16(&sqe.qid, q->q_id); 1608 htolem16(&sqe.cqid, q->q_id); 1609 sqe.qflags = NVM_SQE_Q_PC; 1610 1611 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1612 if (rv != 0) 1613 goto fail; 1614 1615 nvme_ccb_put(sc->sc_admin_q, ccb); 1616 return 0; 1617 1618 fail: 1619 if (sc->sc_use_mq) 1620 sc->sc_intr_disestablish(sc, q->q_id); 1621 1622 nvme_ccb_put(sc->sc_admin_q, ccb); 1623 return rv; 1624 } 1625 1626 static int 1627 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1628 { 1629 struct nvme_sqe_q sqe; 1630 struct nvme_ccb *ccb; 1631 int rv; 1632 1633 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1634 KASSERT(ccb != NULL); 1635 1636 ccb->ccb_done = nvme_empty_done; 1637 ccb->ccb_cookie = &sqe; 1638 1639 memset(&sqe, 0, sizeof(sqe)); 1640 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1641 htolem16(&sqe.qid, q->q_id); 1642 1643 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1644 if (rv != 0) 1645 goto fail; 1646 1647 ccb->ccb_done = nvme_empty_done; 1648 ccb->ccb_cookie = &sqe; 1649 1650 memset(&sqe, 0, sizeof(sqe)); 1651 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1652 htolem16(&sqe.qid, q->q_id); 1653 1654 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1655 if (rv != 0) 1656 goto fail; 1657 1658 fail: 1659 nvme_ccb_put(sc->sc_admin_q, ccb); 1660 1661 if (rv == 0 && sc->sc_use_mq) { 1662 if (sc->sc_intr_disestablish(sc, q->q_id)) 1663 rv = 1; 1664 } 1665 1666 return rv; 1667 } 1668 1669 static void 1670 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1671 { 1672 struct nvme_sqe *sqe = slot; 1673 struct nvme_dmamem *mem = ccb->ccb_cookie; 1674 1675 sqe->opcode = NVM_ADMIN_IDENTIFY; 1676 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1677 htolem32(&sqe->cdw10, 1); 1678 } 1679 1680 static int 1681 nvme_set_number_of_queues(struct nvme_softc *sc, u_int nq, u_int *ncqa, 1682 u_int *nsqa) 1683 { 1684 struct nvme_pt_state state; 1685 struct nvme_pt_command pt; 1686 struct nvme_ccb *ccb; 1687 int rv; 1688 1689 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1690 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1691 1692 memset(&pt, 0, sizeof(pt)); 1693 pt.cmd.opcode = NVM_ADMIN_SET_FEATURES; 1694 htolem32(&pt.cmd.cdw10, NVM_FEATURE_NUMBER_OF_QUEUES); 1695 htolem32(&pt.cmd.cdw11, ((nq - 1) << 16) | (nq - 1)); 1696 1697 memset(&state, 0, sizeof(state)); 1698 state.pt = &pt; 1699 state.finished = false; 1700 1701 ccb->ccb_done = nvme_pt_done; 1702 ccb->ccb_cookie = &state; 1703 1704 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1705 1706 if (rv != 0) { 1707 *ncqa = *nsqa = 0; 1708 return EIO; 1709 } 1710 1711 *ncqa = (pt.cpl.cdw0 >> 16) + 1; 1712 *nsqa = (pt.cpl.cdw0 & 0xffff) + 1; 1713 1714 return 0; 1715 } 1716 1717 static int 1718 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1719 { 1720 struct nvme_softc *sc = q->q_sc; 1721 struct nvme_ccb *ccb; 1722 bus_addr_t off; 1723 uint64_t *prpl; 1724 u_int i; 1725 1726 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1727 cv_init(&q->q_ccb_wait, "nvmeqw"); 1728 q->q_ccb_waiting = false; 1729 SIMPLEQ_INIT(&q->q_ccb_list); 1730 1731 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1732 1733 q->q_nccbs = nccbs; 1734 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1735 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1736 1737 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1738 off = 0; 1739 1740 for (i = 0; i < nccbs; i++) { 1741 ccb = &q->q_ccbs[i]; 1742 1743 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1744 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1745 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1746 &ccb->ccb_dmamap) != 0) 1747 goto free_maps; 1748 1749 ccb->ccb_id = i; 1750 ccb->ccb_prpl = prpl; 1751 ccb->ccb_prpl_off = off; 1752 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1753 1754 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1755 1756 prpl += sc->sc_max_sgl; 1757 off += sizeof(*prpl) * sc->sc_max_sgl; 1758 } 1759 1760 return 0; 1761 1762 free_maps: 1763 nvme_ccbs_free(q); 1764 return 1; 1765 } 1766 1767 static struct nvme_ccb * 1768 nvme_ccb_get(struct nvme_queue *q, bool wait) 1769 { 1770 struct nvme_ccb *ccb = NULL; 1771 1772 mutex_enter(&q->q_ccb_mtx); 1773 again: 1774 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1775 if (ccb != NULL) { 1776 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1777 #ifdef DEBUG 1778 ccb->ccb_cookie = NULL; 1779 #endif 1780 } else { 1781 if (__predict_false(wait)) { 1782 q->q_ccb_waiting = true; 1783 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1784 goto again; 1785 } 1786 } 1787 mutex_exit(&q->q_ccb_mtx); 1788 1789 return ccb; 1790 } 1791 1792 static void 1793 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1794 { 1795 1796 mutex_enter(&q->q_ccb_mtx); 1797 #ifdef DEBUG 1798 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1799 #endif 1800 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1801 1802 /* It's unlikely there are any waiters, it's not used for regular I/O */ 1803 if (__predict_false(q->q_ccb_waiting)) { 1804 q->q_ccb_waiting = false; 1805 cv_broadcast(&q->q_ccb_wait); 1806 } 1807 1808 mutex_exit(&q->q_ccb_mtx); 1809 } 1810 1811 static void 1812 nvme_ccbs_free(struct nvme_queue *q) 1813 { 1814 struct nvme_softc *sc = q->q_sc; 1815 struct nvme_ccb *ccb; 1816 1817 mutex_enter(&q->q_ccb_mtx); 1818 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1819 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1820 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1821 } 1822 mutex_exit(&q->q_ccb_mtx); 1823 1824 nvme_dmamem_free(sc, q->q_ccb_prpls); 1825 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1826 q->q_ccbs = NULL; 1827 cv_destroy(&q->q_ccb_wait); 1828 mutex_destroy(&q->q_ccb_mtx); 1829 } 1830 1831 static struct nvme_queue * 1832 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1833 { 1834 struct nvme_queue *q; 1835 1836 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1837 q->q_sc = sc; 1838 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1839 sizeof(struct nvme_sqe) * entries); 1840 if (q->q_sq_dmamem == NULL) 1841 goto free; 1842 1843 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1844 sizeof(struct nvme_cqe) * entries); 1845 if (q->q_cq_dmamem == NULL) 1846 goto free_sq; 1847 1848 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1849 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1850 1851 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1852 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1853 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1854 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1855 q->q_id = id; 1856 q->q_entries = entries; 1857 q->q_sq_tail = 0; 1858 q->q_cq_head = 0; 1859 q->q_cq_phase = NVME_CQE_PHASE; 1860 1861 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1862 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1863 1864 /* 1865 * Due to definition of full and empty queue (queue is empty 1866 * when head == tail, full when tail is one less then head), 1867 * we can actually only have (entries - 1) in-flight commands. 1868 */ 1869 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1870 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1871 goto free_cq; 1872 } 1873 1874 return q; 1875 1876 free_cq: 1877 nvme_dmamem_free(sc, q->q_cq_dmamem); 1878 free_sq: 1879 nvme_dmamem_free(sc, q->q_sq_dmamem); 1880 free: 1881 kmem_free(q, sizeof(*q)); 1882 1883 return NULL; 1884 } 1885 1886 static void 1887 nvme_q_reset(struct nvme_softc *sc, struct nvme_queue *q) 1888 { 1889 1890 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1891 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1892 1893 q->q_sqtdbl = NVME_SQTDBL(q->q_id, sc->sc_dstrd); 1894 q->q_cqhdbl = NVME_CQHDBL(q->q_id, sc->sc_dstrd); 1895 1896 q->q_sq_tail = 0; 1897 q->q_cq_head = 0; 1898 q->q_cq_phase = NVME_CQE_PHASE; 1899 1900 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1901 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1902 } 1903 1904 static void 1905 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1906 { 1907 nvme_ccbs_free(q); 1908 mutex_destroy(&q->q_sq_mtx); 1909 mutex_destroy(&q->q_cq_mtx); 1910 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1911 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1912 nvme_dmamem_free(sc, q->q_cq_dmamem); 1913 nvme_dmamem_free(sc, q->q_sq_dmamem); 1914 kmem_free(q, sizeof(*q)); 1915 } 1916 1917 int 1918 nvme_intr(void *xsc) 1919 { 1920 struct nvme_softc *sc = xsc; 1921 1922 /* 1923 * INTx is level triggered, controller deasserts the interrupt only 1924 * when we advance command queue head via write to the doorbell. 1925 * Tell the controller to block the interrupts while we process 1926 * the queue(s). 1927 */ 1928 nvme_write4(sc, NVME_INTMS, 1); 1929 1930 softint_schedule(sc->sc_softih[0]); 1931 1932 /* don't know, might not have been for us */ 1933 return 1; 1934 } 1935 1936 void 1937 nvme_softintr_intx(void *xq) 1938 { 1939 struct nvme_queue *q = xq; 1940 struct nvme_softc *sc = q->q_sc; 1941 1942 nvme_q_complete(sc, sc->sc_admin_q); 1943 if (sc->sc_q != NULL) 1944 nvme_q_complete(sc, sc->sc_q[0]); 1945 1946 /* 1947 * Processing done, tell controller to issue interrupts again. There 1948 * is no race, as NVMe spec requires the controller to maintain state, 1949 * and assert the interrupt whenever there are unacknowledged 1950 * completion queue entries. 1951 */ 1952 nvme_write4(sc, NVME_INTMC, 1); 1953 } 1954 1955 int 1956 nvme_intr_msi(void *xq) 1957 { 1958 struct nvme_queue *q = xq; 1959 1960 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1961 && q->q_sc->sc_softih[q->q_id]); 1962 1963 /* 1964 * MSI/MSI-X are edge triggered, so can handover processing to softint 1965 * without masking the interrupt. 1966 */ 1967 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1968 1969 return 1; 1970 } 1971 1972 void 1973 nvme_softintr_msi(void *xq) 1974 { 1975 struct nvme_queue *q = xq; 1976 struct nvme_softc *sc = q->q_sc; 1977 1978 nvme_q_complete(sc, q); 1979 } 1980 1981 static struct nvme_dmamem * 1982 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1983 { 1984 struct nvme_dmamem *ndm; 1985 int nsegs; 1986 1987 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1988 if (ndm == NULL) 1989 return NULL; 1990 1991 ndm->ndm_size = size; 1992 1993 if (bus_dmamap_create(sc->sc_dmat, size, btoc(round_page(size)), size, 0, 1994 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1995 goto ndmfree; 1996 1997 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1998 1, &nsegs, BUS_DMA_WAITOK) != 0) 1999 goto destroy; 2000 2001 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 2002 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 2003 goto free; 2004 memset(ndm->ndm_kva, 0, size); 2005 2006 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 2007 NULL, BUS_DMA_WAITOK) != 0) 2008 goto unmap; 2009 2010 return ndm; 2011 2012 unmap: 2013 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 2014 free: 2015 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 2016 destroy: 2017 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 2018 ndmfree: 2019 kmem_free(ndm, sizeof(*ndm)); 2020 return NULL; 2021 } 2022 2023 static void 2024 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 2025 { 2026 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 2027 0, NVME_DMA_LEN(mem), ops); 2028 } 2029 2030 void 2031 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 2032 { 2033 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 2034 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 2035 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 2036 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 2037 kmem_free(ndm, sizeof(*ndm)); 2038 } 2039 2040 /* 2041 * ioctl 2042 */ 2043 2044 dev_type_open(nvmeopen); 2045 dev_type_close(nvmeclose); 2046 dev_type_ioctl(nvmeioctl); 2047 2048 const struct cdevsw nvme_cdevsw = { 2049 .d_open = nvmeopen, 2050 .d_close = nvmeclose, 2051 .d_read = noread, 2052 .d_write = nowrite, 2053 .d_ioctl = nvmeioctl, 2054 .d_stop = nostop, 2055 .d_tty = notty, 2056 .d_poll = nopoll, 2057 .d_mmap = nommap, 2058 .d_kqfilter = nokqfilter, 2059 .d_discard = nodiscard, 2060 .d_flag = D_OTHER, 2061 }; 2062 2063 /* 2064 * Accept an open operation on the control device. 2065 */ 2066 int 2067 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 2068 { 2069 struct nvme_softc *sc; 2070 int unit = minor(dev) / 0x10000; 2071 int nsid = minor(dev) & 0xffff; 2072 int nsidx; 2073 2074 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 2075 return ENXIO; 2076 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 2077 return ENXIO; 2078 2079 if (nsid == 0) { 2080 /* controller */ 2081 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 2082 return EBUSY; 2083 SET(sc->sc_flags, NVME_F_OPEN); 2084 } else { 2085 /* namespace */ 2086 nsidx = nsid - 1; 2087 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 2088 return ENXIO; 2089 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 2090 return EBUSY; 2091 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 2092 } 2093 return 0; 2094 } 2095 2096 /* 2097 * Accept the last close on the control device. 2098 */ 2099 int 2100 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 2101 { 2102 struct nvme_softc *sc; 2103 int unit = minor(dev) / 0x10000; 2104 int nsid = minor(dev) & 0xffff; 2105 int nsidx; 2106 2107 sc = device_lookup_private(&nvme_cd, unit); 2108 if (sc == NULL) 2109 return ENXIO; 2110 2111 if (nsid == 0) { 2112 /* controller */ 2113 CLR(sc->sc_flags, NVME_F_OPEN); 2114 } else { 2115 /* namespace */ 2116 nsidx = nsid - 1; 2117 if (nsidx >= sc->sc_nn) 2118 return ENXIO; 2119 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 2120 } 2121 2122 return 0; 2123 } 2124 2125 /* 2126 * Handle control operations. 2127 */ 2128 int 2129 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 2130 { 2131 struct nvme_softc *sc; 2132 int unit = minor(dev) / 0x10000; 2133 int nsid = minor(dev) & 0xffff; 2134 struct nvme_pt_command *pt; 2135 2136 sc = device_lookup_private(&nvme_cd, unit); 2137 if (sc == NULL) 2138 return ENXIO; 2139 2140 switch (cmd) { 2141 case NVME_PASSTHROUGH_CMD: 2142 pt = data; 2143 return nvme_command_passthrough(sc, data, 2144 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 2145 } 2146 2147 return ENOTTY; 2148 } 2149
Indexes created Mon Oct 20 16:09:52 GMT 2025