nvme.c revision 1.61
1 /* $NetBSD: nvme.c,v 1.61 2022/07/31 12:02:28 mlelstv Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.61 2022/07/31 12:02:28 mlelstv Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 #include "ioconf.h" 44 #include "locators.h" 45 46 #define B4_CHK_RDY_DELAY_MS 2300 /* workaround controller bug */ 47 48 int nvme_adminq_size = 32; 49 int nvme_ioq_size = 1024; 50 51 static int nvme_print(void *, const char *); 52 53 static int nvme_ready(struct nvme_softc *, uint32_t); 54 static int nvme_enable(struct nvme_softc *, u_int); 55 static int nvme_disable(struct nvme_softc *); 56 static int nvme_shutdown(struct nvme_softc *); 57 58 uint32_t nvme_op_sq_enter(struct nvme_softc *, 59 struct nvme_queue *, struct nvme_ccb *); 60 void nvme_op_sq_leave(struct nvme_softc *, 61 struct nvme_queue *, struct nvme_ccb *); 62 uint32_t nvme_op_sq_enter_locked(struct nvme_softc *, 63 struct nvme_queue *, struct nvme_ccb *); 64 void nvme_op_sq_leave_locked(struct nvme_softc *, 65 struct nvme_queue *, struct nvme_ccb *); 66 67 void nvme_op_cq_done(struct nvme_softc *, 68 struct nvme_queue *, struct nvme_ccb *); 69 70 static const struct nvme_ops nvme_ops = { 71 .op_sq_enter = nvme_op_sq_enter, 72 .op_sq_leave = nvme_op_sq_leave, 73 .op_sq_enter_locked = nvme_op_sq_enter_locked, 74 .op_sq_leave_locked = nvme_op_sq_leave_locked, 75 76 .op_cq_done = nvme_op_cq_done, 77 }; 78 79 #ifdef NVME_DEBUG 80 static void nvme_dumpregs(struct nvme_softc *); 81 #endif 82 static int nvme_identify(struct nvme_softc *, u_int); 83 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 84 void *); 85 86 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 87 static void nvme_ccbs_free(struct nvme_queue *); 88 89 static struct nvme_ccb * 90 nvme_ccb_get(struct nvme_queue *, bool); 91 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 92 93 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 94 struct nvme_ccb *, void (*)(struct nvme_queue *, 95 struct nvme_ccb *, void *), int); 96 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 97 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 98 struct nvme_cqe *); 99 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 100 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 101 struct nvme_cqe *); 102 103 static struct nvme_queue * 104 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 105 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 106 static void nvme_q_reset(struct nvme_softc *, struct nvme_queue *); 107 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 108 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 109 struct nvme_ccb *, void (*)(struct nvme_queue *, 110 struct nvme_ccb *, void *)); 111 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 112 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 113 static void nvme_q_wait_complete(struct nvme_softc *, struct nvme_queue *, 114 bool (*)(void *), void *); 115 116 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 117 void *); 118 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 119 struct nvme_cqe *); 120 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 121 void *); 122 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 123 struct nvme_cqe *); 124 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *, 125 void *); 126 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *, 127 struct nvme_cqe *); 128 129 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 130 void *); 131 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 132 struct nvme_cqe *); 133 static int nvme_command_passthrough(struct nvme_softc *, 134 struct nvme_pt_command *, uint32_t, struct lwp *, bool); 135 136 static int nvme_set_number_of_queues(struct nvme_softc *, u_int, u_int *, 137 u_int *); 138 139 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 140 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 141 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 142 #define NVME_TIMO_SY 60 /* sync cache timeout */ 143 144 /* 145 * Some controllers, at least Apple NVMe, always require split 146 * transfers, so don't use bus_space_{read,write}_8() on LP64. 147 */ 148 uint64_t 149 nvme_read8(struct nvme_softc *sc, bus_size_t r) 150 { 151 uint64_t v; 152 uint32_t *a = (uint32_t *)&v; 153 154 #if _BYTE_ORDER == _LITTLE_ENDIAN 155 a[0] = nvme_read4(sc, r); 156 a[1] = nvme_read4(sc, r + 4); 157 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 158 a[1] = nvme_read4(sc, r); 159 a[0] = nvme_read4(sc, r + 4); 160 #endif 161 162 return v; 163 } 164 165 void 166 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 167 { 168 uint32_t *a = (uint32_t *)&v; 169 170 #if _BYTE_ORDER == _LITTLE_ENDIAN 171 nvme_write4(sc, r, a[0]); 172 nvme_write4(sc, r + 4, a[1]); 173 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 174 nvme_write4(sc, r, a[1]); 175 nvme_write4(sc, r + 4, a[0]); 176 #endif 177 } 178 179 #ifdef NVME_DEBUG 180 static __used void 181 nvme_dumpregs(struct nvme_softc *sc) 182 { 183 uint64_t r8; 184 uint32_t r4; 185 186 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 187 r8 = nvme_read8(sc, NVME_CAP); 188 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 189 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 190 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 191 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 192 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 193 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 194 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 195 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 196 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 197 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 198 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 199 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 200 201 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 202 203 r4 = nvme_read4(sc, NVME_CC); 204 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 205 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 206 (1 << NVME_CC_IOCQES_R(r4))); 207 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 208 (1 << NVME_CC_IOSQES_R(r4))); 209 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 210 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 211 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 212 (1 << NVME_CC_MPS_R(r4))); 213 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 214 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 215 216 r4 = nvme_read4(sc, NVME_CSTS); 217 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 218 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 219 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 220 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 221 222 r4 = nvme_read4(sc, NVME_AQA); 223 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 224 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 225 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 226 227 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 228 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 229 #undef DEVNAME 230 } 231 #endif /* NVME_DEBUG */ 232 233 static int 234 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 235 { 236 u_int i = 0; 237 238 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 239 if (i++ > sc->sc_rdy_to) 240 return ENXIO; 241 242 delay(1000); 243 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 244 } 245 246 return 0; 247 } 248 249 static int 250 nvme_enable(struct nvme_softc *sc, u_int mps) 251 { 252 uint32_t cc, csts; 253 int error; 254 255 cc = nvme_read4(sc, NVME_CC); 256 csts = nvme_read4(sc, NVME_CSTS); 257 258 /* 259 * See note in nvme_disable. Short circuit if we're already enabled. 260 */ 261 if (ISSET(cc, NVME_CC_EN)) { 262 if (ISSET(csts, NVME_CSTS_RDY)) 263 return 0; 264 265 goto waitready; 266 } else { 267 /* EN == 0 already wait for RDY == 0 or fail */ 268 error = nvme_ready(sc, 0); 269 if (error) 270 return error; 271 } 272 273 if (sc->sc_ops->op_enable != NULL) 274 sc->sc_ops->op_enable(sc); 275 276 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 277 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 278 delay(5000); 279 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 280 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 281 delay(5000); 282 283 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 284 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 285 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 286 delay(5000); 287 288 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 289 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 290 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 291 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 292 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 293 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 294 SET(cc, NVME_CC_MPS(mps)); 295 SET(cc, NVME_CC_EN); 296 297 nvme_write4(sc, NVME_CC, cc); 298 nvme_barrier(sc, 0, sc->sc_ios, 299 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 300 301 waitready: 302 return nvme_ready(sc, NVME_CSTS_RDY); 303 } 304 305 static int 306 nvme_disable(struct nvme_softc *sc) 307 { 308 uint32_t cc, csts; 309 int error; 310 311 cc = nvme_read4(sc, NVME_CC); 312 csts = nvme_read4(sc, NVME_CSTS); 313 314 /* 315 * Per 3.1.5 in NVME 1.3 spec, transitioning CC.EN from 0 to 1 316 * when CSTS.RDY is 1 or transitioning CC.EN from 1 to 0 when 317 * CSTS.RDY is 0 "has undefined results" So make sure that CSTS.RDY 318 * isn't the desired value. Short circuit if we're already disabled. 319 */ 320 if (ISSET(cc, NVME_CC_EN)) { 321 if (!ISSET(csts, NVME_CSTS_RDY)) { 322 /* EN == 1, wait for RDY == 1 or fail */ 323 error = nvme_ready(sc, NVME_CSTS_RDY); 324 if (error) 325 return error; 326 } 327 } else { 328 /* EN == 0 already wait for RDY == 0 */ 329 if (!ISSET(csts, NVME_CSTS_RDY)) 330 return 0; 331 332 goto waitready; 333 } 334 335 CLR(cc, NVME_CC_EN); 336 nvme_write4(sc, NVME_CC, cc); 337 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 338 339 /* 340 * Some drives have issues with accessing the mmio after we disable, 341 * so delay for a bit after we write the bit to cope with these issues. 342 */ 343 if (ISSET(sc->sc_quirks, NVME_QUIRK_DELAY_B4_CHK_RDY)) 344 delay(B4_CHK_RDY_DELAY_MS); 345 346 waitready: 347 return nvme_ready(sc, 0); 348 } 349 350 int 351 nvme_attach(struct nvme_softc *sc) 352 { 353 uint64_t cap; 354 uint32_t reg; 355 u_int mps = PAGE_SHIFT; 356 u_int ncq, nsq; 357 uint16_t adminq_entries = nvme_adminq_size; 358 uint16_t ioq_entries = nvme_ioq_size; 359 int i; 360 361 if (sc->sc_ops == NULL) 362 sc->sc_ops = &nvme_ops; 363 364 reg = nvme_read4(sc, NVME_VS); 365 if (reg == 0xffffffff) { 366 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 367 return 1; 368 } 369 370 if (NVME_VS_TER(reg) == 0) 371 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg), 372 NVME_VS_MNR(reg)); 373 else 374 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg), 375 NVME_VS_MNR(reg), NVME_VS_TER(reg)); 376 377 cap = nvme_read8(sc, NVME_CAP); 378 sc->sc_dstrd = NVME_CAP_DSTRD(cap); 379 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 380 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 381 "is greater than CPU page size %u\n", 382 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 383 return 1; 384 } 385 if (NVME_CAP_MPSMAX(cap) < mps) 386 mps = NVME_CAP_MPSMAX(cap); 387 if (ioq_entries > NVME_CAP_MQES(cap)) 388 ioq_entries = NVME_CAP_MQES(cap); 389 390 /* set initial values to be used for admin queue during probe */ 391 sc->sc_rdy_to = NVME_CAP_TO(cap); 392 sc->sc_mps = 1 << mps; 393 sc->sc_mdts = MAXPHYS; 394 sc->sc_max_sgl = btoc(round_page(sc->sc_mdts)); 395 396 if (nvme_disable(sc) != 0) { 397 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 398 return 1; 399 } 400 401 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, 402 sc->sc_dstrd); 403 if (sc->sc_admin_q == NULL) { 404 aprint_error_dev(sc->sc_dev, 405 "unable to allocate admin queue\n"); 406 return 1; 407 } 408 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 409 goto free_admin_q; 410 411 if (nvme_enable(sc, mps) != 0) { 412 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 413 goto disestablish_admin_q; 414 } 415 416 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 417 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 418 goto disable; 419 } 420 if (sc->sc_nn == 0) { 421 aprint_error_dev(sc->sc_dev, "namespace not found\n"); 422 goto disable; 423 } 424 425 /* we know how big things are now */ 426 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 427 428 /* reallocate ccbs of admin queue with new max sgl. */ 429 nvme_ccbs_free(sc->sc_admin_q); 430 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 431 432 if (sc->sc_use_mq) { 433 /* Limit the number of queues to the number allocated in HW */ 434 if (nvme_set_number_of_queues(sc, sc->sc_nq, &ncq, &nsq) != 0) { 435 aprint_error_dev(sc->sc_dev, 436 "unable to get number of queues\n"); 437 goto disable; 438 } 439 if (sc->sc_nq > ncq) 440 sc->sc_nq = ncq; 441 if (sc->sc_nq > nsq) 442 sc->sc_nq = nsq; 443 } 444 445 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 446 for (i = 0; i < sc->sc_nq; i++) { 447 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, 448 sc->sc_dstrd); 449 if (sc->sc_q[i] == NULL) { 450 aprint_error_dev(sc->sc_dev, 451 "unable to allocate io queue\n"); 452 goto free_q; 453 } 454 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 455 aprint_error_dev(sc->sc_dev, 456 "unable to create io queue\n"); 457 nvme_q_free(sc, sc->sc_q[i]); 458 goto free_q; 459 } 460 } 461 462 if (!sc->sc_use_mq) 463 nvme_write4(sc, NVME_INTMC, 1); 464 465 /* probe subdevices */ 466 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 467 KM_SLEEP); 468 nvme_rescan(sc->sc_dev, NULL, NULL); 469 470 return 0; 471 472 free_q: 473 while (--i >= 0) { 474 nvme_q_delete(sc, sc->sc_q[i]); 475 nvme_q_free(sc, sc->sc_q[i]); 476 } 477 disable: 478 nvme_disable(sc); 479 disestablish_admin_q: 480 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 481 free_admin_q: 482 nvme_q_free(sc, sc->sc_admin_q); 483 484 return 1; 485 } 486 487 int 488 nvme_rescan(device_t self, const char *ifattr, const int *locs) 489 { 490 struct nvme_softc *sc = device_private(self); 491 struct nvme_attach_args naa; 492 struct nvm_namespace_format *f; 493 struct nvme_namespace *ns; 494 uint64_t cap; 495 int ioq_entries = nvme_ioq_size; 496 int i, mlocs[NVMECF_NLOCS]; 497 int error; 498 499 cap = nvme_read8(sc, NVME_CAP); 500 if (ioq_entries > NVME_CAP_MQES(cap)) 501 ioq_entries = NVME_CAP_MQES(cap); 502 503 for (i = 1; i <= sc->sc_nn; i++) { 504 if (sc->sc_namespaces[i - 1].dev) 505 continue; 506 507 /* identify to check for availability */ 508 error = nvme_ns_identify(sc, i); 509 if (error) { 510 aprint_error_dev(self, "couldn't identify namespace #%d\n", i); 511 continue; 512 } 513 514 ns = nvme_ns_get(sc, i); 515 KASSERT(ns); 516 517 f = &ns->ident->lbaf[NVME_ID_NS_FLBAS(ns->ident->flbas)]; 518 519 /* 520 * NVME1.0e 6.11 Identify command 521 * 522 * LBADS values smaller than 9 are not supported, a value 523 * of zero means that the format is not used. 524 */ 525 if (f->lbads < 9) { 526 if (f->lbads > 0) 527 aprint_error_dev(self, 528 "unsupported logical data size %u\n", f->lbads); 529 continue; 530 } 531 532 mlocs[NVMECF_NSID] = i; 533 534 memset(&naa, 0, sizeof(naa)); 535 naa.naa_nsid = i; 536 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 537 naa.naa_maxphys = sc->sc_mdts; 538 naa.naa_typename = sc->sc_modelname; 539 sc->sc_namespaces[i - 1].dev = 540 config_found(sc->sc_dev, &naa, nvme_print, 541 CFARGS(.submatch = config_stdsubmatch, 542 .locators = mlocs)); 543 } 544 return 0; 545 } 546 547 static int 548 nvme_print(void *aux, const char *pnp) 549 { 550 struct nvme_attach_args *naa = aux; 551 552 if (pnp) 553 aprint_normal("ld at %s", pnp); 554 555 if (naa->naa_nsid > 0) 556 aprint_normal(" nsid %d", naa->naa_nsid); 557 558 return UNCONF; 559 } 560 561 int 562 nvme_detach(struct nvme_softc *sc, int flags) 563 { 564 int i, error; 565 566 error = config_detach_children(sc->sc_dev, flags); 567 if (error) 568 return error; 569 570 error = nvme_shutdown(sc); 571 if (error) 572 return error; 573 574 /* from now on we are committed to detach, following will never fail */ 575 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 576 for (i = 0; i < sc->sc_nq; i++) 577 nvme_q_free(sc, sc->sc_q[i]); 578 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 579 nvme_q_free(sc, sc->sc_admin_q); 580 581 return 0; 582 } 583 584 int 585 nvme_suspend(struct nvme_softc *sc) 586 { 587 588 return nvme_shutdown(sc); 589 } 590 591 int 592 nvme_resume(struct nvme_softc *sc) 593 { 594 int ioq_entries = nvme_ioq_size; 595 uint64_t cap; 596 int i, error; 597 598 error = nvme_disable(sc); 599 if (error) { 600 device_printf(sc->sc_dev, "unable to disable controller\n"); 601 return error; 602 } 603 604 nvme_q_reset(sc, sc->sc_admin_q); 605 606 error = nvme_enable(sc, ffs(sc->sc_mps) - 1); 607 if (error) { 608 device_printf(sc->sc_dev, "unable to enable controller\n"); 609 return error; 610 } 611 612 for (i = 0; i < sc->sc_nq; i++) { 613 cap = nvme_read8(sc, NVME_CAP); 614 if (ioq_entries > NVME_CAP_MQES(cap)) 615 ioq_entries = NVME_CAP_MQES(cap); 616 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, 617 sc->sc_dstrd); 618 if (sc->sc_q[i] == NULL) { 619 error = ENOMEM; 620 device_printf(sc->sc_dev, "unable to allocate io q %d" 621 "\n", i); 622 goto disable; 623 } 624 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 625 error = EIO; 626 device_printf(sc->sc_dev, "unable to create io q %d" 627 "\n", i); 628 nvme_q_free(sc, sc->sc_q[i]); 629 goto free_q; 630 } 631 } 632 633 nvme_write4(sc, NVME_INTMC, 1); 634 635 return 0; 636 637 free_q: 638 while (i --> 0) 639 nvme_q_free(sc, sc->sc_q[i]); 640 disable: 641 (void)nvme_disable(sc); 642 643 return error; 644 } 645 646 static int 647 nvme_shutdown(struct nvme_softc *sc) 648 { 649 uint32_t cc, csts; 650 bool disabled = false; 651 int i; 652 653 if (!sc->sc_use_mq) 654 nvme_write4(sc, NVME_INTMS, 1); 655 656 for (i = 0; i < sc->sc_nq; i++) { 657 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 658 aprint_error_dev(sc->sc_dev, 659 "unable to delete io queue %d, disabling\n", i + 1); 660 disabled = true; 661 } 662 } 663 if (disabled) 664 goto disable; 665 666 cc = nvme_read4(sc, NVME_CC); 667 CLR(cc, NVME_CC_SHN_MASK); 668 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 669 nvme_write4(sc, NVME_CC, cc); 670 671 for (i = 0; i < 4000; i++) { 672 nvme_barrier(sc, 0, sc->sc_ios, 673 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 674 csts = nvme_read4(sc, NVME_CSTS); 675 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 676 return 0; 677 678 delay(1000); 679 } 680 681 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 682 683 disable: 684 nvme_disable(sc); 685 return 0; 686 } 687 688 void 689 nvme_childdet(device_t self, device_t child) 690 { 691 struct nvme_softc *sc = device_private(self); 692 int i; 693 694 for (i = 0; i < sc->sc_nn; i++) { 695 if (sc->sc_namespaces[i].dev == child) { 696 /* Already freed ns->ident. */ 697 sc->sc_namespaces[i].dev = NULL; 698 break; 699 } 700 } 701 } 702 703 int 704 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 705 { 706 struct nvme_sqe sqe; 707 struct nvm_identify_namespace *identify; 708 struct nvme_dmamem *mem; 709 struct nvme_ccb *ccb; 710 struct nvme_namespace *ns; 711 int rv; 712 713 KASSERT(nsid > 0); 714 715 ns = nvme_ns_get(sc, nsid); 716 KASSERT(ns); 717 718 if (ns->ident != NULL) 719 return 0; 720 721 ccb = nvme_ccb_get(sc->sc_admin_q, false); 722 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 723 724 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 725 if (mem == NULL) { 726 nvme_ccb_put(sc->sc_admin_q, ccb); 727 return ENOMEM; 728 } 729 730 memset(&sqe, 0, sizeof(sqe)); 731 sqe.opcode = NVM_ADMIN_IDENTIFY; 732 htolem32(&sqe.nsid, nsid); 733 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 734 htolem32(&sqe.cdw10, 0); 735 736 ccb->ccb_done = nvme_empty_done; 737 ccb->ccb_cookie = &sqe; 738 739 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 740 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 741 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 742 743 nvme_ccb_put(sc->sc_admin_q, ccb); 744 745 if (rv != 0) { 746 rv = EIO; 747 goto done; 748 } 749 750 /* commit */ 751 752 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 753 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 754 755 /* Convert data to host endian */ 756 nvme_identify_namespace_swapbytes(identify); 757 758 ns->ident = identify; 759 760 done: 761 nvme_dmamem_free(sc, mem); 762 763 return rv; 764 } 765 766 int 767 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 768 struct buf *bp, void *data, size_t datasize, 769 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 770 { 771 struct nvme_queue *q = nvme_get_q(sc, bp, false); 772 struct nvme_ccb *ccb; 773 bus_dmamap_t dmap; 774 int i, error; 775 776 ccb = nvme_ccb_get(q, false); 777 if (ccb == NULL) 778 return EAGAIN; 779 780 ccb->ccb_done = nvme_ns_io_done; 781 ccb->ccb_cookie = cookie; 782 783 /* namespace context */ 784 ccb->nnc_nsid = nsid; 785 ccb->nnc_flags = flags; 786 ccb->nnc_buf = bp; 787 ccb->nnc_datasize = datasize; 788 ccb->nnc_secsize = secsize; 789 ccb->nnc_blkno = blkno; 790 ccb->nnc_done = nnc_done; 791 792 dmap = ccb->ccb_dmamap; 793 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 794 datasize, NULL, 795 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 796 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 797 (ISSET(flags, NVME_NS_CTX_F_READ) ? 798 BUS_DMA_READ : BUS_DMA_WRITE)); 799 if (error) { 800 nvme_ccb_put(q, ccb); 801 return error; 802 } 803 804 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 805 ISSET(flags, NVME_NS_CTX_F_READ) ? 806 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 807 808 if (dmap->dm_nsegs > 2) { 809 for (i = 1; i < dmap->dm_nsegs; i++) { 810 htolem64(&ccb->ccb_prpl[i - 1], 811 dmap->dm_segs[i].ds_addr); 812 } 813 bus_dmamap_sync(sc->sc_dmat, 814 NVME_DMA_MAP(q->q_ccb_prpls), 815 ccb->ccb_prpl_off, 816 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 817 BUS_DMASYNC_PREWRITE); 818 } 819 820 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 821 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 822 return EIO; 823 return 0; 824 } 825 826 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 827 return 0; 828 } 829 830 static void 831 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 832 { 833 struct nvme_sqe_io *sqe = slot; 834 bus_dmamap_t dmap = ccb->ccb_dmamap; 835 836 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 837 NVM_CMD_READ : NVM_CMD_WRITE; 838 htolem32(&sqe->nsid, ccb->nnc_nsid); 839 840 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 841 switch (dmap->dm_nsegs) { 842 case 1: 843 break; 844 case 2: 845 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 846 break; 847 default: 848 /* the prp list is already set up and synced */ 849 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 850 break; 851 } 852 853 htolem64(&sqe->slba, ccb->nnc_blkno); 854 855 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) 856 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA); 857 858 /* guaranteed by upper layers, but check just in case */ 859 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 860 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 861 } 862 863 static void 864 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 865 struct nvme_cqe *cqe) 866 { 867 struct nvme_softc *sc = q->q_sc; 868 bus_dmamap_t dmap = ccb->ccb_dmamap; 869 void *nnc_cookie = ccb->ccb_cookie; 870 nvme_nnc_done nnc_done = ccb->nnc_done; 871 struct buf *bp = ccb->nnc_buf; 872 873 if (dmap->dm_nsegs > 2) { 874 bus_dmamap_sync(sc->sc_dmat, 875 NVME_DMA_MAP(q->q_ccb_prpls), 876 ccb->ccb_prpl_off, 877 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 878 BUS_DMASYNC_POSTWRITE); 879 } 880 881 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 882 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 883 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 884 885 bus_dmamap_unload(sc->sc_dmat, dmap); 886 nvme_ccb_put(q, ccb); 887 888 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 889 } 890 891 /* 892 * If there is no volatile write cache, it makes no sense to issue 893 * flush commands or query for the status. 894 */ 895 static bool 896 nvme_has_volatile_write_cache(struct nvme_softc *sc) 897 { 898 /* sc_identify is filled during attachment */ 899 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0); 900 } 901 902 static bool 903 nvme_ns_sync_finished(void *cookie) 904 { 905 int *result = cookie; 906 907 return (*result != 0); 908 } 909 910 int 911 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, int flags) 912 { 913 struct nvme_queue *q = nvme_get_q(sc, NULL, true); 914 struct nvme_ccb *ccb; 915 int result = 0; 916 917 if (!nvme_has_volatile_write_cache(sc)) { 918 /* cache not present, no value in trying to flush it */ 919 return 0; 920 } 921 922 ccb = nvme_ccb_get(q, true); 923 KASSERT(ccb != NULL); 924 925 ccb->ccb_done = nvme_ns_sync_done; 926 ccb->ccb_cookie = &result; 927 928 /* namespace context */ 929 ccb->nnc_nsid = nsid; 930 ccb->nnc_flags = flags; 931 ccb->nnc_done = NULL; 932 933 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 934 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 935 return EIO; 936 return 0; 937 } 938 939 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 940 941 /* wait for completion */ 942 nvme_q_wait_complete(sc, q, nvme_ns_sync_finished, &result); 943 KASSERT(result != 0); 944 945 return (result > 0) ? 0 : EIO; 946 } 947 948 static void 949 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 950 { 951 struct nvme_sqe *sqe = slot; 952 953 sqe->opcode = NVM_CMD_FLUSH; 954 htolem32(&sqe->nsid, ccb->nnc_nsid); 955 } 956 957 static void 958 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 959 struct nvme_cqe *cqe) 960 { 961 int *result = ccb->ccb_cookie; 962 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 963 964 if (status == NVME_CQE_SC_SUCCESS) 965 *result = 1; 966 else 967 *result = -1; 968 969 nvme_ccb_put(q, ccb); 970 } 971 972 static bool 973 nvme_getcache_finished(void *xc) 974 { 975 int *addr = xc; 976 977 return (*addr != 0); 978 } 979 980 /* 981 * Get status of volatile write cache. Always asynchronous. 982 */ 983 int 984 nvme_admin_getcache(struct nvme_softc *sc, int *addr) 985 { 986 struct nvme_ccb *ccb; 987 struct nvme_queue *q = sc->sc_admin_q; 988 int result = 0, error; 989 990 if (!nvme_has_volatile_write_cache(sc)) { 991 /* cache simply not present */ 992 *addr = 0; 993 return 0; 994 } 995 996 ccb = nvme_ccb_get(q, true); 997 KASSERT(ccb != NULL); 998 999 ccb->ccb_done = nvme_getcache_done; 1000 ccb->ccb_cookie = &result; 1001 1002 /* namespace context */ 1003 ccb->nnc_flags = 0; 1004 ccb->nnc_done = NULL; 1005 1006 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); 1007 1008 /* wait for completion */ 1009 nvme_q_wait_complete(sc, q, nvme_getcache_finished, &result); 1010 KASSERT(result != 0); 1011 1012 if (result > 0) { 1013 *addr = result; 1014 error = 0; 1015 } else 1016 error = EINVAL; 1017 1018 return error; 1019 } 1020 1021 static void 1022 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1023 { 1024 struct nvme_sqe *sqe = slot; 1025 1026 sqe->opcode = NVM_ADMIN_GET_FEATURES; 1027 htolem32(&sqe->cdw10, NVM_FEATURE_VOLATILE_WRITE_CACHE); 1028 htolem32(&sqe->cdw11, NVM_VOLATILE_WRITE_CACHE_WCE); 1029 } 1030 1031 static void 1032 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1033 struct nvme_cqe *cqe) 1034 { 1035 int *addr = ccb->ccb_cookie; 1036 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 1037 uint32_t cdw0 = lemtoh32(&cqe->cdw0); 1038 int result; 1039 1040 if (status == NVME_CQE_SC_SUCCESS) { 1041 result = 0; 1042 1043 /* 1044 * DPO not supported, Dataset Management (DSM) field doesn't 1045 * specify the same semantics. FUA is always supported. 1046 */ 1047 result = DKCACHE_FUA; 1048 1049 if (cdw0 & NVM_VOLATILE_WRITE_CACHE_WCE) 1050 result |= DKCACHE_WRITE; 1051 1052 /* 1053 * If volatile write cache is present, the flag shall also be 1054 * settable. 1055 */ 1056 result |= DKCACHE_WCHANGE; 1057 1058 /* 1059 * ONCS field indicates whether the optional SAVE is also 1060 * supported for Set Features. According to spec v1.3, 1061 * Volatile Write Cache however doesn't support persistency 1062 * across power cycle/reset. 1063 */ 1064 1065 } else { 1066 result = -1; 1067 } 1068 1069 *addr = result; 1070 1071 nvme_ccb_put(q, ccb); 1072 } 1073 1074 struct nvme_setcache_state { 1075 int dkcache; 1076 int result; 1077 }; 1078 1079 static bool 1080 nvme_setcache_finished(void *xc) 1081 { 1082 struct nvme_setcache_state *st = xc; 1083 1084 return (st->result != 0); 1085 } 1086 1087 static void 1088 nvme_setcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1089 { 1090 struct nvme_sqe *sqe = slot; 1091 struct nvme_setcache_state *st = ccb->ccb_cookie; 1092 1093 sqe->opcode = NVM_ADMIN_SET_FEATURES; 1094 htolem32(&sqe->cdw10, NVM_FEATURE_VOLATILE_WRITE_CACHE); 1095 if (st->dkcache & DKCACHE_WRITE) 1096 htolem32(&sqe->cdw11, NVM_VOLATILE_WRITE_CACHE_WCE); 1097 } 1098 1099 static void 1100 nvme_setcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1101 struct nvme_cqe *cqe) 1102 { 1103 struct nvme_setcache_state *st = ccb->ccb_cookie; 1104 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 1105 1106 if (status == NVME_CQE_SC_SUCCESS) { 1107 st->result = 1; 1108 } else { 1109 st->result = -1; 1110 } 1111 1112 nvme_ccb_put(q, ccb); 1113 } 1114 1115 /* 1116 * Set status of volatile write cache. Always asynchronous. 1117 */ 1118 int 1119 nvme_admin_setcache(struct nvme_softc *sc, int dkcache) 1120 { 1121 struct nvme_ccb *ccb; 1122 struct nvme_queue *q = sc->sc_admin_q; 1123 int error; 1124 struct nvme_setcache_state st; 1125 1126 if (!nvme_has_volatile_write_cache(sc)) { 1127 /* cache simply not present */ 1128 return EOPNOTSUPP; 1129 } 1130 1131 if (dkcache & ~(DKCACHE_WRITE)) { 1132 /* unsupported parameters */ 1133 return EOPNOTSUPP; 1134 } 1135 1136 ccb = nvme_ccb_get(q, true); 1137 KASSERT(ccb != NULL); 1138 1139 memset(&st, 0, sizeof(st)); 1140 st.dkcache = dkcache; 1141 1142 ccb->ccb_done = nvme_setcache_done; 1143 ccb->ccb_cookie = &st; 1144 1145 /* namespace context */ 1146 ccb->nnc_flags = 0; 1147 ccb->nnc_done = NULL; 1148 1149 nvme_q_submit(sc, q, ccb, nvme_setcache_fill); 1150 1151 /* wait for completion */ 1152 nvme_q_wait_complete(sc, q, nvme_setcache_finished, &st); 1153 KASSERT(st.result != 0); 1154 1155 if (st.result > 0) 1156 error = 0; 1157 else 1158 error = EINVAL; 1159 1160 return error; 1161 } 1162 1163 void 1164 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 1165 { 1166 struct nvme_namespace *ns; 1167 struct nvm_identify_namespace *identify; 1168 1169 ns = nvme_ns_get(sc, nsid); 1170 KASSERT(ns); 1171 1172 identify = ns->ident; 1173 ns->ident = NULL; 1174 if (identify != NULL) 1175 kmem_free(identify, sizeof(*identify)); 1176 } 1177 1178 struct nvme_pt_state { 1179 struct nvme_pt_command *pt; 1180 bool finished; 1181 }; 1182 1183 static void 1184 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1185 { 1186 struct nvme_softc *sc = q->q_sc; 1187 struct nvme_sqe *sqe = slot; 1188 struct nvme_pt_state *state = ccb->ccb_cookie; 1189 struct nvme_pt_command *pt = state->pt; 1190 bus_dmamap_t dmap = ccb->ccb_dmamap; 1191 int i; 1192 1193 sqe->opcode = pt->cmd.opcode; 1194 htolem32(&sqe->nsid, pt->cmd.nsid); 1195 1196 if (pt->buf != NULL && pt->len > 0) { 1197 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 1198 switch (dmap->dm_nsegs) { 1199 case 1: 1200 break; 1201 case 2: 1202 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 1203 break; 1204 default: 1205 for (i = 1; i < dmap->dm_nsegs; i++) { 1206 htolem64(&ccb->ccb_prpl[i - 1], 1207 dmap->dm_segs[i].ds_addr); 1208 } 1209 bus_dmamap_sync(sc->sc_dmat, 1210 NVME_DMA_MAP(q->q_ccb_prpls), 1211 ccb->ccb_prpl_off, 1212 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 1213 BUS_DMASYNC_PREWRITE); 1214 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 1215 break; 1216 } 1217 } 1218 1219 htolem32(&sqe->cdw10, pt->cmd.cdw10); 1220 htolem32(&sqe->cdw11, pt->cmd.cdw11); 1221 htolem32(&sqe->cdw12, pt->cmd.cdw12); 1222 htolem32(&sqe->cdw13, pt->cmd.cdw13); 1223 htolem32(&sqe->cdw14, pt->cmd.cdw14); 1224 htolem32(&sqe->cdw15, pt->cmd.cdw15); 1225 } 1226 1227 static void 1228 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 1229 { 1230 struct nvme_softc *sc = q->q_sc; 1231 struct nvme_pt_state *state = ccb->ccb_cookie; 1232 struct nvme_pt_command *pt = state->pt; 1233 bus_dmamap_t dmap = ccb->ccb_dmamap; 1234 1235 if (pt->buf != NULL && pt->len > 0) { 1236 if (dmap->dm_nsegs > 2) { 1237 bus_dmamap_sync(sc->sc_dmat, 1238 NVME_DMA_MAP(q->q_ccb_prpls), 1239 ccb->ccb_prpl_off, 1240 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 1241 BUS_DMASYNC_POSTWRITE); 1242 } 1243 1244 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1245 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 1246 bus_dmamap_unload(sc->sc_dmat, dmap); 1247 } 1248 1249 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 1250 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 1251 1252 state->finished = true; 1253 1254 nvme_ccb_put(q, ccb); 1255 } 1256 1257 static bool 1258 nvme_pt_finished(void *cookie) 1259 { 1260 struct nvme_pt_state *state = cookie; 1261 1262 return state->finished; 1263 } 1264 1265 static int 1266 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 1267 uint32_t nsid, struct lwp *l, bool is_adminq) 1268 { 1269 struct nvme_queue *q; 1270 struct nvme_ccb *ccb; 1271 void *buf = NULL; 1272 struct nvme_pt_state state; 1273 int error; 1274 1275 /* limit command size to maximum data transfer size */ 1276 if ((pt->buf == NULL && pt->len > 0) || 1277 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 1278 return EINVAL; 1279 1280 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc, NULL, true); 1281 ccb = nvme_ccb_get(q, true); 1282 KASSERT(ccb != NULL); 1283 1284 if (pt->buf != NULL) { 1285 KASSERT(pt->len > 0); 1286 buf = kmem_alloc(pt->len, KM_SLEEP); 1287 if (!pt->is_read) { 1288 error = copyin(pt->buf, buf, pt->len); 1289 if (error) 1290 goto kmem_free; 1291 } 1292 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 1293 pt->len, NULL, 1294 BUS_DMA_WAITOK | 1295 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 1296 if (error) 1297 goto kmem_free; 1298 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 1299 0, ccb->ccb_dmamap->dm_mapsize, 1300 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 1301 } 1302 1303 memset(&state, 0, sizeof(state)); 1304 state.pt = pt; 1305 state.finished = false; 1306 1307 ccb->ccb_done = nvme_pt_done; 1308 ccb->ccb_cookie = &state; 1309 1310 pt->cmd.nsid = nsid; 1311 1312 nvme_q_submit(sc, q, ccb, nvme_pt_fill); 1313 1314 /* wait for completion */ 1315 nvme_q_wait_complete(sc, q, nvme_pt_finished, &state); 1316 KASSERT(state.finished); 1317 1318 error = 0; 1319 1320 if (buf != NULL) { 1321 if (error == 0 && pt->is_read) 1322 error = copyout(buf, pt->buf, pt->len); 1323 kmem_free: 1324 kmem_free(buf, pt->len); 1325 } 1326 1327 return error; 1328 } 1329 1330 uint32_t 1331 nvme_op_sq_enter(struct nvme_softc *sc, 1332 struct nvme_queue *q, struct nvme_ccb *ccb) 1333 { 1334 mutex_enter(&q->q_sq_mtx); 1335 1336 return nvme_op_sq_enter_locked(sc, q, ccb); 1337 } 1338 1339 uint32_t 1340 nvme_op_sq_enter_locked(struct nvme_softc *sc, 1341 struct nvme_queue *q, struct nvme_ccb *ccb) 1342 { 1343 return q->q_sq_tail; 1344 } 1345 1346 void 1347 nvme_op_sq_leave_locked(struct nvme_softc *sc, 1348 struct nvme_queue *q, struct nvme_ccb *ccb) 1349 { 1350 uint32_t tail; 1351 1352 tail = ++q->q_sq_tail; 1353 if (tail >= q->q_entries) 1354 tail = 0; 1355 q->q_sq_tail = tail; 1356 nvme_write4(sc, q->q_sqtdbl, tail); 1357 } 1358 1359 void 1360 nvme_op_sq_leave(struct nvme_softc *sc, 1361 struct nvme_queue *q, struct nvme_ccb *ccb) 1362 { 1363 nvme_op_sq_leave_locked(sc, q, ccb); 1364 1365 mutex_exit(&q->q_sq_mtx); 1366 } 1367 1368 static void 1369 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1370 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 1371 { 1372 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 1373 uint32_t tail; 1374 1375 tail = sc->sc_ops->op_sq_enter(sc, q, ccb); 1376 1377 sqe += tail; 1378 1379 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1380 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 1381 memset(sqe, 0, sizeof(*sqe)); 1382 (*fill)(q, ccb, sqe); 1383 htolem16(&sqe->cid, ccb->ccb_id); 1384 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1385 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 1386 1387 sc->sc_ops->op_sq_leave(sc, q, ccb); 1388 } 1389 1390 struct nvme_poll_state { 1391 struct nvme_sqe s; 1392 struct nvme_cqe c; 1393 void *cookie; 1394 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 1395 }; 1396 1397 static int 1398 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1399 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 1400 { 1401 struct nvme_poll_state state; 1402 uint16_t flags; 1403 int step = 10; 1404 int maxloop = timo_sec * 1000000 / step; 1405 int error = 0; 1406 1407 memset(&state, 0, sizeof(state)); 1408 (*fill)(q, ccb, &state.s); 1409 1410 state.done = ccb->ccb_done; 1411 state.cookie = ccb->ccb_cookie; 1412 1413 ccb->ccb_done = nvme_poll_done; 1414 ccb->ccb_cookie = &state; 1415 1416 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1417 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1418 if (nvme_q_complete(sc, q) == 0) 1419 delay(step); 1420 1421 if (timo_sec >= 0 && --maxloop <= 0) { 1422 error = ETIMEDOUT; 1423 break; 1424 } 1425 } 1426 1427 if (error == 0) { 1428 flags = lemtoh16(&state.c.flags); 1429 return flags & ~NVME_CQE_PHASE; 1430 } else { 1431 /* 1432 * If it succeds later, it would hit ccb which will have been 1433 * already reused for something else. Not good. Cross 1434 * fingers and hope for best. XXX do controller reset? 1435 */ 1436 aprint_error_dev(sc->sc_dev, "polled command timed out\n"); 1437 1438 /* Invoke the callback to clean state anyway */ 1439 struct nvme_cqe cqe; 1440 memset(&cqe, 0, sizeof(cqe)); 1441 ccb->ccb_done(q, ccb, &cqe); 1442 1443 return 1; 1444 } 1445 } 1446 1447 static void 1448 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1449 { 1450 struct nvme_sqe *sqe = slot; 1451 struct nvme_poll_state *state = ccb->ccb_cookie; 1452 1453 *sqe = state->s; 1454 } 1455 1456 static void 1457 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1458 struct nvme_cqe *cqe) 1459 { 1460 struct nvme_poll_state *state = ccb->ccb_cookie; 1461 1462 state->c = *cqe; 1463 SET(state->c.flags, htole16(NVME_CQE_PHASE)); 1464 1465 ccb->ccb_cookie = state->cookie; 1466 state->done(q, ccb, &state->c); 1467 } 1468 1469 static void 1470 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1471 { 1472 struct nvme_sqe *src = ccb->ccb_cookie; 1473 struct nvme_sqe *dst = slot; 1474 1475 *dst = *src; 1476 } 1477 1478 static void 1479 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1480 struct nvme_cqe *cqe) 1481 { 1482 } 1483 1484 void 1485 nvme_op_cq_done(struct nvme_softc *sc, 1486 struct nvme_queue *q, struct nvme_ccb *ccb) 1487 { 1488 /* nop */ 1489 } 1490 1491 static int 1492 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1493 { 1494 struct nvme_ccb *ccb; 1495 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1496 uint16_t flags; 1497 int rv = 0; 1498 1499 mutex_enter(&q->q_cq_mtx); 1500 1501 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1502 for (;;) { 1503 cqe = &ring[q->q_cq_head]; 1504 flags = lemtoh16(&cqe->flags); 1505 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1506 break; 1507 1508 ccb = &q->q_ccbs[lemtoh16(&cqe->cid)]; 1509 1510 if (++q->q_cq_head >= q->q_entries) { 1511 q->q_cq_head = 0; 1512 q->q_cq_phase ^= NVME_CQE_PHASE; 1513 } 1514 1515 #ifdef DEBUG 1516 /* 1517 * If we get spurious completion notification, something 1518 * is seriously hosed up. Very likely DMA to some random 1519 * memory place happened, so just bail out. 1520 */ 1521 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1522 panic("%s: invalid ccb detected", 1523 device_xname(sc->sc_dev)); 1524 /* NOTREACHED */ 1525 } 1526 #endif 1527 1528 rv++; 1529 1530 sc->sc_ops->op_cq_done(sc, q, ccb); 1531 1532 /* 1533 * Unlock the mutex before calling the ccb_done callback 1534 * and re-lock afterwards. The callback triggers lddone() 1535 * which schedules another i/o, and also calls nvme_ccb_put(). 1536 * Unlock/relock avoids possibility of deadlock. 1537 */ 1538 mutex_exit(&q->q_cq_mtx); 1539 ccb->ccb_done(q, ccb, cqe); 1540 mutex_enter(&q->q_cq_mtx); 1541 } 1542 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1543 1544 if (rv) 1545 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1546 1547 mutex_exit(&q->q_cq_mtx); 1548 1549 return rv; 1550 } 1551 1552 static void 1553 nvme_q_wait_complete(struct nvme_softc *sc, 1554 struct nvme_queue *q, bool (*finished)(void *), void *cookie) 1555 { 1556 mutex_enter(&q->q_ccb_mtx); 1557 if (finished(cookie)) 1558 goto out; 1559 1560 for(;;) { 1561 q->q_ccb_waiting = true; 1562 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1563 1564 if (finished(cookie)) 1565 break; 1566 } 1567 1568 out: 1569 mutex_exit(&q->q_ccb_mtx); 1570 } 1571 1572 static int 1573 nvme_identify(struct nvme_softc *sc, u_int mps) 1574 { 1575 char sn[41], mn[81], fr[17]; 1576 struct nvm_identify_controller *identify; 1577 struct nvme_dmamem *mem; 1578 struct nvme_ccb *ccb; 1579 u_int mdts; 1580 int rv = 1; 1581 1582 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1583 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1584 1585 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1586 if (mem == NULL) 1587 return 1; 1588 1589 ccb->ccb_done = nvme_empty_done; 1590 ccb->ccb_cookie = mem; 1591 1592 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1593 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1594 NVME_TIMO_IDENT); 1595 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1596 1597 nvme_ccb_put(sc->sc_admin_q, ccb); 1598 1599 if (rv != 0) 1600 goto done; 1601 1602 identify = NVME_DMA_KVA(mem); 1603 sc->sc_identify = *identify; 1604 identify = NULL; 1605 1606 /* Convert data to host endian */ 1607 nvme_identify_controller_swapbytes(&sc->sc_identify); 1608 1609 strnvisx(sn, sizeof(sn), (const char *)sc->sc_identify.sn, 1610 sizeof(sc->sc_identify.sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1611 strnvisx(mn, sizeof(mn), (const char *)sc->sc_identify.mn, 1612 sizeof(sc->sc_identify.mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1613 strnvisx(fr, sizeof(fr), (const char *)sc->sc_identify.fr, 1614 sizeof(sc->sc_identify.fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1615 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1616 sn); 1617 1618 strlcpy(sc->sc_modelname, mn, sizeof(sc->sc_modelname)); 1619 1620 if (sc->sc_identify.mdts > 0) { 1621 mdts = (1 << sc->sc_identify.mdts) * (1 << mps); 1622 if (mdts < sc->sc_mdts) 1623 sc->sc_mdts = mdts; 1624 } 1625 1626 sc->sc_nn = sc->sc_identify.nn; 1627 1628 done: 1629 nvme_dmamem_free(sc, mem); 1630 1631 return rv; 1632 } 1633 1634 static int 1635 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1636 { 1637 struct nvme_sqe_q sqe; 1638 struct nvme_ccb *ccb; 1639 int rv; 1640 1641 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1642 return 1; 1643 1644 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1645 KASSERT(ccb != NULL); 1646 1647 ccb->ccb_done = nvme_empty_done; 1648 ccb->ccb_cookie = &sqe; 1649 1650 memset(&sqe, 0, sizeof(sqe)); 1651 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1652 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1653 htolem16(&sqe.qsize, q->q_entries - 1); 1654 htolem16(&sqe.qid, q->q_id); 1655 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1656 if (sc->sc_use_mq) 1657 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1658 1659 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1660 if (rv != 0) 1661 goto fail; 1662 1663 ccb->ccb_done = nvme_empty_done; 1664 ccb->ccb_cookie = &sqe; 1665 1666 memset(&sqe, 0, sizeof(sqe)); 1667 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1668 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1669 htolem16(&sqe.qsize, q->q_entries - 1); 1670 htolem16(&sqe.qid, q->q_id); 1671 htolem16(&sqe.cqid, q->q_id); 1672 sqe.qflags = NVM_SQE_Q_PC; 1673 1674 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1675 if (rv != 0) 1676 goto fail; 1677 1678 nvme_ccb_put(sc->sc_admin_q, ccb); 1679 return 0; 1680 1681 fail: 1682 if (sc->sc_use_mq) 1683 sc->sc_intr_disestablish(sc, q->q_id); 1684 1685 nvme_ccb_put(sc->sc_admin_q, ccb); 1686 return rv; 1687 } 1688 1689 static int 1690 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1691 { 1692 struct nvme_sqe_q sqe; 1693 struct nvme_ccb *ccb; 1694 int rv; 1695 1696 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1697 KASSERT(ccb != NULL); 1698 1699 ccb->ccb_done = nvme_empty_done; 1700 ccb->ccb_cookie = &sqe; 1701 1702 memset(&sqe, 0, sizeof(sqe)); 1703 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1704 htolem16(&sqe.qid, q->q_id); 1705 1706 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1707 if (rv != 0) 1708 goto fail; 1709 1710 ccb->ccb_done = nvme_empty_done; 1711 ccb->ccb_cookie = &sqe; 1712 1713 memset(&sqe, 0, sizeof(sqe)); 1714 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1715 htolem16(&sqe.qid, q->q_id); 1716 1717 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1718 if (rv != 0) 1719 goto fail; 1720 1721 fail: 1722 nvme_ccb_put(sc->sc_admin_q, ccb); 1723 1724 if (rv == 0 && sc->sc_use_mq) { 1725 if (sc->sc_intr_disestablish(sc, q->q_id)) 1726 rv = 1; 1727 } 1728 1729 return rv; 1730 } 1731 1732 static void 1733 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1734 { 1735 struct nvme_sqe *sqe = slot; 1736 struct nvme_dmamem *mem = ccb->ccb_cookie; 1737 1738 sqe->opcode = NVM_ADMIN_IDENTIFY; 1739 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1740 htolem32(&sqe->cdw10, 1); 1741 } 1742 1743 static int 1744 nvme_set_number_of_queues(struct nvme_softc *sc, u_int nq, u_int *ncqa, 1745 u_int *nsqa) 1746 { 1747 struct nvme_pt_state state; 1748 struct nvme_pt_command pt; 1749 struct nvme_ccb *ccb; 1750 int rv; 1751 1752 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1753 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1754 1755 memset(&pt, 0, sizeof(pt)); 1756 pt.cmd.opcode = NVM_ADMIN_SET_FEATURES; 1757 pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES; 1758 pt.cmd.cdw11 = ((nq - 1) << 16) | (nq - 1); 1759 1760 memset(&state, 0, sizeof(state)); 1761 state.pt = &pt; 1762 state.finished = false; 1763 1764 ccb->ccb_done = nvme_pt_done; 1765 ccb->ccb_cookie = &state; 1766 1767 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1768 1769 if (rv != 0) { 1770 *ncqa = *nsqa = 0; 1771 return EIO; 1772 } 1773 1774 *ncqa = (pt.cpl.cdw0 >> 16) + 1; 1775 *nsqa = (pt.cpl.cdw0 & 0xffff) + 1; 1776 1777 return 0; 1778 } 1779 1780 static int 1781 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1782 { 1783 struct nvme_softc *sc = q->q_sc; 1784 struct nvme_ccb *ccb; 1785 bus_addr_t off; 1786 uint64_t *prpl; 1787 u_int i; 1788 1789 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1790 cv_init(&q->q_ccb_wait, "nvmeqw"); 1791 q->q_ccb_waiting = false; 1792 SIMPLEQ_INIT(&q->q_ccb_list); 1793 1794 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1795 1796 q->q_nccbs = nccbs; 1797 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1798 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1799 1800 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1801 off = 0; 1802 1803 for (i = 0; i < nccbs; i++) { 1804 ccb = &q->q_ccbs[i]; 1805 1806 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1807 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1808 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1809 &ccb->ccb_dmamap) != 0) 1810 goto free_maps; 1811 1812 ccb->ccb_id = i; 1813 ccb->ccb_prpl = prpl; 1814 ccb->ccb_prpl_off = off; 1815 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1816 1817 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1818 1819 prpl += sc->sc_max_sgl; 1820 off += sizeof(*prpl) * sc->sc_max_sgl; 1821 } 1822 1823 return 0; 1824 1825 free_maps: 1826 nvme_ccbs_free(q); 1827 return 1; 1828 } 1829 1830 static struct nvme_ccb * 1831 nvme_ccb_get(struct nvme_queue *q, bool wait) 1832 { 1833 struct nvme_ccb *ccb = NULL; 1834 1835 mutex_enter(&q->q_ccb_mtx); 1836 again: 1837 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1838 if (ccb != NULL) { 1839 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1840 #ifdef DEBUG 1841 ccb->ccb_cookie = NULL; 1842 #endif 1843 } else { 1844 if (__predict_false(wait)) { 1845 q->q_ccb_waiting = true; 1846 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1847 goto again; 1848 } 1849 } 1850 mutex_exit(&q->q_ccb_mtx); 1851 1852 return ccb; 1853 } 1854 1855 static void 1856 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1857 { 1858 1859 mutex_enter(&q->q_ccb_mtx); 1860 #ifdef DEBUG 1861 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1862 #endif 1863 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1864 1865 /* It's unlikely there are any waiters, it's not used for regular I/O */ 1866 if (__predict_false(q->q_ccb_waiting)) { 1867 q->q_ccb_waiting = false; 1868 cv_broadcast(&q->q_ccb_wait); 1869 } 1870 1871 mutex_exit(&q->q_ccb_mtx); 1872 } 1873 1874 static void 1875 nvme_ccbs_free(struct nvme_queue *q) 1876 { 1877 struct nvme_softc *sc = q->q_sc; 1878 struct nvme_ccb *ccb; 1879 1880 mutex_enter(&q->q_ccb_mtx); 1881 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1882 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1883 /* 1884 * bus_dmamap_destroy() may call vm_map_lock() and rw_enter() 1885 * internally. don't hold spin mutex 1886 */ 1887 mutex_exit(&q->q_ccb_mtx); 1888 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1889 mutex_enter(&q->q_ccb_mtx); 1890 } 1891 mutex_exit(&q->q_ccb_mtx); 1892 1893 nvme_dmamem_free(sc, q->q_ccb_prpls); 1894 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1895 q->q_ccbs = NULL; 1896 cv_destroy(&q->q_ccb_wait); 1897 mutex_destroy(&q->q_ccb_mtx); 1898 } 1899 1900 static struct nvme_queue * 1901 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1902 { 1903 struct nvme_queue *q; 1904 1905 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1906 q->q_sc = sc; 1907 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1908 sizeof(struct nvme_sqe) * entries); 1909 if (q->q_sq_dmamem == NULL) 1910 goto free; 1911 1912 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1913 sizeof(struct nvme_cqe) * entries); 1914 if (q->q_cq_dmamem == NULL) 1915 goto free_sq; 1916 1917 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1918 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1919 1920 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1921 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1922 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1923 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1924 q->q_id = id; 1925 q->q_entries = entries; 1926 q->q_sq_tail = 0; 1927 q->q_cq_head = 0; 1928 q->q_cq_phase = NVME_CQE_PHASE; 1929 1930 if (sc->sc_ops->op_q_alloc != NULL) { 1931 if (sc->sc_ops->op_q_alloc(sc, q) != 0) 1932 goto free_cq; 1933 } 1934 1935 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1936 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1937 1938 /* 1939 * Due to definition of full and empty queue (queue is empty 1940 * when head == tail, full when tail is one less then head), 1941 * we can actually only have (entries - 1) in-flight commands. 1942 */ 1943 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1944 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1945 goto free_cq; 1946 } 1947 1948 return q; 1949 1950 free_cq: 1951 nvme_dmamem_free(sc, q->q_cq_dmamem); 1952 free_sq: 1953 nvme_dmamem_free(sc, q->q_sq_dmamem); 1954 free: 1955 kmem_free(q, sizeof(*q)); 1956 1957 return NULL; 1958 } 1959 1960 static void 1961 nvme_q_reset(struct nvme_softc *sc, struct nvme_queue *q) 1962 { 1963 1964 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1965 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1966 1967 q->q_sq_tail = 0; 1968 q->q_cq_head = 0; 1969 q->q_cq_phase = NVME_CQE_PHASE; 1970 1971 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1972 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1973 } 1974 1975 static void 1976 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1977 { 1978 nvme_ccbs_free(q); 1979 mutex_destroy(&q->q_sq_mtx); 1980 mutex_destroy(&q->q_cq_mtx); 1981 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1982 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1983 1984 if (sc->sc_ops->op_q_alloc != NULL) 1985 sc->sc_ops->op_q_free(sc, q); 1986 1987 nvme_dmamem_free(sc, q->q_cq_dmamem); 1988 nvme_dmamem_free(sc, q->q_sq_dmamem); 1989 kmem_free(q, sizeof(*q)); 1990 } 1991 1992 int 1993 nvme_intr(void *xsc) 1994 { 1995 struct nvme_softc *sc = xsc; 1996 1997 /* 1998 * INTx is level triggered, controller deasserts the interrupt only 1999 * when we advance command queue head via write to the doorbell. 2000 * Tell the controller to block the interrupts while we process 2001 * the queue(s). 2002 */ 2003 nvme_write4(sc, NVME_INTMS, 1); 2004 2005 softint_schedule(sc->sc_softih[0]); 2006 2007 /* don't know, might not have been for us */ 2008 return 1; 2009 } 2010 2011 void 2012 nvme_softintr_intx(void *xq) 2013 { 2014 struct nvme_queue *q = xq; 2015 struct nvme_softc *sc = q->q_sc; 2016 2017 nvme_q_complete(sc, sc->sc_admin_q); 2018 if (sc->sc_q != NULL) 2019 nvme_q_complete(sc, sc->sc_q[0]); 2020 2021 /* 2022 * Processing done, tell controller to issue interrupts again. There 2023 * is no race, as NVMe spec requires the controller to maintain state, 2024 * and assert the interrupt whenever there are unacknowledged 2025 * completion queue entries. 2026 */ 2027 nvme_write4(sc, NVME_INTMC, 1); 2028 } 2029 2030 int 2031 nvme_intr_msi(void *xq) 2032 { 2033 struct nvme_queue *q = xq; 2034 2035 KASSERT(q && q->q_sc && q->q_sc->sc_softih 2036 && q->q_sc->sc_softih[q->q_id]); 2037 2038 /* 2039 * MSI/MSI-X are edge triggered, so can handover processing to softint 2040 * without masking the interrupt. 2041 */ 2042 softint_schedule(q->q_sc->sc_softih[q->q_id]); 2043 2044 return 1; 2045 } 2046 2047 void 2048 nvme_softintr_msi(void *xq) 2049 { 2050 struct nvme_queue *q = xq; 2051 struct nvme_softc *sc = q->q_sc; 2052 2053 nvme_q_complete(sc, q); 2054 } 2055 2056 struct nvme_dmamem * 2057 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 2058 { 2059 struct nvme_dmamem *ndm; 2060 int nsegs; 2061 2062 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 2063 if (ndm == NULL) 2064 return NULL; 2065 2066 ndm->ndm_size = size; 2067 2068 if (bus_dmamap_create(sc->sc_dmat, size, btoc(round_page(size)), size, 0, 2069 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 2070 goto ndmfree; 2071 2072 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 2073 1, &nsegs, BUS_DMA_WAITOK) != 0) 2074 goto destroy; 2075 2076 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 2077 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 2078 goto free; 2079 2080 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 2081 NULL, BUS_DMA_WAITOK) != 0) 2082 goto unmap; 2083 2084 memset(ndm->ndm_kva, 0, size); 2085 bus_dmamap_sync(sc->sc_dmat, ndm->ndm_map, 0, size, BUS_DMASYNC_PREREAD); 2086 2087 return ndm; 2088 2089 unmap: 2090 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 2091 free: 2092 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 2093 destroy: 2094 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 2095 ndmfree: 2096 kmem_free(ndm, sizeof(*ndm)); 2097 return NULL; 2098 } 2099 2100 void 2101 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 2102 { 2103 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 2104 0, NVME_DMA_LEN(mem), ops); 2105 } 2106 2107 void 2108 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 2109 { 2110 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 2111 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 2112 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 2113 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 2114 kmem_free(ndm, sizeof(*ndm)); 2115 } 2116 2117 /* 2118 * ioctl 2119 */ 2120 2121 dev_type_open(nvmeopen); 2122 dev_type_close(nvmeclose); 2123 dev_type_ioctl(nvmeioctl); 2124 2125 const struct cdevsw nvme_cdevsw = { 2126 .d_open = nvmeopen, 2127 .d_close = nvmeclose, 2128 .d_read = noread, 2129 .d_write = nowrite, 2130 .d_ioctl = nvmeioctl, 2131 .d_stop = nostop, 2132 .d_tty = notty, 2133 .d_poll = nopoll, 2134 .d_mmap = nommap, 2135 .d_kqfilter = nokqfilter, 2136 .d_discard = nodiscard, 2137 .d_flag = D_OTHER, 2138 }; 2139 2140 /* 2141 * Accept an open operation on the control device. 2142 */ 2143 int 2144 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 2145 { 2146 struct nvme_softc *sc; 2147 int unit = minor(dev) / 0x10000; 2148 int nsid = minor(dev) & 0xffff; 2149 int nsidx; 2150 2151 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 2152 return ENXIO; 2153 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 2154 return ENXIO; 2155 2156 if (nsid == 0) { 2157 /* controller */ 2158 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 2159 return EBUSY; 2160 SET(sc->sc_flags, NVME_F_OPEN); 2161 } else { 2162 /* namespace */ 2163 nsidx = nsid - 1; 2164 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 2165 return ENXIO; 2166 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 2167 return EBUSY; 2168 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 2169 } 2170 return 0; 2171 } 2172 2173 /* 2174 * Accept the last close on the control device. 2175 */ 2176 int 2177 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 2178 { 2179 struct nvme_softc *sc; 2180 int unit = minor(dev) / 0x10000; 2181 int nsid = minor(dev) & 0xffff; 2182 int nsidx; 2183 2184 sc = device_lookup_private(&nvme_cd, unit); 2185 if (sc == NULL) 2186 return ENXIO; 2187 2188 if (nsid == 0) { 2189 /* controller */ 2190 CLR(sc->sc_flags, NVME_F_OPEN); 2191 } else { 2192 /* namespace */ 2193 nsidx = nsid - 1; 2194 if (nsidx >= sc->sc_nn) 2195 return ENXIO; 2196 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 2197 } 2198 2199 return 0; 2200 } 2201 2202 /* 2203 * Handle control operations. 2204 */ 2205 int 2206 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 2207 { 2208 struct nvme_softc *sc; 2209 int unit = minor(dev) / 0x10000; 2210 int nsid = minor(dev) & 0xffff; 2211 struct nvme_pt_command *pt; 2212 2213 sc = device_lookup_private(&nvme_cd, unit); 2214 if (sc == NULL) 2215 return ENXIO; 2216 2217 switch (cmd) { 2218 case NVME_PASSTHROUGH_CMD: 2219 pt = data; 2220 return nvme_command_passthrough(sc, data, 2221 nsid == 0 ? pt->cmd.nsid : (uint32_t)nsid, l, nsid == 0); 2222 } 2223 2224 return ENOTTY; 2225 } 2226
Indexes created Mon Sep 29 21:09:56 GMT 2025