nvme.c revision 1.7
1 /* $NetBSD: nvme.c,v 1.7 2016/09/16 12:57:26 jdolecek Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.7 2016/09/16 12:57:26 jdolecek Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 int nvme_adminq_size = 128; 44 int nvme_ioq_size = 128; 45 46 static int nvme_print(void *, const char *); 47 48 static int nvme_ready(struct nvme_softc *, uint32_t); 49 static int nvme_enable(struct nvme_softc *, u_int); 50 static int nvme_disable(struct nvme_softc *); 51 static int nvme_shutdown(struct nvme_softc *); 52 53 static void nvme_version(struct nvme_softc *, uint32_t); 54 #ifdef NVME_DEBUG 55 static void nvme_dumpregs(struct nvme_softc *); 56 #endif 57 static int nvme_identify(struct nvme_softc *, u_int); 58 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 59 void *); 60 61 static int nvme_ccbs_alloc(struct nvme_queue *, u_int); 62 static void nvme_ccbs_free(struct nvme_queue *); 63 64 static struct nvme_ccb * 65 nvme_ccb_get(struct nvme_queue *); 66 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 67 68 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 69 struct nvme_ccb *, void (*)(struct nvme_queue *, 70 struct nvme_ccb *, void *), int); 71 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 72 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 73 struct nvme_cqe *); 74 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 76 struct nvme_cqe *); 77 78 static struct nvme_queue * 79 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 80 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 81 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 82 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 83 struct nvme_ccb *, void (*)(struct nvme_queue *, 84 struct nvme_ccb *, void *)); 85 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 86 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 87 88 static struct nvme_dmamem * 89 nvme_dmamem_alloc(struct nvme_softc *, size_t); 90 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 91 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 92 int); 93 94 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 95 void *); 96 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 97 struct nvme_cqe *); 98 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 99 void *); 100 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 101 struct nvme_cqe *); 102 103 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 104 void *); 105 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 106 struct nvme_cqe *); 107 static int nvme_command_passthrough(struct nvme_softc *, 108 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 109 110 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 111 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 112 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 113 #define NVME_TIMO_SY -1 /* sync cache timeout */ 114 115 #define nvme_read4(_s, _r) \ 116 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 117 #define nvme_write4(_s, _r, _v) \ 118 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 119 #ifdef __LP64__ 120 #define nvme_read8(_s, _r) \ 121 bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r)) 122 #define nvme_write8(_s, _r, _v) \ 123 bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 124 #else /* __LP64__ */ 125 static inline uint64_t 126 nvme_read8(struct nvme_softc *sc, bus_size_t r) 127 { 128 uint64_t v; 129 uint32_t *a = (uint32_t *)&v; 130 131 #if _BYTE_ORDER == _LITTLE_ENDIAN 132 a[0] = nvme_read4(sc, r); 133 a[1] = nvme_read4(sc, r + 4); 134 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 135 a[1] = nvme_read4(sc, r); 136 a[0] = nvme_read4(sc, r + 4); 137 #endif 138 139 return v; 140 } 141 142 static inline void 143 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 144 { 145 uint32_t *a = (uint32_t *)&v; 146 147 #if _BYTE_ORDER == _LITTLE_ENDIAN 148 nvme_write4(sc, r, a[0]); 149 nvme_write4(sc, r + 4, a[1]); 150 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 151 nvme_write4(sc, r, a[1]); 152 nvme_write4(sc, r + 4, a[0]); 153 #endif 154 } 155 #endif /* __LP64__ */ 156 #define nvme_barrier(_s, _r, _l, _f) \ 157 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 158 159 pool_cache_t nvme_ns_ctx_cache; 160 ONCE_DECL(nvme_init_once); 161 162 static int 163 nvme_init(void) 164 { 165 nvme_ns_ctx_cache = pool_cache_init(sizeof(struct nvme_ns_context), 166 0, 0, 0, "nvme_ns_ctx", NULL, IPL_BIO, NULL, NULL, NULL); 167 KASSERT(nvme_ns_ctx_cache != NULL); 168 return 0; 169 } 170 171 static void 172 nvme_version(struct nvme_softc *sc, uint32_t ver) 173 { 174 const char *v = NULL; 175 176 switch (ver) { 177 case NVME_VS_1_0: 178 v = "1.0"; 179 break; 180 case NVME_VS_1_1: 181 v = "1.1"; 182 break; 183 case NVME_VS_1_2: 184 v = "1.2"; 185 break; 186 default: 187 aprint_error_dev(sc->sc_dev, "unknown version 0x%08x\n", ver); 188 return; 189 } 190 191 aprint_normal_dev(sc->sc_dev, "NVMe %s\n", v); 192 } 193 194 #ifdef NVME_DEBUG 195 static __used void 196 nvme_dumpregs(struct nvme_softc *sc) 197 { 198 uint64_t r8; 199 uint32_t r4; 200 201 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 202 r8 = nvme_read8(sc, NVME_CAP); 203 printf("%s: cap 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 204 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 205 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 206 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 207 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 208 printf("%s: css %llu\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 209 printf("%s: nssrs %llu\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 210 printf("%s: dstrd %llu\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 211 printf("%s: to %llu msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 212 printf("%s: ams %llu\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 213 printf("%s: cqr %llu\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 214 printf("%s: mqes %llu\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 215 216 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 217 218 r4 = nvme_read4(sc, NVME_CC); 219 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 220 printf("%s: iocqes %u\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4)); 221 printf("%s: iosqes %u\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4)); 222 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 223 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 224 printf("%s: mps %u\n", DEVNAME(sc), NVME_CC_MPS_R(r4)); 225 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 226 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 227 228 printf("%s: csts 0x%08x\n", DEVNAME(sc), nvme_read4(sc, NVME_CSTS)); 229 printf("%s: aqa 0x%08x\n", DEVNAME(sc), nvme_read4(sc, NVME_AQA)); 230 printf("%s: asq 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 231 printf("%s: acq 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 232 #undef DEVNAME 233 } 234 #endif /* NVME_DEBUG */ 235 236 static int 237 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 238 { 239 u_int i = 0; 240 241 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 242 if (i++ > sc->sc_rdy_to) 243 return 1; 244 245 delay(1000); 246 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 247 } 248 249 return 0; 250 } 251 252 static int 253 nvme_enable(struct nvme_softc *sc, u_int mps) 254 { 255 uint32_t cc; 256 257 cc = nvme_read4(sc, NVME_CC); 258 if (ISSET(cc, NVME_CC_EN)) { 259 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 260 return 0; 261 } 262 263 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 264 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 265 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 266 267 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 268 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 269 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 270 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 271 272 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 273 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 274 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 275 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 276 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 277 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 278 SET(cc, NVME_CC_MPS(mps)); 279 SET(cc, NVME_CC_EN); 280 281 nvme_write4(sc, NVME_CC, cc); 282 nvme_barrier(sc, 0, sc->sc_ios, 283 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 284 285 return nvme_ready(sc, NVME_CSTS_RDY); 286 } 287 288 static int 289 nvme_disable(struct nvme_softc *sc) 290 { 291 uint32_t cc, csts; 292 293 cc = nvme_read4(sc, NVME_CC); 294 if (ISSET(cc, NVME_CC_EN)) { 295 csts = nvme_read4(sc, NVME_CSTS); 296 if (!ISSET(csts, NVME_CSTS_CFS) && 297 nvme_ready(sc, NVME_CSTS_RDY) != 0) 298 return 1; 299 } 300 301 CLR(cc, NVME_CC_EN); 302 303 nvme_write4(sc, NVME_CC, cc); 304 nvme_barrier(sc, 0, sc->sc_ios, 305 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 306 307 return nvme_ready(sc, 0); 308 } 309 310 int 311 nvme_attach(struct nvme_softc *sc) 312 { 313 struct nvme_attach_args naa; 314 uint64_t cap; 315 uint32_t reg; 316 u_int dstrd; 317 u_int mps = PAGE_SHIFT; 318 int adminq_entries = nvme_adminq_size; 319 int ioq_entries = nvme_ioq_size; 320 int i; 321 322 RUN_ONCE(&nvme_init_once, nvme_init); 323 324 reg = nvme_read4(sc, NVME_VS); 325 if (reg == 0xffffffff) { 326 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 327 return 1; 328 } 329 330 nvme_version(sc, reg); 331 332 cap = nvme_read8(sc, NVME_CAP); 333 dstrd = NVME_CAP_DSTRD(cap); 334 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 335 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 336 "is greater than CPU page size %u\n", 337 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 338 return 1; 339 } 340 if (NVME_CAP_MPSMAX(cap) < mps) 341 mps = NVME_CAP_MPSMAX(cap); 342 343 sc->sc_rdy_to = NVME_CAP_TO(cap); 344 sc->sc_mps = 1 << mps; 345 sc->sc_mdts = MAXPHYS; 346 sc->sc_max_sgl = 2; 347 348 if (nvme_disable(sc) != 0) { 349 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 350 return 1; 351 } 352 353 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 354 if (sc->sc_admin_q == NULL) { 355 aprint_error_dev(sc->sc_dev, 356 "unable to allocate admin queue\n"); 357 return 1; 358 } 359 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 360 goto free_admin_q; 361 362 if (nvme_enable(sc, mps) != 0) { 363 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 364 goto disestablish_admin_q; 365 } 366 367 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 368 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 369 goto disable; 370 } 371 372 /* we know how big things are now */ 373 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 374 375 /* reallocate ccbs of admin queue with new max sgl. */ 376 nvme_ccbs_free(sc->sc_admin_q); 377 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 378 379 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 380 if (sc->sc_q == NULL) { 381 aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n"); 382 goto disable; 383 } 384 for (i = 0; i < sc->sc_nq; i++) { 385 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 386 if (sc->sc_q[i] == NULL) { 387 aprint_error_dev(sc->sc_dev, 388 "unable to allocate io queue\n"); 389 goto free_q; 390 } 391 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 392 aprint_error_dev(sc->sc_dev, 393 "unable to create io queue\n"); 394 nvme_q_free(sc, sc->sc_q[i]); 395 goto free_q; 396 } 397 } 398 399 if (!sc->sc_use_mq) 400 nvme_write4(sc, NVME_INTMC, 1); 401 402 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 403 KM_SLEEP); 404 for (i = 0; i < sc->sc_nn; i++) { 405 memset(&naa, 0, sizeof(naa)); 406 naa.naa_nsid = i + 1; 407 naa.naa_qentries = ioq_entries; 408 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 409 nvme_print); 410 } 411 412 return 0; 413 414 free_q: 415 while (--i >= 0) { 416 nvme_q_delete(sc, sc->sc_q[i]); 417 nvme_q_free(sc, sc->sc_q[i]); 418 } 419 disable: 420 nvme_disable(sc); 421 disestablish_admin_q: 422 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 423 free_admin_q: 424 nvme_q_free(sc, sc->sc_admin_q); 425 426 return 1; 427 } 428 429 static int 430 nvme_print(void *aux, const char *pnp) 431 { 432 struct nvme_attach_args *naa = aux; 433 434 if (pnp) 435 aprint_normal("at %s", pnp); 436 437 if (naa->naa_nsid > 0) 438 aprint_normal(" nsid %d", naa->naa_nsid); 439 440 return UNCONF; 441 } 442 443 int 444 nvme_detach(struct nvme_softc *sc, int flags) 445 { 446 int i, error; 447 448 error = config_detach_children(sc->sc_dev, flags); 449 if (error) 450 return error; 451 452 error = nvme_shutdown(sc); 453 if (error) 454 return error; 455 456 for (i = 0; i < sc->sc_nq; i++) 457 nvme_q_free(sc, sc->sc_q[i]); 458 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 459 nvme_q_free(sc, sc->sc_admin_q); 460 461 return 0; 462 } 463 464 static int 465 nvme_shutdown(struct nvme_softc *sc) 466 { 467 uint32_t cc, csts; 468 bool disabled = false; 469 int i; 470 471 if (!sc->sc_use_mq) 472 nvme_write4(sc, NVME_INTMS, 1); 473 474 for (i = 0; i < sc->sc_nq; i++) { 475 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 476 aprint_error_dev(sc->sc_dev, 477 "unable to delete io queue %d, disabling\n", i + 1); 478 disabled = true; 479 } 480 } 481 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 482 if (disabled) 483 goto disable; 484 485 cc = nvme_read4(sc, NVME_CC); 486 CLR(cc, NVME_CC_SHN_MASK); 487 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 488 nvme_write4(sc, NVME_CC, cc); 489 490 for (i = 0; i < 4000; i++) { 491 nvme_barrier(sc, 0, sc->sc_ios, 492 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 493 csts = nvme_read4(sc, NVME_CSTS); 494 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 495 return 0; 496 497 delay(1000); 498 } 499 500 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 501 502 disable: 503 nvme_disable(sc); 504 return 0; 505 } 506 507 void 508 nvme_childdet(device_t self, device_t child) 509 { 510 struct nvme_softc *sc = device_private(self); 511 int i; 512 513 for (i = 0; i < sc->sc_nn; i++) { 514 if (sc->sc_namespaces[i].dev == child) { 515 /* Already freed ns->ident. */ 516 sc->sc_namespaces[i].dev = NULL; 517 break; 518 } 519 } 520 } 521 522 int 523 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 524 { 525 struct nvme_sqe sqe; 526 struct nvm_identify_namespace *identify; 527 struct nvme_dmamem *mem; 528 struct nvme_ccb *ccb; 529 struct nvme_namespace *ns; 530 int rv; 531 532 KASSERT(nsid > 0); 533 534 ccb = nvme_ccb_get(sc->sc_admin_q); 535 KASSERT(ccb != NULL); 536 537 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 538 if (mem == NULL) 539 return ENOMEM; 540 541 memset(&sqe, 0, sizeof(sqe)); 542 sqe.opcode = NVM_ADMIN_IDENTIFY; 543 htolem32(&sqe.nsid, nsid); 544 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 545 htolem32(&sqe.cdw10, 0); 546 547 ccb->ccb_done = nvme_empty_done; 548 ccb->ccb_cookie = &sqe; 549 550 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 551 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 552 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 553 554 nvme_ccb_put(sc->sc_admin_q, ccb); 555 556 if (rv != 0) { 557 rv = EIO; 558 goto done; 559 } 560 561 /* commit */ 562 563 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 564 memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 565 566 ns = nvme_ns_get(sc, nsid); 567 KASSERT(ns); 568 ns->ident = identify; 569 570 done: 571 nvme_dmamem_free(sc, mem); 572 573 return rv; 574 } 575 576 int 577 nvme_ns_dobio(struct nvme_softc *sc, struct nvme_ns_context *ctx) 578 { 579 struct nvme_queue *q = nvme_get_q(sc); 580 struct nvme_ccb *ccb; 581 bus_dmamap_t dmap; 582 int i, error; 583 584 ccb = nvme_ccb_get(q); 585 if (ccb == NULL) 586 return EAGAIN; 587 588 ccb->ccb_done = nvme_ns_io_done; 589 ccb->ccb_cookie = ctx; 590 591 dmap = ccb->ccb_dmamap; 592 error = bus_dmamap_load(sc->sc_dmat, dmap, ctx->nnc_data, 593 ctx->nnc_datasize, NULL, 594 (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL) ? 595 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 596 (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 597 BUS_DMA_READ : BUS_DMA_WRITE)); 598 if (error) { 599 nvme_ccb_put(q, ccb); 600 return error; 601 } 602 603 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 604 ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 605 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 606 607 if (dmap->dm_nsegs > 2) { 608 for (i = 1; i < dmap->dm_nsegs; i++) { 609 htolem64(&ccb->ccb_prpl[i - 1], 610 dmap->dm_segs[i].ds_addr); 611 } 612 bus_dmamap_sync(sc->sc_dmat, 613 NVME_DMA_MAP(q->q_ccb_prpls), 614 ccb->ccb_prpl_off, 615 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 616 BUS_DMASYNC_PREWRITE); 617 } 618 619 if (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL)) { 620 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 621 return EIO; 622 return 0; 623 } 624 625 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 626 return 0; 627 } 628 629 static void 630 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 631 { 632 struct nvme_sqe_io *sqe = slot; 633 struct nvme_ns_context *ctx = ccb->ccb_cookie; 634 bus_dmamap_t dmap = ccb->ccb_dmamap; 635 636 sqe->opcode = ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 637 NVM_CMD_READ : NVM_CMD_WRITE; 638 htolem32(&sqe->nsid, ctx->nnc_nsid); 639 640 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 641 switch (dmap->dm_nsegs) { 642 case 1: 643 break; 644 case 2: 645 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 646 break; 647 default: 648 /* the prp list is already set up and synced */ 649 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 650 break; 651 } 652 653 htolem64(&sqe->slba, ctx->nnc_blkno); 654 htolem16(&sqe->nlb, (ctx->nnc_datasize / ctx->nnc_secsize) - 1); 655 } 656 657 static void 658 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 659 struct nvme_cqe *cqe) 660 { 661 struct nvme_softc *sc = q->q_sc; 662 struct nvme_ns_context *ctx = ccb->ccb_cookie; 663 bus_dmamap_t dmap = ccb->ccb_dmamap; 664 uint16_t flags; 665 666 if (dmap->dm_nsegs > 2) { 667 bus_dmamap_sync(sc->sc_dmat, 668 NVME_DMA_MAP(q->q_ccb_prpls), 669 ccb->ccb_prpl_off, 670 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 671 BUS_DMASYNC_POSTWRITE); 672 } 673 674 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 675 ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 676 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 677 678 bus_dmamap_unload(sc->sc_dmat, dmap); 679 nvme_ccb_put(q, ccb); 680 681 flags = lemtoh16(&cqe->flags); 682 683 ctx->nnc_status = flags; 684 (*ctx->nnc_done)(ctx); 685 } 686 687 int 688 nvme_ns_sync(struct nvme_softc *sc, struct nvme_ns_context *ctx) 689 { 690 struct nvme_queue *q = nvme_get_q(sc); 691 struct nvme_ccb *ccb; 692 693 ccb = nvme_ccb_get(q); 694 if (ccb == NULL) 695 return EAGAIN; 696 697 ccb->ccb_done = nvme_ns_sync_done; 698 ccb->ccb_cookie = ctx; 699 700 if (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL)) { 701 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 702 return EIO; 703 return 0; 704 } 705 706 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 707 return 0; 708 } 709 710 static void 711 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 712 { 713 struct nvme_sqe *sqe = slot; 714 struct nvme_ns_context *ctx = ccb->ccb_cookie; 715 716 sqe->opcode = NVM_CMD_FLUSH; 717 htolem32(&sqe->nsid, ctx->nnc_nsid); 718 } 719 720 static void 721 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 722 struct nvme_cqe *cqe) 723 { 724 struct nvme_ns_context *ctx = ccb->ccb_cookie; 725 uint16_t flags; 726 727 nvme_ccb_put(q, ccb); 728 729 flags = lemtoh16(&cqe->flags); 730 731 ctx->nnc_status = flags; 732 (*ctx->nnc_done)(ctx); 733 } 734 735 void 736 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 737 { 738 struct nvme_namespace *ns; 739 struct nvm_identify_namespace *identify; 740 741 ns = nvme_ns_get(sc, nsid); 742 KASSERT(ns); 743 744 identify = ns->ident; 745 ns->ident = NULL; 746 if (identify != NULL) 747 kmem_free(identify, sizeof(*identify)); 748 } 749 750 static void 751 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 752 { 753 struct nvme_softc *sc = q->q_sc; 754 struct nvme_sqe *sqe = slot; 755 struct nvme_pt_command *pt = ccb->ccb_cookie; 756 bus_dmamap_t dmap = ccb->ccb_dmamap; 757 int i; 758 759 sqe->opcode = pt->cmd.opcode; 760 htolem32(&sqe->nsid, pt->cmd.nsid); 761 762 if (pt->buf != NULL && pt->len > 0) { 763 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 764 switch (dmap->dm_nsegs) { 765 case 1: 766 break; 767 case 2: 768 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 769 break; 770 default: 771 for (i = 1; i < dmap->dm_nsegs; i++) { 772 htolem64(&ccb->ccb_prpl[i - 1], 773 dmap->dm_segs[i].ds_addr); 774 } 775 bus_dmamap_sync(sc->sc_dmat, 776 NVME_DMA_MAP(q->q_ccb_prpls), 777 ccb->ccb_prpl_off, 778 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 779 BUS_DMASYNC_PREWRITE); 780 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 781 break; 782 } 783 } 784 785 htolem32(&sqe->cdw10, pt->cmd.cdw10); 786 htolem32(&sqe->cdw11, pt->cmd.cdw11); 787 htolem32(&sqe->cdw12, pt->cmd.cdw12); 788 htolem32(&sqe->cdw13, pt->cmd.cdw13); 789 htolem32(&sqe->cdw14, pt->cmd.cdw14); 790 htolem32(&sqe->cdw15, pt->cmd.cdw15); 791 } 792 793 static void 794 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 795 { 796 struct nvme_softc *sc = q->q_sc; 797 struct nvme_pt_command *pt = ccb->ccb_cookie; 798 bus_dmamap_t dmap = ccb->ccb_dmamap; 799 800 if (pt->buf != NULL && pt->len > 0) { 801 if (dmap->dm_nsegs > 2) { 802 bus_dmamap_sync(sc->sc_dmat, 803 NVME_DMA_MAP(q->q_ccb_prpls), 804 ccb->ccb_prpl_off, 805 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 806 BUS_DMASYNC_POSTWRITE); 807 } 808 809 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 810 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 811 bus_dmamap_unload(sc->sc_dmat, dmap); 812 } 813 814 pt->cpl.cdw0 = cqe->cdw0; 815 pt->cpl.flags = cqe->flags & ~NVME_CQE_PHASE; 816 } 817 818 static int 819 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 820 uint16_t nsid, struct lwp *l, bool is_adminq) 821 { 822 struct nvme_queue *q; 823 struct nvme_ccb *ccb; 824 void *buf = NULL; 825 int error; 826 827 if ((pt->buf == NULL && pt->len > 0) || 828 (pt->buf != NULL && pt->len == 0)) 829 return EINVAL; 830 831 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 832 ccb = nvme_ccb_get(q); 833 if (ccb == NULL) 834 return EBUSY; 835 836 if (pt->buf != NULL && pt->len > 0) { 837 buf = kmem_alloc(pt->len, KM_SLEEP); 838 if (buf == NULL) { 839 error = ENOMEM; 840 goto ccb_put; 841 } 842 if (!pt->is_read) { 843 error = copyin(pt->buf, buf, pt->len); 844 if (error) 845 goto kmem_free; 846 } 847 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 848 pt->len, NULL, 849 BUS_DMA_WAITOK | 850 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 851 if (error) 852 goto kmem_free; 853 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 854 0, ccb->ccb_dmamap->dm_mapsize, 855 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 856 } 857 858 ccb->ccb_done = nvme_pt_done; 859 ccb->ccb_cookie = pt; 860 861 pt->cmd.nsid = nsid; 862 if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) { 863 error = EIO; 864 goto out; 865 } 866 867 error = 0; 868 out: 869 if (buf != NULL) { 870 if (error == 0 && pt->is_read) 871 error = copyout(buf, pt->buf, pt->len); 872 kmem_free: 873 kmem_free(buf, pt->len); 874 } 875 ccb_put: 876 nvme_ccb_put(q, ccb); 877 return error; 878 } 879 880 static void 881 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 882 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 883 { 884 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 885 uint32_t tail; 886 887 mutex_enter(&q->q_sq_mtx); 888 tail = q->q_sq_tail; 889 if (++q->q_sq_tail >= q->q_entries) 890 q->q_sq_tail = 0; 891 892 sqe += tail; 893 894 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 895 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 896 memset(sqe, 0, sizeof(*sqe)); 897 (*fill)(q, ccb, sqe); 898 sqe->cid = ccb->ccb_id; 899 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 900 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 901 902 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 903 mutex_exit(&q->q_sq_mtx); 904 } 905 906 struct nvme_poll_state { 907 struct nvme_sqe s; 908 struct nvme_cqe c; 909 }; 910 911 static int 912 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 913 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 914 { 915 struct nvme_poll_state state; 916 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 917 void *cookie; 918 uint16_t flags; 919 int step = 10; 920 int maxloop = timo_sec * 1000000 / step; 921 int error = 0; 922 923 memset(&state, 0, sizeof(state)); 924 (*fill)(q, ccb, &state.s); 925 926 done = ccb->ccb_done; 927 cookie = ccb->ccb_cookie; 928 929 ccb->ccb_done = nvme_poll_done; 930 ccb->ccb_cookie = &state; 931 932 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 933 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 934 if (nvme_q_complete(sc, q) == 0) 935 delay(step); 936 937 if (timo_sec >= 0 && --maxloop <= 0) { 938 error = ETIMEDOUT; 939 break; 940 } 941 } 942 943 ccb->ccb_cookie = cookie; 944 done(q, ccb, &state.c); 945 946 if (error == 0) { 947 flags = lemtoh16(&state.c.flags); 948 return flags & ~NVME_CQE_PHASE; 949 } else { 950 return 1; 951 } 952 } 953 954 static void 955 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 956 { 957 struct nvme_sqe *sqe = slot; 958 struct nvme_poll_state *state = ccb->ccb_cookie; 959 960 *sqe = state->s; 961 } 962 963 static void 964 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 965 struct nvme_cqe *cqe) 966 { 967 struct nvme_poll_state *state = ccb->ccb_cookie; 968 969 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 970 state->c = *cqe; 971 } 972 973 static void 974 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 975 { 976 struct nvme_sqe *src = ccb->ccb_cookie; 977 struct nvme_sqe *dst = slot; 978 979 *dst = *src; 980 } 981 982 static void 983 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 984 struct nvme_cqe *cqe) 985 { 986 } 987 988 static int 989 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 990 { 991 struct nvme_ccb *ccb; 992 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 993 uint32_t head; 994 uint16_t flags; 995 int rv = 0; 996 997 if (!mutex_tryenter(&q->q_cq_mtx)) 998 return -1; 999 1000 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1001 head = q->q_cq_head; 1002 for (;;) { 1003 cqe = &ring[head]; 1004 flags = lemtoh16(&cqe->flags); 1005 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1006 break; 1007 1008 ccb = &q->q_ccbs[cqe->cid]; 1009 ccb->ccb_done(q, ccb, cqe); 1010 1011 if (++head >= q->q_entries) { 1012 head = 0; 1013 q->q_cq_phase ^= NVME_CQE_PHASE; 1014 } 1015 1016 rv = 1; 1017 } 1018 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1019 1020 if (rv) 1021 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head = head); 1022 mutex_exit(&q->q_cq_mtx); 1023 1024 return rv; 1025 } 1026 1027 static int 1028 nvme_identify(struct nvme_softc *sc, u_int mps) 1029 { 1030 char sn[41], mn[81], fr[17]; 1031 struct nvm_identify_controller *identify; 1032 struct nvme_dmamem *mem; 1033 struct nvme_ccb *ccb; 1034 u_int mdts; 1035 int rv = 1; 1036 1037 ccb = nvme_ccb_get(sc->sc_admin_q); 1038 if (ccb == NULL) 1039 panic("%s: nvme_ccb_get returned NULL", __func__); 1040 1041 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1042 if (mem == NULL) 1043 return 1; 1044 1045 ccb->ccb_done = nvme_empty_done; 1046 ccb->ccb_cookie = mem; 1047 1048 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1049 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1050 NVME_TIMO_IDENT); 1051 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1052 1053 nvme_ccb_put(sc->sc_admin_q, ccb); 1054 1055 if (rv != 0) 1056 goto done; 1057 1058 identify = NVME_DMA_KVA(mem); 1059 1060 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1061 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1062 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1063 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1064 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1065 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1066 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1067 sn); 1068 1069 if (identify->mdts > 0) { 1070 mdts = (1 << identify->mdts) * (1 << mps); 1071 if (mdts < sc->sc_mdts) 1072 sc->sc_mdts = mdts; 1073 } 1074 1075 sc->sc_nn = lemtoh32(&identify->nn); 1076 1077 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1078 1079 done: 1080 nvme_dmamem_free(sc, mem); 1081 1082 return rv; 1083 } 1084 1085 static int 1086 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1087 { 1088 struct nvme_sqe_q sqe; 1089 struct nvme_ccb *ccb; 1090 int rv; 1091 1092 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q)) 1093 return 1; 1094 1095 ccb = nvme_ccb_get(sc->sc_admin_q); 1096 KASSERT(ccb != NULL); 1097 1098 ccb->ccb_done = nvme_empty_done; 1099 ccb->ccb_cookie = &sqe; 1100 1101 memset(&sqe, 0, sizeof(sqe)); 1102 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1103 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1104 htolem16(&sqe.qsize, q->q_entries - 1); 1105 htolem16(&sqe.qid, q->q_id); 1106 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1107 if (sc->sc_use_mq) 1108 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1109 1110 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1111 if (rv != 0) 1112 goto fail; 1113 1114 ccb->ccb_done = nvme_empty_done; 1115 ccb->ccb_cookie = &sqe; 1116 1117 memset(&sqe, 0, sizeof(sqe)); 1118 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1119 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1120 htolem16(&sqe.qsize, q->q_entries - 1); 1121 htolem16(&sqe.qid, q->q_id); 1122 htolem16(&sqe.cqid, q->q_id); 1123 sqe.qflags = NVM_SQE_Q_PC; 1124 1125 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1126 if (rv != 0) 1127 goto fail; 1128 1129 fail: 1130 nvme_ccb_put(sc->sc_admin_q, ccb); 1131 return rv; 1132 } 1133 1134 static int 1135 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1136 { 1137 struct nvme_sqe_q sqe; 1138 struct nvme_ccb *ccb; 1139 int rv; 1140 1141 ccb = nvme_ccb_get(sc->sc_admin_q); 1142 KASSERT(ccb != NULL); 1143 1144 ccb->ccb_done = nvme_empty_done; 1145 ccb->ccb_cookie = &sqe; 1146 1147 memset(&sqe, 0, sizeof(sqe)); 1148 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1149 htolem16(&sqe.qid, q->q_id); 1150 1151 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1152 if (rv != 0) 1153 goto fail; 1154 1155 ccb->ccb_done = nvme_empty_done; 1156 ccb->ccb_cookie = &sqe; 1157 1158 memset(&sqe, 0, sizeof(sqe)); 1159 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1160 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1161 htolem16(&sqe.qid, q->q_id); 1162 1163 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1164 if (rv != 0) 1165 goto fail; 1166 1167 fail: 1168 nvme_ccb_put(sc->sc_admin_q, ccb); 1169 1170 if (rv == 0 && sc->sc_use_mq) { 1171 if (sc->sc_intr_disestablish(sc, q->q_id)) 1172 rv = 1; 1173 } 1174 1175 return rv; 1176 } 1177 1178 static void 1179 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1180 { 1181 struct nvme_sqe *sqe = slot; 1182 struct nvme_dmamem *mem = ccb->ccb_cookie; 1183 1184 sqe->opcode = NVM_ADMIN_IDENTIFY; 1185 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1186 htolem32(&sqe->cdw10, 1); 1187 } 1188 1189 static int 1190 nvme_ccbs_alloc(struct nvme_queue *q, u_int nccbs) 1191 { 1192 struct nvme_softc *sc = q->q_sc; 1193 struct nvme_ccb *ccb; 1194 bus_addr_t off; 1195 uint64_t *prpl; 1196 u_int i; 1197 1198 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1199 SIMPLEQ_INIT(&q->q_ccb_list); 1200 1201 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1202 if (q->q_ccbs == NULL) 1203 return 1; 1204 1205 q->q_nccbs = nccbs; 1206 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1207 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1208 1209 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1210 off = 0; 1211 1212 for (i = 0; i < nccbs; i++) { 1213 ccb = &q->q_ccbs[i]; 1214 1215 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1216 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1217 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1218 &ccb->ccb_dmamap) != 0) 1219 goto free_maps; 1220 1221 ccb->ccb_id = i; 1222 ccb->ccb_prpl = prpl; 1223 ccb->ccb_prpl_off = off; 1224 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1225 1226 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1227 1228 prpl += sc->sc_max_sgl; 1229 off += sizeof(*prpl) * sc->sc_max_sgl; 1230 } 1231 1232 return 0; 1233 1234 free_maps: 1235 nvme_ccbs_free(q); 1236 return 1; 1237 } 1238 1239 static struct nvme_ccb * 1240 nvme_ccb_get(struct nvme_queue *q) 1241 { 1242 struct nvme_ccb *ccb; 1243 1244 mutex_enter(&q->q_ccb_mtx); 1245 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1246 if (ccb != NULL) 1247 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1248 mutex_exit(&q->q_ccb_mtx); 1249 1250 return ccb; 1251 } 1252 1253 static void 1254 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1255 { 1256 1257 mutex_enter(&q->q_ccb_mtx); 1258 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1259 mutex_exit(&q->q_ccb_mtx); 1260 } 1261 1262 static void 1263 nvme_ccbs_free(struct nvme_queue *q) 1264 { 1265 struct nvme_softc *sc = q->q_sc; 1266 struct nvme_ccb *ccb; 1267 1268 mutex_enter(&q->q_ccb_mtx); 1269 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1270 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1271 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1272 } 1273 mutex_exit(&q->q_ccb_mtx); 1274 1275 nvme_dmamem_free(sc, q->q_ccb_prpls); 1276 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1277 q->q_ccbs = NULL; 1278 mutex_destroy(&q->q_ccb_mtx); 1279 } 1280 1281 static struct nvme_queue * 1282 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1283 { 1284 struct nvme_queue *q; 1285 1286 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1287 if (q == NULL) 1288 return NULL; 1289 1290 q->q_sc = sc; 1291 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1292 sizeof(struct nvme_sqe) * entries); 1293 if (q->q_sq_dmamem == NULL) 1294 goto free; 1295 1296 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1297 sizeof(struct nvme_cqe) * entries); 1298 if (q->q_cq_dmamem == NULL) 1299 goto free_sq; 1300 1301 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1302 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1303 1304 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1305 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1306 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1307 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1308 q->q_id = id; 1309 q->q_entries = entries; 1310 q->q_sq_tail = 0; 1311 q->q_cq_head = 0; 1312 q->q_cq_phase = NVME_CQE_PHASE; 1313 1314 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1315 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1316 1317 if (nvme_ccbs_alloc(q, entries) != 0) { 1318 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1319 goto free_cq; 1320 } 1321 1322 return q; 1323 1324 free_cq: 1325 nvme_dmamem_free(sc, q->q_cq_dmamem); 1326 free_sq: 1327 nvme_dmamem_free(sc, q->q_sq_dmamem); 1328 free: 1329 kmem_free(q, sizeof(*q)); 1330 1331 return NULL; 1332 } 1333 1334 static void 1335 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1336 { 1337 nvme_ccbs_free(q); 1338 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1339 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1340 nvme_dmamem_free(sc, q->q_cq_dmamem); 1341 nvme_dmamem_free(sc, q->q_sq_dmamem); 1342 kmem_free(q, sizeof(*q)); 1343 } 1344 1345 int 1346 nvme_intr(void *xsc) 1347 { 1348 struct nvme_softc *sc = xsc; 1349 int rv = 0; 1350 1351 nvme_write4(sc, NVME_INTMS, 1); 1352 1353 if (nvme_q_complete(sc, sc->sc_admin_q)) 1354 rv = 1; 1355 if (sc->sc_q != NULL) 1356 if (nvme_q_complete(sc, sc->sc_q[0])) 1357 rv = 1; 1358 1359 nvme_write4(sc, NVME_INTMC, 1); 1360 1361 return rv; 1362 } 1363 1364 int 1365 nvme_mq_msi_intr(void *xq) 1366 { 1367 struct nvme_queue *q = xq; 1368 struct nvme_softc *sc = q->q_sc; 1369 int rv = 0; 1370 1371 nvme_write4(sc, NVME_INTMS, 1U << q->q_id); 1372 1373 if (nvme_q_complete(sc, q)) 1374 rv = 1; 1375 1376 nvme_write4(sc, NVME_INTMC, 1U << q->q_id); 1377 1378 return rv; 1379 } 1380 1381 int 1382 nvme_mq_msix_intr(void *xq) 1383 { 1384 struct nvme_queue *q = xq; 1385 int rv = 0; 1386 1387 if (nvme_q_complete(q->q_sc, q)) 1388 rv = 1; 1389 1390 return rv; 1391 } 1392 1393 static struct nvme_dmamem * 1394 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1395 { 1396 struct nvme_dmamem *ndm; 1397 int nsegs; 1398 1399 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1400 if (ndm == NULL) 1401 return NULL; 1402 1403 ndm->ndm_size = size; 1404 1405 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1406 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1407 goto ndmfree; 1408 1409 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1410 1, &nsegs, BUS_DMA_WAITOK) != 0) 1411 goto destroy; 1412 1413 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1414 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1415 goto free; 1416 memset(ndm->ndm_kva, 0, size); 1417 1418 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1419 NULL, BUS_DMA_WAITOK) != 0) 1420 goto unmap; 1421 1422 return ndm; 1423 1424 unmap: 1425 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1426 free: 1427 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1428 destroy: 1429 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1430 ndmfree: 1431 kmem_free(ndm, sizeof(*ndm)); 1432 return NULL; 1433 } 1434 1435 static void 1436 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1437 { 1438 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1439 0, NVME_DMA_LEN(mem), ops); 1440 } 1441 1442 void 1443 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1444 { 1445 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1446 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1447 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1448 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1449 kmem_free(ndm, sizeof(*ndm)); 1450 } 1451 1452 /* 1453 * ioctl 1454 */ 1455 1456 dev_type_open(nvmeopen); 1457 dev_type_close(nvmeclose); 1458 dev_type_ioctl(nvmeioctl); 1459 1460 const struct cdevsw nvme_cdevsw = { 1461 .d_open = nvmeopen, 1462 .d_close = nvmeclose, 1463 .d_read = noread, 1464 .d_write = nowrite, 1465 .d_ioctl = nvmeioctl, 1466 .d_stop = nostop, 1467 .d_tty = notty, 1468 .d_poll = nopoll, 1469 .d_mmap = nommap, 1470 .d_kqfilter = nokqfilter, 1471 .d_discard = nodiscard, 1472 .d_flag = D_OTHER, 1473 }; 1474 1475 extern struct cfdriver nvme_cd; 1476 1477 /* 1478 * Accept an open operation on the control device. 1479 */ 1480 int 1481 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1482 { 1483 struct nvme_softc *sc; 1484 int unit = minor(dev) / 0x10000; 1485 int nsid = minor(dev) & 0xffff; 1486 int nsidx; 1487 1488 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1489 return ENXIO; 1490 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1491 return ENXIO; 1492 1493 if (nsid == 0) { 1494 /* controller */ 1495 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1496 return EBUSY; 1497 SET(sc->sc_flags, NVME_F_OPEN); 1498 } else { 1499 /* namespace */ 1500 nsidx = nsid - 1; 1501 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1502 return ENXIO; 1503 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1504 return EBUSY; 1505 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1506 } 1507 return 0; 1508 } 1509 1510 /* 1511 * Accept the last close on the control device. 1512 */ 1513 int 1514 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1515 { 1516 struct nvme_softc *sc; 1517 int unit = minor(dev) / 0x10000; 1518 int nsid = minor(dev) & 0xffff; 1519 int nsidx; 1520 1521 sc = device_lookup_private(&nvme_cd, unit); 1522 if (sc == NULL) 1523 return ENXIO; 1524 1525 if (nsid == 0) { 1526 /* controller */ 1527 CLR(sc->sc_flags, NVME_F_OPEN); 1528 } else { 1529 /* namespace */ 1530 nsidx = nsid - 1; 1531 if (nsidx >= sc->sc_nn) 1532 return ENXIO; 1533 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1534 } 1535 1536 return 0; 1537 } 1538 1539 /* 1540 * Handle control operations. 1541 */ 1542 int 1543 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1544 { 1545 struct nvme_softc *sc; 1546 int unit = minor(dev) / 0x10000; 1547 int nsid = minor(dev) & 0xffff; 1548 struct nvme_pt_command *pt; 1549 1550 sc = device_lookup_private(&nvme_cd, unit); 1551 if (sc == NULL) 1552 return ENXIO; 1553 1554 switch (cmd) { 1555 case NVME_PASSTHROUGH_CMD: 1556 pt = data; 1557 return nvme_command_passthrough(sc, data, 1558 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1559 } 1560 1561 return ENOTTY; 1562 } 1563
Indexes created Thu Sep 25 16:09:42 GMT 2025