nvme.c revision 1.6
1 /* $NetBSD: nvme.c,v 1.6 2016/09/16 11:41:40 jdolecek Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.6 2016/09/16 11:41:40 jdolecek Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 int nvme_adminq_size = 128; 44 int nvme_ioq_size = 128; 45 46 static int nvme_print(void *, const char *); 47 48 static int nvme_ready(struct nvme_softc *, uint32_t); 49 static int nvme_enable(struct nvme_softc *, u_int); 50 static int nvme_disable(struct nvme_softc *); 51 static int nvme_shutdown(struct nvme_softc *); 52 53 static void nvme_version(struct nvme_softc *, uint32_t); 54 #ifdef NVME_DEBUG 55 static void nvme_dumpregs(struct nvme_softc *); 56 #endif 57 static int nvme_identify(struct nvme_softc *, u_int); 58 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 59 void *); 60 61 static int nvme_ccbs_alloc(struct nvme_queue *, u_int); 62 static void nvme_ccbs_free(struct nvme_queue *); 63 64 static struct nvme_ccb * 65 nvme_ccb_get(struct nvme_queue *); 66 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 67 68 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 69 struct nvme_ccb *, void (*)(struct nvme_queue *, 70 struct nvme_ccb *, void *)); 71 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 72 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 73 struct nvme_cqe *); 74 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 76 struct nvme_cqe *); 77 78 static struct nvme_queue * 79 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 80 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 81 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 82 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 83 struct nvme_ccb *, void (*)(struct nvme_queue *, 84 struct nvme_ccb *, void *)); 85 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 86 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 87 88 static struct nvme_dmamem * 89 nvme_dmamem_alloc(struct nvme_softc *, size_t); 90 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 91 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 92 int); 93 94 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 95 void *); 96 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 97 struct nvme_cqe *); 98 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 99 void *); 100 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 101 struct nvme_cqe *); 102 103 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 104 void *); 105 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 106 struct nvme_cqe *); 107 static int nvme_command_passthrough(struct nvme_softc *, 108 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 109 110 #define nvme_read4(_s, _r) \ 111 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 112 #define nvme_write4(_s, _r, _v) \ 113 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 114 #ifdef __LP64__ 115 #define nvme_read8(_s, _r) \ 116 bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r)) 117 #define nvme_write8(_s, _r, _v) \ 118 bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 119 #else /* __LP64__ */ 120 static inline uint64_t 121 nvme_read8(struct nvme_softc *sc, bus_size_t r) 122 { 123 uint64_t v; 124 uint32_t *a = (uint32_t *)&v; 125 126 #if _BYTE_ORDER == _LITTLE_ENDIAN 127 a[0] = nvme_read4(sc, r); 128 a[1] = nvme_read4(sc, r + 4); 129 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 130 a[1] = nvme_read4(sc, r); 131 a[0] = nvme_read4(sc, r + 4); 132 #endif 133 134 return v; 135 } 136 137 static inline void 138 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 139 { 140 uint32_t *a = (uint32_t *)&v; 141 142 #if _BYTE_ORDER == _LITTLE_ENDIAN 143 nvme_write4(sc, r, a[0]); 144 nvme_write4(sc, r + 4, a[1]); 145 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 146 nvme_write4(sc, r, a[1]); 147 nvme_write4(sc, r + 4, a[0]); 148 #endif 149 } 150 #endif /* __LP64__ */ 151 #define nvme_barrier(_s, _r, _l, _f) \ 152 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 153 154 pool_cache_t nvme_ns_ctx_cache; 155 ONCE_DECL(nvme_init_once); 156 157 static int 158 nvme_init(void) 159 { 160 nvme_ns_ctx_cache = pool_cache_init(sizeof(struct nvme_ns_context), 161 0, 0, 0, "nvme_ns_ctx", NULL, IPL_BIO, NULL, NULL, NULL); 162 KASSERT(nvme_ns_ctx_cache != NULL); 163 return 0; 164 } 165 166 static void 167 nvme_version(struct nvme_softc *sc, uint32_t ver) 168 { 169 const char *v = NULL; 170 171 switch (ver) { 172 case NVME_VS_1_0: 173 v = "1.0"; 174 break; 175 case NVME_VS_1_1: 176 v = "1.1"; 177 break; 178 case NVME_VS_1_2: 179 v = "1.2"; 180 break; 181 default: 182 aprint_error_dev(sc->sc_dev, "unknown version 0x%08x\n", ver); 183 return; 184 } 185 186 aprint_normal_dev(sc->sc_dev, "NVMe %s\n", v); 187 } 188 189 #ifdef NVME_DEBUG 190 static __used void 191 nvme_dumpregs(struct nvme_softc *sc) 192 { 193 uint64_t r8; 194 uint32_t r4; 195 196 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 197 r8 = nvme_read8(sc, NVME_CAP); 198 printf("%s: cap 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 199 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 200 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 201 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 202 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 203 printf("%s: css %llu\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 204 printf("%s: nssrs %llu\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 205 printf("%s: dstrd %llu\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 206 printf("%s: to %llu msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 207 printf("%s: ams %llu\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 208 printf("%s: cqr %llu\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 209 printf("%s: mqes %llu\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 210 211 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 212 213 r4 = nvme_read4(sc, NVME_CC); 214 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 215 printf("%s: iocqes %u\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4)); 216 printf("%s: iosqes %u\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4)); 217 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 218 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 219 printf("%s: mps %u\n", DEVNAME(sc), NVME_CC_MPS_R(r4)); 220 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 221 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 222 223 printf("%s: csts 0x%08x\n", DEVNAME(sc), nvme_read4(sc, NVME_CSTS)); 224 printf("%s: aqa 0x%08x\n", DEVNAME(sc), nvme_read4(sc, NVME_AQA)); 225 printf("%s: asq 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 226 printf("%s: acq 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 227 #undef DEVNAME 228 } 229 #endif /* NVME_DEBUG */ 230 231 static int 232 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 233 { 234 u_int i = 0; 235 236 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 237 if (i++ > sc->sc_rdy_to) 238 return 1; 239 240 delay(1000); 241 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 242 } 243 244 return 0; 245 } 246 247 static int 248 nvme_enable(struct nvme_softc *sc, u_int mps) 249 { 250 uint32_t cc; 251 252 cc = nvme_read4(sc, NVME_CC); 253 if (ISSET(cc, NVME_CC_EN)) 254 return nvme_ready(sc, NVME_CSTS_RDY); 255 256 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 257 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 258 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 259 260 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 261 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 262 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 263 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 264 265 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 266 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 267 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 268 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 269 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 270 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 271 SET(cc, NVME_CC_MPS(mps)); 272 SET(cc, NVME_CC_EN); 273 274 nvme_write4(sc, NVME_CC, cc); 275 nvme_barrier(sc, 0, sc->sc_ios, 276 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 277 278 return nvme_ready(sc, NVME_CSTS_RDY); 279 } 280 281 static int 282 nvme_disable(struct nvme_softc *sc) 283 { 284 uint32_t cc, csts; 285 286 cc = nvme_read4(sc, NVME_CC); 287 if (ISSET(cc, NVME_CC_EN)) { 288 csts = nvme_read4(sc, NVME_CSTS); 289 if (!ISSET(csts, NVME_CSTS_CFS) && 290 nvme_ready(sc, NVME_CSTS_RDY) != 0) 291 return 1; 292 } 293 294 CLR(cc, NVME_CC_EN); 295 296 nvme_write4(sc, NVME_CC, cc); 297 nvme_barrier(sc, 0, sc->sc_ios, 298 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 299 300 return nvme_ready(sc, 0); 301 } 302 303 int 304 nvme_attach(struct nvme_softc *sc) 305 { 306 struct nvme_attach_args naa; 307 uint64_t cap; 308 uint32_t reg; 309 u_int dstrd; 310 u_int mps = PAGE_SHIFT; 311 int adminq_entries = nvme_adminq_size; 312 int ioq_entries = nvme_ioq_size; 313 int i; 314 315 RUN_ONCE(&nvme_init_once, nvme_init); 316 317 reg = nvme_read4(sc, NVME_VS); 318 if (reg == 0xffffffff) { 319 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 320 return 1; 321 } 322 323 nvme_version(sc, reg); 324 325 cap = nvme_read8(sc, NVME_CAP); 326 dstrd = NVME_CAP_DSTRD(cap); 327 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 328 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 329 "is greater than CPU page size %u\n", 330 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 331 return 1; 332 } 333 if (NVME_CAP_MPSMAX(cap) < mps) 334 mps = NVME_CAP_MPSMAX(cap); 335 336 sc->sc_rdy_to = NVME_CAP_TO(cap); 337 sc->sc_mps = 1 << mps; 338 sc->sc_mdts = MAXPHYS; 339 sc->sc_max_sgl = 2; 340 341 if (nvme_disable(sc) != 0) { 342 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 343 return 1; 344 } 345 346 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 347 if (sc->sc_admin_q == NULL) { 348 aprint_error_dev(sc->sc_dev, 349 "unable to allocate admin queue\n"); 350 return 1; 351 } 352 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 353 goto free_admin_q; 354 355 if (nvme_enable(sc, mps) != 0) { 356 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 357 goto disestablish_admin_q; 358 } 359 360 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 361 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 362 goto disable; 363 } 364 365 /* we know how big things are now */ 366 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 367 368 /* reallocate ccbs of admin queue with new max sgl. */ 369 nvme_ccbs_free(sc->sc_admin_q); 370 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 371 372 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 373 if (sc->sc_q == NULL) { 374 aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n"); 375 goto disable; 376 } 377 for (i = 0; i < sc->sc_nq; i++) { 378 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 379 if (sc->sc_q[i] == NULL) { 380 aprint_error_dev(sc->sc_dev, 381 "unable to allocate io queue\n"); 382 goto free_q; 383 } 384 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 385 aprint_error_dev(sc->sc_dev, 386 "unable to create io queue\n"); 387 nvme_q_free(sc, sc->sc_q[i]); 388 goto free_q; 389 } 390 } 391 392 if (!sc->sc_use_mq) 393 nvme_write4(sc, NVME_INTMC, 1); 394 395 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 396 KM_SLEEP); 397 for (i = 0; i < sc->sc_nn; i++) { 398 memset(&naa, 0, sizeof(naa)); 399 naa.naa_nsid = i + 1; 400 naa.naa_qentries = ioq_entries; 401 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 402 nvme_print); 403 } 404 405 return 0; 406 407 free_q: 408 while (--i >= 0) { 409 nvme_q_delete(sc, sc->sc_q[i]); 410 nvme_q_free(sc, sc->sc_q[i]); 411 } 412 disable: 413 nvme_disable(sc); 414 disestablish_admin_q: 415 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 416 free_admin_q: 417 nvme_q_free(sc, sc->sc_admin_q); 418 419 return 1; 420 } 421 422 static int 423 nvme_print(void *aux, const char *pnp) 424 { 425 struct nvme_attach_args *naa = aux; 426 427 if (pnp) 428 aprint_normal("at %s", pnp); 429 430 if (naa->naa_nsid > 0) 431 aprint_normal(" nsid %d", naa->naa_nsid); 432 433 return UNCONF; 434 } 435 436 int 437 nvme_detach(struct nvme_softc *sc, int flags) 438 { 439 int i, error; 440 441 error = config_detach_children(sc->sc_dev, flags); 442 if (error) 443 return error; 444 445 error = nvme_shutdown(sc); 446 if (error) 447 return error; 448 449 for (i = 0; i < sc->sc_nq; i++) 450 nvme_q_free(sc, sc->sc_q[i]); 451 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 452 nvme_q_free(sc, sc->sc_admin_q); 453 454 return 0; 455 } 456 457 static int 458 nvme_shutdown(struct nvme_softc *sc) 459 { 460 uint32_t cc, csts; 461 bool disabled = false; 462 int i; 463 464 if (!sc->sc_use_mq) 465 nvme_write4(sc, NVME_INTMS, 1); 466 467 for (i = 0; i < sc->sc_nq; i++) { 468 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 469 aprint_error_dev(sc->sc_dev, 470 "unable to delete io queue %d, disabling\n", i + 1); 471 disabled = true; 472 } 473 } 474 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 475 if (disabled) 476 goto disable; 477 478 cc = nvme_read4(sc, NVME_CC); 479 CLR(cc, NVME_CC_SHN_MASK); 480 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 481 nvme_write4(sc, NVME_CC, cc); 482 483 for (i = 0; i < 4000; i++) { 484 nvme_barrier(sc, 0, sc->sc_ios, 485 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 486 csts = nvme_read4(sc, NVME_CSTS); 487 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 488 return 0; 489 490 delay(1000); 491 } 492 493 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 494 495 disable: 496 nvme_disable(sc); 497 return 0; 498 } 499 500 void 501 nvme_childdet(device_t self, device_t child) 502 { 503 struct nvme_softc *sc = device_private(self); 504 int i; 505 506 for (i = 0; i < sc->sc_nn; i++) { 507 if (sc->sc_namespaces[i].dev == child) { 508 /* Already freed ns->ident. */ 509 sc->sc_namespaces[i].dev = NULL; 510 break; 511 } 512 } 513 } 514 515 int 516 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 517 { 518 struct nvme_sqe sqe; 519 struct nvm_identify_namespace *identify; 520 struct nvme_dmamem *mem; 521 struct nvme_ccb *ccb; 522 struct nvme_namespace *ns; 523 int rv; 524 525 KASSERT(nsid > 0); 526 527 ccb = nvme_ccb_get(sc->sc_admin_q); 528 KASSERT(ccb != NULL); 529 530 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 531 if (mem == NULL) 532 return ENOMEM; 533 534 memset(&sqe, 0, sizeof(sqe)); 535 sqe.opcode = NVM_ADMIN_IDENTIFY; 536 htolem32(&sqe.nsid, nsid); 537 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 538 htolem32(&sqe.cdw10, 0); 539 540 ccb->ccb_done = nvme_empty_done; 541 ccb->ccb_cookie = &sqe; 542 543 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 544 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 545 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 546 547 nvme_ccb_put(sc->sc_admin_q, ccb); 548 549 if (rv != 0) { 550 rv = EIO; 551 goto done; 552 } 553 554 /* commit */ 555 556 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 557 memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 558 559 ns = nvme_ns_get(sc, nsid); 560 KASSERT(ns); 561 ns->ident = identify; 562 563 done: 564 nvme_dmamem_free(sc, mem); 565 566 return rv; 567 } 568 569 int 570 nvme_ns_dobio(struct nvme_softc *sc, struct nvme_ns_context *ctx) 571 { 572 struct nvme_queue *q = nvme_get_q(sc); 573 struct nvme_ccb *ccb; 574 bus_dmamap_t dmap; 575 int i, error; 576 577 ccb = nvme_ccb_get(q); 578 if (ccb == NULL) 579 return EAGAIN; 580 581 ccb->ccb_done = nvme_ns_io_done; 582 ccb->ccb_cookie = ctx; 583 584 dmap = ccb->ccb_dmamap; 585 error = bus_dmamap_load(sc->sc_dmat, dmap, ctx->nnc_data, 586 ctx->nnc_datasize, NULL, 587 (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL) ? 588 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 589 (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 590 BUS_DMA_READ : BUS_DMA_WRITE)); 591 if (error) { 592 nvme_ccb_put(q, ccb); 593 return error; 594 } 595 596 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 597 ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 598 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 599 600 if (dmap->dm_nsegs > 2) { 601 for (i = 1; i < dmap->dm_nsegs; i++) { 602 htolem64(&ccb->ccb_prpl[i - 1], 603 dmap->dm_segs[i].ds_addr); 604 } 605 bus_dmamap_sync(sc->sc_dmat, 606 NVME_DMA_MAP(q->q_ccb_prpls), 607 ccb->ccb_prpl_off, 608 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 609 BUS_DMASYNC_PREWRITE); 610 } 611 612 if (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL)) { 613 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill) != 0) 614 return EIO; 615 return 0; 616 } 617 618 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 619 return 0; 620 } 621 622 static void 623 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 624 { 625 struct nvme_sqe_io *sqe = slot; 626 struct nvme_ns_context *ctx = ccb->ccb_cookie; 627 bus_dmamap_t dmap = ccb->ccb_dmamap; 628 629 sqe->opcode = ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 630 NVM_CMD_READ : NVM_CMD_WRITE; 631 htolem32(&sqe->nsid, ctx->nnc_nsid); 632 633 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 634 switch (dmap->dm_nsegs) { 635 case 1: 636 break; 637 case 2: 638 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 639 break; 640 default: 641 /* the prp list is already set up and synced */ 642 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 643 break; 644 } 645 646 htolem64(&sqe->slba, ctx->nnc_blkno); 647 htolem16(&sqe->nlb, (ctx->nnc_datasize / ctx->nnc_secsize) - 1); 648 } 649 650 static void 651 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 652 struct nvme_cqe *cqe) 653 { 654 struct nvme_softc *sc = q->q_sc; 655 struct nvme_ns_context *ctx = ccb->ccb_cookie; 656 bus_dmamap_t dmap = ccb->ccb_dmamap; 657 uint16_t flags; 658 659 if (dmap->dm_nsegs > 2) { 660 bus_dmamap_sync(sc->sc_dmat, 661 NVME_DMA_MAP(q->q_ccb_prpls), 662 ccb->ccb_prpl_off, 663 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 664 BUS_DMASYNC_POSTWRITE); 665 } 666 667 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 668 ISSET(ctx->nnc_flags, NVME_NS_CTX_F_READ) ? 669 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 670 671 bus_dmamap_unload(sc->sc_dmat, dmap); 672 nvme_ccb_put(q, ccb); 673 674 flags = lemtoh16(&cqe->flags); 675 676 ctx->nnc_status = flags; 677 (*ctx->nnc_done)(ctx); 678 } 679 680 int 681 nvme_ns_sync(struct nvme_softc *sc, struct nvme_ns_context *ctx) 682 { 683 struct nvme_queue *q = nvme_get_q(sc); 684 struct nvme_ccb *ccb; 685 686 ccb = nvme_ccb_get(q); 687 if (ccb == NULL) 688 return EAGAIN; 689 690 ccb->ccb_done = nvme_ns_sync_done; 691 ccb->ccb_cookie = ctx; 692 693 if (ISSET(ctx->nnc_flags, NVME_NS_CTX_F_POLL)) { 694 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill) != 0) 695 return EIO; 696 return 0; 697 } 698 699 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 700 return 0; 701 } 702 703 static void 704 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 705 { 706 struct nvme_sqe *sqe = slot; 707 struct nvme_ns_context *ctx = ccb->ccb_cookie; 708 709 sqe->opcode = NVM_CMD_FLUSH; 710 htolem32(&sqe->nsid, ctx->nnc_nsid); 711 } 712 713 static void 714 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 715 struct nvme_cqe *cqe) 716 { 717 struct nvme_ns_context *ctx = ccb->ccb_cookie; 718 uint16_t flags; 719 720 nvme_ccb_put(q, ccb); 721 722 flags = lemtoh16(&cqe->flags); 723 724 ctx->nnc_status = flags; 725 (*ctx->nnc_done)(ctx); 726 } 727 728 void 729 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 730 { 731 struct nvme_namespace *ns; 732 struct nvm_identify_namespace *identify; 733 734 ns = nvme_ns_get(sc, nsid); 735 KASSERT(ns); 736 737 identify = ns->ident; 738 ns->ident = NULL; 739 if (identify != NULL) 740 kmem_free(identify, sizeof(*identify)); 741 } 742 743 static void 744 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 745 { 746 struct nvme_softc *sc = q->q_sc; 747 struct nvme_sqe *sqe = slot; 748 struct nvme_pt_command *pt = ccb->ccb_cookie; 749 bus_dmamap_t dmap = ccb->ccb_dmamap; 750 int i; 751 752 sqe->opcode = pt->cmd.opcode; 753 htolem32(&sqe->nsid, pt->cmd.nsid); 754 755 if (pt->buf != NULL && pt->len > 0) { 756 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 757 switch (dmap->dm_nsegs) { 758 case 1: 759 break; 760 case 2: 761 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 762 break; 763 default: 764 for (i = 1; i < dmap->dm_nsegs; i++) { 765 htolem64(&ccb->ccb_prpl[i - 1], 766 dmap->dm_segs[i].ds_addr); 767 } 768 bus_dmamap_sync(sc->sc_dmat, 769 NVME_DMA_MAP(q->q_ccb_prpls), 770 ccb->ccb_prpl_off, 771 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 772 BUS_DMASYNC_PREWRITE); 773 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 774 break; 775 } 776 } 777 778 htolem32(&sqe->cdw10, pt->cmd.cdw10); 779 htolem32(&sqe->cdw11, pt->cmd.cdw11); 780 htolem32(&sqe->cdw12, pt->cmd.cdw12); 781 htolem32(&sqe->cdw13, pt->cmd.cdw13); 782 htolem32(&sqe->cdw14, pt->cmd.cdw14); 783 htolem32(&sqe->cdw15, pt->cmd.cdw15); 784 } 785 786 static void 787 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 788 { 789 struct nvme_softc *sc = q->q_sc; 790 struct nvme_pt_command *pt = ccb->ccb_cookie; 791 bus_dmamap_t dmap = ccb->ccb_dmamap; 792 793 if (pt->buf != NULL && pt->len > 0) { 794 if (dmap->dm_nsegs > 2) { 795 bus_dmamap_sync(sc->sc_dmat, 796 NVME_DMA_MAP(q->q_ccb_prpls), 797 ccb->ccb_prpl_off, 798 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 799 BUS_DMASYNC_POSTWRITE); 800 } 801 802 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 803 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 804 bus_dmamap_unload(sc->sc_dmat, dmap); 805 } 806 807 pt->cpl.cdw0 = cqe->cdw0; 808 pt->cpl.flags = cqe->flags & ~NVME_CQE_PHASE; 809 } 810 811 static int 812 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 813 uint16_t nsid, struct lwp *l, bool is_adminq) 814 { 815 struct nvme_queue *q; 816 struct nvme_ccb *ccb; 817 void *buf = NULL; 818 int error; 819 820 if ((pt->buf == NULL && pt->len > 0) || 821 (pt->buf != NULL && pt->len == 0)) 822 return EINVAL; 823 824 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 825 ccb = nvme_ccb_get(q); 826 if (ccb == NULL) 827 return EBUSY; 828 829 if (pt->buf != NULL && pt->len > 0) { 830 buf = kmem_alloc(pt->len, KM_SLEEP); 831 if (buf == NULL) { 832 error = ENOMEM; 833 goto ccb_put; 834 } 835 if (!pt->is_read) { 836 error = copyin(pt->buf, buf, pt->len); 837 if (error) 838 goto kmem_free; 839 } 840 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 841 pt->len, NULL, 842 BUS_DMA_WAITOK | 843 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 844 if (error) 845 goto kmem_free; 846 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 847 0, ccb->ccb_dmamap->dm_mapsize, 848 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 849 } 850 851 ccb->ccb_done = nvme_pt_done; 852 ccb->ccb_cookie = pt; 853 854 pt->cmd.nsid = nsid; 855 if (nvme_poll(sc, q, ccb, nvme_pt_fill)) { 856 error = EIO; 857 goto out; 858 } 859 860 error = 0; 861 out: 862 if (buf != NULL) { 863 if (error == 0 && pt->is_read) 864 error = copyout(buf, pt->buf, pt->len); 865 kmem_free: 866 kmem_free(buf, pt->len); 867 } 868 ccb_put: 869 nvme_ccb_put(q, ccb); 870 return error; 871 } 872 873 static void 874 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 875 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 876 { 877 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 878 uint32_t tail; 879 880 mutex_enter(&q->q_sq_mtx); 881 tail = q->q_sq_tail; 882 if (++q->q_sq_tail >= q->q_entries) 883 q->q_sq_tail = 0; 884 885 sqe += tail; 886 887 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 888 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 889 memset(sqe, 0, sizeof(*sqe)); 890 (*fill)(q, ccb, sqe); 891 sqe->cid = ccb->ccb_id; 892 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 893 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 894 895 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 896 mutex_exit(&q->q_sq_mtx); 897 } 898 899 struct nvme_poll_state { 900 struct nvme_sqe s; 901 struct nvme_cqe c; 902 }; 903 904 static int 905 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 906 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 907 { 908 struct nvme_poll_state state; 909 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 910 void *cookie; 911 uint16_t flags; 912 913 memset(&state, 0, sizeof(state)); 914 (*fill)(q, ccb, &state.s); 915 916 done = ccb->ccb_done; 917 cookie = ccb->ccb_cookie; 918 919 ccb->ccb_done = nvme_poll_done; 920 ccb->ccb_cookie = &state; 921 922 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 923 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 924 if (nvme_q_complete(sc, q) == 0) 925 delay(10); 926 927 /* XXX no timeout? */ 928 } 929 930 ccb->ccb_cookie = cookie; 931 done(q, ccb, &state.c); 932 933 flags = lemtoh16(&state.c.flags); 934 935 return flags & ~NVME_CQE_PHASE; 936 } 937 938 static void 939 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 940 { 941 struct nvme_sqe *sqe = slot; 942 struct nvme_poll_state *state = ccb->ccb_cookie; 943 944 *sqe = state->s; 945 } 946 947 static void 948 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 949 struct nvme_cqe *cqe) 950 { 951 struct nvme_poll_state *state = ccb->ccb_cookie; 952 953 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 954 state->c = *cqe; 955 } 956 957 static void 958 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 959 { 960 struct nvme_sqe *src = ccb->ccb_cookie; 961 struct nvme_sqe *dst = slot; 962 963 *dst = *src; 964 } 965 966 static void 967 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 968 struct nvme_cqe *cqe) 969 { 970 } 971 972 static int 973 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 974 { 975 struct nvme_ccb *ccb; 976 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 977 uint32_t head; 978 uint16_t flags; 979 int rv = 0; 980 981 if (!mutex_tryenter(&q->q_cq_mtx)) 982 return -1; 983 984 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 985 head = q->q_cq_head; 986 for (;;) { 987 cqe = &ring[head]; 988 flags = lemtoh16(&cqe->flags); 989 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 990 break; 991 992 ccb = &q->q_ccbs[cqe->cid]; 993 ccb->ccb_done(q, ccb, cqe); 994 995 if (++head >= q->q_entries) { 996 head = 0; 997 q->q_cq_phase ^= NVME_CQE_PHASE; 998 } 999 1000 rv = 1; 1001 } 1002 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1003 1004 if (rv) 1005 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head = head); 1006 mutex_exit(&q->q_cq_mtx); 1007 1008 return rv; 1009 } 1010 1011 static int 1012 nvme_identify(struct nvme_softc *sc, u_int mps) 1013 { 1014 char sn[41], mn[81], fr[17]; 1015 struct nvm_identify_controller *identify; 1016 struct nvme_dmamem *mem; 1017 struct nvme_ccb *ccb; 1018 u_int mdts; 1019 int rv = 1; 1020 1021 ccb = nvme_ccb_get(sc->sc_admin_q); 1022 if (ccb == NULL) 1023 panic("%s: nvme_ccb_get returned NULL", __func__); 1024 1025 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1026 if (mem == NULL) 1027 return 1; 1028 1029 ccb->ccb_done = nvme_empty_done; 1030 ccb->ccb_cookie = mem; 1031 1032 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1033 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify); 1034 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1035 1036 nvme_ccb_put(sc->sc_admin_q, ccb); 1037 1038 if (rv != 0) 1039 goto done; 1040 1041 identify = NVME_DMA_KVA(mem); 1042 1043 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1044 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1045 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1046 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1047 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1048 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1049 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1050 sn); 1051 1052 if (identify->mdts > 0) { 1053 mdts = (1 << identify->mdts) * (1 << mps); 1054 if (mdts < sc->sc_mdts) 1055 sc->sc_mdts = mdts; 1056 } 1057 1058 sc->sc_nn = lemtoh32(&identify->nn); 1059 1060 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1061 1062 done: 1063 nvme_dmamem_free(sc, mem); 1064 1065 return rv; 1066 } 1067 1068 static int 1069 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1070 { 1071 struct nvme_sqe_q sqe; 1072 struct nvme_ccb *ccb; 1073 int rv; 1074 1075 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q)) 1076 return 1; 1077 1078 ccb = nvme_ccb_get(sc->sc_admin_q); 1079 KASSERT(ccb != NULL); 1080 1081 ccb->ccb_done = nvme_empty_done; 1082 ccb->ccb_cookie = &sqe; 1083 1084 memset(&sqe, 0, sizeof(sqe)); 1085 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1086 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1087 htolem16(&sqe.qsize, q->q_entries - 1); 1088 htolem16(&sqe.qid, q->q_id); 1089 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1090 if (sc->sc_use_mq) 1091 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1092 1093 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1094 if (rv != 0) 1095 goto fail; 1096 1097 ccb->ccb_done = nvme_empty_done; 1098 ccb->ccb_cookie = &sqe; 1099 1100 memset(&sqe, 0, sizeof(sqe)); 1101 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1102 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1103 htolem16(&sqe.qsize, q->q_entries - 1); 1104 htolem16(&sqe.qid, q->q_id); 1105 htolem16(&sqe.cqid, q->q_id); 1106 sqe.qflags = NVM_SQE_Q_PC; 1107 1108 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1109 if (rv != 0) 1110 goto fail; 1111 1112 fail: 1113 nvme_ccb_put(sc->sc_admin_q, ccb); 1114 return rv; 1115 } 1116 1117 static int 1118 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1119 { 1120 struct nvme_sqe_q sqe; 1121 struct nvme_ccb *ccb; 1122 int rv; 1123 1124 ccb = nvme_ccb_get(sc->sc_admin_q); 1125 KASSERT(ccb != NULL); 1126 1127 ccb->ccb_done = nvme_empty_done; 1128 ccb->ccb_cookie = &sqe; 1129 1130 memset(&sqe, 0, sizeof(sqe)); 1131 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1132 htolem16(&sqe.qid, q->q_id); 1133 1134 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1135 if (rv != 0) 1136 goto fail; 1137 1138 ccb->ccb_done = nvme_empty_done; 1139 ccb->ccb_cookie = &sqe; 1140 1141 memset(&sqe, 0, sizeof(sqe)); 1142 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1143 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1144 htolem16(&sqe.qid, q->q_id); 1145 1146 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1147 if (rv != 0) 1148 goto fail; 1149 1150 fail: 1151 nvme_ccb_put(sc->sc_admin_q, ccb); 1152 1153 if (rv == 0 && sc->sc_use_mq) { 1154 if (sc->sc_intr_disestablish(sc, q->q_id)) 1155 rv = 1; 1156 } 1157 1158 return rv; 1159 } 1160 1161 static void 1162 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1163 { 1164 struct nvme_sqe *sqe = slot; 1165 struct nvme_dmamem *mem = ccb->ccb_cookie; 1166 1167 sqe->opcode = NVM_ADMIN_IDENTIFY; 1168 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1169 htolem32(&sqe->cdw10, 1); 1170 } 1171 1172 static int 1173 nvme_ccbs_alloc(struct nvme_queue *q, u_int nccbs) 1174 { 1175 struct nvme_softc *sc = q->q_sc; 1176 struct nvme_ccb *ccb; 1177 bus_addr_t off; 1178 uint64_t *prpl; 1179 u_int i; 1180 1181 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1182 SIMPLEQ_INIT(&q->q_ccb_list); 1183 1184 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1185 if (q->q_ccbs == NULL) 1186 return 1; 1187 1188 q->q_nccbs = nccbs; 1189 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1190 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1191 1192 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1193 off = 0; 1194 1195 for (i = 0; i < nccbs; i++) { 1196 ccb = &q->q_ccbs[i]; 1197 1198 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1199 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1200 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1201 &ccb->ccb_dmamap) != 0) 1202 goto free_maps; 1203 1204 ccb->ccb_id = i; 1205 ccb->ccb_prpl = prpl; 1206 ccb->ccb_prpl_off = off; 1207 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1208 1209 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1210 1211 prpl += sc->sc_max_sgl; 1212 off += sizeof(*prpl) * sc->sc_max_sgl; 1213 } 1214 1215 return 0; 1216 1217 free_maps: 1218 nvme_ccbs_free(q); 1219 return 1; 1220 } 1221 1222 static struct nvme_ccb * 1223 nvme_ccb_get(struct nvme_queue *q) 1224 { 1225 struct nvme_ccb *ccb; 1226 1227 mutex_enter(&q->q_ccb_mtx); 1228 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1229 if (ccb != NULL) 1230 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1231 mutex_exit(&q->q_ccb_mtx); 1232 1233 return ccb; 1234 } 1235 1236 static void 1237 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1238 { 1239 1240 mutex_enter(&q->q_ccb_mtx); 1241 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1242 mutex_exit(&q->q_ccb_mtx); 1243 } 1244 1245 static void 1246 nvme_ccbs_free(struct nvme_queue *q) 1247 { 1248 struct nvme_softc *sc = q->q_sc; 1249 struct nvme_ccb *ccb; 1250 1251 mutex_enter(&q->q_ccb_mtx); 1252 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1253 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1254 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1255 } 1256 mutex_exit(&q->q_ccb_mtx); 1257 1258 nvme_dmamem_free(sc, q->q_ccb_prpls); 1259 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1260 q->q_ccbs = NULL; 1261 mutex_destroy(&q->q_ccb_mtx); 1262 } 1263 1264 static struct nvme_queue * 1265 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1266 { 1267 struct nvme_queue *q; 1268 1269 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1270 if (q == NULL) 1271 return NULL; 1272 1273 q->q_sc = sc; 1274 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1275 sizeof(struct nvme_sqe) * entries); 1276 if (q->q_sq_dmamem == NULL) 1277 goto free; 1278 1279 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1280 sizeof(struct nvme_cqe) * entries); 1281 if (q->q_cq_dmamem == NULL) 1282 goto free_sq; 1283 1284 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1285 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1286 1287 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1288 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1289 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1290 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1291 q->q_id = id; 1292 q->q_entries = entries; 1293 q->q_sq_tail = 0; 1294 q->q_cq_head = 0; 1295 q->q_cq_phase = NVME_CQE_PHASE; 1296 1297 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1298 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1299 1300 if (nvme_ccbs_alloc(q, entries) != 0) { 1301 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1302 goto free_cq; 1303 } 1304 1305 return q; 1306 1307 free_cq: 1308 nvme_dmamem_free(sc, q->q_cq_dmamem); 1309 free_sq: 1310 nvme_dmamem_free(sc, q->q_sq_dmamem); 1311 free: 1312 kmem_free(q, sizeof(*q)); 1313 1314 return NULL; 1315 } 1316 1317 static void 1318 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1319 { 1320 nvme_ccbs_free(q); 1321 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1322 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1323 nvme_dmamem_free(sc, q->q_cq_dmamem); 1324 nvme_dmamem_free(sc, q->q_sq_dmamem); 1325 kmem_free(q, sizeof(*q)); 1326 } 1327 1328 int 1329 nvme_intr(void *xsc) 1330 { 1331 struct nvme_softc *sc = xsc; 1332 int rv = 0; 1333 1334 nvme_write4(sc, NVME_INTMS, 1); 1335 1336 if (nvme_q_complete(sc, sc->sc_admin_q)) 1337 rv = 1; 1338 if (sc->sc_q != NULL) 1339 if (nvme_q_complete(sc, sc->sc_q[0])) 1340 rv = 1; 1341 1342 nvme_write4(sc, NVME_INTMC, 1); 1343 1344 return rv; 1345 } 1346 1347 int 1348 nvme_mq_msi_intr(void *xq) 1349 { 1350 struct nvme_queue *q = xq; 1351 struct nvme_softc *sc = q->q_sc; 1352 int rv = 0; 1353 1354 nvme_write4(sc, NVME_INTMS, 1U << q->q_id); 1355 1356 if (nvme_q_complete(sc, q)) 1357 rv = 1; 1358 1359 nvme_write4(sc, NVME_INTMC, 1U << q->q_id); 1360 1361 return rv; 1362 } 1363 1364 int 1365 nvme_mq_msix_intr(void *xq) 1366 { 1367 struct nvme_queue *q = xq; 1368 int rv = 0; 1369 1370 if (nvme_q_complete(q->q_sc, q)) 1371 rv = 1; 1372 1373 return rv; 1374 } 1375 1376 static struct nvme_dmamem * 1377 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1378 { 1379 struct nvme_dmamem *ndm; 1380 int nsegs; 1381 1382 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1383 if (ndm == NULL) 1384 return NULL; 1385 1386 ndm->ndm_size = size; 1387 1388 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1389 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1390 goto ndmfree; 1391 1392 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1393 1, &nsegs, BUS_DMA_WAITOK) != 0) 1394 goto destroy; 1395 1396 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1397 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1398 goto free; 1399 memset(ndm->ndm_kva, 0, size); 1400 1401 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1402 NULL, BUS_DMA_WAITOK) != 0) 1403 goto unmap; 1404 1405 return ndm; 1406 1407 unmap: 1408 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1409 free: 1410 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1411 destroy: 1412 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1413 ndmfree: 1414 kmem_free(ndm, sizeof(*ndm)); 1415 return NULL; 1416 } 1417 1418 static void 1419 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1420 { 1421 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1422 0, NVME_DMA_LEN(mem), ops); 1423 } 1424 1425 void 1426 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1427 { 1428 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1429 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1430 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1431 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1432 kmem_free(ndm, sizeof(*ndm)); 1433 } 1434 1435 /* 1436 * ioctl 1437 */ 1438 1439 dev_type_open(nvmeopen); 1440 dev_type_close(nvmeclose); 1441 dev_type_ioctl(nvmeioctl); 1442 1443 const struct cdevsw nvme_cdevsw = { 1444 .d_open = nvmeopen, 1445 .d_close = nvmeclose, 1446 .d_read = noread, 1447 .d_write = nowrite, 1448 .d_ioctl = nvmeioctl, 1449 .d_stop = nostop, 1450 .d_tty = notty, 1451 .d_poll = nopoll, 1452 .d_mmap = nommap, 1453 .d_kqfilter = nokqfilter, 1454 .d_discard = nodiscard, 1455 .d_flag = D_OTHER, 1456 }; 1457 1458 extern struct cfdriver nvme_cd; 1459 1460 /* 1461 * Accept an open operation on the control device. 1462 */ 1463 int 1464 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1465 { 1466 struct nvme_softc *sc; 1467 int unit = minor(dev) / 0x10000; 1468 int nsid = minor(dev) & 0xffff; 1469 int nsidx; 1470 1471 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1472 return ENXIO; 1473 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1474 return ENXIO; 1475 1476 if (nsid == 0) { 1477 /* controller */ 1478 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1479 return EBUSY; 1480 SET(sc->sc_flags, NVME_F_OPEN); 1481 } else { 1482 /* namespace */ 1483 nsidx = nsid - 1; 1484 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1485 return ENXIO; 1486 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1487 return EBUSY; 1488 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1489 } 1490 return 0; 1491 } 1492 1493 /* 1494 * Accept the last close on the control device. 1495 */ 1496 int 1497 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1498 { 1499 struct nvme_softc *sc; 1500 int unit = minor(dev) / 0x10000; 1501 int nsid = minor(dev) & 0xffff; 1502 int nsidx; 1503 1504 sc = device_lookup_private(&nvme_cd, unit); 1505 if (sc == NULL) 1506 return ENXIO; 1507 1508 if (nsid == 0) { 1509 /* controller */ 1510 CLR(sc->sc_flags, NVME_F_OPEN); 1511 } else { 1512 /* namespace */ 1513 nsidx = nsid - 1; 1514 if (nsidx >= sc->sc_nn) 1515 return ENXIO; 1516 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1517 } 1518 1519 return 0; 1520 } 1521 1522 /* 1523 * Handle control operations. 1524 */ 1525 int 1526 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1527 { 1528 struct nvme_softc *sc; 1529 int unit = minor(dev) / 0x10000; 1530 int nsid = minor(dev) & 0xffff; 1531 struct nvme_pt_command *pt; 1532 1533 sc = device_lookup_private(&nvme_cd, unit); 1534 if (sc == NULL) 1535 return ENXIO; 1536 1537 switch (cmd) { 1538 case NVME_PASSTHROUGH_CMD: 1539 pt = data; 1540 return nvme_command_passthrough(sc, data, 1541 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1542 } 1543 1544 return ENOTTY; 1545 } 1546
Indexes created Wed Sep 24 05:09:52 GMT 2025