nvme.c revision 1.13
1 /* $NetBSD: nvme.c,v 1.13 2016/09/20 21:18:08 jdolecek Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.13 2016/09/20 21:18:08 jdolecek Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 int nvme_adminq_size = 128; 44 int nvme_ioq_size = 1024; 45 46 static int nvme_print(void *, const char *); 47 48 static int nvme_ready(struct nvme_softc *, uint32_t); 49 static int nvme_enable(struct nvme_softc *, u_int); 50 static int nvme_disable(struct nvme_softc *); 51 static int nvme_shutdown(struct nvme_softc *); 52 53 static void nvme_version(struct nvme_softc *, uint32_t); 54 #ifdef NVME_DEBUG 55 static void nvme_dumpregs(struct nvme_softc *); 56 #endif 57 static int nvme_identify(struct nvme_softc *, u_int); 58 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 59 void *); 60 61 static int nvme_ccbs_alloc(struct nvme_queue *, u_int); 62 static void nvme_ccbs_free(struct nvme_queue *); 63 64 static struct nvme_ccb * 65 nvme_ccb_get(struct nvme_queue *); 66 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 67 68 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 69 struct nvme_ccb *, void (*)(struct nvme_queue *, 70 struct nvme_ccb *, void *), int); 71 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 72 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 73 struct nvme_cqe *); 74 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 76 struct nvme_cqe *); 77 78 static struct nvme_queue * 79 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 80 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 81 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 82 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 83 struct nvme_ccb *, void (*)(struct nvme_queue *, 84 struct nvme_ccb *, void *)); 85 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 86 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 87 88 static int nvme_dmamem_alloc(struct nvme_softc *, size_t, 89 struct nvme_dmamem *); 90 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 91 92 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 93 void *); 94 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 95 struct nvme_cqe *); 96 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 97 void *); 98 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 99 struct nvme_cqe *); 100 101 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 102 void *); 103 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 104 struct nvme_cqe *); 105 static int nvme_command_passthrough(struct nvme_softc *, 106 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 107 108 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 109 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 110 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 111 #define NVME_TIMO_SY 60 /* sync cache timeout */ 112 113 #define nvme_read4(_s, _r) \ 114 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 115 #define nvme_write4(_s, _r, _v) \ 116 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 117 #ifdef __LP64__ 118 #define nvme_read8(_s, _r) \ 119 bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r)) 120 #define nvme_write8(_s, _r, _v) \ 121 bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 122 #else /* __LP64__ */ 123 static inline uint64_t 124 nvme_read8(struct nvme_softc *sc, bus_size_t r) 125 { 126 uint64_t v; 127 uint32_t *a = (uint32_t *)&v; 128 129 #if _BYTE_ORDER == _LITTLE_ENDIAN 130 a[0] = nvme_read4(sc, r); 131 a[1] = nvme_read4(sc, r + 4); 132 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 133 a[1] = nvme_read4(sc, r); 134 a[0] = nvme_read4(sc, r + 4); 135 #endif 136 137 return v; 138 } 139 140 static inline void 141 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 142 { 143 uint32_t *a = (uint32_t *)&v; 144 145 #if _BYTE_ORDER == _LITTLE_ENDIAN 146 nvme_write4(sc, r, a[0]); 147 nvme_write4(sc, r + 4, a[1]); 148 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 149 nvme_write4(sc, r, a[1]); 150 nvme_write4(sc, r + 4, a[0]); 151 #endif 152 } 153 #endif /* __LP64__ */ 154 155 #define nvme_barrier(_s, _r, _l, _f) \ 156 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 157 #define nvme_dmamem_sync(sc, mem, ops) \ 158 bus_dmamap_sync((sc)->sc_dmat, NVME_DMA_MAP(mem), \ 159 0, NVME_DMA_LEN(mem), (ops)); 160 161 static void 162 nvme_version(struct nvme_softc *sc, uint32_t ver) 163 { 164 const char *v = NULL; 165 166 switch (ver) { 167 case NVME_VS_1_0: 168 v = "1.0"; 169 break; 170 case NVME_VS_1_1: 171 v = "1.1"; 172 break; 173 case NVME_VS_1_2: 174 v = "1.2"; 175 break; 176 default: 177 aprint_error_dev(sc->sc_dev, "unknown version 0x%08x\n", ver); 178 return; 179 } 180 181 aprint_normal_dev(sc->sc_dev, "NVMe %s\n", v); 182 } 183 184 #ifdef NVME_DEBUG 185 static __used void 186 nvme_dumpregs(struct nvme_softc *sc) 187 { 188 uint64_t r8; 189 uint32_t r4; 190 191 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 192 r8 = nvme_read8(sc, NVME_CAP); 193 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 194 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 195 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 196 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 197 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 198 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 199 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 200 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 201 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 202 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 203 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 204 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 205 206 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 207 208 r4 = nvme_read4(sc, NVME_CC); 209 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 210 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 211 (1 << NVME_CC_IOCQES_R(r4))); 212 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 213 (1 << NVME_CC_IOSQES_R(r4))); 214 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 215 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 216 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 217 (1 << NVME_CC_MPS_R(r4))); 218 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 219 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 220 221 r4 = nvme_read4(sc, NVME_CSTS); 222 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 223 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 224 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 225 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 226 227 r4 = nvme_read4(sc, NVME_AQA); 228 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 229 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 230 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 231 232 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 233 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 234 #undef DEVNAME 235 } 236 #endif /* NVME_DEBUG */ 237 238 static int 239 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 240 { 241 u_int i = 0; 242 uint32_t cc; 243 244 cc = nvme_read4(sc, NVME_CC); 245 if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) { 246 aprint_error_dev(sc->sc_dev, 247 "controller enabled status expected %d, found to be %d\n", 248 (rdy != 0), ((cc & NVME_CC_EN) != 0)); 249 return ENXIO; 250 } 251 252 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 253 if (i++ > sc->sc_rdy_to) 254 return ENXIO; 255 256 delay(1000); 257 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 258 } 259 260 return 0; 261 } 262 263 static int 264 nvme_enable(struct nvme_softc *sc, u_int mps) 265 { 266 uint32_t cc, csts; 267 268 cc = nvme_read4(sc, NVME_CC); 269 csts = nvme_read4(sc, NVME_CSTS); 270 271 if (ISSET(cc, NVME_CC_EN)) { 272 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 273 274 if (ISSET(csts, NVME_CSTS_RDY)) 275 return 1; 276 277 goto waitready; 278 } 279 280 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 281 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 282 delay(5000); 283 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 284 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 285 delay(5000); 286 287 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 288 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 289 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 290 delay(5000); 291 292 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 293 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 294 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 295 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 296 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 297 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 298 SET(cc, NVME_CC_MPS(mps)); 299 SET(cc, NVME_CC_EN); 300 301 nvme_write4(sc, NVME_CC, cc); 302 nvme_barrier(sc, 0, sc->sc_ios, 303 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 304 delay(5000); 305 306 waitready: 307 return nvme_ready(sc, NVME_CSTS_RDY); 308 } 309 310 static int 311 nvme_disable(struct nvme_softc *sc) 312 { 313 uint32_t cc, csts; 314 315 cc = nvme_read4(sc, NVME_CC); 316 csts = nvme_read4(sc, NVME_CSTS); 317 318 if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY)) 319 nvme_ready(sc, NVME_CSTS_RDY); 320 321 CLR(cc, NVME_CC_EN); 322 323 nvme_write4(sc, NVME_CC, cc); 324 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 325 326 delay(5000); 327 328 return nvme_ready(sc, 0); 329 } 330 331 int 332 nvme_attach(struct nvme_softc *sc) 333 { 334 struct nvme_attach_args naa; 335 uint64_t cap; 336 uint32_t reg; 337 u_int dstrd; 338 u_int mps = PAGE_SHIFT; 339 int adminq_entries = nvme_adminq_size; 340 int ioq_entries = nvme_ioq_size; 341 int i; 342 343 reg = nvme_read4(sc, NVME_VS); 344 if (reg == 0xffffffff) { 345 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 346 return 1; 347 } 348 349 nvme_version(sc, reg); 350 351 cap = nvme_read8(sc, NVME_CAP); 352 dstrd = NVME_CAP_DSTRD(cap); 353 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 354 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 355 "is greater than CPU page size %u\n", 356 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 357 return 1; 358 } 359 if (NVME_CAP_MPSMAX(cap) < mps) 360 mps = NVME_CAP_MPSMAX(cap); 361 362 /* set initial values to be used for admin queue during probe */ 363 sc->sc_rdy_to = NVME_CAP_TO(cap); 364 sc->sc_mps = 1 << mps; 365 sc->sc_mdts = MAXPHYS; 366 sc->sc_max_sgl = 2; 367 368 if (nvme_disable(sc) != 0) { 369 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 370 return 1; 371 } 372 373 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 374 if (sc->sc_admin_q == NULL) { 375 aprint_error_dev(sc->sc_dev, 376 "unable to allocate admin queue\n"); 377 return 1; 378 } 379 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 380 goto free_admin_q; 381 382 if (nvme_enable(sc, mps) != 0) { 383 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 384 goto disestablish_admin_q; 385 } 386 387 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 388 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 389 goto disable; 390 } 391 392 /* we know how big things are now */ 393 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 394 395 /* reallocate ccbs of admin queue with new max sgl. */ 396 nvme_ccbs_free(sc->sc_admin_q); 397 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 398 399 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 400 if (sc->sc_q == NULL) { 401 aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n"); 402 goto disable; 403 } 404 for (i = 0; i < sc->sc_nq; i++) { 405 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 406 if (sc->sc_q[i] == NULL) { 407 aprint_error_dev(sc->sc_dev, 408 "unable to allocate io queue\n"); 409 goto free_q; 410 } 411 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 412 aprint_error_dev(sc->sc_dev, 413 "unable to create io queue\n"); 414 nvme_q_free(sc, sc->sc_q[i]); 415 goto free_q; 416 } 417 } 418 419 if (!sc->sc_use_mq) 420 nvme_write4(sc, NVME_INTMC, 1); 421 422 /* probe subdevices */ 423 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 424 KM_SLEEP); 425 if (sc->sc_namespaces == NULL) 426 goto free_q; 427 for (i = 0; i < sc->sc_nn; i++) { 428 memset(&naa, 0, sizeof(naa)); 429 naa.naa_nsid = i + 1; 430 naa.naa_qentries = ioq_entries; 431 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 432 nvme_print); 433 } 434 435 return 0; 436 437 free_q: 438 while (--i >= 0) { 439 nvme_q_delete(sc, sc->sc_q[i]); 440 nvme_q_free(sc, sc->sc_q[i]); 441 } 442 disable: 443 nvme_disable(sc); 444 disestablish_admin_q: 445 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 446 free_admin_q: 447 nvme_q_free(sc, sc->sc_admin_q); 448 449 return 1; 450 } 451 452 static int 453 nvme_print(void *aux, const char *pnp) 454 { 455 struct nvme_attach_args *naa = aux; 456 457 if (pnp) 458 aprint_normal("at %s", pnp); 459 460 if (naa->naa_nsid > 0) 461 aprint_normal(" nsid %d", naa->naa_nsid); 462 463 return UNCONF; 464 } 465 466 int 467 nvme_detach(struct nvme_softc *sc, int flags) 468 { 469 int i, error; 470 471 error = config_detach_children(sc->sc_dev, flags); 472 if (error) 473 return error; 474 475 error = nvme_shutdown(sc); 476 if (error) 477 return error; 478 479 /* from now on we are committed to detach, following will never fail */ 480 for (i = 0; i < sc->sc_nq; i++) 481 nvme_q_free(sc, sc->sc_q[i]); 482 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 483 nvme_q_free(sc, sc->sc_admin_q); 484 485 return 0; 486 } 487 488 static int 489 nvme_shutdown(struct nvme_softc *sc) 490 { 491 uint32_t cc, csts; 492 bool disabled = false; 493 int i; 494 495 if (!sc->sc_use_mq) 496 nvme_write4(sc, NVME_INTMS, 1); 497 498 for (i = 0; i < sc->sc_nq; i++) { 499 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 500 aprint_error_dev(sc->sc_dev, 501 "unable to delete io queue %d, disabling\n", i + 1); 502 disabled = true; 503 } 504 } 505 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 506 if (disabled) 507 goto disable; 508 509 cc = nvme_read4(sc, NVME_CC); 510 CLR(cc, NVME_CC_SHN_MASK); 511 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 512 nvme_write4(sc, NVME_CC, cc); 513 514 for (i = 0; i < 4000; i++) { 515 nvme_barrier(sc, 0, sc->sc_ios, 516 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 517 csts = nvme_read4(sc, NVME_CSTS); 518 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 519 return 0; 520 521 delay(1000); 522 } 523 524 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 525 526 disable: 527 nvme_disable(sc); 528 return 0; 529 } 530 531 void 532 nvme_childdet(device_t self, device_t child) 533 { 534 struct nvme_softc *sc = device_private(self); 535 int i; 536 537 for (i = 0; i < sc->sc_nn; i++) { 538 if (sc->sc_namespaces[i].dev == child) { 539 /* Already freed ns->ident. */ 540 sc->sc_namespaces[i].dev = NULL; 541 break; 542 } 543 } 544 } 545 546 int 547 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 548 { 549 struct nvme_sqe sqe; 550 struct nvm_identify_namespace *identify; 551 struct nvme_dmamem mem; 552 struct nvme_ccb *ccb; 553 struct nvme_namespace *ns; 554 int error; 555 556 KASSERT(nsid > 0); 557 558 ccb = nvme_ccb_get(sc->sc_admin_q); 559 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 560 561 error = nvme_dmamem_alloc(sc, sizeof(*identify), &mem); 562 if (error) 563 return error; 564 565 memset(&sqe, 0, sizeof(sqe)); 566 sqe.opcode = NVM_ADMIN_IDENTIFY; 567 htolem32(&sqe.nsid, nsid); 568 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 569 htolem32(&sqe.cdw10, 0); 570 571 ccb->ccb_done = nvme_empty_done; 572 ccb->ccb_cookie = &sqe; 573 574 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 575 error = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, 576 NVME_TIMO_IDENT); 577 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 578 579 nvme_ccb_put(sc->sc_admin_q, ccb); 580 581 if (error != 0) { 582 error = EIO; 583 goto done; 584 } 585 586 /* commit */ 587 588 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 589 memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 590 591 ns = nvme_ns_get(sc, nsid); 592 KASSERT(ns); 593 ns->ident = identify; 594 595 done: 596 nvme_dmamem_free(sc, &mem); 597 598 return error; 599 } 600 601 int 602 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 603 struct buf *bp, void *data, size_t datasize, 604 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 605 { 606 struct nvme_queue *q = nvme_get_q(sc); 607 struct nvme_ccb *ccb; 608 bus_dmamap_t dmap; 609 int i, error; 610 611 ccb = nvme_ccb_get(q); 612 if (ccb == NULL) 613 return EAGAIN; 614 615 ccb->ccb_done = nvme_ns_io_done; 616 ccb->ccb_cookie = cookie; 617 618 /* namespace context */ 619 ccb->nnc_nsid = nsid; 620 ccb->nnc_flags = flags; 621 ccb->nnc_buf = bp; 622 ccb->nnc_datasize = datasize; 623 ccb->nnc_secsize = secsize; 624 ccb->nnc_blkno = blkno; 625 ccb->nnc_done = nnc_done; 626 627 dmap = ccb->ccb_dmamap; 628 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 629 datasize, NULL, 630 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 631 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 632 (ISSET(flags, NVME_NS_CTX_F_READ) ? 633 BUS_DMA_READ : BUS_DMA_WRITE)); 634 if (error) { 635 nvme_ccb_put(q, ccb); 636 return error; 637 } 638 639 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 640 ISSET(flags, NVME_NS_CTX_F_READ) ? 641 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 642 643 if (dmap->dm_nsegs > 2) { 644 for (i = 1; i < dmap->dm_nsegs; i++) { 645 htolem64(&ccb->ccb_prpl[i - 1], 646 dmap->dm_segs[i].ds_addr); 647 } 648 bus_dmamap_sync(sc->sc_dmat, 649 NVME_DMA_MAP(q->q_ccb_prpls), 650 ccb->ccb_prpl_off, 651 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 652 BUS_DMASYNC_PREWRITE); 653 } 654 655 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 656 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 657 return EIO; 658 return 0; 659 } 660 661 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 662 return 0; 663 } 664 665 static void 666 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 667 { 668 struct nvme_sqe_io *sqe = slot; 669 bus_dmamap_t dmap = ccb->ccb_dmamap; 670 671 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 672 NVM_CMD_READ : NVM_CMD_WRITE; 673 htolem32(&sqe->nsid, ccb->nnc_nsid); 674 675 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 676 switch (dmap->dm_nsegs) { 677 case 1: 678 break; 679 case 2: 680 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 681 break; 682 default: 683 /* the prp list is already set up and synced */ 684 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 685 break; 686 } 687 688 htolem64(&sqe->slba, ccb->nnc_blkno); 689 690 /* guaranteed by upper layers, but check just in case */ 691 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 692 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 693 } 694 695 static void 696 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 697 struct nvme_cqe *cqe) 698 { 699 struct nvme_softc *sc = q->q_sc; 700 bus_dmamap_t dmap = ccb->ccb_dmamap; 701 void *nnc_cookie = ccb->ccb_cookie; 702 nvme_nnc_done nnc_done = ccb->nnc_done; 703 struct buf *bp = ccb->nnc_buf; 704 705 if (dmap->dm_nsegs > 2) { 706 bus_dmamap_sync(sc->sc_dmat, 707 NVME_DMA_MAP(q->q_ccb_prpls), 708 ccb->ccb_prpl_off, 709 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 710 BUS_DMASYNC_POSTWRITE); 711 } 712 713 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 714 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 715 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 716 717 bus_dmamap_unload(sc->sc_dmat, dmap); 718 nvme_ccb_put(q, ccb); 719 720 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags)); 721 } 722 723 int 724 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie, 725 int flags, nvme_nnc_done nnc_done) 726 { 727 struct nvme_queue *q = nvme_get_q(sc); 728 struct nvme_ccb *ccb; 729 730 ccb = nvme_ccb_get(q); 731 if (ccb == NULL) 732 return EAGAIN; 733 734 ccb->ccb_done = nvme_ns_sync_done; 735 ccb->ccb_cookie = cookie; 736 737 /* namespace context */ 738 ccb->nnc_nsid = nsid; 739 ccb->nnc_flags = flags; 740 ccb->nnc_done = nnc_done; 741 742 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 743 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 744 return EIO; 745 return 0; 746 } 747 748 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 749 return 0; 750 } 751 752 static void 753 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 754 { 755 struct nvme_sqe *sqe = slot; 756 757 sqe->opcode = NVM_CMD_FLUSH; 758 htolem32(&sqe->nsid, ccb->nnc_nsid); 759 } 760 761 static void 762 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 763 struct nvme_cqe *cqe) 764 { 765 void *cookie = ccb->ccb_cookie; 766 nvme_nnc_done nnc_done = ccb->nnc_done; 767 768 nvme_ccb_put(q, ccb); 769 770 nnc_done(cookie, NULL, lemtoh16(&cqe->flags)); 771 } 772 773 void 774 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 775 { 776 struct nvme_namespace *ns; 777 struct nvm_identify_namespace *identify; 778 779 ns = nvme_ns_get(sc, nsid); 780 KASSERT(ns); 781 782 identify = ns->ident; 783 ns->ident = NULL; 784 if (identify != NULL) 785 kmem_free(identify, sizeof(*identify)); 786 } 787 788 static void 789 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 790 { 791 struct nvme_softc *sc = q->q_sc; 792 struct nvme_sqe *sqe = slot; 793 struct nvme_pt_command *pt = ccb->ccb_cookie; 794 bus_dmamap_t dmap = ccb->ccb_dmamap; 795 int i; 796 797 sqe->opcode = pt->cmd.opcode; 798 htolem32(&sqe->nsid, pt->cmd.nsid); 799 800 if (pt->buf != NULL && pt->len > 0) { 801 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 802 switch (dmap->dm_nsegs) { 803 case 1: 804 break; 805 case 2: 806 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 807 break; 808 default: 809 for (i = 1; i < dmap->dm_nsegs; i++) { 810 htolem64(&ccb->ccb_prpl[i - 1], 811 dmap->dm_segs[i].ds_addr); 812 } 813 bus_dmamap_sync(sc->sc_dmat, 814 NVME_DMA_MAP(q->q_ccb_prpls), 815 ccb->ccb_prpl_off, 816 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 817 BUS_DMASYNC_PREWRITE); 818 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 819 break; 820 } 821 } 822 823 htolem32(&sqe->cdw10, pt->cmd.cdw10); 824 htolem32(&sqe->cdw11, pt->cmd.cdw11); 825 htolem32(&sqe->cdw12, pt->cmd.cdw12); 826 htolem32(&sqe->cdw13, pt->cmd.cdw13); 827 htolem32(&sqe->cdw14, pt->cmd.cdw14); 828 htolem32(&sqe->cdw15, pt->cmd.cdw15); 829 } 830 831 static void 832 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 833 { 834 struct nvme_softc *sc = q->q_sc; 835 struct nvme_pt_command *pt = ccb->ccb_cookie; 836 bus_dmamap_t dmap = ccb->ccb_dmamap; 837 838 if (pt->buf != NULL && pt->len > 0) { 839 if (dmap->dm_nsegs > 2) { 840 bus_dmamap_sync(sc->sc_dmat, 841 NVME_DMA_MAP(q->q_ccb_prpls), 842 ccb->ccb_prpl_off, 843 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 844 BUS_DMASYNC_POSTWRITE); 845 } 846 847 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 848 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 849 bus_dmamap_unload(sc->sc_dmat, dmap); 850 } 851 852 pt->cpl.cdw0 = cqe->cdw0; 853 pt->cpl.flags = cqe->flags & ~NVME_CQE_PHASE; 854 } 855 856 static int 857 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 858 uint16_t nsid, struct lwp *l, bool is_adminq) 859 { 860 struct nvme_queue *q; 861 struct nvme_ccb *ccb; 862 void *buf = NULL; 863 int error; 864 865 /* limit command size to maximum data transfer size */ 866 if ((pt->buf == NULL && pt->len > 0) || 867 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 868 return EINVAL; 869 870 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 871 ccb = nvme_ccb_get(q); 872 if (ccb == NULL) 873 return EBUSY; 874 875 if (pt->buf != NULL) { 876 KASSERT(pt->len > 0); 877 buf = kmem_alloc(pt->len, KM_SLEEP); 878 if (buf == NULL) { 879 error = ENOMEM; 880 goto ccb_put; 881 } 882 if (!pt->is_read) { 883 error = copyin(pt->buf, buf, pt->len); 884 if (error) 885 goto kmem_free; 886 } 887 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 888 pt->len, NULL, 889 BUS_DMA_WAITOK | 890 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 891 if (error) 892 goto kmem_free; 893 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 894 0, ccb->ccb_dmamap->dm_mapsize, 895 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 896 } 897 898 ccb->ccb_done = nvme_pt_done; 899 ccb->ccb_cookie = pt; 900 901 pt->cmd.nsid = nsid; 902 if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) { 903 error = EIO; 904 goto out; 905 } 906 907 error = 0; 908 out: 909 if (buf != NULL) { 910 if (error == 0 && pt->is_read) 911 error = copyout(buf, pt->buf, pt->len); 912 kmem_free: 913 kmem_free(buf, pt->len); 914 } 915 ccb_put: 916 nvme_ccb_put(q, ccb); 917 return error; 918 } 919 920 static void 921 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 922 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 923 { 924 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 925 uint32_t tail; 926 927 mutex_enter(&q->q_sq_mtx); 928 tail = q->q_sq_tail; 929 if (++q->q_sq_tail >= q->q_entries) 930 q->q_sq_tail = 0; 931 932 sqe += tail; 933 934 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 935 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 936 memset(sqe, 0, sizeof(*sqe)); 937 (*fill)(q, ccb, sqe); 938 sqe->cid = ccb->ccb_id; 939 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 940 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 941 942 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 943 mutex_exit(&q->q_sq_mtx); 944 } 945 946 struct nvme_poll_state { 947 struct nvme_sqe s; 948 struct nvme_cqe c; 949 }; 950 951 static int 952 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 953 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 954 { 955 struct nvme_poll_state state; 956 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 957 void *cookie; 958 uint16_t flags; 959 int step = 10; 960 int maxloop = timo_sec * 1000000 / step; 961 int error = 0; 962 963 memset(&state, 0, sizeof(state)); 964 (*fill)(q, ccb, &state.s); 965 966 done = ccb->ccb_done; 967 cookie = ccb->ccb_cookie; 968 969 ccb->ccb_done = nvme_poll_done; 970 ccb->ccb_cookie = &state; 971 972 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 973 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 974 if (nvme_q_complete(sc, q) == 0) 975 delay(step); 976 977 if (timo_sec >= 0 && --maxloop <= 0) { 978 error = ETIMEDOUT; 979 break; 980 } 981 } 982 983 ccb->ccb_cookie = cookie; 984 done(q, ccb, &state.c); 985 986 if (error == 0) { 987 flags = lemtoh16(&state.c.flags); 988 return flags & ~NVME_CQE_PHASE; 989 } else { 990 return 1; 991 } 992 } 993 994 static void 995 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 996 { 997 struct nvme_sqe *sqe = slot; 998 struct nvme_poll_state *state = ccb->ccb_cookie; 999 1000 *sqe = state->s; 1001 } 1002 1003 static void 1004 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1005 struct nvme_cqe *cqe) 1006 { 1007 struct nvme_poll_state *state = ccb->ccb_cookie; 1008 1009 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1010 state->c = *cqe; 1011 } 1012 1013 static void 1014 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1015 { 1016 struct nvme_sqe *src = ccb->ccb_cookie; 1017 struct nvme_sqe *dst = slot; 1018 1019 *dst = *src; 1020 } 1021 1022 static void 1023 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1024 struct nvme_cqe *cqe) 1025 { 1026 } 1027 1028 static int 1029 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1030 { 1031 struct nvme_ccb *ccb; 1032 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1033 uint16_t flags; 1034 int rv = 0; 1035 1036 mutex_enter(&q->q_cq_mtx); 1037 1038 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1039 for (;;) { 1040 cqe = &ring[q->q_cq_head]; 1041 flags = lemtoh16(&cqe->flags); 1042 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1043 break; 1044 1045 ccb = &q->q_ccbs[cqe->cid]; 1046 1047 if (++q->q_cq_head >= q->q_entries) { 1048 q->q_cq_head = 0; 1049 q->q_cq_phase ^= NVME_CQE_PHASE; 1050 } 1051 1052 rv = 1; 1053 1054 /* 1055 * Unlock the mutex before calling the ccb_done callback 1056 * and re-lock afterwards. The callback triggers lddone() 1057 * which schedules another i/o, and also calls nvme_ccb_put(). 1058 * Unlock/relock avoids possibility of deadlock. 1059 */ 1060 mutex_exit(&q->q_cq_mtx); 1061 ccb->ccb_done(q, ccb, cqe); 1062 mutex_enter(&q->q_cq_mtx); 1063 } 1064 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1065 1066 if (rv) 1067 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1068 1069 mutex_exit(&q->q_cq_mtx); 1070 1071 return rv; 1072 } 1073 1074 static int 1075 nvme_identify(struct nvme_softc *sc, u_int mps) 1076 { 1077 char sn[41], mn[81], fr[17]; 1078 struct nvm_identify_controller *identify; 1079 struct nvme_dmamem mem; 1080 struct nvme_ccb *ccb; 1081 u_int mdts; 1082 int error; 1083 1084 ccb = nvme_ccb_get(sc->sc_admin_q); 1085 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1086 1087 error = nvme_dmamem_alloc(sc, sizeof(*identify), &mem); 1088 if (error) 1089 return error; 1090 1091 ccb->ccb_done = nvme_empty_done; 1092 ccb->ccb_cookie = &mem; 1093 1094 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1095 error = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1096 NVME_TIMO_IDENT); 1097 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1098 1099 nvme_ccb_put(sc->sc_admin_q, ccb); 1100 1101 if (error != 0) 1102 goto done; 1103 1104 identify = NVME_DMA_KVA(mem); 1105 1106 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1107 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1108 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1109 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1110 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1111 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1112 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1113 sn); 1114 1115 if (identify->mdts > 0) { 1116 mdts = (1 << identify->mdts) * (1 << mps); 1117 if (mdts < sc->sc_mdts) 1118 sc->sc_mdts = mdts; 1119 } 1120 1121 sc->sc_nn = lemtoh32(&identify->nn); 1122 1123 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1124 1125 done: 1126 nvme_dmamem_free(sc, &mem); 1127 1128 return error; 1129 } 1130 1131 static int 1132 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1133 { 1134 struct nvme_sqe_q sqe; 1135 struct nvme_ccb *ccb; 1136 int rv; 1137 1138 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1139 return 1; 1140 1141 ccb = nvme_ccb_get(sc->sc_admin_q); 1142 KASSERT(ccb != NULL); 1143 1144 ccb->ccb_done = nvme_empty_done; 1145 ccb->ccb_cookie = &sqe; 1146 1147 memset(&sqe, 0, sizeof(sqe)); 1148 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1149 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1150 htolem16(&sqe.qsize, q->q_entries - 1); 1151 htolem16(&sqe.qid, q->q_id); 1152 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1153 if (sc->sc_use_mq) 1154 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1155 1156 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1157 if (rv != 0) 1158 goto fail; 1159 1160 ccb->ccb_done = nvme_empty_done; 1161 ccb->ccb_cookie = &sqe; 1162 1163 memset(&sqe, 0, sizeof(sqe)); 1164 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1165 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1166 htolem16(&sqe.qsize, q->q_entries - 1); 1167 htolem16(&sqe.qid, q->q_id); 1168 htolem16(&sqe.cqid, q->q_id); 1169 sqe.qflags = NVM_SQE_Q_PC; 1170 1171 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1172 if (rv != 0) 1173 goto fail; 1174 1175 fail: 1176 nvme_ccb_put(sc->sc_admin_q, ccb); 1177 return rv; 1178 } 1179 1180 static int 1181 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1182 { 1183 struct nvme_sqe_q sqe; 1184 struct nvme_ccb *ccb; 1185 int rv; 1186 1187 ccb = nvme_ccb_get(sc->sc_admin_q); 1188 KASSERT(ccb != NULL); 1189 1190 ccb->ccb_done = nvme_empty_done; 1191 ccb->ccb_cookie = &sqe; 1192 1193 memset(&sqe, 0, sizeof(sqe)); 1194 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1195 htolem16(&sqe.qid, q->q_id); 1196 1197 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1198 if (rv != 0) 1199 goto fail; 1200 1201 ccb->ccb_done = nvme_empty_done; 1202 ccb->ccb_cookie = &sqe; 1203 1204 memset(&sqe, 0, sizeof(sqe)); 1205 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1206 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1207 htolem16(&sqe.qid, q->q_id); 1208 1209 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1210 if (rv != 0) 1211 goto fail; 1212 1213 fail: 1214 nvme_ccb_put(sc->sc_admin_q, ccb); 1215 1216 if (rv == 0 && sc->sc_use_mq) { 1217 if (sc->sc_intr_disestablish(sc, q->q_id)) 1218 rv = 1; 1219 } 1220 1221 return rv; 1222 } 1223 1224 static void 1225 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1226 { 1227 struct nvme_sqe *sqe = slot; 1228 struct nvme_dmamem *mem = ccb->ccb_cookie; 1229 1230 sqe->opcode = NVM_ADMIN_IDENTIFY; 1231 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(*mem)); 1232 htolem32(&sqe->cdw10, 1); 1233 } 1234 1235 static int 1236 nvme_ccbs_alloc(struct nvme_queue *q, u_int nccbs) 1237 { 1238 struct nvme_softc *sc = q->q_sc; 1239 struct nvme_ccb *ccb; 1240 bus_addr_t off; 1241 uint64_t *prpl; 1242 u_int i; 1243 int error; 1244 1245 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1246 SIMPLEQ_INIT(&q->q_ccb_list); 1247 1248 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1249 if (q->q_ccbs == NULL) 1250 return 1; 1251 1252 q->q_nccbs = nccbs; 1253 error = nvme_dmamem_alloc(sc, sizeof(*prpl) * sc->sc_max_sgl * nccbs, 1254 &q->q_ccb_prpls); 1255 if (error) { 1256 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1257 return error; 1258 } 1259 1260 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1261 off = 0; 1262 1263 for (i = 0; i < nccbs; i++) { 1264 ccb = &q->q_ccbs[i]; 1265 1266 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1267 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1268 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1269 &ccb->ccb_dmamap) != 0) 1270 goto free_maps; 1271 1272 ccb->ccb_id = i; 1273 ccb->ccb_prpl = prpl; 1274 ccb->ccb_prpl_off = off; 1275 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1276 1277 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1278 1279 prpl += sc->sc_max_sgl; 1280 off += sizeof(*prpl) * sc->sc_max_sgl; 1281 } 1282 1283 return 0; 1284 1285 free_maps: 1286 nvme_ccbs_free(q); 1287 return 1; 1288 } 1289 1290 static struct nvme_ccb * 1291 nvme_ccb_get(struct nvme_queue *q) 1292 { 1293 struct nvme_ccb *ccb; 1294 1295 mutex_enter(&q->q_ccb_mtx); 1296 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1297 if (ccb != NULL) 1298 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1299 mutex_exit(&q->q_ccb_mtx); 1300 1301 return ccb; 1302 } 1303 1304 static void 1305 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1306 { 1307 1308 mutex_enter(&q->q_ccb_mtx); 1309 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1310 mutex_exit(&q->q_ccb_mtx); 1311 } 1312 1313 static void 1314 nvme_ccbs_free(struct nvme_queue *q) 1315 { 1316 struct nvme_softc *sc = q->q_sc; 1317 struct nvme_ccb *ccb; 1318 1319 mutex_enter(&q->q_ccb_mtx); 1320 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1321 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1322 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1323 } 1324 mutex_exit(&q->q_ccb_mtx); 1325 1326 nvme_dmamem_free(sc, &q->q_ccb_prpls); 1327 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1328 q->q_ccbs = NULL; 1329 mutex_destroy(&q->q_ccb_mtx); 1330 } 1331 1332 static struct nvme_queue * 1333 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1334 { 1335 struct nvme_queue *q; 1336 int error; 1337 1338 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1339 if (q == NULL) 1340 return NULL; 1341 1342 q->q_sc = sc; 1343 error = nvme_dmamem_alloc(sc, sizeof(struct nvme_sqe) * entries, 1344 &q->q_sq_dmamem); 1345 if (error) 1346 goto free; 1347 1348 error = nvme_dmamem_alloc(sc, sizeof(struct nvme_cqe) * entries, 1349 &q->q_cq_dmamem); 1350 if (error) 1351 goto free_sq; 1352 1353 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1354 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1355 1356 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1357 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1358 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1359 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1360 q->q_id = id; 1361 q->q_entries = entries; 1362 q->q_sq_tail = 0; 1363 q->q_cq_head = 0; 1364 q->q_cq_phase = NVME_CQE_PHASE; 1365 1366 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1367 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1368 1369 if (nvme_ccbs_alloc(q, entries) != 0) { 1370 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1371 goto free_cq; 1372 } 1373 1374 return q; 1375 1376 free_cq: 1377 nvme_dmamem_free(sc, &q->q_cq_dmamem); 1378 free_sq: 1379 nvme_dmamem_free(sc, &q->q_sq_dmamem); 1380 free: 1381 kmem_free(q, sizeof(*q)); 1382 1383 return NULL; 1384 } 1385 1386 static void 1387 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1388 { 1389 nvme_ccbs_free(q); 1390 mutex_destroy(&q->q_sq_mtx); 1391 mutex_destroy(&q->q_cq_mtx); 1392 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1393 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1394 nvme_dmamem_free(sc, &q->q_cq_dmamem); 1395 nvme_dmamem_free(sc, &q->q_sq_dmamem); 1396 kmem_free(q, sizeof(*q)); 1397 } 1398 1399 int 1400 nvme_intr(void *xsc) 1401 { 1402 struct nvme_softc *sc = xsc; 1403 int rv = 0; 1404 1405 /* 1406 * INTx is level triggered, controller deasserts the interrupt only 1407 * when we advance command queue head via write to the doorbell. 1408 */ 1409 if (nvme_q_complete(sc, sc->sc_admin_q)) 1410 rv = 1; 1411 if (sc->sc_q != NULL) 1412 if (nvme_q_complete(sc, sc->sc_q[0])) 1413 rv = 1; 1414 1415 return rv; 1416 } 1417 1418 int 1419 nvme_intr_msi(void *xq) 1420 { 1421 struct nvme_queue *q = xq; 1422 1423 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1424 && q->q_sc->sc_softih[q->q_id]); 1425 1426 /* MSI are edge triggered, so can handover processing to softint */ 1427 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1428 1429 return 1; 1430 } 1431 1432 void 1433 nvme_softintr_msi(void *xq) 1434 { 1435 struct nvme_queue *q = xq; 1436 struct nvme_softc *sc = q->q_sc; 1437 1438 nvme_q_complete(sc, q); 1439 } 1440 1441 static int 1442 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size, struct nvme_dmamem *ndm) 1443 { 1444 int nsegs; 1445 1446 memset(ndm, 0, sizeof(*ndm)); 1447 ndm->ndm_size = size; 1448 1449 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1450 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1451 goto ndmfree; 1452 1453 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1454 1, &nsegs, BUS_DMA_WAITOK) != 0) 1455 goto destroy; 1456 1457 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1458 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1459 goto free; 1460 memset(ndm->ndm_kva, 0, size); 1461 1462 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1463 NULL, BUS_DMA_WAITOK) != 0) 1464 goto unmap; 1465 1466 return 0; 1467 1468 unmap: 1469 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1470 free: 1471 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1472 destroy: 1473 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1474 ndmfree: 1475 return ENOMEM; 1476 } 1477 1478 void 1479 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1480 { 1481 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1482 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1483 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1484 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1485 } 1486 1487 /* 1488 * ioctl 1489 */ 1490 1491 dev_type_open(nvmeopen); 1492 dev_type_close(nvmeclose); 1493 dev_type_ioctl(nvmeioctl); 1494 1495 const struct cdevsw nvme_cdevsw = { 1496 .d_open = nvmeopen, 1497 .d_close = nvmeclose, 1498 .d_read = noread, 1499 .d_write = nowrite, 1500 .d_ioctl = nvmeioctl, 1501 .d_stop = nostop, 1502 .d_tty = notty, 1503 .d_poll = nopoll, 1504 .d_mmap = nommap, 1505 .d_kqfilter = nokqfilter, 1506 .d_discard = nodiscard, 1507 .d_flag = D_OTHER, 1508 }; 1509 1510 extern struct cfdriver nvme_cd; 1511 1512 /* 1513 * Accept an open operation on the control device. 1514 */ 1515 int 1516 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1517 { 1518 struct nvme_softc *sc; 1519 int unit = minor(dev) / 0x10000; 1520 int nsid = minor(dev) & 0xffff; 1521 int nsidx; 1522 1523 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1524 return ENXIO; 1525 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1526 return ENXIO; 1527 1528 if (nsid == 0) { 1529 /* controller */ 1530 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1531 return EBUSY; 1532 SET(sc->sc_flags, NVME_F_OPEN); 1533 } else { 1534 /* namespace */ 1535 nsidx = nsid - 1; 1536 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1537 return ENXIO; 1538 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1539 return EBUSY; 1540 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1541 } 1542 return 0; 1543 } 1544 1545 /* 1546 * Accept the last close on the control device. 1547 */ 1548 int 1549 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1550 { 1551 struct nvme_softc *sc; 1552 int unit = minor(dev) / 0x10000; 1553 int nsid = minor(dev) & 0xffff; 1554 int nsidx; 1555 1556 sc = device_lookup_private(&nvme_cd, unit); 1557 if (sc == NULL) 1558 return ENXIO; 1559 1560 if (nsid == 0) { 1561 /* controller */ 1562 CLR(sc->sc_flags, NVME_F_OPEN); 1563 } else { 1564 /* namespace */ 1565 nsidx = nsid - 1; 1566 if (nsidx >= sc->sc_nn) 1567 return ENXIO; 1568 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1569 } 1570 1571 return 0; 1572 } 1573 1574 /* 1575 * Handle control operations. 1576 */ 1577 int 1578 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1579 { 1580 struct nvme_softc *sc; 1581 int unit = minor(dev) / 0x10000; 1582 int nsid = minor(dev) & 0xffff; 1583 struct nvme_pt_command *pt; 1584 1585 sc = device_lookup_private(&nvme_cd, unit); 1586 if (sc == NULL) 1587 return ENXIO; 1588 1589 switch (cmd) { 1590 case NVME_PASSTHROUGH_CMD: 1591 pt = data; 1592 return nvme_command_passthrough(sc, data, 1593 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1594 } 1595 1596 return ENOTTY; 1597 } 1598
Indexes created Tue Sep 23 07:09:52 GMT 2025