nvme.c revision 1.14
1 /* $NetBSD: nvme.c,v 1.14 2016/09/27 03:33:32 pgoyette Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.14 2016/09/27 03:33:32 pgoyette Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 int nvme_adminq_size = 128; 44 int nvme_ioq_size = 1024; 45 46 static int nvme_print(void *, const char *); 47 48 static int nvme_ready(struct nvme_softc *, uint32_t); 49 static int nvme_enable(struct nvme_softc *, u_int); 50 static int nvme_disable(struct nvme_softc *); 51 static int nvme_shutdown(struct nvme_softc *); 52 53 static void nvme_version(struct nvme_softc *, uint32_t); 54 #ifdef NVME_DEBUG 55 static void nvme_dumpregs(struct nvme_softc *); 56 #endif 57 static int nvme_identify(struct nvme_softc *, u_int); 58 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 59 void *); 60 61 static int nvme_ccbs_alloc(struct nvme_queue *, u_int); 62 static void nvme_ccbs_free(struct nvme_queue *); 63 64 static struct nvme_ccb * 65 nvme_ccb_get(struct nvme_queue *); 66 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 67 68 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 69 struct nvme_ccb *, void (*)(struct nvme_queue *, 70 struct nvme_ccb *, void *), int); 71 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 72 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 73 struct nvme_cqe *); 74 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 76 struct nvme_cqe *); 77 78 static struct nvme_queue * 79 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 80 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 81 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 82 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 83 struct nvme_ccb *, void (*)(struct nvme_queue *, 84 struct nvme_ccb *, void *)); 85 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 86 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 87 88 static int nvme_dmamem_alloc(struct nvme_softc *, size_t, 89 struct nvme_dmamem *); 90 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 91 92 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 93 void *); 94 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 95 struct nvme_cqe *); 96 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 97 void *); 98 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 99 struct nvme_cqe *); 100 101 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 102 void *); 103 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 104 struct nvme_cqe *); 105 static int nvme_command_passthrough(struct nvme_softc *, 106 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 107 108 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 109 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 110 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 111 #define NVME_TIMO_SY 60 /* sync cache timeout */ 112 113 #define nvme_read4(_s, _r) \ 114 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 115 #define nvme_write4(_s, _r, _v) \ 116 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 117 #ifdef __LP64__ 118 #define nvme_read8(_s, _r) \ 119 bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r)) 120 #define nvme_write8(_s, _r, _v) \ 121 bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 122 #else /* __LP64__ */ 123 static inline uint64_t 124 nvme_read8(struct nvme_softc *sc, bus_size_t r) 125 { 126 uint64_t v; 127 uint32_t *a = (uint32_t *)&v; 128 129 #if _BYTE_ORDER == _LITTLE_ENDIAN 130 a[0] = nvme_read4(sc, r); 131 a[1] = nvme_read4(sc, r + 4); 132 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 133 a[1] = nvme_read4(sc, r); 134 a[0] = nvme_read4(sc, r + 4); 135 #endif 136 137 return v; 138 } 139 140 static inline void 141 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 142 { 143 uint32_t *a = (uint32_t *)&v; 144 145 #if _BYTE_ORDER == _LITTLE_ENDIAN 146 nvme_write4(sc, r, a[0]); 147 nvme_write4(sc, r + 4, a[1]); 148 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 149 nvme_write4(sc, r, a[1]); 150 nvme_write4(sc, r + 4, a[0]); 151 #endif 152 } 153 #endif /* __LP64__ */ 154 155 #define nvme_barrier(_s, _r, _l, _f) \ 156 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 157 #define nvme_dmamem_sync(sc, mem, ops) \ 158 bus_dmamap_sync((sc)->sc_dmat, NVME_DMA_MAP(mem), \ 159 0, NVME_DMA_LEN(mem), (ops)); 160 161 static void 162 nvme_version(struct nvme_softc *sc, uint32_t ver) 163 { 164 const char *v = NULL; 165 166 switch (ver) { 167 case NVME_VS_1_0: 168 v = "1.0"; 169 break; 170 case NVME_VS_1_1: 171 v = "1.1"; 172 break; 173 case NVME_VS_1_2: 174 v = "1.2"; 175 break; 176 default: 177 aprint_error_dev(sc->sc_dev, "unknown version 0x%08x\n", ver); 178 return; 179 } 180 181 aprint_normal_dev(sc->sc_dev, "NVMe %s\n", v); 182 } 183 184 #ifdef NVME_DEBUG 185 static __used void 186 nvme_dumpregs(struct nvme_softc *sc) 187 { 188 uint64_t r8; 189 uint32_t r4; 190 191 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 192 r8 = nvme_read8(sc, NVME_CAP); 193 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 194 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 195 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 196 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 197 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 198 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 199 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 200 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 201 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 202 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 203 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 204 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 205 206 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 207 208 r4 = nvme_read4(sc, NVME_CC); 209 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 210 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 211 (1 << NVME_CC_IOCQES_R(r4))); 212 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 213 (1 << NVME_CC_IOSQES_R(r4))); 214 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 215 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 216 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 217 (1 << NVME_CC_MPS_R(r4))); 218 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 219 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 220 221 r4 = nvme_read4(sc, NVME_CSTS); 222 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 223 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 224 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 225 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 226 227 r4 = nvme_read4(sc, NVME_AQA); 228 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 229 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 230 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 231 232 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 233 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 234 #undef DEVNAME 235 } 236 #endif /* NVME_DEBUG */ 237 238 static int 239 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 240 { 241 u_int i = 0; 242 uint32_t cc; 243 244 cc = nvme_read4(sc, NVME_CC); 245 if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) { 246 aprint_error_dev(sc->sc_dev, 247 "controller enabled status expected %d, found to be %d\n", 248 (rdy != 0), ((cc & NVME_CC_EN) != 0)); 249 return ENXIO; 250 } 251 252 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 253 if (i++ > sc->sc_rdy_to) 254 return ENXIO; 255 256 delay(1000); 257 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 258 } 259 260 return 0; 261 } 262 263 static int 264 nvme_enable(struct nvme_softc *sc, u_int mps) 265 { 266 uint32_t cc, csts; 267 268 cc = nvme_read4(sc, NVME_CC); 269 csts = nvme_read4(sc, NVME_CSTS); 270 271 if (ISSET(cc, NVME_CC_EN)) { 272 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 273 274 if (ISSET(csts, NVME_CSTS_RDY)) 275 return 1; 276 277 goto waitready; 278 } 279 280 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 281 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 282 delay(5000); 283 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 284 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 285 delay(5000); 286 287 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 288 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 289 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 290 delay(5000); 291 292 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 293 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 294 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 295 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 296 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 297 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 298 SET(cc, NVME_CC_MPS(mps)); 299 SET(cc, NVME_CC_EN); 300 301 nvme_write4(sc, NVME_CC, cc); 302 nvme_barrier(sc, 0, sc->sc_ios, 303 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 304 delay(5000); 305 306 waitready: 307 return nvme_ready(sc, NVME_CSTS_RDY); 308 } 309 310 static int 311 nvme_disable(struct nvme_softc *sc) 312 { 313 uint32_t cc, csts; 314 315 cc = nvme_read4(sc, NVME_CC); 316 csts = nvme_read4(sc, NVME_CSTS); 317 318 if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY)) 319 nvme_ready(sc, NVME_CSTS_RDY); 320 321 CLR(cc, NVME_CC_EN); 322 323 nvme_write4(sc, NVME_CC, cc); 324 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 325 326 delay(5000); 327 328 return nvme_ready(sc, 0); 329 } 330 331 int 332 nvme_attach(struct nvme_softc *sc) 333 { 334 uint64_t cap; 335 uint32_t reg; 336 u_int dstrd; 337 u_int mps = PAGE_SHIFT; 338 int adminq_entries = nvme_adminq_size; 339 int ioq_entries = nvme_ioq_size; 340 int i; 341 342 reg = nvme_read4(sc, NVME_VS); 343 if (reg == 0xffffffff) { 344 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 345 return 1; 346 } 347 348 nvme_version(sc, reg); 349 350 cap = nvme_read8(sc, NVME_CAP); 351 dstrd = NVME_CAP_DSTRD(cap); 352 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 353 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 354 "is greater than CPU page size %u\n", 355 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 356 return 1; 357 } 358 if (NVME_CAP_MPSMAX(cap) < mps) 359 mps = NVME_CAP_MPSMAX(cap); 360 361 /* set initial values to be used for admin queue during probe */ 362 sc->sc_rdy_to = NVME_CAP_TO(cap); 363 sc->sc_mps = 1 << mps; 364 sc->sc_mdts = MAXPHYS; 365 sc->sc_max_sgl = 2; 366 367 if (nvme_disable(sc) != 0) { 368 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 369 return 1; 370 } 371 372 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 373 if (sc->sc_admin_q == NULL) { 374 aprint_error_dev(sc->sc_dev, 375 "unable to allocate admin queue\n"); 376 return 1; 377 } 378 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 379 goto free_admin_q; 380 381 if (nvme_enable(sc, mps) != 0) { 382 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 383 goto disestablish_admin_q; 384 } 385 386 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 387 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 388 goto disable; 389 } 390 391 /* we know how big things are now */ 392 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 393 394 /* reallocate ccbs of admin queue with new max sgl. */ 395 nvme_ccbs_free(sc->sc_admin_q); 396 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 397 398 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 399 if (sc->sc_q == NULL) { 400 aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n"); 401 goto disable; 402 } 403 for (i = 0; i < sc->sc_nq; i++) { 404 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 405 if (sc->sc_q[i] == NULL) { 406 aprint_error_dev(sc->sc_dev, 407 "unable to allocate io queue\n"); 408 goto free_q; 409 } 410 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 411 aprint_error_dev(sc->sc_dev, 412 "unable to create io queue\n"); 413 nvme_q_free(sc, sc->sc_q[i]); 414 goto free_q; 415 } 416 } 417 418 if (!sc->sc_use_mq) 419 nvme_write4(sc, NVME_INTMC, 1); 420 421 /* probe subdevices */ 422 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 423 KM_SLEEP); 424 if (sc->sc_namespaces == NULL) 425 goto free_q; 426 nvme_rescan(sc->sc_dev, "nvme", &i); 427 428 return 0; 429 430 free_q: 431 while (--i >= 0) { 432 nvme_q_delete(sc, sc->sc_q[i]); 433 nvme_q_free(sc, sc->sc_q[i]); 434 } 435 disable: 436 nvme_disable(sc); 437 disestablish_admin_q: 438 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 439 free_admin_q: 440 nvme_q_free(sc, sc->sc_admin_q); 441 442 return 1; 443 } 444 445 int 446 nvme_rescan(device_t self, const char *attr, const int *flags) 447 { 448 int i; 449 struct nvme_softc *sc = device_private(self); 450 struct nvme_attach_args naa; 451 452 for (i = 0; i < sc->sc_nn; i++) { 453 if (sc->sc_namespaces[i].dev) 454 continue; 455 memset(&naa, 0, sizeof(naa)); 456 naa.naa_nsid = i + 1; 457 naa.naa_qentries = nvme_ioq_size; 458 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 459 nvme_print); 460 } 461 return 0; 462 } 463 464 static int 465 nvme_print(void *aux, const char *pnp) 466 { 467 struct nvme_attach_args *naa = aux; 468 469 if (pnp) 470 aprint_normal("at %s", pnp); 471 472 if (naa->naa_nsid > 0) 473 aprint_normal(" nsid %d", naa->naa_nsid); 474 475 return UNCONF; 476 } 477 478 int 479 nvme_detach(struct nvme_softc *sc, int flags) 480 { 481 int i, error; 482 483 error = config_detach_children(sc->sc_dev, flags); 484 if (error) 485 return error; 486 487 error = nvme_shutdown(sc); 488 if (error) 489 return error; 490 491 /* from now on we are committed to detach, following will never fail */ 492 for (i = 0; i < sc->sc_nq; i++) 493 nvme_q_free(sc, sc->sc_q[i]); 494 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 495 nvme_q_free(sc, sc->sc_admin_q); 496 497 return 0; 498 } 499 500 static int 501 nvme_shutdown(struct nvme_softc *sc) 502 { 503 uint32_t cc, csts; 504 bool disabled = false; 505 int i; 506 507 if (!sc->sc_use_mq) 508 nvme_write4(sc, NVME_INTMS, 1); 509 510 for (i = 0; i < sc->sc_nq; i++) { 511 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 512 aprint_error_dev(sc->sc_dev, 513 "unable to delete io queue %d, disabling\n", i + 1); 514 disabled = true; 515 } 516 } 517 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 518 if (disabled) 519 goto disable; 520 521 cc = nvme_read4(sc, NVME_CC); 522 CLR(cc, NVME_CC_SHN_MASK); 523 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 524 nvme_write4(sc, NVME_CC, cc); 525 526 for (i = 0; i < 4000; i++) { 527 nvme_barrier(sc, 0, sc->sc_ios, 528 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 529 csts = nvme_read4(sc, NVME_CSTS); 530 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 531 return 0; 532 533 delay(1000); 534 } 535 536 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 537 538 disable: 539 nvme_disable(sc); 540 return 0; 541 } 542 543 void 544 nvme_childdet(device_t self, device_t child) 545 { 546 struct nvme_softc *sc = device_private(self); 547 int i; 548 549 for (i = 0; i < sc->sc_nn; i++) { 550 if (sc->sc_namespaces[i].dev == child) { 551 /* Already freed ns->ident. */ 552 sc->sc_namespaces[i].dev = NULL; 553 break; 554 } 555 } 556 } 557 558 int 559 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 560 { 561 struct nvme_sqe sqe; 562 struct nvm_identify_namespace *identify; 563 struct nvme_dmamem mem; 564 struct nvme_ccb *ccb; 565 struct nvme_namespace *ns; 566 int error; 567 568 KASSERT(nsid > 0); 569 570 ccb = nvme_ccb_get(sc->sc_admin_q); 571 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 572 573 error = nvme_dmamem_alloc(sc, sizeof(*identify), &mem); 574 if (error) 575 return error; 576 577 memset(&sqe, 0, sizeof(sqe)); 578 sqe.opcode = NVM_ADMIN_IDENTIFY; 579 htolem32(&sqe.nsid, nsid); 580 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 581 htolem32(&sqe.cdw10, 0); 582 583 ccb->ccb_done = nvme_empty_done; 584 ccb->ccb_cookie = &sqe; 585 586 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 587 error = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, 588 NVME_TIMO_IDENT); 589 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 590 591 nvme_ccb_put(sc->sc_admin_q, ccb); 592 593 if (error != 0) { 594 error = EIO; 595 goto done; 596 } 597 598 /* commit */ 599 600 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 601 memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 602 603 ns = nvme_ns_get(sc, nsid); 604 KASSERT(ns); 605 ns->ident = identify; 606 607 done: 608 nvme_dmamem_free(sc, &mem); 609 610 return error; 611 } 612 613 int 614 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 615 struct buf *bp, void *data, size_t datasize, 616 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 617 { 618 struct nvme_queue *q = nvme_get_q(sc); 619 struct nvme_ccb *ccb; 620 bus_dmamap_t dmap; 621 int i, error; 622 623 ccb = nvme_ccb_get(q); 624 if (ccb == NULL) 625 return EAGAIN; 626 627 ccb->ccb_done = nvme_ns_io_done; 628 ccb->ccb_cookie = cookie; 629 630 /* namespace context */ 631 ccb->nnc_nsid = nsid; 632 ccb->nnc_flags = flags; 633 ccb->nnc_buf = bp; 634 ccb->nnc_datasize = datasize; 635 ccb->nnc_secsize = secsize; 636 ccb->nnc_blkno = blkno; 637 ccb->nnc_done = nnc_done; 638 639 dmap = ccb->ccb_dmamap; 640 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 641 datasize, NULL, 642 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 643 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 644 (ISSET(flags, NVME_NS_CTX_F_READ) ? 645 BUS_DMA_READ : BUS_DMA_WRITE)); 646 if (error) { 647 nvme_ccb_put(q, ccb); 648 return error; 649 } 650 651 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 652 ISSET(flags, NVME_NS_CTX_F_READ) ? 653 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 654 655 if (dmap->dm_nsegs > 2) { 656 for (i = 1; i < dmap->dm_nsegs; i++) { 657 htolem64(&ccb->ccb_prpl[i - 1], 658 dmap->dm_segs[i].ds_addr); 659 } 660 bus_dmamap_sync(sc->sc_dmat, 661 NVME_DMA_MAP(q->q_ccb_prpls), 662 ccb->ccb_prpl_off, 663 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 664 BUS_DMASYNC_PREWRITE); 665 } 666 667 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 668 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 669 return EIO; 670 return 0; 671 } 672 673 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 674 return 0; 675 } 676 677 static void 678 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 679 { 680 struct nvme_sqe_io *sqe = slot; 681 bus_dmamap_t dmap = ccb->ccb_dmamap; 682 683 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 684 NVM_CMD_READ : NVM_CMD_WRITE; 685 htolem32(&sqe->nsid, ccb->nnc_nsid); 686 687 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 688 switch (dmap->dm_nsegs) { 689 case 1: 690 break; 691 case 2: 692 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 693 break; 694 default: 695 /* the prp list is already set up and synced */ 696 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 697 break; 698 } 699 700 htolem64(&sqe->slba, ccb->nnc_blkno); 701 702 /* guaranteed by upper layers, but check just in case */ 703 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 704 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 705 } 706 707 static void 708 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 709 struct nvme_cqe *cqe) 710 { 711 struct nvme_softc *sc = q->q_sc; 712 bus_dmamap_t dmap = ccb->ccb_dmamap; 713 void *nnc_cookie = ccb->ccb_cookie; 714 nvme_nnc_done nnc_done = ccb->nnc_done; 715 struct buf *bp = ccb->nnc_buf; 716 717 if (dmap->dm_nsegs > 2) { 718 bus_dmamap_sync(sc->sc_dmat, 719 NVME_DMA_MAP(q->q_ccb_prpls), 720 ccb->ccb_prpl_off, 721 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 722 BUS_DMASYNC_POSTWRITE); 723 } 724 725 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 726 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 727 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 728 729 bus_dmamap_unload(sc->sc_dmat, dmap); 730 nvme_ccb_put(q, ccb); 731 732 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags)); 733 } 734 735 int 736 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie, 737 int flags, nvme_nnc_done nnc_done) 738 { 739 struct nvme_queue *q = nvme_get_q(sc); 740 struct nvme_ccb *ccb; 741 742 ccb = nvme_ccb_get(q); 743 if (ccb == NULL) 744 return EAGAIN; 745 746 ccb->ccb_done = nvme_ns_sync_done; 747 ccb->ccb_cookie = cookie; 748 749 /* namespace context */ 750 ccb->nnc_nsid = nsid; 751 ccb->nnc_flags = flags; 752 ccb->nnc_done = nnc_done; 753 754 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 755 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 756 return EIO; 757 return 0; 758 } 759 760 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 761 return 0; 762 } 763 764 static void 765 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 766 { 767 struct nvme_sqe *sqe = slot; 768 769 sqe->opcode = NVM_CMD_FLUSH; 770 htolem32(&sqe->nsid, ccb->nnc_nsid); 771 } 772 773 static void 774 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 775 struct nvme_cqe *cqe) 776 { 777 void *cookie = ccb->ccb_cookie; 778 nvme_nnc_done nnc_done = ccb->nnc_done; 779 780 nvme_ccb_put(q, ccb); 781 782 nnc_done(cookie, NULL, lemtoh16(&cqe->flags)); 783 } 784 785 void 786 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 787 { 788 struct nvme_namespace *ns; 789 struct nvm_identify_namespace *identify; 790 791 ns = nvme_ns_get(sc, nsid); 792 KASSERT(ns); 793 794 identify = ns->ident; 795 ns->ident = NULL; 796 if (identify != NULL) 797 kmem_free(identify, sizeof(*identify)); 798 } 799 800 static void 801 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 802 { 803 struct nvme_softc *sc = q->q_sc; 804 struct nvme_sqe *sqe = slot; 805 struct nvme_pt_command *pt = ccb->ccb_cookie; 806 bus_dmamap_t dmap = ccb->ccb_dmamap; 807 int i; 808 809 sqe->opcode = pt->cmd.opcode; 810 htolem32(&sqe->nsid, pt->cmd.nsid); 811 812 if (pt->buf != NULL && pt->len > 0) { 813 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 814 switch (dmap->dm_nsegs) { 815 case 1: 816 break; 817 case 2: 818 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 819 break; 820 default: 821 for (i = 1; i < dmap->dm_nsegs; i++) { 822 htolem64(&ccb->ccb_prpl[i - 1], 823 dmap->dm_segs[i].ds_addr); 824 } 825 bus_dmamap_sync(sc->sc_dmat, 826 NVME_DMA_MAP(q->q_ccb_prpls), 827 ccb->ccb_prpl_off, 828 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 829 BUS_DMASYNC_PREWRITE); 830 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 831 break; 832 } 833 } 834 835 htolem32(&sqe->cdw10, pt->cmd.cdw10); 836 htolem32(&sqe->cdw11, pt->cmd.cdw11); 837 htolem32(&sqe->cdw12, pt->cmd.cdw12); 838 htolem32(&sqe->cdw13, pt->cmd.cdw13); 839 htolem32(&sqe->cdw14, pt->cmd.cdw14); 840 htolem32(&sqe->cdw15, pt->cmd.cdw15); 841 } 842 843 static void 844 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 845 { 846 struct nvme_softc *sc = q->q_sc; 847 struct nvme_pt_command *pt = ccb->ccb_cookie; 848 bus_dmamap_t dmap = ccb->ccb_dmamap; 849 850 if (pt->buf != NULL && pt->len > 0) { 851 if (dmap->dm_nsegs > 2) { 852 bus_dmamap_sync(sc->sc_dmat, 853 NVME_DMA_MAP(q->q_ccb_prpls), 854 ccb->ccb_prpl_off, 855 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 856 BUS_DMASYNC_POSTWRITE); 857 } 858 859 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 860 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 861 bus_dmamap_unload(sc->sc_dmat, dmap); 862 } 863 864 pt->cpl.cdw0 = cqe->cdw0; 865 pt->cpl.flags = cqe->flags & ~NVME_CQE_PHASE; 866 } 867 868 static int 869 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 870 uint16_t nsid, struct lwp *l, bool is_adminq) 871 { 872 struct nvme_queue *q; 873 struct nvme_ccb *ccb; 874 void *buf = NULL; 875 int error; 876 877 /* limit command size to maximum data transfer size */ 878 if ((pt->buf == NULL && pt->len > 0) || 879 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 880 return EINVAL; 881 882 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 883 ccb = nvme_ccb_get(q); 884 if (ccb == NULL) 885 return EBUSY; 886 887 if (pt->buf != NULL) { 888 KASSERT(pt->len > 0); 889 buf = kmem_alloc(pt->len, KM_SLEEP); 890 if (buf == NULL) { 891 error = ENOMEM; 892 goto ccb_put; 893 } 894 if (!pt->is_read) { 895 error = copyin(pt->buf, buf, pt->len); 896 if (error) 897 goto kmem_free; 898 } 899 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 900 pt->len, NULL, 901 BUS_DMA_WAITOK | 902 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 903 if (error) 904 goto kmem_free; 905 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 906 0, ccb->ccb_dmamap->dm_mapsize, 907 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 908 } 909 910 ccb->ccb_done = nvme_pt_done; 911 ccb->ccb_cookie = pt; 912 913 pt->cmd.nsid = nsid; 914 if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) { 915 error = EIO; 916 goto out; 917 } 918 919 error = 0; 920 out: 921 if (buf != NULL) { 922 if (error == 0 && pt->is_read) 923 error = copyout(buf, pt->buf, pt->len); 924 kmem_free: 925 kmem_free(buf, pt->len); 926 } 927 ccb_put: 928 nvme_ccb_put(q, ccb); 929 return error; 930 } 931 932 static void 933 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 934 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 935 { 936 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 937 uint32_t tail; 938 939 mutex_enter(&q->q_sq_mtx); 940 tail = q->q_sq_tail; 941 if (++q->q_sq_tail >= q->q_entries) 942 q->q_sq_tail = 0; 943 944 sqe += tail; 945 946 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 947 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 948 memset(sqe, 0, sizeof(*sqe)); 949 (*fill)(q, ccb, sqe); 950 sqe->cid = ccb->ccb_id; 951 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 952 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 953 954 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 955 mutex_exit(&q->q_sq_mtx); 956 } 957 958 struct nvme_poll_state { 959 struct nvme_sqe s; 960 struct nvme_cqe c; 961 }; 962 963 static int 964 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 965 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 966 { 967 struct nvme_poll_state state; 968 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 969 void *cookie; 970 uint16_t flags; 971 int step = 10; 972 int maxloop = timo_sec * 1000000 / step; 973 int error = 0; 974 975 memset(&state, 0, sizeof(state)); 976 (*fill)(q, ccb, &state.s); 977 978 done = ccb->ccb_done; 979 cookie = ccb->ccb_cookie; 980 981 ccb->ccb_done = nvme_poll_done; 982 ccb->ccb_cookie = &state; 983 984 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 985 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 986 if (nvme_q_complete(sc, q) == 0) 987 delay(step); 988 989 if (timo_sec >= 0 && --maxloop <= 0) { 990 error = ETIMEDOUT; 991 break; 992 } 993 } 994 995 ccb->ccb_cookie = cookie; 996 done(q, ccb, &state.c); 997 998 if (error == 0) { 999 flags = lemtoh16(&state.c.flags); 1000 return flags & ~NVME_CQE_PHASE; 1001 } else { 1002 return 1; 1003 } 1004 } 1005 1006 static void 1007 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1008 { 1009 struct nvme_sqe *sqe = slot; 1010 struct nvme_poll_state *state = ccb->ccb_cookie; 1011 1012 *sqe = state->s; 1013 } 1014 1015 static void 1016 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1017 struct nvme_cqe *cqe) 1018 { 1019 struct nvme_poll_state *state = ccb->ccb_cookie; 1020 1021 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1022 state->c = *cqe; 1023 } 1024 1025 static void 1026 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1027 { 1028 struct nvme_sqe *src = ccb->ccb_cookie; 1029 struct nvme_sqe *dst = slot; 1030 1031 *dst = *src; 1032 } 1033 1034 static void 1035 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1036 struct nvme_cqe *cqe) 1037 { 1038 } 1039 1040 static int 1041 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1042 { 1043 struct nvme_ccb *ccb; 1044 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1045 uint16_t flags; 1046 int rv = 0; 1047 1048 mutex_enter(&q->q_cq_mtx); 1049 1050 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1051 for (;;) { 1052 cqe = &ring[q->q_cq_head]; 1053 flags = lemtoh16(&cqe->flags); 1054 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1055 break; 1056 1057 ccb = &q->q_ccbs[cqe->cid]; 1058 1059 if (++q->q_cq_head >= q->q_entries) { 1060 q->q_cq_head = 0; 1061 q->q_cq_phase ^= NVME_CQE_PHASE; 1062 } 1063 1064 rv = 1; 1065 1066 /* 1067 * Unlock the mutex before calling the ccb_done callback 1068 * and re-lock afterwards. The callback triggers lddone() 1069 * which schedules another i/o, and also calls nvme_ccb_put(). 1070 * Unlock/relock avoids possibility of deadlock. 1071 */ 1072 mutex_exit(&q->q_cq_mtx); 1073 ccb->ccb_done(q, ccb, cqe); 1074 mutex_enter(&q->q_cq_mtx); 1075 } 1076 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1077 1078 if (rv) 1079 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1080 1081 mutex_exit(&q->q_cq_mtx); 1082 1083 return rv; 1084 } 1085 1086 static int 1087 nvme_identify(struct nvme_softc *sc, u_int mps) 1088 { 1089 char sn[41], mn[81], fr[17]; 1090 struct nvm_identify_controller *identify; 1091 struct nvme_dmamem mem; 1092 struct nvme_ccb *ccb; 1093 u_int mdts; 1094 int error; 1095 1096 ccb = nvme_ccb_get(sc->sc_admin_q); 1097 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1098 1099 error = nvme_dmamem_alloc(sc, sizeof(*identify), &mem); 1100 if (error) 1101 return error; 1102 1103 ccb->ccb_done = nvme_empty_done; 1104 ccb->ccb_cookie = &mem; 1105 1106 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1107 error = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1108 NVME_TIMO_IDENT); 1109 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1110 1111 nvme_ccb_put(sc->sc_admin_q, ccb); 1112 1113 if (error != 0) 1114 goto done; 1115 1116 identify = NVME_DMA_KVA(mem); 1117 1118 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1119 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1120 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1121 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1122 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1123 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1124 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1125 sn); 1126 1127 if (identify->mdts > 0) { 1128 mdts = (1 << identify->mdts) * (1 << mps); 1129 if (mdts < sc->sc_mdts) 1130 sc->sc_mdts = mdts; 1131 } 1132 1133 sc->sc_nn = lemtoh32(&identify->nn); 1134 1135 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1136 1137 done: 1138 nvme_dmamem_free(sc, &mem); 1139 1140 return error; 1141 } 1142 1143 static int 1144 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1145 { 1146 struct nvme_sqe_q sqe; 1147 struct nvme_ccb *ccb; 1148 int rv; 1149 1150 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1151 return 1; 1152 1153 ccb = nvme_ccb_get(sc->sc_admin_q); 1154 KASSERT(ccb != NULL); 1155 1156 ccb->ccb_done = nvme_empty_done; 1157 ccb->ccb_cookie = &sqe; 1158 1159 memset(&sqe, 0, sizeof(sqe)); 1160 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1161 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1162 htolem16(&sqe.qsize, q->q_entries - 1); 1163 htolem16(&sqe.qid, q->q_id); 1164 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1165 if (sc->sc_use_mq) 1166 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1167 1168 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1169 if (rv != 0) 1170 goto fail; 1171 1172 ccb->ccb_done = nvme_empty_done; 1173 ccb->ccb_cookie = &sqe; 1174 1175 memset(&sqe, 0, sizeof(sqe)); 1176 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1177 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1178 htolem16(&sqe.qsize, q->q_entries - 1); 1179 htolem16(&sqe.qid, q->q_id); 1180 htolem16(&sqe.cqid, q->q_id); 1181 sqe.qflags = NVM_SQE_Q_PC; 1182 1183 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1184 if (rv != 0) 1185 goto fail; 1186 1187 fail: 1188 nvme_ccb_put(sc->sc_admin_q, ccb); 1189 return rv; 1190 } 1191 1192 static int 1193 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1194 { 1195 struct nvme_sqe_q sqe; 1196 struct nvme_ccb *ccb; 1197 int rv; 1198 1199 ccb = nvme_ccb_get(sc->sc_admin_q); 1200 KASSERT(ccb != NULL); 1201 1202 ccb->ccb_done = nvme_empty_done; 1203 ccb->ccb_cookie = &sqe; 1204 1205 memset(&sqe, 0, sizeof(sqe)); 1206 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1207 htolem16(&sqe.qid, q->q_id); 1208 1209 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1210 if (rv != 0) 1211 goto fail; 1212 1213 ccb->ccb_done = nvme_empty_done; 1214 ccb->ccb_cookie = &sqe; 1215 1216 memset(&sqe, 0, sizeof(sqe)); 1217 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1218 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1219 htolem16(&sqe.qid, q->q_id); 1220 1221 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1222 if (rv != 0) 1223 goto fail; 1224 1225 fail: 1226 nvme_ccb_put(sc->sc_admin_q, ccb); 1227 1228 if (rv == 0 && sc->sc_use_mq) { 1229 if (sc->sc_intr_disestablish(sc, q->q_id)) 1230 rv = 1; 1231 } 1232 1233 return rv; 1234 } 1235 1236 static void 1237 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1238 { 1239 struct nvme_sqe *sqe = slot; 1240 struct nvme_dmamem *mem = ccb->ccb_cookie; 1241 1242 sqe->opcode = NVM_ADMIN_IDENTIFY; 1243 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(*mem)); 1244 htolem32(&sqe->cdw10, 1); 1245 } 1246 1247 static int 1248 nvme_ccbs_alloc(struct nvme_queue *q, u_int nccbs) 1249 { 1250 struct nvme_softc *sc = q->q_sc; 1251 struct nvme_ccb *ccb; 1252 bus_addr_t off; 1253 uint64_t *prpl; 1254 u_int i; 1255 int error; 1256 1257 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1258 SIMPLEQ_INIT(&q->q_ccb_list); 1259 1260 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1261 if (q->q_ccbs == NULL) 1262 return 1; 1263 1264 q->q_nccbs = nccbs; 1265 error = nvme_dmamem_alloc(sc, sizeof(*prpl) * sc->sc_max_sgl * nccbs, 1266 &q->q_ccb_prpls); 1267 if (error) { 1268 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1269 return error; 1270 } 1271 1272 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1273 off = 0; 1274 1275 for (i = 0; i < nccbs; i++) { 1276 ccb = &q->q_ccbs[i]; 1277 1278 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1279 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1280 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1281 &ccb->ccb_dmamap) != 0) 1282 goto free_maps; 1283 1284 ccb->ccb_id = i; 1285 ccb->ccb_prpl = prpl; 1286 ccb->ccb_prpl_off = off; 1287 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1288 1289 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1290 1291 prpl += sc->sc_max_sgl; 1292 off += sizeof(*prpl) * sc->sc_max_sgl; 1293 } 1294 1295 return 0; 1296 1297 free_maps: 1298 nvme_ccbs_free(q); 1299 return 1; 1300 } 1301 1302 static struct nvme_ccb * 1303 nvme_ccb_get(struct nvme_queue *q) 1304 { 1305 struct nvme_ccb *ccb; 1306 1307 mutex_enter(&q->q_ccb_mtx); 1308 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1309 if (ccb != NULL) 1310 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1311 mutex_exit(&q->q_ccb_mtx); 1312 1313 return ccb; 1314 } 1315 1316 static void 1317 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1318 { 1319 1320 mutex_enter(&q->q_ccb_mtx); 1321 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1322 mutex_exit(&q->q_ccb_mtx); 1323 } 1324 1325 static void 1326 nvme_ccbs_free(struct nvme_queue *q) 1327 { 1328 struct nvme_softc *sc = q->q_sc; 1329 struct nvme_ccb *ccb; 1330 1331 mutex_enter(&q->q_ccb_mtx); 1332 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1333 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1334 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1335 } 1336 mutex_exit(&q->q_ccb_mtx); 1337 1338 nvme_dmamem_free(sc, &q->q_ccb_prpls); 1339 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1340 q->q_ccbs = NULL; 1341 mutex_destroy(&q->q_ccb_mtx); 1342 } 1343 1344 static struct nvme_queue * 1345 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1346 { 1347 struct nvme_queue *q; 1348 int error; 1349 1350 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1351 if (q == NULL) 1352 return NULL; 1353 1354 q->q_sc = sc; 1355 error = nvme_dmamem_alloc(sc, sizeof(struct nvme_sqe) * entries, 1356 &q->q_sq_dmamem); 1357 if (error) 1358 goto free; 1359 1360 error = nvme_dmamem_alloc(sc, sizeof(struct nvme_cqe) * entries, 1361 &q->q_cq_dmamem); 1362 if (error) 1363 goto free_sq; 1364 1365 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1366 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1367 1368 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1369 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1370 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1371 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1372 q->q_id = id; 1373 q->q_entries = entries; 1374 q->q_sq_tail = 0; 1375 q->q_cq_head = 0; 1376 q->q_cq_phase = NVME_CQE_PHASE; 1377 1378 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1379 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1380 1381 if (nvme_ccbs_alloc(q, entries) != 0) { 1382 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1383 goto free_cq; 1384 } 1385 1386 return q; 1387 1388 free_cq: 1389 nvme_dmamem_free(sc, &q->q_cq_dmamem); 1390 free_sq: 1391 nvme_dmamem_free(sc, &q->q_sq_dmamem); 1392 free: 1393 kmem_free(q, sizeof(*q)); 1394 1395 return NULL; 1396 } 1397 1398 static void 1399 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1400 { 1401 nvme_ccbs_free(q); 1402 mutex_destroy(&q->q_sq_mtx); 1403 mutex_destroy(&q->q_cq_mtx); 1404 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1405 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1406 nvme_dmamem_free(sc, &q->q_cq_dmamem); 1407 nvme_dmamem_free(sc, &q->q_sq_dmamem); 1408 kmem_free(q, sizeof(*q)); 1409 } 1410 1411 int 1412 nvme_intr(void *xsc) 1413 { 1414 struct nvme_softc *sc = xsc; 1415 int rv = 0; 1416 1417 /* 1418 * INTx is level triggered, controller deasserts the interrupt only 1419 * when we advance command queue head via write to the doorbell. 1420 */ 1421 if (nvme_q_complete(sc, sc->sc_admin_q)) 1422 rv = 1; 1423 if (sc->sc_q != NULL) 1424 if (nvme_q_complete(sc, sc->sc_q[0])) 1425 rv = 1; 1426 1427 return rv; 1428 } 1429 1430 int 1431 nvme_intr_msi(void *xq) 1432 { 1433 struct nvme_queue *q = xq; 1434 1435 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1436 && q->q_sc->sc_softih[q->q_id]); 1437 1438 /* MSI are edge triggered, so can handover processing to softint */ 1439 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1440 1441 return 1; 1442 } 1443 1444 void 1445 nvme_softintr_msi(void *xq) 1446 { 1447 struct nvme_queue *q = xq; 1448 struct nvme_softc *sc = q->q_sc; 1449 1450 nvme_q_complete(sc, q); 1451 } 1452 1453 static int 1454 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size, struct nvme_dmamem *ndm) 1455 { 1456 int nsegs; 1457 1458 memset(ndm, 0, sizeof(*ndm)); 1459 ndm->ndm_size = size; 1460 1461 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1462 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1463 goto ndmfree; 1464 1465 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1466 1, &nsegs, BUS_DMA_WAITOK) != 0) 1467 goto destroy; 1468 1469 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1470 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1471 goto free; 1472 memset(ndm->ndm_kva, 0, size); 1473 1474 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1475 NULL, BUS_DMA_WAITOK) != 0) 1476 goto unmap; 1477 1478 return 0; 1479 1480 unmap: 1481 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1482 free: 1483 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1484 destroy: 1485 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1486 ndmfree: 1487 return ENOMEM; 1488 } 1489 1490 void 1491 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1492 { 1493 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1494 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1495 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1496 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1497 } 1498 1499 /* 1500 * ioctl 1501 */ 1502 1503 dev_type_open(nvmeopen); 1504 dev_type_close(nvmeclose); 1505 dev_type_ioctl(nvmeioctl); 1506 1507 const struct cdevsw nvme_cdevsw = { 1508 .d_open = nvmeopen, 1509 .d_close = nvmeclose, 1510 .d_read = noread, 1511 .d_write = nowrite, 1512 .d_ioctl = nvmeioctl, 1513 .d_stop = nostop, 1514 .d_tty = notty, 1515 .d_poll = nopoll, 1516 .d_mmap = nommap, 1517 .d_kqfilter = nokqfilter, 1518 .d_discard = nodiscard, 1519 .d_flag = D_OTHER, 1520 }; 1521 1522 extern struct cfdriver nvme_cd; 1523 1524 /* 1525 * Accept an open operation on the control device. 1526 */ 1527 int 1528 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1529 { 1530 struct nvme_softc *sc; 1531 int unit = minor(dev) / 0x10000; 1532 int nsid = minor(dev) & 0xffff; 1533 int nsidx; 1534 1535 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1536 return ENXIO; 1537 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1538 return ENXIO; 1539 1540 if (nsid == 0) { 1541 /* controller */ 1542 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1543 return EBUSY; 1544 SET(sc->sc_flags, NVME_F_OPEN); 1545 } else { 1546 /* namespace */ 1547 nsidx = nsid - 1; 1548 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1549 return ENXIO; 1550 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1551 return EBUSY; 1552 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1553 } 1554 return 0; 1555 } 1556 1557 /* 1558 * Accept the last close on the control device. 1559 */ 1560 int 1561 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1562 { 1563 struct nvme_softc *sc; 1564 int unit = minor(dev) / 0x10000; 1565 int nsid = minor(dev) & 0xffff; 1566 int nsidx; 1567 1568 sc = device_lookup_private(&nvme_cd, unit); 1569 if (sc == NULL) 1570 return ENXIO; 1571 1572 if (nsid == 0) { 1573 /* controller */ 1574 CLR(sc->sc_flags, NVME_F_OPEN); 1575 } else { 1576 /* namespace */ 1577 nsidx = nsid - 1; 1578 if (nsidx >= sc->sc_nn) 1579 return ENXIO; 1580 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1581 } 1582 1583 return 0; 1584 } 1585 1586 /* 1587 * Handle control operations. 1588 */ 1589 int 1590 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1591 { 1592 struct nvme_softc *sc; 1593 int unit = minor(dev) / 0x10000; 1594 int nsid = minor(dev) & 0xffff; 1595 struct nvme_pt_command *pt; 1596 1597 sc = device_lookup_private(&nvme_cd, unit); 1598 if (sc == NULL) 1599 return ENXIO; 1600 1601 switch (cmd) { 1602 case NVME_PASSTHROUGH_CMD: 1603 pt = data; 1604 return nvme_command_passthrough(sc, data, 1605 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1606 } 1607 1608 return ENOTTY; 1609 } 1610
Indexes created Wed Sep 24 22:09:59 GMT 2025