nvme.c revision 1.17
1 /* $NetBSD: nvme.c,v 1.17 2016/10/19 19:31:23 jdolecek Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.17 2016/10/19 19:31:23 jdolecek Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 int nvme_adminq_size = 128; 44 int nvme_ioq_size = 1024; 45 46 static int nvme_print(void *, const char *); 47 48 static int nvme_ready(struct nvme_softc *, uint32_t); 49 static int nvme_enable(struct nvme_softc *, u_int); 50 static int nvme_disable(struct nvme_softc *); 51 static int nvme_shutdown(struct nvme_softc *); 52 53 static void nvme_version(struct nvme_softc *, uint32_t); 54 #ifdef NVME_DEBUG 55 static void nvme_dumpregs(struct nvme_softc *); 56 #endif 57 static int nvme_identify(struct nvme_softc *, u_int); 58 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 59 void *); 60 61 static int nvme_ccbs_alloc(struct nvme_queue *, u_int); 62 static void nvme_ccbs_free(struct nvme_queue *); 63 64 static struct nvme_ccb * 65 nvme_ccb_get(struct nvme_queue *); 66 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 67 68 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 69 struct nvme_ccb *, void (*)(struct nvme_queue *, 70 struct nvme_ccb *, void *), int); 71 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 72 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 73 struct nvme_cqe *); 74 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 76 struct nvme_cqe *); 77 78 static struct nvme_queue * 79 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 80 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 81 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 82 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 83 struct nvme_ccb *, void (*)(struct nvme_queue *, 84 struct nvme_ccb *, void *)); 85 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 86 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 87 88 static int nvme_dmamem_alloc(struct nvme_softc *, size_t, 89 struct nvme_dmamem *); 90 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 91 92 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 93 void *); 94 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 95 struct nvme_cqe *); 96 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 97 void *); 98 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 99 struct nvme_cqe *); 100 101 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 102 void *); 103 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 104 struct nvme_cqe *); 105 static int nvme_command_passthrough(struct nvme_softc *, 106 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 107 108 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 109 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 110 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 111 #define NVME_TIMO_SY 60 /* sync cache timeout */ 112 113 #define nvme_read4(_s, _r) \ 114 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 115 #define nvme_write4(_s, _r, _v) \ 116 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 117 #ifdef __LP64__ 118 #define nvme_read8(_s, _r) \ 119 bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r)) 120 #define nvme_write8(_s, _r, _v) \ 121 bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 122 #else /* __LP64__ */ 123 static inline uint64_t 124 nvme_read8(struct nvme_softc *sc, bus_size_t r) 125 { 126 uint64_t v; 127 uint32_t *a = (uint32_t *)&v; 128 129 #if _BYTE_ORDER == _LITTLE_ENDIAN 130 a[0] = nvme_read4(sc, r); 131 a[1] = nvme_read4(sc, r + 4); 132 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 133 a[1] = nvme_read4(sc, r); 134 a[0] = nvme_read4(sc, r + 4); 135 #endif 136 137 return v; 138 } 139 140 static inline void 141 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 142 { 143 uint32_t *a = (uint32_t *)&v; 144 145 #if _BYTE_ORDER == _LITTLE_ENDIAN 146 nvme_write4(sc, r, a[0]); 147 nvme_write4(sc, r + 4, a[1]); 148 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 149 nvme_write4(sc, r, a[1]); 150 nvme_write4(sc, r + 4, a[0]); 151 #endif 152 } 153 #endif /* __LP64__ */ 154 155 #define nvme_barrier(_s, _r, _l, _f) \ 156 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 157 #define nvme_dmamem_sync(sc, mem, ops) \ 158 bus_dmamap_sync((sc)->sc_dmat, NVME_DMA_MAP(mem), \ 159 0, NVME_DMA_LEN(mem), (ops)); 160 161 static void 162 nvme_version(struct nvme_softc *sc, uint32_t ver) 163 { 164 const char *v = NULL; 165 166 switch (ver) { 167 case NVME_VS_1_0: 168 v = "1.0"; 169 break; 170 case NVME_VS_1_1: 171 v = "1.1"; 172 break; 173 case NVME_VS_1_2: 174 v = "1.2"; 175 break; 176 default: 177 aprint_error_dev(sc->sc_dev, "unknown version 0x%08x\n", ver); 178 return; 179 } 180 181 aprint_normal_dev(sc->sc_dev, "NVMe %s\n", v); 182 } 183 184 #ifdef NVME_DEBUG 185 static __used void 186 nvme_dumpregs(struct nvme_softc *sc) 187 { 188 uint64_t r8; 189 uint32_t r4; 190 191 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 192 r8 = nvme_read8(sc, NVME_CAP); 193 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 194 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 195 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 196 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 197 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 198 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 199 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 200 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 201 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 202 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 203 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 204 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 205 206 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 207 208 r4 = nvme_read4(sc, NVME_CC); 209 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 210 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 211 (1 << NVME_CC_IOCQES_R(r4))); 212 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 213 (1 << NVME_CC_IOSQES_R(r4))); 214 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 215 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 216 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 217 (1 << NVME_CC_MPS_R(r4))); 218 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 219 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 220 221 r4 = nvme_read4(sc, NVME_CSTS); 222 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 223 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 224 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 225 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 226 227 r4 = nvme_read4(sc, NVME_AQA); 228 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 229 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 230 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 231 232 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 233 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 234 #undef DEVNAME 235 } 236 #endif /* NVME_DEBUG */ 237 238 static int 239 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 240 { 241 u_int i = 0; 242 uint32_t cc; 243 244 cc = nvme_read4(sc, NVME_CC); 245 if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) { 246 aprint_error_dev(sc->sc_dev, 247 "controller enabled status expected %d, found to be %d\n", 248 (rdy != 0), ((cc & NVME_CC_EN) != 0)); 249 return ENXIO; 250 } 251 252 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 253 if (i++ > sc->sc_rdy_to) 254 return ENXIO; 255 256 delay(1000); 257 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 258 } 259 260 return 0; 261 } 262 263 static int 264 nvme_enable(struct nvme_softc *sc, u_int mps) 265 { 266 uint32_t cc, csts; 267 268 cc = nvme_read4(sc, NVME_CC); 269 csts = nvme_read4(sc, NVME_CSTS); 270 271 if (ISSET(cc, NVME_CC_EN)) { 272 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 273 274 if (ISSET(csts, NVME_CSTS_RDY)) 275 return 1; 276 277 goto waitready; 278 } 279 280 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 281 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 282 delay(5000); 283 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 284 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 285 delay(5000); 286 287 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 288 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 289 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 290 delay(5000); 291 292 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 293 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 294 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 295 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 296 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 297 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 298 SET(cc, NVME_CC_MPS(mps)); 299 SET(cc, NVME_CC_EN); 300 301 nvme_write4(sc, NVME_CC, cc); 302 nvme_barrier(sc, 0, sc->sc_ios, 303 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 304 delay(5000); 305 306 waitready: 307 return nvme_ready(sc, NVME_CSTS_RDY); 308 } 309 310 static int 311 nvme_disable(struct nvme_softc *sc) 312 { 313 uint32_t cc, csts; 314 315 cc = nvme_read4(sc, NVME_CC); 316 csts = nvme_read4(sc, NVME_CSTS); 317 318 if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY)) 319 nvme_ready(sc, NVME_CSTS_RDY); 320 321 CLR(cc, NVME_CC_EN); 322 323 nvme_write4(sc, NVME_CC, cc); 324 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 325 326 delay(5000); 327 328 return nvme_ready(sc, 0); 329 } 330 331 int 332 nvme_attach(struct nvme_softc *sc) 333 { 334 uint64_t cap; 335 uint32_t reg; 336 u_int dstrd; 337 u_int mps = PAGE_SHIFT; 338 int adminq_entries = nvme_adminq_size; 339 int ioq_entries = nvme_ioq_size; 340 int i; 341 342 reg = nvme_read4(sc, NVME_VS); 343 if (reg == 0xffffffff) { 344 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 345 return 1; 346 } 347 348 nvme_version(sc, reg); 349 350 cap = nvme_read8(sc, NVME_CAP); 351 dstrd = NVME_CAP_DSTRD(cap); 352 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 353 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 354 "is greater than CPU page size %u\n", 355 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 356 return 1; 357 } 358 if (NVME_CAP_MPSMAX(cap) < mps) 359 mps = NVME_CAP_MPSMAX(cap); 360 if (ioq_entries > NVME_CAP_MQES(cap)) 361 ioq_entries = NVME_CAP_MQES(cap); 362 363 /* set initial values to be used for admin queue during probe */ 364 sc->sc_rdy_to = NVME_CAP_TO(cap); 365 sc->sc_mps = 1 << mps; 366 sc->sc_mdts = MAXPHYS; 367 sc->sc_max_sgl = 2; 368 369 if (nvme_disable(sc) != 0) { 370 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 371 return 1; 372 } 373 374 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 375 if (sc->sc_admin_q == NULL) { 376 aprint_error_dev(sc->sc_dev, 377 "unable to allocate admin queue\n"); 378 return 1; 379 } 380 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 381 goto free_admin_q; 382 383 if (nvme_enable(sc, mps) != 0) { 384 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 385 goto disestablish_admin_q; 386 } 387 388 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 389 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 390 goto disable; 391 } 392 393 /* we know how big things are now */ 394 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 395 396 /* reallocate ccbs of admin queue with new max sgl. */ 397 nvme_ccbs_free(sc->sc_admin_q); 398 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 399 400 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 401 if (sc->sc_q == NULL) { 402 aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n"); 403 goto disable; 404 } 405 for (i = 0; i < sc->sc_nq; i++) { 406 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 407 if (sc->sc_q[i] == NULL) { 408 aprint_error_dev(sc->sc_dev, 409 "unable to allocate io queue\n"); 410 goto free_q; 411 } 412 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 413 aprint_error_dev(sc->sc_dev, 414 "unable to create io queue\n"); 415 nvme_q_free(sc, sc->sc_q[i]); 416 goto free_q; 417 } 418 } 419 420 if (!sc->sc_use_mq) 421 nvme_write4(sc, NVME_INTMC, 1); 422 423 /* probe subdevices */ 424 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 425 KM_SLEEP); 426 if (sc->sc_namespaces == NULL) 427 goto free_q; 428 nvme_rescan(sc->sc_dev, "nvme", &i); 429 430 return 0; 431 432 free_q: 433 while (--i >= 0) { 434 nvme_q_delete(sc, sc->sc_q[i]); 435 nvme_q_free(sc, sc->sc_q[i]); 436 } 437 disable: 438 nvme_disable(sc); 439 disestablish_admin_q: 440 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 441 free_admin_q: 442 nvme_q_free(sc, sc->sc_admin_q); 443 444 return 1; 445 } 446 447 int 448 nvme_rescan(device_t self, const char *attr, const int *flags) 449 { 450 struct nvme_softc *sc = device_private(self); 451 struct nvme_attach_args naa; 452 uint64_t cap; 453 int ioq_entries = nvme_ioq_size; 454 int i; 455 456 cap = nvme_read8(sc, NVME_CAP); 457 if (ioq_entries > NVME_CAP_MQES(cap)) 458 ioq_entries = NVME_CAP_MQES(cap); 459 460 for (i = 0; i < sc->sc_nn; i++) { 461 if (sc->sc_namespaces[i].dev) 462 continue; 463 memset(&naa, 0, sizeof(naa)); 464 naa.naa_nsid = i + 1; 465 naa.naa_qentries = ioq_entries; 466 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 467 nvme_print); 468 } 469 return 0; 470 } 471 472 static int 473 nvme_print(void *aux, const char *pnp) 474 { 475 struct nvme_attach_args *naa = aux; 476 477 if (pnp) 478 aprint_normal("at %s", pnp); 479 480 if (naa->naa_nsid > 0) 481 aprint_normal(" nsid %d", naa->naa_nsid); 482 483 return UNCONF; 484 } 485 486 int 487 nvme_detach(struct nvme_softc *sc, int flags) 488 { 489 int i, error; 490 491 error = config_detach_children(sc->sc_dev, flags); 492 if (error) 493 return error; 494 495 error = nvme_shutdown(sc); 496 if (error) 497 return error; 498 499 /* from now on we are committed to detach, following will never fail */ 500 for (i = 0; i < sc->sc_nq; i++) 501 nvme_q_free(sc, sc->sc_q[i]); 502 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 503 nvme_q_free(sc, sc->sc_admin_q); 504 505 return 0; 506 } 507 508 static int 509 nvme_shutdown(struct nvme_softc *sc) 510 { 511 uint32_t cc, csts; 512 bool disabled = false; 513 int i; 514 515 if (!sc->sc_use_mq) 516 nvme_write4(sc, NVME_INTMS, 1); 517 518 for (i = 0; i < sc->sc_nq; i++) { 519 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 520 aprint_error_dev(sc->sc_dev, 521 "unable to delete io queue %d, disabling\n", i + 1); 522 disabled = true; 523 } 524 } 525 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 526 if (disabled) 527 goto disable; 528 529 cc = nvme_read4(sc, NVME_CC); 530 CLR(cc, NVME_CC_SHN_MASK); 531 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 532 nvme_write4(sc, NVME_CC, cc); 533 534 for (i = 0; i < 4000; i++) { 535 nvme_barrier(sc, 0, sc->sc_ios, 536 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 537 csts = nvme_read4(sc, NVME_CSTS); 538 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 539 return 0; 540 541 delay(1000); 542 } 543 544 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 545 546 disable: 547 nvme_disable(sc); 548 return 0; 549 } 550 551 void 552 nvme_childdet(device_t self, device_t child) 553 { 554 struct nvme_softc *sc = device_private(self); 555 int i; 556 557 for (i = 0; i < sc->sc_nn; i++) { 558 if (sc->sc_namespaces[i].dev == child) { 559 /* Already freed ns->ident. */ 560 sc->sc_namespaces[i].dev = NULL; 561 break; 562 } 563 } 564 } 565 566 int 567 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 568 { 569 struct nvme_sqe sqe; 570 struct nvm_identify_namespace *identify; 571 struct nvme_dmamem mem; 572 struct nvme_ccb *ccb; 573 struct nvme_namespace *ns; 574 int error; 575 576 KASSERT(nsid > 0); 577 578 ccb = nvme_ccb_get(sc->sc_admin_q); 579 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 580 581 error = nvme_dmamem_alloc(sc, sizeof(*identify), &mem); 582 if (error) 583 return error; 584 585 memset(&sqe, 0, sizeof(sqe)); 586 sqe.opcode = NVM_ADMIN_IDENTIFY; 587 htolem32(&sqe.nsid, nsid); 588 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 589 htolem32(&sqe.cdw10, 0); 590 591 ccb->ccb_done = nvme_empty_done; 592 ccb->ccb_cookie = &sqe; 593 594 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 595 error = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, 596 NVME_TIMO_IDENT); 597 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 598 599 nvme_ccb_put(sc->sc_admin_q, ccb); 600 601 if (error != 0) { 602 error = EIO; 603 goto done; 604 } 605 606 /* commit */ 607 608 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 609 memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 610 611 ns = nvme_ns_get(sc, nsid); 612 KASSERT(ns); 613 ns->ident = identify; 614 615 done: 616 nvme_dmamem_free(sc, &mem); 617 618 return error; 619 } 620 621 int 622 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 623 struct buf *bp, void *data, size_t datasize, 624 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 625 { 626 struct nvme_queue *q = nvme_get_q(sc); 627 struct nvme_ccb *ccb; 628 bus_dmamap_t dmap; 629 int i, error; 630 631 ccb = nvme_ccb_get(q); 632 if (ccb == NULL) 633 return EAGAIN; 634 635 ccb->ccb_done = nvme_ns_io_done; 636 ccb->ccb_cookie = cookie; 637 638 /* namespace context */ 639 ccb->nnc_nsid = nsid; 640 ccb->nnc_flags = flags; 641 ccb->nnc_buf = bp; 642 ccb->nnc_datasize = datasize; 643 ccb->nnc_secsize = secsize; 644 ccb->nnc_blkno = blkno; 645 ccb->nnc_done = nnc_done; 646 647 dmap = ccb->ccb_dmamap; 648 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 649 datasize, NULL, 650 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 651 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 652 (ISSET(flags, NVME_NS_CTX_F_READ) ? 653 BUS_DMA_READ : BUS_DMA_WRITE)); 654 if (error) { 655 nvme_ccb_put(q, ccb); 656 return error; 657 } 658 659 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 660 ISSET(flags, NVME_NS_CTX_F_READ) ? 661 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 662 663 if (dmap->dm_nsegs > 2) { 664 for (i = 1; i < dmap->dm_nsegs; i++) { 665 htolem64(&ccb->ccb_prpl[i - 1], 666 dmap->dm_segs[i].ds_addr); 667 } 668 bus_dmamap_sync(sc->sc_dmat, 669 NVME_DMA_MAP(q->q_ccb_prpls), 670 ccb->ccb_prpl_off, 671 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 672 BUS_DMASYNC_PREWRITE); 673 } 674 675 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 676 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 677 return EIO; 678 return 0; 679 } 680 681 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 682 return 0; 683 } 684 685 static void 686 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 687 { 688 struct nvme_sqe_io *sqe = slot; 689 bus_dmamap_t dmap = ccb->ccb_dmamap; 690 691 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 692 NVM_CMD_READ : NVM_CMD_WRITE; 693 htolem32(&sqe->nsid, ccb->nnc_nsid); 694 695 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 696 switch (dmap->dm_nsegs) { 697 case 1: 698 break; 699 case 2: 700 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 701 break; 702 default: 703 /* the prp list is already set up and synced */ 704 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 705 break; 706 } 707 708 htolem64(&sqe->slba, ccb->nnc_blkno); 709 710 /* guaranteed by upper layers, but check just in case */ 711 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 712 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 713 } 714 715 static void 716 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 717 struct nvme_cqe *cqe) 718 { 719 struct nvme_softc *sc = q->q_sc; 720 bus_dmamap_t dmap = ccb->ccb_dmamap; 721 void *nnc_cookie = ccb->ccb_cookie; 722 nvme_nnc_done nnc_done = ccb->nnc_done; 723 struct buf *bp = ccb->nnc_buf; 724 725 if (dmap->dm_nsegs > 2) { 726 bus_dmamap_sync(sc->sc_dmat, 727 NVME_DMA_MAP(q->q_ccb_prpls), 728 ccb->ccb_prpl_off, 729 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 730 BUS_DMASYNC_POSTWRITE); 731 } 732 733 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 734 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 735 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 736 737 bus_dmamap_unload(sc->sc_dmat, dmap); 738 nvme_ccb_put(q, ccb); 739 740 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags)); 741 } 742 743 int 744 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie, 745 int flags, nvme_nnc_done nnc_done) 746 { 747 struct nvme_queue *q = nvme_get_q(sc); 748 struct nvme_ccb *ccb; 749 750 ccb = nvme_ccb_get(q); 751 if (ccb == NULL) 752 return EAGAIN; 753 754 ccb->ccb_done = nvme_ns_sync_done; 755 ccb->ccb_cookie = cookie; 756 757 /* namespace context */ 758 ccb->nnc_nsid = nsid; 759 ccb->nnc_flags = flags; 760 ccb->nnc_done = nnc_done; 761 762 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 763 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 764 return EIO; 765 return 0; 766 } 767 768 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 769 return 0; 770 } 771 772 static void 773 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 774 { 775 struct nvme_sqe *sqe = slot; 776 777 sqe->opcode = NVM_CMD_FLUSH; 778 htolem32(&sqe->nsid, ccb->nnc_nsid); 779 } 780 781 static void 782 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 783 struct nvme_cqe *cqe) 784 { 785 void *cookie = ccb->ccb_cookie; 786 nvme_nnc_done nnc_done = ccb->nnc_done; 787 788 nvme_ccb_put(q, ccb); 789 790 nnc_done(cookie, NULL, lemtoh16(&cqe->flags)); 791 } 792 793 void 794 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 795 { 796 struct nvme_namespace *ns; 797 struct nvm_identify_namespace *identify; 798 799 ns = nvme_ns_get(sc, nsid); 800 KASSERT(ns); 801 802 identify = ns->ident; 803 ns->ident = NULL; 804 if (identify != NULL) 805 kmem_free(identify, sizeof(*identify)); 806 } 807 808 static void 809 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 810 { 811 struct nvme_softc *sc = q->q_sc; 812 struct nvme_sqe *sqe = slot; 813 struct nvme_pt_command *pt = ccb->ccb_cookie; 814 bus_dmamap_t dmap = ccb->ccb_dmamap; 815 int i; 816 817 sqe->opcode = pt->cmd.opcode; 818 htolem32(&sqe->nsid, pt->cmd.nsid); 819 820 if (pt->buf != NULL && pt->len > 0) { 821 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 822 switch (dmap->dm_nsegs) { 823 case 1: 824 break; 825 case 2: 826 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 827 break; 828 default: 829 for (i = 1; i < dmap->dm_nsegs; i++) { 830 htolem64(&ccb->ccb_prpl[i - 1], 831 dmap->dm_segs[i].ds_addr); 832 } 833 bus_dmamap_sync(sc->sc_dmat, 834 NVME_DMA_MAP(q->q_ccb_prpls), 835 ccb->ccb_prpl_off, 836 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 837 BUS_DMASYNC_PREWRITE); 838 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 839 break; 840 } 841 } 842 843 htolem32(&sqe->cdw10, pt->cmd.cdw10); 844 htolem32(&sqe->cdw11, pt->cmd.cdw11); 845 htolem32(&sqe->cdw12, pt->cmd.cdw12); 846 htolem32(&sqe->cdw13, pt->cmd.cdw13); 847 htolem32(&sqe->cdw14, pt->cmd.cdw14); 848 htolem32(&sqe->cdw15, pt->cmd.cdw15); 849 } 850 851 static void 852 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 853 { 854 struct nvme_softc *sc = q->q_sc; 855 struct nvme_pt_command *pt = ccb->ccb_cookie; 856 bus_dmamap_t dmap = ccb->ccb_dmamap; 857 858 if (pt->buf != NULL && pt->len > 0) { 859 if (dmap->dm_nsegs > 2) { 860 bus_dmamap_sync(sc->sc_dmat, 861 NVME_DMA_MAP(q->q_ccb_prpls), 862 ccb->ccb_prpl_off, 863 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 864 BUS_DMASYNC_POSTWRITE); 865 } 866 867 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 868 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 869 bus_dmamap_unload(sc->sc_dmat, dmap); 870 } 871 872 pt->cpl.cdw0 = cqe->cdw0; 873 pt->cpl.flags = cqe->flags & ~NVME_CQE_PHASE; 874 } 875 876 static int 877 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 878 uint16_t nsid, struct lwp *l, bool is_adminq) 879 { 880 struct nvme_queue *q; 881 struct nvme_ccb *ccb; 882 void *buf = NULL; 883 int error; 884 885 /* limit command size to maximum data transfer size */ 886 if ((pt->buf == NULL && pt->len > 0) || 887 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 888 return EINVAL; 889 890 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 891 ccb = nvme_ccb_get(q); 892 if (ccb == NULL) 893 return EBUSY; 894 895 if (pt->buf != NULL) { 896 KASSERT(pt->len > 0); 897 buf = kmem_alloc(pt->len, KM_SLEEP); 898 if (buf == NULL) { 899 error = ENOMEM; 900 goto ccb_put; 901 } 902 if (!pt->is_read) { 903 error = copyin(pt->buf, buf, pt->len); 904 if (error) 905 goto kmem_free; 906 } 907 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 908 pt->len, NULL, 909 BUS_DMA_WAITOK | 910 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 911 if (error) 912 goto kmem_free; 913 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 914 0, ccb->ccb_dmamap->dm_mapsize, 915 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 916 } 917 918 ccb->ccb_done = nvme_pt_done; 919 ccb->ccb_cookie = pt; 920 921 pt->cmd.nsid = nsid; 922 if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) { 923 error = EIO; 924 goto out; 925 } 926 927 error = 0; 928 out: 929 if (buf != NULL) { 930 if (error == 0 && pt->is_read) 931 error = copyout(buf, pt->buf, pt->len); 932 kmem_free: 933 kmem_free(buf, pt->len); 934 } 935 ccb_put: 936 nvme_ccb_put(q, ccb); 937 return error; 938 } 939 940 static void 941 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 942 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 943 { 944 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 945 uint32_t tail; 946 947 mutex_enter(&q->q_sq_mtx); 948 tail = q->q_sq_tail; 949 if (++q->q_sq_tail >= q->q_entries) 950 q->q_sq_tail = 0; 951 952 sqe += tail; 953 954 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 955 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 956 memset(sqe, 0, sizeof(*sqe)); 957 (*fill)(q, ccb, sqe); 958 sqe->cid = ccb->ccb_id; 959 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 960 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 961 962 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 963 mutex_exit(&q->q_sq_mtx); 964 } 965 966 struct nvme_poll_state { 967 struct nvme_sqe s; 968 struct nvme_cqe c; 969 }; 970 971 static int 972 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 973 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 974 { 975 struct nvme_poll_state state; 976 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 977 void *cookie; 978 uint16_t flags; 979 int step = 10; 980 int maxloop = timo_sec * 1000000 / step; 981 int error = 0; 982 983 memset(&state, 0, sizeof(state)); 984 (*fill)(q, ccb, &state.s); 985 986 done = ccb->ccb_done; 987 cookie = ccb->ccb_cookie; 988 989 ccb->ccb_done = nvme_poll_done; 990 ccb->ccb_cookie = &state; 991 992 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 993 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 994 if (nvme_q_complete(sc, q) == 0) 995 delay(step); 996 997 if (timo_sec >= 0 && --maxloop <= 0) { 998 error = ETIMEDOUT; 999 break; 1000 } 1001 } 1002 1003 ccb->ccb_cookie = cookie; 1004 done(q, ccb, &state.c); 1005 1006 if (error == 0) { 1007 flags = lemtoh16(&state.c.flags); 1008 return flags & ~NVME_CQE_PHASE; 1009 } else { 1010 return 1; 1011 } 1012 } 1013 1014 static void 1015 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1016 { 1017 struct nvme_sqe *sqe = slot; 1018 struct nvme_poll_state *state = ccb->ccb_cookie; 1019 1020 *sqe = state->s; 1021 } 1022 1023 static void 1024 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1025 struct nvme_cqe *cqe) 1026 { 1027 struct nvme_poll_state *state = ccb->ccb_cookie; 1028 1029 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1030 state->c = *cqe; 1031 } 1032 1033 static void 1034 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1035 { 1036 struct nvme_sqe *src = ccb->ccb_cookie; 1037 struct nvme_sqe *dst = slot; 1038 1039 *dst = *src; 1040 } 1041 1042 static void 1043 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1044 struct nvme_cqe *cqe) 1045 { 1046 } 1047 1048 static int 1049 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1050 { 1051 struct nvme_ccb *ccb; 1052 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1053 uint16_t flags; 1054 int rv = 0; 1055 1056 mutex_enter(&q->q_cq_mtx); 1057 1058 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1059 for (;;) { 1060 cqe = &ring[q->q_cq_head]; 1061 flags = lemtoh16(&cqe->flags); 1062 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1063 break; 1064 1065 ccb = &q->q_ccbs[cqe->cid]; 1066 1067 if (++q->q_cq_head >= q->q_entries) { 1068 q->q_cq_head = 0; 1069 q->q_cq_phase ^= NVME_CQE_PHASE; 1070 } 1071 1072 rv = 1; 1073 1074 /* 1075 * Unlock the mutex before calling the ccb_done callback 1076 * and re-lock afterwards. The callback triggers lddone() 1077 * which schedules another i/o, and also calls nvme_ccb_put(). 1078 * Unlock/relock avoids possibility of deadlock. 1079 */ 1080 mutex_exit(&q->q_cq_mtx); 1081 ccb->ccb_done(q, ccb, cqe); 1082 mutex_enter(&q->q_cq_mtx); 1083 } 1084 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1085 1086 if (rv) 1087 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1088 1089 mutex_exit(&q->q_cq_mtx); 1090 1091 return rv; 1092 } 1093 1094 static int 1095 nvme_identify(struct nvme_softc *sc, u_int mps) 1096 { 1097 char sn[41], mn[81], fr[17]; 1098 struct nvm_identify_controller *identify; 1099 struct nvme_dmamem mem; 1100 struct nvme_ccb *ccb; 1101 u_int mdts; 1102 int error; 1103 1104 ccb = nvme_ccb_get(sc->sc_admin_q); 1105 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1106 1107 error = nvme_dmamem_alloc(sc, sizeof(*identify), &mem); 1108 if (error) 1109 return error; 1110 1111 ccb->ccb_done = nvme_empty_done; 1112 ccb->ccb_cookie = &mem; 1113 1114 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1115 error = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1116 NVME_TIMO_IDENT); 1117 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1118 1119 nvme_ccb_put(sc->sc_admin_q, ccb); 1120 1121 if (error != 0) 1122 goto done; 1123 1124 identify = NVME_DMA_KVA(mem); 1125 1126 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1127 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1128 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1129 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1130 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1131 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1132 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1133 sn); 1134 1135 if (identify->mdts > 0) { 1136 mdts = (1 << identify->mdts) * (1 << mps); 1137 if (mdts < sc->sc_mdts) 1138 sc->sc_mdts = mdts; 1139 } 1140 1141 sc->sc_nn = lemtoh32(&identify->nn); 1142 1143 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1144 1145 done: 1146 nvme_dmamem_free(sc, &mem); 1147 1148 return error; 1149 } 1150 1151 static int 1152 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1153 { 1154 struct nvme_sqe_q sqe; 1155 struct nvme_ccb *ccb; 1156 int rv; 1157 1158 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1159 return 1; 1160 1161 ccb = nvme_ccb_get(sc->sc_admin_q); 1162 KASSERT(ccb != NULL); 1163 1164 ccb->ccb_done = nvme_empty_done; 1165 ccb->ccb_cookie = &sqe; 1166 1167 memset(&sqe, 0, sizeof(sqe)); 1168 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1169 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1170 htolem16(&sqe.qsize, q->q_entries - 1); 1171 htolem16(&sqe.qid, q->q_id); 1172 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1173 if (sc->sc_use_mq) 1174 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1175 1176 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1177 if (rv != 0) 1178 goto fail; 1179 1180 ccb->ccb_done = nvme_empty_done; 1181 ccb->ccb_cookie = &sqe; 1182 1183 memset(&sqe, 0, sizeof(sqe)); 1184 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1185 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1186 htolem16(&sqe.qsize, q->q_entries - 1); 1187 htolem16(&sqe.qid, q->q_id); 1188 htolem16(&sqe.cqid, q->q_id); 1189 sqe.qflags = NVM_SQE_Q_PC; 1190 1191 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1192 if (rv != 0) 1193 goto fail; 1194 1195 fail: 1196 nvme_ccb_put(sc->sc_admin_q, ccb); 1197 return rv; 1198 } 1199 1200 static int 1201 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1202 { 1203 struct nvme_sqe_q sqe; 1204 struct nvme_ccb *ccb; 1205 int rv; 1206 1207 ccb = nvme_ccb_get(sc->sc_admin_q); 1208 KASSERT(ccb != NULL); 1209 1210 ccb->ccb_done = nvme_empty_done; 1211 ccb->ccb_cookie = &sqe; 1212 1213 memset(&sqe, 0, sizeof(sqe)); 1214 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1215 htolem16(&sqe.qid, q->q_id); 1216 1217 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1218 if (rv != 0) 1219 goto fail; 1220 1221 ccb->ccb_done = nvme_empty_done; 1222 ccb->ccb_cookie = &sqe; 1223 1224 memset(&sqe, 0, sizeof(sqe)); 1225 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1226 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1227 htolem16(&sqe.qid, q->q_id); 1228 1229 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1230 if (rv != 0) 1231 goto fail; 1232 1233 fail: 1234 nvme_ccb_put(sc->sc_admin_q, ccb); 1235 1236 if (rv == 0 && sc->sc_use_mq) { 1237 if (sc->sc_intr_disestablish(sc, q->q_id)) 1238 rv = 1; 1239 } 1240 1241 return rv; 1242 } 1243 1244 static void 1245 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1246 { 1247 struct nvme_sqe *sqe = slot; 1248 struct nvme_dmamem *mem = ccb->ccb_cookie; 1249 1250 sqe->opcode = NVM_ADMIN_IDENTIFY; 1251 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(*mem)); 1252 htolem32(&sqe->cdw10, 1); 1253 } 1254 1255 static int 1256 nvme_ccbs_alloc(struct nvme_queue *q, u_int nccbs) 1257 { 1258 struct nvme_softc *sc = q->q_sc; 1259 struct nvme_ccb *ccb; 1260 bus_addr_t off; 1261 uint64_t *prpl; 1262 u_int i; 1263 int error; 1264 1265 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1266 SIMPLEQ_INIT(&q->q_ccb_list); 1267 1268 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1269 if (q->q_ccbs == NULL) 1270 return 1; 1271 1272 q->q_nccbs = nccbs; 1273 error = nvme_dmamem_alloc(sc, sizeof(*prpl) * sc->sc_max_sgl * nccbs, 1274 &q->q_ccb_prpls); 1275 if (error) { 1276 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1277 return error; 1278 } 1279 1280 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1281 off = 0; 1282 1283 for (i = 0; i < nccbs; i++) { 1284 ccb = &q->q_ccbs[i]; 1285 1286 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1287 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1288 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1289 &ccb->ccb_dmamap) != 0) 1290 goto free_maps; 1291 1292 ccb->ccb_id = i; 1293 ccb->ccb_prpl = prpl; 1294 ccb->ccb_prpl_off = off; 1295 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1296 1297 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1298 1299 prpl += sc->sc_max_sgl; 1300 off += sizeof(*prpl) * sc->sc_max_sgl; 1301 } 1302 1303 return 0; 1304 1305 free_maps: 1306 nvme_ccbs_free(q); 1307 return 1; 1308 } 1309 1310 static struct nvme_ccb * 1311 nvme_ccb_get(struct nvme_queue *q) 1312 { 1313 struct nvme_ccb *ccb; 1314 1315 mutex_enter(&q->q_ccb_mtx); 1316 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1317 if (ccb != NULL) 1318 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1319 mutex_exit(&q->q_ccb_mtx); 1320 1321 return ccb; 1322 } 1323 1324 static void 1325 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1326 { 1327 1328 mutex_enter(&q->q_ccb_mtx); 1329 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1330 mutex_exit(&q->q_ccb_mtx); 1331 } 1332 1333 static void 1334 nvme_ccbs_free(struct nvme_queue *q) 1335 { 1336 struct nvme_softc *sc = q->q_sc; 1337 struct nvme_ccb *ccb; 1338 1339 mutex_enter(&q->q_ccb_mtx); 1340 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1341 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1342 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1343 } 1344 mutex_exit(&q->q_ccb_mtx); 1345 1346 nvme_dmamem_free(sc, &q->q_ccb_prpls); 1347 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1348 q->q_ccbs = NULL; 1349 mutex_destroy(&q->q_ccb_mtx); 1350 } 1351 1352 static struct nvme_queue * 1353 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1354 { 1355 struct nvme_queue *q; 1356 int error; 1357 1358 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1359 if (q == NULL) 1360 return NULL; 1361 1362 q->q_sc = sc; 1363 error = nvme_dmamem_alloc(sc, sizeof(struct nvme_sqe) * entries, 1364 &q->q_sq_dmamem); 1365 if (error) 1366 goto free; 1367 1368 error = nvme_dmamem_alloc(sc, sizeof(struct nvme_cqe) * entries, 1369 &q->q_cq_dmamem); 1370 if (error) 1371 goto free_sq; 1372 1373 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1374 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1375 1376 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1377 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1378 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1379 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1380 q->q_id = id; 1381 q->q_entries = entries; 1382 q->q_sq_tail = 0; 1383 q->q_cq_head = 0; 1384 q->q_cq_phase = NVME_CQE_PHASE; 1385 1386 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1387 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1388 1389 if (nvme_ccbs_alloc(q, entries) != 0) { 1390 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1391 goto free_cq; 1392 } 1393 1394 return q; 1395 1396 free_cq: 1397 nvme_dmamem_free(sc, &q->q_cq_dmamem); 1398 free_sq: 1399 nvme_dmamem_free(sc, &q->q_sq_dmamem); 1400 free: 1401 kmem_free(q, sizeof(*q)); 1402 1403 return NULL; 1404 } 1405 1406 static void 1407 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1408 { 1409 nvme_ccbs_free(q); 1410 mutex_destroy(&q->q_sq_mtx); 1411 mutex_destroy(&q->q_cq_mtx); 1412 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1413 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1414 nvme_dmamem_free(sc, &q->q_cq_dmamem); 1415 nvme_dmamem_free(sc, &q->q_sq_dmamem); 1416 kmem_free(q, sizeof(*q)); 1417 } 1418 1419 int 1420 nvme_intr(void *xsc) 1421 { 1422 struct nvme_softc *sc = xsc; 1423 1424 /* 1425 * INTx is level triggered, controller deasserts the interrupt only 1426 * when we advance command queue head via write to the doorbell. 1427 * Tell the controller to block the interrupts while we process 1428 * the queue(s). 1429 */ 1430 nvme_write4(sc, NVME_INTMS, 1); 1431 1432 softint_schedule(sc->sc_softih[0]); 1433 1434 /* don't know, might not have been for us */ 1435 return 1; 1436 } 1437 1438 void 1439 nvme_softintr_intx(void *xq) 1440 { 1441 struct nvme_queue *q = xq; 1442 struct nvme_softc *sc = q->q_sc; 1443 1444 nvme_q_complete(sc, sc->sc_admin_q); 1445 if (sc->sc_q != NULL) 1446 nvme_q_complete(sc, sc->sc_q[0]); 1447 1448 /* 1449 * Processing done, tell controller to issue interrupts again. There 1450 * is no race, as NVMe spec requires the controller to maintain state, 1451 * and assert the interrupt whenever there are unacknowledged 1452 * completion queue entries. 1453 */ 1454 nvme_write4(sc, NVME_INTMC, 1); 1455 } 1456 1457 int 1458 nvme_intr_msi(void *xq) 1459 { 1460 struct nvme_queue *q = xq; 1461 1462 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1463 && q->q_sc->sc_softih[q->q_id]); 1464 1465 /* 1466 * MSI/MSI-X are edge triggered, so can handover processing to softint 1467 * without masking the interrupt. 1468 */ 1469 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1470 1471 return 1; 1472 } 1473 1474 void 1475 nvme_softintr_msi(void *xq) 1476 { 1477 struct nvme_queue *q = xq; 1478 struct nvme_softc *sc = q->q_sc; 1479 1480 nvme_q_complete(sc, q); 1481 } 1482 1483 static int 1484 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size, struct nvme_dmamem *ndm) 1485 { 1486 int nsegs; 1487 1488 memset(ndm, 0, sizeof(*ndm)); 1489 ndm->ndm_size = size; 1490 1491 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1492 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1493 goto ndmfree; 1494 1495 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1496 1, &nsegs, BUS_DMA_WAITOK) != 0) 1497 goto destroy; 1498 1499 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1500 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1501 goto free; 1502 memset(ndm->ndm_kva, 0, size); 1503 1504 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1505 NULL, BUS_DMA_WAITOK) != 0) 1506 goto unmap; 1507 1508 return 0; 1509 1510 unmap: 1511 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1512 free: 1513 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1514 destroy: 1515 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1516 ndmfree: 1517 return ENOMEM; 1518 } 1519 1520 void 1521 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1522 { 1523 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1524 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1525 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1526 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1527 } 1528 1529 /* 1530 * ioctl 1531 */ 1532 1533 dev_type_open(nvmeopen); 1534 dev_type_close(nvmeclose); 1535 dev_type_ioctl(nvmeioctl); 1536 1537 const struct cdevsw nvme_cdevsw = { 1538 .d_open = nvmeopen, 1539 .d_close = nvmeclose, 1540 .d_read = noread, 1541 .d_write = nowrite, 1542 .d_ioctl = nvmeioctl, 1543 .d_stop = nostop, 1544 .d_tty = notty, 1545 .d_poll = nopoll, 1546 .d_mmap = nommap, 1547 .d_kqfilter = nokqfilter, 1548 .d_discard = nodiscard, 1549 .d_flag = D_OTHER, 1550 }; 1551 1552 extern struct cfdriver nvme_cd; 1553 1554 /* 1555 * Accept an open operation on the control device. 1556 */ 1557 int 1558 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1559 { 1560 struct nvme_softc *sc; 1561 int unit = minor(dev) / 0x10000; 1562 int nsid = minor(dev) & 0xffff; 1563 int nsidx; 1564 1565 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1566 return ENXIO; 1567 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1568 return ENXIO; 1569 1570 if (nsid == 0) { 1571 /* controller */ 1572 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1573 return EBUSY; 1574 SET(sc->sc_flags, NVME_F_OPEN); 1575 } else { 1576 /* namespace */ 1577 nsidx = nsid - 1; 1578 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1579 return ENXIO; 1580 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1581 return EBUSY; 1582 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1583 } 1584 return 0; 1585 } 1586 1587 /* 1588 * Accept the last close on the control device. 1589 */ 1590 int 1591 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1592 { 1593 struct nvme_softc *sc; 1594 int unit = minor(dev) / 0x10000; 1595 int nsid = minor(dev) & 0xffff; 1596 int nsidx; 1597 1598 sc = device_lookup_private(&nvme_cd, unit); 1599 if (sc == NULL) 1600 return ENXIO; 1601 1602 if (nsid == 0) { 1603 /* controller */ 1604 CLR(sc->sc_flags, NVME_F_OPEN); 1605 } else { 1606 /* namespace */ 1607 nsidx = nsid - 1; 1608 if (nsidx >= sc->sc_nn) 1609 return ENXIO; 1610 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1611 } 1612 1613 return 0; 1614 } 1615 1616 /* 1617 * Handle control operations. 1618 */ 1619 int 1620 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1621 { 1622 struct nvme_softc *sc; 1623 int unit = minor(dev) / 0x10000; 1624 int nsid = minor(dev) & 0xffff; 1625 struct nvme_pt_command *pt; 1626 1627 sc = device_lookup_private(&nvme_cd, unit); 1628 if (sc == NULL) 1629 return ENXIO; 1630 1631 switch (cmd) { 1632 case NVME_PASSTHROUGH_CMD: 1633 pt = data; 1634 return nvme_command_passthrough(sc, data, 1635 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1636 } 1637 1638 return ENOTTY; 1639 } 1640
Indexes created Fri Sep 26 08:10:20 GMT 2025