nvme.c revision 1.24
1 /* $NetBSD: nvme.c,v 1.24 2017/02/13 11:11:32 nonaka Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.24 2017/02/13 11:11:32 nonaka Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 int nvme_adminq_size = 32; 44 int nvme_ioq_size = 1024; 45 46 static int nvme_print(void *, const char *); 47 48 static int nvme_ready(struct nvme_softc *, uint32_t); 49 static int nvme_enable(struct nvme_softc *, u_int); 50 static int nvme_disable(struct nvme_softc *); 51 static int nvme_shutdown(struct nvme_softc *); 52 53 static void nvme_version(struct nvme_softc *, uint32_t); 54 #ifdef NVME_DEBUG 55 static void nvme_dumpregs(struct nvme_softc *); 56 #endif 57 static int nvme_identify(struct nvme_softc *, u_int); 58 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 59 void *); 60 61 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 62 static void nvme_ccbs_free(struct nvme_queue *); 63 64 static struct nvme_ccb * 65 nvme_ccb_get(struct nvme_queue *); 66 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 67 68 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 69 struct nvme_ccb *, void (*)(struct nvme_queue *, 70 struct nvme_ccb *, void *), int); 71 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 72 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 73 struct nvme_cqe *); 74 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 76 struct nvme_cqe *); 77 78 static struct nvme_queue * 79 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 80 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 81 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 82 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 83 struct nvme_ccb *, void (*)(struct nvme_queue *, 84 struct nvme_ccb *, void *)); 85 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 86 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 87 88 static struct nvme_dmamem * 89 nvme_dmamem_alloc(struct nvme_softc *, size_t); 90 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 91 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 92 int); 93 94 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 95 void *); 96 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 97 struct nvme_cqe *); 98 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 99 void *); 100 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 101 struct nvme_cqe *); 102 103 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 104 void *); 105 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 106 struct nvme_cqe *); 107 static int nvme_command_passthrough(struct nvme_softc *, 108 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 109 110 static int nvme_get_number_of_queues(struct nvme_softc *, u_int *); 111 112 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 113 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 114 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 115 #define NVME_TIMO_SY 60 /* sync cache timeout */ 116 117 #define nvme_read4(_s, _r) \ 118 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 119 #define nvme_write4(_s, _r, _v) \ 120 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 121 #ifdef __LP64__ 122 #define nvme_read8(_s, _r) \ 123 bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r)) 124 #define nvme_write8(_s, _r, _v) \ 125 bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 126 #else /* __LP64__ */ 127 static inline uint64_t 128 nvme_read8(struct nvme_softc *sc, bus_size_t r) 129 { 130 uint64_t v; 131 uint32_t *a = (uint32_t *)&v; 132 133 #if _BYTE_ORDER == _LITTLE_ENDIAN 134 a[0] = nvme_read4(sc, r); 135 a[1] = nvme_read4(sc, r + 4); 136 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 137 a[1] = nvme_read4(sc, r); 138 a[0] = nvme_read4(sc, r + 4); 139 #endif 140 141 return v; 142 } 143 144 static inline void 145 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 146 { 147 uint32_t *a = (uint32_t *)&v; 148 149 #if _BYTE_ORDER == _LITTLE_ENDIAN 150 nvme_write4(sc, r, a[0]); 151 nvme_write4(sc, r + 4, a[1]); 152 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 153 nvme_write4(sc, r, a[1]); 154 nvme_write4(sc, r + 4, a[0]); 155 #endif 156 } 157 #endif /* __LP64__ */ 158 #define nvme_barrier(_s, _r, _l, _f) \ 159 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 160 161 static void 162 nvme_version(struct nvme_softc *sc, uint32_t ver) 163 { 164 const char *v = NULL; 165 166 switch (ver) { 167 case NVME_VS_1_0: 168 v = "1.0"; 169 break; 170 case NVME_VS_1_1: 171 v = "1.1"; 172 break; 173 case NVME_VS_1_2: 174 v = "1.2"; 175 break; 176 case NVME_VS_1_2_1: 177 v = "1.2.1"; 178 break; 179 default: 180 aprint_error_dev(sc->sc_dev, "unknown version 0x%08x\n", ver); 181 return; 182 } 183 184 aprint_normal_dev(sc->sc_dev, "NVMe %s\n", v); 185 } 186 187 #ifdef NVME_DEBUG 188 static __used void 189 nvme_dumpregs(struct nvme_softc *sc) 190 { 191 uint64_t r8; 192 uint32_t r4; 193 194 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 195 r8 = nvme_read8(sc, NVME_CAP); 196 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 197 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 198 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 199 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 200 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 201 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 202 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 203 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 204 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 205 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 206 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 207 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 208 209 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 210 211 r4 = nvme_read4(sc, NVME_CC); 212 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 213 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 214 (1 << NVME_CC_IOCQES_R(r4))); 215 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 216 (1 << NVME_CC_IOSQES_R(r4))); 217 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 218 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 219 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 220 (1 << NVME_CC_MPS_R(r4))); 221 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 222 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 223 224 r4 = nvme_read4(sc, NVME_CSTS); 225 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 226 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 227 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 228 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 229 230 r4 = nvme_read4(sc, NVME_AQA); 231 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 232 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 233 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 234 235 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 236 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 237 #undef DEVNAME 238 } 239 #endif /* NVME_DEBUG */ 240 241 static int 242 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 243 { 244 u_int i = 0; 245 uint32_t cc; 246 247 cc = nvme_read4(sc, NVME_CC); 248 if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) { 249 aprint_error_dev(sc->sc_dev, 250 "controller enabled status expected %d, found to be %d\n", 251 (rdy != 0), ((cc & NVME_CC_EN) != 0)); 252 return ENXIO; 253 } 254 255 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 256 if (i++ > sc->sc_rdy_to) 257 return ENXIO; 258 259 delay(1000); 260 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 261 } 262 263 return 0; 264 } 265 266 static int 267 nvme_enable(struct nvme_softc *sc, u_int mps) 268 { 269 uint32_t cc, csts; 270 271 cc = nvme_read4(sc, NVME_CC); 272 csts = nvme_read4(sc, NVME_CSTS); 273 274 if (ISSET(cc, NVME_CC_EN)) { 275 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 276 277 if (ISSET(csts, NVME_CSTS_RDY)) 278 return 1; 279 280 goto waitready; 281 } 282 283 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 284 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 285 delay(5000); 286 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 287 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 288 delay(5000); 289 290 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 291 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 292 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 293 delay(5000); 294 295 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 296 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 297 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 298 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 299 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 300 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 301 SET(cc, NVME_CC_MPS(mps)); 302 SET(cc, NVME_CC_EN); 303 304 nvme_write4(sc, NVME_CC, cc); 305 nvme_barrier(sc, 0, sc->sc_ios, 306 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 307 delay(5000); 308 309 waitready: 310 return nvme_ready(sc, NVME_CSTS_RDY); 311 } 312 313 static int 314 nvme_disable(struct nvme_softc *sc) 315 { 316 uint32_t cc, csts; 317 318 cc = nvme_read4(sc, NVME_CC); 319 csts = nvme_read4(sc, NVME_CSTS); 320 321 if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY)) 322 nvme_ready(sc, NVME_CSTS_RDY); 323 324 CLR(cc, NVME_CC_EN); 325 326 nvme_write4(sc, NVME_CC, cc); 327 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 328 329 delay(5000); 330 331 return nvme_ready(sc, 0); 332 } 333 334 int 335 nvme_attach(struct nvme_softc *sc) 336 { 337 uint64_t cap; 338 uint32_t reg; 339 u_int dstrd; 340 u_int mps = PAGE_SHIFT; 341 u_int ioq_allocated; 342 uint16_t adminq_entries = nvme_adminq_size; 343 uint16_t ioq_entries = nvme_ioq_size; 344 int i; 345 346 reg = nvme_read4(sc, NVME_VS); 347 if (reg == 0xffffffff) { 348 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 349 return 1; 350 } 351 352 nvme_version(sc, reg); 353 354 cap = nvme_read8(sc, NVME_CAP); 355 dstrd = NVME_CAP_DSTRD(cap); 356 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 357 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 358 "is greater than CPU page size %u\n", 359 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 360 return 1; 361 } 362 if (NVME_CAP_MPSMAX(cap) < mps) 363 mps = NVME_CAP_MPSMAX(cap); 364 if (ioq_entries > NVME_CAP_MQES(cap)) 365 ioq_entries = NVME_CAP_MQES(cap); 366 367 /* set initial values to be used for admin queue during probe */ 368 sc->sc_rdy_to = NVME_CAP_TO(cap); 369 sc->sc_mps = 1 << mps; 370 sc->sc_mdts = MAXPHYS; 371 sc->sc_max_sgl = 2; 372 373 if (nvme_disable(sc) != 0) { 374 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 375 return 1; 376 } 377 378 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 379 if (sc->sc_admin_q == NULL) { 380 aprint_error_dev(sc->sc_dev, 381 "unable to allocate admin queue\n"); 382 return 1; 383 } 384 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 385 goto free_admin_q; 386 387 if (nvme_enable(sc, mps) != 0) { 388 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 389 goto disestablish_admin_q; 390 } 391 392 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 393 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 394 goto disable; 395 } 396 397 /* we know how big things are now */ 398 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 399 400 /* reallocate ccbs of admin queue with new max sgl. */ 401 nvme_ccbs_free(sc->sc_admin_q); 402 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 403 404 if (sc->sc_use_mq) { 405 /* Limit the number of queues to the number allocated in HW */ 406 if (nvme_get_number_of_queues(sc, &ioq_allocated) != 0) { 407 aprint_error_dev(sc->sc_dev, 408 "unable to get number of queues\n"); 409 goto disable; 410 } 411 if (sc->sc_nq > ioq_allocated) 412 sc->sc_nq = ioq_allocated; 413 } 414 415 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 416 if (sc->sc_q == NULL) { 417 aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n"); 418 goto disable; 419 } 420 for (i = 0; i < sc->sc_nq; i++) { 421 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 422 if (sc->sc_q[i] == NULL) { 423 aprint_error_dev(sc->sc_dev, 424 "unable to allocate io queue\n"); 425 goto free_q; 426 } 427 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 428 aprint_error_dev(sc->sc_dev, 429 "unable to create io queue\n"); 430 nvme_q_free(sc, sc->sc_q[i]); 431 goto free_q; 432 } 433 } 434 435 if (!sc->sc_use_mq) 436 nvme_write4(sc, NVME_INTMC, 1); 437 438 /* probe subdevices */ 439 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 440 KM_SLEEP); 441 if (sc->sc_namespaces == NULL) 442 goto free_q; 443 nvme_rescan(sc->sc_dev, "nvme", &i); 444 445 return 0; 446 447 free_q: 448 while (--i >= 0) { 449 nvme_q_delete(sc, sc->sc_q[i]); 450 nvme_q_free(sc, sc->sc_q[i]); 451 } 452 disable: 453 nvme_disable(sc); 454 disestablish_admin_q: 455 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 456 free_admin_q: 457 nvme_q_free(sc, sc->sc_admin_q); 458 459 return 1; 460 } 461 462 int 463 nvme_rescan(device_t self, const char *attr, const int *flags) 464 { 465 struct nvme_softc *sc = device_private(self); 466 struct nvme_attach_args naa; 467 uint64_t cap; 468 int ioq_entries = nvme_ioq_size; 469 int i; 470 471 cap = nvme_read8(sc, NVME_CAP); 472 if (ioq_entries > NVME_CAP_MQES(cap)) 473 ioq_entries = NVME_CAP_MQES(cap); 474 475 for (i = 0; i < sc->sc_nn; i++) { 476 if (sc->sc_namespaces[i].dev) 477 continue; 478 memset(&naa, 0, sizeof(naa)); 479 naa.naa_nsid = i + 1; 480 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 481 naa.naa_maxphys = sc->sc_mdts; 482 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 483 nvme_print); 484 } 485 return 0; 486 } 487 488 static int 489 nvme_print(void *aux, const char *pnp) 490 { 491 struct nvme_attach_args *naa = aux; 492 493 if (pnp) 494 aprint_normal("at %s", pnp); 495 496 if (naa->naa_nsid > 0) 497 aprint_normal(" nsid %d", naa->naa_nsid); 498 499 return UNCONF; 500 } 501 502 int 503 nvme_detach(struct nvme_softc *sc, int flags) 504 { 505 int i, error; 506 507 error = config_detach_children(sc->sc_dev, flags); 508 if (error) 509 return error; 510 511 error = nvme_shutdown(sc); 512 if (error) 513 return error; 514 515 /* from now on we are committed to detach, following will never fail */ 516 for (i = 0; i < sc->sc_nq; i++) 517 nvme_q_free(sc, sc->sc_q[i]); 518 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 519 nvme_q_free(sc, sc->sc_admin_q); 520 521 return 0; 522 } 523 524 static int 525 nvme_shutdown(struct nvme_softc *sc) 526 { 527 uint32_t cc, csts; 528 bool disabled = false; 529 int i; 530 531 if (!sc->sc_use_mq) 532 nvme_write4(sc, NVME_INTMS, 1); 533 534 for (i = 0; i < sc->sc_nq; i++) { 535 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 536 aprint_error_dev(sc->sc_dev, 537 "unable to delete io queue %d, disabling\n", i + 1); 538 disabled = true; 539 } 540 } 541 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 542 if (disabled) 543 goto disable; 544 545 cc = nvme_read4(sc, NVME_CC); 546 CLR(cc, NVME_CC_SHN_MASK); 547 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 548 nvme_write4(sc, NVME_CC, cc); 549 550 for (i = 0; i < 4000; i++) { 551 nvme_barrier(sc, 0, sc->sc_ios, 552 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 553 csts = nvme_read4(sc, NVME_CSTS); 554 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 555 return 0; 556 557 delay(1000); 558 } 559 560 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 561 562 disable: 563 nvme_disable(sc); 564 return 0; 565 } 566 567 void 568 nvme_childdet(device_t self, device_t child) 569 { 570 struct nvme_softc *sc = device_private(self); 571 int i; 572 573 for (i = 0; i < sc->sc_nn; i++) { 574 if (sc->sc_namespaces[i].dev == child) { 575 /* Already freed ns->ident. */ 576 sc->sc_namespaces[i].dev = NULL; 577 break; 578 } 579 } 580 } 581 582 int 583 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 584 { 585 struct nvme_sqe sqe; 586 struct nvm_identify_namespace *identify; 587 struct nvme_dmamem *mem; 588 struct nvme_ccb *ccb; 589 struct nvme_namespace *ns; 590 int rv; 591 592 KASSERT(nsid > 0); 593 594 ccb = nvme_ccb_get(sc->sc_admin_q); 595 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 596 597 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 598 if (mem == NULL) 599 return ENOMEM; 600 601 memset(&sqe, 0, sizeof(sqe)); 602 sqe.opcode = NVM_ADMIN_IDENTIFY; 603 htolem32(&sqe.nsid, nsid); 604 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 605 htolem32(&sqe.cdw10, 0); 606 607 ccb->ccb_done = nvme_empty_done; 608 ccb->ccb_cookie = &sqe; 609 610 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 611 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 612 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 613 614 nvme_ccb_put(sc->sc_admin_q, ccb); 615 616 if (rv != 0) { 617 rv = EIO; 618 goto done; 619 } 620 621 /* commit */ 622 623 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 624 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 625 //memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 626 627 ns = nvme_ns_get(sc, nsid); 628 KASSERT(ns); 629 ns->ident = identify; 630 631 done: 632 nvme_dmamem_free(sc, mem); 633 634 return rv; 635 } 636 637 int 638 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 639 struct buf *bp, void *data, size_t datasize, 640 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 641 { 642 struct nvme_queue *q = nvme_get_q(sc); 643 struct nvme_ccb *ccb; 644 bus_dmamap_t dmap; 645 int i, error; 646 647 ccb = nvme_ccb_get(q); 648 if (ccb == NULL) 649 return EAGAIN; 650 651 ccb->ccb_done = nvme_ns_io_done; 652 ccb->ccb_cookie = cookie; 653 654 /* namespace context */ 655 ccb->nnc_nsid = nsid; 656 ccb->nnc_flags = flags; 657 ccb->nnc_buf = bp; 658 ccb->nnc_datasize = datasize; 659 ccb->nnc_secsize = secsize; 660 ccb->nnc_blkno = blkno; 661 ccb->nnc_done = nnc_done; 662 663 dmap = ccb->ccb_dmamap; 664 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 665 datasize, NULL, 666 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 667 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 668 (ISSET(flags, NVME_NS_CTX_F_READ) ? 669 BUS_DMA_READ : BUS_DMA_WRITE)); 670 if (error) { 671 nvme_ccb_put(q, ccb); 672 return error; 673 } 674 675 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 676 ISSET(flags, NVME_NS_CTX_F_READ) ? 677 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 678 679 if (dmap->dm_nsegs > 2) { 680 for (i = 1; i < dmap->dm_nsegs; i++) { 681 htolem64(&ccb->ccb_prpl[i - 1], 682 dmap->dm_segs[i].ds_addr); 683 } 684 bus_dmamap_sync(sc->sc_dmat, 685 NVME_DMA_MAP(q->q_ccb_prpls), 686 ccb->ccb_prpl_off, 687 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 688 BUS_DMASYNC_PREWRITE); 689 } 690 691 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 692 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 693 return EIO; 694 return 0; 695 } 696 697 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 698 return 0; 699 } 700 701 static void 702 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 703 { 704 struct nvme_sqe_io *sqe = slot; 705 bus_dmamap_t dmap = ccb->ccb_dmamap; 706 707 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 708 NVM_CMD_READ : NVM_CMD_WRITE; 709 htolem32(&sqe->nsid, ccb->nnc_nsid); 710 711 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 712 switch (dmap->dm_nsegs) { 713 case 1: 714 break; 715 case 2: 716 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 717 break; 718 default: 719 /* the prp list is already set up and synced */ 720 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 721 break; 722 } 723 724 htolem64(&sqe->slba, ccb->nnc_blkno); 725 726 /* guaranteed by upper layers, but check just in case */ 727 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 728 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 729 } 730 731 static void 732 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 733 struct nvme_cqe *cqe) 734 { 735 struct nvme_softc *sc = q->q_sc; 736 bus_dmamap_t dmap = ccb->ccb_dmamap; 737 void *nnc_cookie = ccb->ccb_cookie; 738 nvme_nnc_done nnc_done = ccb->nnc_done; 739 struct buf *bp = ccb->nnc_buf; 740 741 if (dmap->dm_nsegs > 2) { 742 bus_dmamap_sync(sc->sc_dmat, 743 NVME_DMA_MAP(q->q_ccb_prpls), 744 ccb->ccb_prpl_off, 745 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 746 BUS_DMASYNC_POSTWRITE); 747 } 748 749 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 750 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 751 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 752 753 bus_dmamap_unload(sc->sc_dmat, dmap); 754 nvme_ccb_put(q, ccb); 755 756 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags)); 757 } 758 759 int 760 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie, 761 int flags, nvme_nnc_done nnc_done) 762 { 763 struct nvme_queue *q = nvme_get_q(sc); 764 struct nvme_ccb *ccb; 765 766 ccb = nvme_ccb_get(q); 767 if (ccb == NULL) 768 return EAGAIN; 769 770 ccb->ccb_done = nvme_ns_sync_done; 771 ccb->ccb_cookie = cookie; 772 773 /* namespace context */ 774 ccb->nnc_nsid = nsid; 775 ccb->nnc_flags = flags; 776 ccb->nnc_done = nnc_done; 777 778 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 779 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 780 return EIO; 781 return 0; 782 } 783 784 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 785 return 0; 786 } 787 788 static void 789 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 790 { 791 struct nvme_sqe *sqe = slot; 792 793 sqe->opcode = NVM_CMD_FLUSH; 794 htolem32(&sqe->nsid, ccb->nnc_nsid); 795 } 796 797 static void 798 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 799 struct nvme_cqe *cqe) 800 { 801 void *cookie = ccb->ccb_cookie; 802 nvme_nnc_done nnc_done = ccb->nnc_done; 803 804 nvme_ccb_put(q, ccb); 805 806 nnc_done(cookie, NULL, lemtoh16(&cqe->flags)); 807 } 808 809 void 810 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 811 { 812 struct nvme_namespace *ns; 813 struct nvm_identify_namespace *identify; 814 815 ns = nvme_ns_get(sc, nsid); 816 KASSERT(ns); 817 818 identify = ns->ident; 819 ns->ident = NULL; 820 if (identify != NULL) 821 kmem_free(identify, sizeof(*identify)); 822 } 823 824 static void 825 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 826 { 827 struct nvme_softc *sc = q->q_sc; 828 struct nvme_sqe *sqe = slot; 829 struct nvme_pt_command *pt = ccb->ccb_cookie; 830 bus_dmamap_t dmap = ccb->ccb_dmamap; 831 int i; 832 833 sqe->opcode = pt->cmd.opcode; 834 htolem32(&sqe->nsid, pt->cmd.nsid); 835 836 if (pt->buf != NULL && pt->len > 0) { 837 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 838 switch (dmap->dm_nsegs) { 839 case 1: 840 break; 841 case 2: 842 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 843 break; 844 default: 845 for (i = 1; i < dmap->dm_nsegs; i++) { 846 htolem64(&ccb->ccb_prpl[i - 1], 847 dmap->dm_segs[i].ds_addr); 848 } 849 bus_dmamap_sync(sc->sc_dmat, 850 NVME_DMA_MAP(q->q_ccb_prpls), 851 ccb->ccb_prpl_off, 852 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 853 BUS_DMASYNC_PREWRITE); 854 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 855 break; 856 } 857 } 858 859 htolem32(&sqe->cdw10, pt->cmd.cdw10); 860 htolem32(&sqe->cdw11, pt->cmd.cdw11); 861 htolem32(&sqe->cdw12, pt->cmd.cdw12); 862 htolem32(&sqe->cdw13, pt->cmd.cdw13); 863 htolem32(&sqe->cdw14, pt->cmd.cdw14); 864 htolem32(&sqe->cdw15, pt->cmd.cdw15); 865 } 866 867 static void 868 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 869 { 870 struct nvme_softc *sc = q->q_sc; 871 struct nvme_pt_command *pt = ccb->ccb_cookie; 872 bus_dmamap_t dmap = ccb->ccb_dmamap; 873 874 if (pt->buf != NULL && pt->len > 0) { 875 if (dmap->dm_nsegs > 2) { 876 bus_dmamap_sync(sc->sc_dmat, 877 NVME_DMA_MAP(q->q_ccb_prpls), 878 ccb->ccb_prpl_off, 879 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 880 BUS_DMASYNC_POSTWRITE); 881 } 882 883 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 884 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 885 bus_dmamap_unload(sc->sc_dmat, dmap); 886 } 887 888 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 889 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 890 } 891 892 static int 893 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 894 uint16_t nsid, struct lwp *l, bool is_adminq) 895 { 896 struct nvme_queue *q; 897 struct nvme_ccb *ccb; 898 void *buf = NULL; 899 int error; 900 901 /* limit command size to maximum data transfer size */ 902 if ((pt->buf == NULL && pt->len > 0) || 903 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 904 return EINVAL; 905 906 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 907 ccb = nvme_ccb_get(q); 908 if (ccb == NULL) 909 return EBUSY; 910 911 if (pt->buf != NULL) { 912 KASSERT(pt->len > 0); 913 buf = kmem_alloc(pt->len, KM_SLEEP); 914 if (buf == NULL) { 915 error = ENOMEM; 916 goto ccb_put; 917 } 918 if (!pt->is_read) { 919 error = copyin(pt->buf, buf, pt->len); 920 if (error) 921 goto kmem_free; 922 } 923 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 924 pt->len, NULL, 925 BUS_DMA_WAITOK | 926 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 927 if (error) 928 goto kmem_free; 929 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 930 0, ccb->ccb_dmamap->dm_mapsize, 931 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 932 } 933 934 ccb->ccb_done = nvme_pt_done; 935 ccb->ccb_cookie = pt; 936 937 pt->cmd.nsid = nsid; 938 if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) { 939 error = EIO; 940 goto out; 941 } 942 943 error = 0; 944 out: 945 if (buf != NULL) { 946 if (error == 0 && pt->is_read) 947 error = copyout(buf, pt->buf, pt->len); 948 kmem_free: 949 kmem_free(buf, pt->len); 950 } 951 ccb_put: 952 nvme_ccb_put(q, ccb); 953 return error; 954 } 955 956 static void 957 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 958 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 959 { 960 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 961 uint32_t tail; 962 963 mutex_enter(&q->q_sq_mtx); 964 tail = q->q_sq_tail; 965 if (++q->q_sq_tail >= q->q_entries) 966 q->q_sq_tail = 0; 967 968 sqe += tail; 969 970 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 971 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 972 memset(sqe, 0, sizeof(*sqe)); 973 (*fill)(q, ccb, sqe); 974 sqe->cid = ccb->ccb_id; 975 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 976 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 977 978 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 979 mutex_exit(&q->q_sq_mtx); 980 } 981 982 struct nvme_poll_state { 983 struct nvme_sqe s; 984 struct nvme_cqe c; 985 }; 986 987 static int 988 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 989 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 990 { 991 struct nvme_poll_state state; 992 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 993 void *cookie; 994 uint16_t flags; 995 int step = 10; 996 int maxloop = timo_sec * 1000000 / step; 997 int error = 0; 998 999 memset(&state, 0, sizeof(state)); 1000 (*fill)(q, ccb, &state.s); 1001 1002 done = ccb->ccb_done; 1003 cookie = ccb->ccb_cookie; 1004 1005 ccb->ccb_done = nvme_poll_done; 1006 ccb->ccb_cookie = &state; 1007 1008 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1009 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1010 if (nvme_q_complete(sc, q) == 0) 1011 delay(step); 1012 1013 if (timo_sec >= 0 && --maxloop <= 0) { 1014 error = ETIMEDOUT; 1015 break; 1016 } 1017 } 1018 1019 ccb->ccb_cookie = cookie; 1020 done(q, ccb, &state.c); 1021 1022 if (error == 0) { 1023 flags = lemtoh16(&state.c.flags); 1024 return flags & ~NVME_CQE_PHASE; 1025 } else { 1026 return 1; 1027 } 1028 } 1029 1030 static void 1031 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1032 { 1033 struct nvme_sqe *sqe = slot; 1034 struct nvme_poll_state *state = ccb->ccb_cookie; 1035 1036 *sqe = state->s; 1037 } 1038 1039 static void 1040 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1041 struct nvme_cqe *cqe) 1042 { 1043 struct nvme_poll_state *state = ccb->ccb_cookie; 1044 1045 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1046 state->c = *cqe; 1047 } 1048 1049 static void 1050 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1051 { 1052 struct nvme_sqe *src = ccb->ccb_cookie; 1053 struct nvme_sqe *dst = slot; 1054 1055 *dst = *src; 1056 } 1057 1058 static void 1059 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1060 struct nvme_cqe *cqe) 1061 { 1062 } 1063 1064 static int 1065 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1066 { 1067 struct nvme_ccb *ccb; 1068 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1069 uint16_t flags; 1070 int rv = 0; 1071 1072 mutex_enter(&q->q_cq_mtx); 1073 1074 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1075 for (;;) { 1076 cqe = &ring[q->q_cq_head]; 1077 flags = lemtoh16(&cqe->flags); 1078 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1079 break; 1080 1081 ccb = &q->q_ccbs[cqe->cid]; 1082 1083 if (++q->q_cq_head >= q->q_entries) { 1084 q->q_cq_head = 0; 1085 q->q_cq_phase ^= NVME_CQE_PHASE; 1086 } 1087 1088 #ifdef DEBUG 1089 /* 1090 * If we get spurious completion notification, something 1091 * is seriously hosed up. Very likely DMA to some random 1092 * memory place happened, so just bail out. 1093 */ 1094 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1095 panic("%s: invalid ccb detected", 1096 device_xname(sc->sc_dev)); 1097 /* NOTREACHED */ 1098 } 1099 #endif 1100 1101 rv++; 1102 1103 /* 1104 * Unlock the mutex before calling the ccb_done callback 1105 * and re-lock afterwards. The callback triggers lddone() 1106 * which schedules another i/o, and also calls nvme_ccb_put(). 1107 * Unlock/relock avoids possibility of deadlock. 1108 */ 1109 mutex_exit(&q->q_cq_mtx); 1110 ccb->ccb_done(q, ccb, cqe); 1111 mutex_enter(&q->q_cq_mtx); 1112 } 1113 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1114 1115 if (rv) 1116 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1117 1118 mutex_exit(&q->q_cq_mtx); 1119 1120 if (rv) { 1121 mutex_enter(&q->q_ccb_mtx); 1122 q->q_nccbs_avail += rv; 1123 mutex_exit(&q->q_ccb_mtx); 1124 } 1125 1126 return rv; 1127 } 1128 1129 static int 1130 nvme_identify(struct nvme_softc *sc, u_int mps) 1131 { 1132 char sn[41], mn[81], fr[17]; 1133 struct nvm_identify_controller *identify; 1134 struct nvme_dmamem *mem; 1135 struct nvme_ccb *ccb; 1136 u_int mdts; 1137 int rv = 1; 1138 1139 ccb = nvme_ccb_get(sc->sc_admin_q); 1140 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1141 1142 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1143 if (mem == NULL) 1144 return 1; 1145 1146 ccb->ccb_done = nvme_empty_done; 1147 ccb->ccb_cookie = mem; 1148 1149 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1150 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1151 NVME_TIMO_IDENT); 1152 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1153 1154 nvme_ccb_put(sc->sc_admin_q, ccb); 1155 1156 if (rv != 0) 1157 goto done; 1158 1159 identify = NVME_DMA_KVA(mem); 1160 1161 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1162 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1163 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1164 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1165 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1166 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1167 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1168 sn); 1169 1170 if (identify->mdts > 0) { 1171 mdts = (1 << identify->mdts) * (1 << mps); 1172 if (mdts < sc->sc_mdts) 1173 sc->sc_mdts = mdts; 1174 } 1175 1176 sc->sc_nn = lemtoh32(&identify->nn); 1177 1178 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1179 1180 done: 1181 nvme_dmamem_free(sc, mem); 1182 1183 return rv; 1184 } 1185 1186 static int 1187 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1188 { 1189 struct nvme_sqe_q sqe; 1190 struct nvme_ccb *ccb; 1191 int rv; 1192 1193 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1194 return 1; 1195 1196 ccb = nvme_ccb_get(sc->sc_admin_q); 1197 KASSERT(ccb != NULL); 1198 1199 ccb->ccb_done = nvme_empty_done; 1200 ccb->ccb_cookie = &sqe; 1201 1202 memset(&sqe, 0, sizeof(sqe)); 1203 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1204 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1205 htolem16(&sqe.qsize, q->q_entries - 1); 1206 htolem16(&sqe.qid, q->q_id); 1207 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1208 if (sc->sc_use_mq) 1209 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1210 1211 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1212 if (rv != 0) 1213 goto fail; 1214 1215 ccb->ccb_done = nvme_empty_done; 1216 ccb->ccb_cookie = &sqe; 1217 1218 memset(&sqe, 0, sizeof(sqe)); 1219 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1220 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1221 htolem16(&sqe.qsize, q->q_entries - 1); 1222 htolem16(&sqe.qid, q->q_id); 1223 htolem16(&sqe.cqid, q->q_id); 1224 sqe.qflags = NVM_SQE_Q_PC; 1225 1226 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1227 if (rv != 0) 1228 goto fail; 1229 1230 fail: 1231 nvme_ccb_put(sc->sc_admin_q, ccb); 1232 return rv; 1233 } 1234 1235 static int 1236 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1237 { 1238 struct nvme_sqe_q sqe; 1239 struct nvme_ccb *ccb; 1240 int rv; 1241 1242 ccb = nvme_ccb_get(sc->sc_admin_q); 1243 KASSERT(ccb != NULL); 1244 1245 ccb->ccb_done = nvme_empty_done; 1246 ccb->ccb_cookie = &sqe; 1247 1248 memset(&sqe, 0, sizeof(sqe)); 1249 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1250 htolem16(&sqe.qid, q->q_id); 1251 1252 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1253 if (rv != 0) 1254 goto fail; 1255 1256 ccb->ccb_done = nvme_empty_done; 1257 ccb->ccb_cookie = &sqe; 1258 1259 memset(&sqe, 0, sizeof(sqe)); 1260 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1261 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1262 htolem16(&sqe.qid, q->q_id); 1263 1264 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1265 if (rv != 0) 1266 goto fail; 1267 1268 fail: 1269 nvme_ccb_put(sc->sc_admin_q, ccb); 1270 1271 if (rv == 0 && sc->sc_use_mq) { 1272 if (sc->sc_intr_disestablish(sc, q->q_id)) 1273 rv = 1; 1274 } 1275 1276 return rv; 1277 } 1278 1279 static void 1280 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1281 { 1282 struct nvme_sqe *sqe = slot; 1283 struct nvme_dmamem *mem = ccb->ccb_cookie; 1284 1285 sqe->opcode = NVM_ADMIN_IDENTIFY; 1286 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1287 htolem32(&sqe->cdw10, 1); 1288 } 1289 1290 static int 1291 nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap) 1292 { 1293 struct nvme_pt_command pt; 1294 struct nvme_ccb *ccb; 1295 uint16_t ncqa, nsqa; 1296 int rv; 1297 1298 ccb = nvme_ccb_get(sc->sc_admin_q); 1299 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1300 1301 memset(&pt, 0, sizeof(pt)); 1302 pt.cmd.opcode = NVM_ADMIN_GET_FEATURES; 1303 pt.cmd.cdw10 = 7 /*NVME_FEAT_NUMBER_OF_QUEUES*/; 1304 1305 ccb->ccb_done = nvme_pt_done; 1306 ccb->ccb_cookie = &pt; 1307 1308 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1309 1310 nvme_ccb_put(sc->sc_admin_q, ccb); 1311 1312 if (rv != 0) { 1313 *nqap = 0; 1314 return EIO; 1315 } 1316 1317 ncqa = pt.cpl.cdw0 >> 16; 1318 nsqa = pt.cpl.cdw0 & 0xffff; 1319 *nqap = MIN(ncqa, nsqa) + 1; 1320 1321 return 0; 1322 } 1323 1324 static int 1325 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1326 { 1327 struct nvme_softc *sc = q->q_sc; 1328 struct nvme_ccb *ccb; 1329 bus_addr_t off; 1330 uint64_t *prpl; 1331 u_int i; 1332 1333 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1334 SIMPLEQ_INIT(&q->q_ccb_list); 1335 1336 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1337 if (q->q_ccbs == NULL) 1338 return 1; 1339 1340 q->q_nccbs = nccbs; 1341 q->q_nccbs_avail = nccbs; 1342 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1343 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1344 1345 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1346 off = 0; 1347 1348 for (i = 0; i < nccbs; i++) { 1349 ccb = &q->q_ccbs[i]; 1350 1351 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1352 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1353 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1354 &ccb->ccb_dmamap) != 0) 1355 goto free_maps; 1356 1357 ccb->ccb_id = i; 1358 ccb->ccb_prpl = prpl; 1359 ccb->ccb_prpl_off = off; 1360 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1361 1362 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1363 1364 prpl += sc->sc_max_sgl; 1365 off += sizeof(*prpl) * sc->sc_max_sgl; 1366 } 1367 1368 return 0; 1369 1370 free_maps: 1371 nvme_ccbs_free(q); 1372 return 1; 1373 } 1374 1375 static struct nvme_ccb * 1376 nvme_ccb_get(struct nvme_queue *q) 1377 { 1378 struct nvme_ccb *ccb = NULL; 1379 1380 mutex_enter(&q->q_ccb_mtx); 1381 if (q->q_nccbs_avail > 0) { 1382 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1383 KASSERT(ccb != NULL); 1384 q->q_nccbs_avail--; 1385 1386 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1387 #ifdef DEBUG 1388 ccb->ccb_cookie = NULL; 1389 #endif 1390 } 1391 mutex_exit(&q->q_ccb_mtx); 1392 1393 return ccb; 1394 } 1395 1396 static void 1397 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1398 { 1399 1400 mutex_enter(&q->q_ccb_mtx); 1401 #ifdef DEBUG 1402 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1403 #endif 1404 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1405 mutex_exit(&q->q_ccb_mtx); 1406 } 1407 1408 static void 1409 nvme_ccbs_free(struct nvme_queue *q) 1410 { 1411 struct nvme_softc *sc = q->q_sc; 1412 struct nvme_ccb *ccb; 1413 1414 mutex_enter(&q->q_ccb_mtx); 1415 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1416 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1417 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1418 } 1419 mutex_exit(&q->q_ccb_mtx); 1420 1421 nvme_dmamem_free(sc, q->q_ccb_prpls); 1422 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1423 q->q_ccbs = NULL; 1424 mutex_destroy(&q->q_ccb_mtx); 1425 } 1426 1427 static struct nvme_queue * 1428 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1429 { 1430 struct nvme_queue *q; 1431 1432 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1433 if (q == NULL) 1434 return NULL; 1435 1436 q->q_sc = sc; 1437 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1438 sizeof(struct nvme_sqe) * entries); 1439 if (q->q_sq_dmamem == NULL) 1440 goto free; 1441 1442 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1443 sizeof(struct nvme_cqe) * entries); 1444 if (q->q_cq_dmamem == NULL) 1445 goto free_sq; 1446 1447 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1448 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1449 1450 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1451 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1452 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1453 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1454 q->q_id = id; 1455 q->q_entries = entries; 1456 q->q_sq_tail = 0; 1457 q->q_cq_head = 0; 1458 q->q_cq_phase = NVME_CQE_PHASE; 1459 1460 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1461 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1462 1463 /* 1464 * Due to definition of full and empty queue (queue is empty 1465 * when head == tail, full when tail is one less then head), 1466 * we can actually only have (entries - 1) in-flight commands. 1467 */ 1468 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1469 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1470 goto free_cq; 1471 } 1472 1473 return q; 1474 1475 free_cq: 1476 nvme_dmamem_free(sc, q->q_cq_dmamem); 1477 free_sq: 1478 nvme_dmamem_free(sc, q->q_sq_dmamem); 1479 free: 1480 kmem_free(q, sizeof(*q)); 1481 1482 return NULL; 1483 } 1484 1485 static void 1486 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1487 { 1488 nvme_ccbs_free(q); 1489 mutex_destroy(&q->q_sq_mtx); 1490 mutex_destroy(&q->q_cq_mtx); 1491 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1492 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1493 nvme_dmamem_free(sc, q->q_cq_dmamem); 1494 nvme_dmamem_free(sc, q->q_sq_dmamem); 1495 kmem_free(q, sizeof(*q)); 1496 } 1497 1498 int 1499 nvme_intr(void *xsc) 1500 { 1501 struct nvme_softc *sc = xsc; 1502 1503 /* 1504 * INTx is level triggered, controller deasserts the interrupt only 1505 * when we advance command queue head via write to the doorbell. 1506 * Tell the controller to block the interrupts while we process 1507 * the queue(s). 1508 */ 1509 nvme_write4(sc, NVME_INTMS, 1); 1510 1511 softint_schedule(sc->sc_softih[0]); 1512 1513 /* don't know, might not have been for us */ 1514 return 1; 1515 } 1516 1517 void 1518 nvme_softintr_intx(void *xq) 1519 { 1520 struct nvme_queue *q = xq; 1521 struct nvme_softc *sc = q->q_sc; 1522 1523 nvme_q_complete(sc, sc->sc_admin_q); 1524 if (sc->sc_q != NULL) 1525 nvme_q_complete(sc, sc->sc_q[0]); 1526 1527 /* 1528 * Processing done, tell controller to issue interrupts again. There 1529 * is no race, as NVMe spec requires the controller to maintain state, 1530 * and assert the interrupt whenever there are unacknowledged 1531 * completion queue entries. 1532 */ 1533 nvme_write4(sc, NVME_INTMC, 1); 1534 } 1535 1536 int 1537 nvme_intr_msi(void *xq) 1538 { 1539 struct nvme_queue *q = xq; 1540 1541 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1542 && q->q_sc->sc_softih[q->q_id]); 1543 1544 /* 1545 * MSI/MSI-X are edge triggered, so can handover processing to softint 1546 * without masking the interrupt. 1547 */ 1548 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1549 1550 return 1; 1551 } 1552 1553 void 1554 nvme_softintr_msi(void *xq) 1555 { 1556 struct nvme_queue *q = xq; 1557 struct nvme_softc *sc = q->q_sc; 1558 1559 nvme_q_complete(sc, q); 1560 } 1561 1562 static struct nvme_dmamem * 1563 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1564 { 1565 struct nvme_dmamem *ndm; 1566 int nsegs; 1567 1568 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1569 if (ndm == NULL) 1570 return NULL; 1571 1572 ndm->ndm_size = size; 1573 1574 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1575 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1576 goto ndmfree; 1577 1578 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1579 1, &nsegs, BUS_DMA_WAITOK) != 0) 1580 goto destroy; 1581 1582 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1583 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1584 goto free; 1585 memset(ndm->ndm_kva, 0, size); 1586 1587 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1588 NULL, BUS_DMA_WAITOK) != 0) 1589 goto unmap; 1590 1591 return ndm; 1592 1593 unmap: 1594 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1595 free: 1596 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1597 destroy: 1598 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1599 ndmfree: 1600 kmem_free(ndm, sizeof(*ndm)); 1601 return NULL; 1602 } 1603 1604 static void 1605 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1606 { 1607 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1608 0, NVME_DMA_LEN(mem), ops); 1609 } 1610 1611 void 1612 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1613 { 1614 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1615 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1616 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1617 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1618 kmem_free(ndm, sizeof(*ndm)); 1619 } 1620 1621 /* 1622 * ioctl 1623 */ 1624 1625 dev_type_open(nvmeopen); 1626 dev_type_close(nvmeclose); 1627 dev_type_ioctl(nvmeioctl); 1628 1629 const struct cdevsw nvme_cdevsw = { 1630 .d_open = nvmeopen, 1631 .d_close = nvmeclose, 1632 .d_read = noread, 1633 .d_write = nowrite, 1634 .d_ioctl = nvmeioctl, 1635 .d_stop = nostop, 1636 .d_tty = notty, 1637 .d_poll = nopoll, 1638 .d_mmap = nommap, 1639 .d_kqfilter = nokqfilter, 1640 .d_discard = nodiscard, 1641 .d_flag = D_OTHER, 1642 }; 1643 1644 extern struct cfdriver nvme_cd; 1645 1646 /* 1647 * Accept an open operation on the control device. 1648 */ 1649 int 1650 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1651 { 1652 struct nvme_softc *sc; 1653 int unit = minor(dev) / 0x10000; 1654 int nsid = minor(dev) & 0xffff; 1655 int nsidx; 1656 1657 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1658 return ENXIO; 1659 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1660 return ENXIO; 1661 1662 if (nsid == 0) { 1663 /* controller */ 1664 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1665 return EBUSY; 1666 SET(sc->sc_flags, NVME_F_OPEN); 1667 } else { 1668 /* namespace */ 1669 nsidx = nsid - 1; 1670 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1671 return ENXIO; 1672 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1673 return EBUSY; 1674 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1675 } 1676 return 0; 1677 } 1678 1679 /* 1680 * Accept the last close on the control device. 1681 */ 1682 int 1683 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1684 { 1685 struct nvme_softc *sc; 1686 int unit = minor(dev) / 0x10000; 1687 int nsid = minor(dev) & 0xffff; 1688 int nsidx; 1689 1690 sc = device_lookup_private(&nvme_cd, unit); 1691 if (sc == NULL) 1692 return ENXIO; 1693 1694 if (nsid == 0) { 1695 /* controller */ 1696 CLR(sc->sc_flags, NVME_F_OPEN); 1697 } else { 1698 /* namespace */ 1699 nsidx = nsid - 1; 1700 if (nsidx >= sc->sc_nn) 1701 return ENXIO; 1702 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1703 } 1704 1705 return 0; 1706 } 1707 1708 /* 1709 * Handle control operations. 1710 */ 1711 int 1712 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1713 { 1714 struct nvme_softc *sc; 1715 int unit = minor(dev) / 0x10000; 1716 int nsid = minor(dev) & 0xffff; 1717 struct nvme_pt_command *pt; 1718 1719 sc = device_lookup_private(&nvme_cd, unit); 1720 if (sc == NULL) 1721 return ENXIO; 1722 1723 switch (cmd) { 1724 case NVME_PASSTHROUGH_CMD: 1725 pt = data; 1726 return nvme_command_passthrough(sc, data, 1727 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1728 } 1729 1730 return ENOTTY; 1731 } 1732
Indexes created Mon Sep 29 13:09:55 GMT 2025