nvme.c revision 1.28
1 /* $NetBSD: nvme.c,v 1.28 2017/05/29 02:24:00 nonaka Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.28 2017/05/29 02:24:00 nonaka Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 int nvme_adminq_size = 32; 44 int nvme_ioq_size = 1024; 45 46 static int nvme_print(void *, const char *); 47 48 static int nvme_ready(struct nvme_softc *, uint32_t); 49 static int nvme_enable(struct nvme_softc *, u_int); 50 static int nvme_disable(struct nvme_softc *); 51 static int nvme_shutdown(struct nvme_softc *); 52 53 #ifdef NVME_DEBUG 54 static void nvme_dumpregs(struct nvme_softc *); 55 #endif 56 static int nvme_identify(struct nvme_softc *, u_int); 57 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 58 void *); 59 60 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 61 static void nvme_ccbs_free(struct nvme_queue *); 62 63 static struct nvme_ccb * 64 nvme_ccb_get(struct nvme_queue *); 65 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 66 67 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 68 struct nvme_ccb *, void (*)(struct nvme_queue *, 69 struct nvme_ccb *, void *), int); 70 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 71 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 72 struct nvme_cqe *); 73 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 74 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 75 struct nvme_cqe *); 76 77 static struct nvme_queue * 78 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 79 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 80 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 81 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 82 struct nvme_ccb *, void (*)(struct nvme_queue *, 83 struct nvme_ccb *, void *)); 84 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 85 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 86 87 static struct nvme_dmamem * 88 nvme_dmamem_alloc(struct nvme_softc *, size_t); 89 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 90 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 91 int); 92 93 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 94 void *); 95 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 96 struct nvme_cqe *); 97 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 98 void *); 99 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 100 struct nvme_cqe *); 101 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *, 102 void *); 103 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *, 104 struct nvme_cqe *); 105 106 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 107 void *); 108 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 109 struct nvme_cqe *); 110 static int nvme_command_passthrough(struct nvme_softc *, 111 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 112 113 static int nvme_get_number_of_queues(struct nvme_softc *, u_int *); 114 115 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 116 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 117 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 118 #define NVME_TIMO_SY 60 /* sync cache timeout */ 119 120 #define nvme_read4(_s, _r) \ 121 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 122 #define nvme_write4(_s, _r, _v) \ 123 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 124 /* 125 * Some controllers, at least Apple NVMe, always require split 126 * transfers, so don't use bus_space_{read,write}_8() on LP64. 127 */ 128 static inline uint64_t 129 nvme_read8(struct nvme_softc *sc, bus_size_t r) 130 { 131 uint64_t v; 132 uint32_t *a = (uint32_t *)&v; 133 134 #if _BYTE_ORDER == _LITTLE_ENDIAN 135 a[0] = nvme_read4(sc, r); 136 a[1] = nvme_read4(sc, r + 4); 137 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 138 a[1] = nvme_read4(sc, r); 139 a[0] = nvme_read4(sc, r + 4); 140 #endif 141 142 return v; 143 } 144 145 static inline void 146 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 147 { 148 uint32_t *a = (uint32_t *)&v; 149 150 #if _BYTE_ORDER == _LITTLE_ENDIAN 151 nvme_write4(sc, r, a[0]); 152 nvme_write4(sc, r + 4, a[1]); 153 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 154 nvme_write4(sc, r, a[1]); 155 nvme_write4(sc, r + 4, a[0]); 156 #endif 157 } 158 #define nvme_barrier(_s, _r, _l, _f) \ 159 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 160 161 #ifdef NVME_DEBUG 162 static __used void 163 nvme_dumpregs(struct nvme_softc *sc) 164 { 165 uint64_t r8; 166 uint32_t r4; 167 168 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 169 r8 = nvme_read8(sc, NVME_CAP); 170 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 171 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 172 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 173 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 174 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 175 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 176 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 177 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 178 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 179 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 180 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 181 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 182 183 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 184 185 r4 = nvme_read4(sc, NVME_CC); 186 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 187 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 188 (1 << NVME_CC_IOCQES_R(r4))); 189 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 190 (1 << NVME_CC_IOSQES_R(r4))); 191 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 192 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 193 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 194 (1 << NVME_CC_MPS_R(r4))); 195 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 196 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 197 198 r4 = nvme_read4(sc, NVME_CSTS); 199 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 200 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 201 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 202 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 203 204 r4 = nvme_read4(sc, NVME_AQA); 205 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 206 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 207 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 208 209 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 210 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 211 #undef DEVNAME 212 } 213 #endif /* NVME_DEBUG */ 214 215 static int 216 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 217 { 218 u_int i = 0; 219 uint32_t cc; 220 221 cc = nvme_read4(sc, NVME_CC); 222 if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) { 223 aprint_error_dev(sc->sc_dev, 224 "controller enabled status expected %d, found to be %d\n", 225 (rdy != 0), ((cc & NVME_CC_EN) != 0)); 226 return ENXIO; 227 } 228 229 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 230 if (i++ > sc->sc_rdy_to) 231 return ENXIO; 232 233 delay(1000); 234 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 235 } 236 237 return 0; 238 } 239 240 static int 241 nvme_enable(struct nvme_softc *sc, u_int mps) 242 { 243 uint32_t cc, csts; 244 245 cc = nvme_read4(sc, NVME_CC); 246 csts = nvme_read4(sc, NVME_CSTS); 247 248 if (ISSET(cc, NVME_CC_EN)) { 249 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 250 251 if (ISSET(csts, NVME_CSTS_RDY)) 252 return 1; 253 254 goto waitready; 255 } 256 257 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 258 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 259 delay(5000); 260 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 261 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 262 delay(5000); 263 264 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 265 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 266 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 267 delay(5000); 268 269 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 270 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 271 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 272 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 273 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 274 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 275 SET(cc, NVME_CC_MPS(mps)); 276 SET(cc, NVME_CC_EN); 277 278 nvme_write4(sc, NVME_CC, cc); 279 nvme_barrier(sc, 0, sc->sc_ios, 280 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 281 delay(5000); 282 283 waitready: 284 return nvme_ready(sc, NVME_CSTS_RDY); 285 } 286 287 static int 288 nvme_disable(struct nvme_softc *sc) 289 { 290 uint32_t cc, csts; 291 292 cc = nvme_read4(sc, NVME_CC); 293 csts = nvme_read4(sc, NVME_CSTS); 294 295 if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY)) 296 nvme_ready(sc, NVME_CSTS_RDY); 297 298 CLR(cc, NVME_CC_EN); 299 300 nvme_write4(sc, NVME_CC, cc); 301 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 302 303 delay(5000); 304 305 return nvme_ready(sc, 0); 306 } 307 308 int 309 nvme_attach(struct nvme_softc *sc) 310 { 311 uint64_t cap; 312 uint32_t reg; 313 u_int dstrd; 314 u_int mps = PAGE_SHIFT; 315 u_int ioq_allocated; 316 uint16_t adminq_entries = nvme_adminq_size; 317 uint16_t ioq_entries = nvme_ioq_size; 318 int i; 319 320 reg = nvme_read4(sc, NVME_VS); 321 if (reg == 0xffffffff) { 322 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 323 return 1; 324 } 325 326 if (NVME_VS_TER(reg) == 0) 327 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg), 328 NVME_VS_MNR(reg)); 329 else 330 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg), 331 NVME_VS_MNR(reg), NVME_VS_TER(reg)); 332 333 cap = nvme_read8(sc, NVME_CAP); 334 dstrd = NVME_CAP_DSTRD(cap); 335 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 336 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 337 "is greater than CPU page size %u\n", 338 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 339 return 1; 340 } 341 if (NVME_CAP_MPSMAX(cap) < mps) 342 mps = NVME_CAP_MPSMAX(cap); 343 if (ioq_entries > NVME_CAP_MQES(cap)) 344 ioq_entries = NVME_CAP_MQES(cap); 345 346 /* set initial values to be used for admin queue during probe */ 347 sc->sc_rdy_to = NVME_CAP_TO(cap); 348 sc->sc_mps = 1 << mps; 349 sc->sc_mdts = MAXPHYS; 350 sc->sc_max_sgl = 2; 351 352 if (nvme_disable(sc) != 0) { 353 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 354 return 1; 355 } 356 357 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 358 if (sc->sc_admin_q == NULL) { 359 aprint_error_dev(sc->sc_dev, 360 "unable to allocate admin queue\n"); 361 return 1; 362 } 363 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 364 goto free_admin_q; 365 366 if (nvme_enable(sc, mps) != 0) { 367 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 368 goto disestablish_admin_q; 369 } 370 371 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 372 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 373 goto disable; 374 } 375 376 /* we know how big things are now */ 377 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 378 379 /* reallocate ccbs of admin queue with new max sgl. */ 380 nvme_ccbs_free(sc->sc_admin_q); 381 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 382 383 if (sc->sc_use_mq) { 384 /* Limit the number of queues to the number allocated in HW */ 385 if (nvme_get_number_of_queues(sc, &ioq_allocated) != 0) { 386 aprint_error_dev(sc->sc_dev, 387 "unable to get number of queues\n"); 388 goto disable; 389 } 390 if (sc->sc_nq > ioq_allocated) 391 sc->sc_nq = ioq_allocated; 392 } 393 394 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 395 if (sc->sc_q == NULL) { 396 aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n"); 397 goto disable; 398 } 399 for (i = 0; i < sc->sc_nq; i++) { 400 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 401 if (sc->sc_q[i] == NULL) { 402 aprint_error_dev(sc->sc_dev, 403 "unable to allocate io queue\n"); 404 goto free_q; 405 } 406 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 407 aprint_error_dev(sc->sc_dev, 408 "unable to create io queue\n"); 409 nvme_q_free(sc, sc->sc_q[i]); 410 goto free_q; 411 } 412 } 413 414 if (!sc->sc_use_mq) 415 nvme_write4(sc, NVME_INTMC, 1); 416 417 /* probe subdevices */ 418 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 419 KM_SLEEP); 420 if (sc->sc_namespaces == NULL) 421 goto free_q; 422 nvme_rescan(sc->sc_dev, "nvme", &i); 423 424 return 0; 425 426 free_q: 427 while (--i >= 0) { 428 nvme_q_delete(sc, sc->sc_q[i]); 429 nvme_q_free(sc, sc->sc_q[i]); 430 } 431 disable: 432 nvme_disable(sc); 433 disestablish_admin_q: 434 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 435 free_admin_q: 436 nvme_q_free(sc, sc->sc_admin_q); 437 438 return 1; 439 } 440 441 int 442 nvme_rescan(device_t self, const char *attr, const int *flags) 443 { 444 struct nvme_softc *sc = device_private(self); 445 struct nvme_attach_args naa; 446 uint64_t cap; 447 int ioq_entries = nvme_ioq_size; 448 int i; 449 450 cap = nvme_read8(sc, NVME_CAP); 451 if (ioq_entries > NVME_CAP_MQES(cap)) 452 ioq_entries = NVME_CAP_MQES(cap); 453 454 for (i = 0; i < sc->sc_nn; i++) { 455 if (sc->sc_namespaces[i].dev) 456 continue; 457 memset(&naa, 0, sizeof(naa)); 458 naa.naa_nsid = i + 1; 459 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 460 naa.naa_maxphys = sc->sc_mdts; 461 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 462 nvme_print); 463 } 464 return 0; 465 } 466 467 static int 468 nvme_print(void *aux, const char *pnp) 469 { 470 struct nvme_attach_args *naa = aux; 471 472 if (pnp) 473 aprint_normal("at %s", pnp); 474 475 if (naa->naa_nsid > 0) 476 aprint_normal(" nsid %d", naa->naa_nsid); 477 478 return UNCONF; 479 } 480 481 int 482 nvme_detach(struct nvme_softc *sc, int flags) 483 { 484 int i, error; 485 486 error = config_detach_children(sc->sc_dev, flags); 487 if (error) 488 return error; 489 490 error = nvme_shutdown(sc); 491 if (error) 492 return error; 493 494 /* from now on we are committed to detach, following will never fail */ 495 for (i = 0; i < sc->sc_nq; i++) 496 nvme_q_free(sc, sc->sc_q[i]); 497 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 498 nvme_q_free(sc, sc->sc_admin_q); 499 500 return 0; 501 } 502 503 static int 504 nvme_shutdown(struct nvme_softc *sc) 505 { 506 uint32_t cc, csts; 507 bool disabled = false; 508 int i; 509 510 if (!sc->sc_use_mq) 511 nvme_write4(sc, NVME_INTMS, 1); 512 513 for (i = 0; i < sc->sc_nq; i++) { 514 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 515 aprint_error_dev(sc->sc_dev, 516 "unable to delete io queue %d, disabling\n", i + 1); 517 disabled = true; 518 } 519 } 520 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 521 if (disabled) 522 goto disable; 523 524 cc = nvme_read4(sc, NVME_CC); 525 CLR(cc, NVME_CC_SHN_MASK); 526 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 527 nvme_write4(sc, NVME_CC, cc); 528 529 for (i = 0; i < 4000; i++) { 530 nvme_barrier(sc, 0, sc->sc_ios, 531 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 532 csts = nvme_read4(sc, NVME_CSTS); 533 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 534 return 0; 535 536 delay(1000); 537 } 538 539 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 540 541 disable: 542 nvme_disable(sc); 543 return 0; 544 } 545 546 void 547 nvme_childdet(device_t self, device_t child) 548 { 549 struct nvme_softc *sc = device_private(self); 550 int i; 551 552 for (i = 0; i < sc->sc_nn; i++) { 553 if (sc->sc_namespaces[i].dev == child) { 554 /* Already freed ns->ident. */ 555 sc->sc_namespaces[i].dev = NULL; 556 break; 557 } 558 } 559 } 560 561 int 562 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 563 { 564 struct nvme_sqe sqe; 565 struct nvm_identify_namespace *identify; 566 struct nvme_dmamem *mem; 567 struct nvme_ccb *ccb; 568 struct nvme_namespace *ns; 569 int rv; 570 571 KASSERT(nsid > 0); 572 573 ccb = nvme_ccb_get(sc->sc_admin_q); 574 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 575 576 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 577 if (mem == NULL) 578 return ENOMEM; 579 580 memset(&sqe, 0, sizeof(sqe)); 581 sqe.opcode = NVM_ADMIN_IDENTIFY; 582 htolem32(&sqe.nsid, nsid); 583 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 584 htolem32(&sqe.cdw10, 0); 585 586 ccb->ccb_done = nvme_empty_done; 587 ccb->ccb_cookie = &sqe; 588 589 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 590 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 591 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 592 593 nvme_ccb_put(sc->sc_admin_q, ccb); 594 595 if (rv != 0) { 596 rv = EIO; 597 goto done; 598 } 599 600 /* commit */ 601 602 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 603 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 604 //memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 605 606 ns = nvme_ns_get(sc, nsid); 607 KASSERT(ns); 608 ns->ident = identify; 609 610 done: 611 nvme_dmamem_free(sc, mem); 612 613 return rv; 614 } 615 616 int 617 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 618 struct buf *bp, void *data, size_t datasize, 619 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 620 { 621 struct nvme_queue *q = nvme_get_q(sc); 622 struct nvme_ccb *ccb; 623 bus_dmamap_t dmap; 624 int i, error; 625 626 ccb = nvme_ccb_get(q); 627 if (ccb == NULL) 628 return EAGAIN; 629 630 ccb->ccb_done = nvme_ns_io_done; 631 ccb->ccb_cookie = cookie; 632 633 /* namespace context */ 634 ccb->nnc_nsid = nsid; 635 ccb->nnc_flags = flags; 636 ccb->nnc_buf = bp; 637 ccb->nnc_datasize = datasize; 638 ccb->nnc_secsize = secsize; 639 ccb->nnc_blkno = blkno; 640 ccb->nnc_done = nnc_done; 641 642 dmap = ccb->ccb_dmamap; 643 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 644 datasize, NULL, 645 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 646 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 647 (ISSET(flags, NVME_NS_CTX_F_READ) ? 648 BUS_DMA_READ : BUS_DMA_WRITE)); 649 if (error) { 650 nvme_ccb_put(q, ccb); 651 return error; 652 } 653 654 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 655 ISSET(flags, NVME_NS_CTX_F_READ) ? 656 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 657 658 if (dmap->dm_nsegs > 2) { 659 for (i = 1; i < dmap->dm_nsegs; i++) { 660 htolem64(&ccb->ccb_prpl[i - 1], 661 dmap->dm_segs[i].ds_addr); 662 } 663 bus_dmamap_sync(sc->sc_dmat, 664 NVME_DMA_MAP(q->q_ccb_prpls), 665 ccb->ccb_prpl_off, 666 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 667 BUS_DMASYNC_PREWRITE); 668 } 669 670 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 671 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 672 return EIO; 673 return 0; 674 } 675 676 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 677 return 0; 678 } 679 680 static void 681 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 682 { 683 struct nvme_sqe_io *sqe = slot; 684 bus_dmamap_t dmap = ccb->ccb_dmamap; 685 686 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 687 NVM_CMD_READ : NVM_CMD_WRITE; 688 htolem32(&sqe->nsid, ccb->nnc_nsid); 689 690 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 691 switch (dmap->dm_nsegs) { 692 case 1: 693 break; 694 case 2: 695 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 696 break; 697 default: 698 /* the prp list is already set up and synced */ 699 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 700 break; 701 } 702 703 htolem64(&sqe->slba, ccb->nnc_blkno); 704 705 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) 706 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA); 707 708 /* guaranteed by upper layers, but check just in case */ 709 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 710 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 711 } 712 713 static void 714 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 715 struct nvme_cqe *cqe) 716 { 717 struct nvme_softc *sc = q->q_sc; 718 bus_dmamap_t dmap = ccb->ccb_dmamap; 719 void *nnc_cookie = ccb->ccb_cookie; 720 nvme_nnc_done nnc_done = ccb->nnc_done; 721 struct buf *bp = ccb->nnc_buf; 722 723 if (dmap->dm_nsegs > 2) { 724 bus_dmamap_sync(sc->sc_dmat, 725 NVME_DMA_MAP(q->q_ccb_prpls), 726 ccb->ccb_prpl_off, 727 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 728 BUS_DMASYNC_POSTWRITE); 729 } 730 731 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 732 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 733 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 734 735 bus_dmamap_unload(sc->sc_dmat, dmap); 736 nvme_ccb_put(q, ccb); 737 738 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 739 } 740 741 /* 742 * If there is no volatile write cache, it makes no sense to issue 743 * flush commands or query for the status. 744 */ 745 bool 746 nvme_has_volatile_write_cache(struct nvme_softc *sc) 747 { 748 /* sc_identify is filled during attachment */ 749 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0); 750 } 751 752 int 753 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie, 754 int flags, nvme_nnc_done nnc_done) 755 { 756 struct nvme_queue *q = nvme_get_q(sc); 757 struct nvme_ccb *ccb; 758 759 ccb = nvme_ccb_get(q); 760 if (ccb == NULL) 761 return EAGAIN; 762 763 ccb->ccb_done = nvme_ns_sync_done; 764 ccb->ccb_cookie = cookie; 765 766 /* namespace context */ 767 ccb->nnc_nsid = nsid; 768 ccb->nnc_flags = flags; 769 ccb->nnc_done = nnc_done; 770 771 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 772 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 773 return EIO; 774 return 0; 775 } 776 777 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 778 return 0; 779 } 780 781 static void 782 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 783 { 784 struct nvme_sqe *sqe = slot; 785 786 sqe->opcode = NVM_CMD_FLUSH; 787 htolem32(&sqe->nsid, ccb->nnc_nsid); 788 } 789 790 static void 791 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 792 struct nvme_cqe *cqe) 793 { 794 void *cookie = ccb->ccb_cookie; 795 nvme_nnc_done nnc_done = ccb->nnc_done; 796 797 nvme_ccb_put(q, ccb); 798 799 nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 800 } 801 802 /* 803 * Get status of volatile write cache. Always asynchronous. 804 */ 805 int 806 nvme_admin_getcache(struct nvme_softc *sc, void *cookie, nvme_nnc_done nnc_done) 807 { 808 struct nvme_ccb *ccb; 809 struct nvme_queue *q = sc->sc_admin_q; 810 811 ccb = nvme_ccb_get(q); 812 if (ccb == NULL) 813 return EAGAIN; 814 815 ccb->ccb_done = nvme_getcache_done; 816 ccb->ccb_cookie = cookie; 817 818 /* namespace context */ 819 ccb->nnc_flags = 0; 820 ccb->nnc_done = nnc_done; 821 822 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); 823 return 0; 824 } 825 826 static void 827 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 828 { 829 struct nvme_sqe *sqe = slot; 830 831 sqe->opcode = NVM_ADMIN_GET_FEATURES; 832 sqe->cdw10 = NVM_FEATURE_VOLATILE_WRITE_CACHE; 833 } 834 835 static void 836 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 837 struct nvme_cqe *cqe) 838 { 839 void *cookie = ccb->ccb_cookie; 840 nvme_nnc_done nnc_done = ccb->nnc_done; 841 842 nvme_ccb_put(q, ccb); 843 844 nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 845 } 846 847 void 848 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 849 { 850 struct nvme_namespace *ns; 851 struct nvm_identify_namespace *identify; 852 853 ns = nvme_ns_get(sc, nsid); 854 KASSERT(ns); 855 856 identify = ns->ident; 857 ns->ident = NULL; 858 if (identify != NULL) 859 kmem_free(identify, sizeof(*identify)); 860 } 861 862 static void 863 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 864 { 865 struct nvme_softc *sc = q->q_sc; 866 struct nvme_sqe *sqe = slot; 867 struct nvme_pt_command *pt = ccb->ccb_cookie; 868 bus_dmamap_t dmap = ccb->ccb_dmamap; 869 int i; 870 871 sqe->opcode = pt->cmd.opcode; 872 htolem32(&sqe->nsid, pt->cmd.nsid); 873 874 if (pt->buf != NULL && pt->len > 0) { 875 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 876 switch (dmap->dm_nsegs) { 877 case 1: 878 break; 879 case 2: 880 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 881 break; 882 default: 883 for (i = 1; i < dmap->dm_nsegs; i++) { 884 htolem64(&ccb->ccb_prpl[i - 1], 885 dmap->dm_segs[i].ds_addr); 886 } 887 bus_dmamap_sync(sc->sc_dmat, 888 NVME_DMA_MAP(q->q_ccb_prpls), 889 ccb->ccb_prpl_off, 890 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 891 BUS_DMASYNC_PREWRITE); 892 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 893 break; 894 } 895 } 896 897 htolem32(&sqe->cdw10, pt->cmd.cdw10); 898 htolem32(&sqe->cdw11, pt->cmd.cdw11); 899 htolem32(&sqe->cdw12, pt->cmd.cdw12); 900 htolem32(&sqe->cdw13, pt->cmd.cdw13); 901 htolem32(&sqe->cdw14, pt->cmd.cdw14); 902 htolem32(&sqe->cdw15, pt->cmd.cdw15); 903 } 904 905 static void 906 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 907 { 908 struct nvme_softc *sc = q->q_sc; 909 struct nvme_pt_command *pt = ccb->ccb_cookie; 910 bus_dmamap_t dmap = ccb->ccb_dmamap; 911 912 if (pt->buf != NULL && pt->len > 0) { 913 if (dmap->dm_nsegs > 2) { 914 bus_dmamap_sync(sc->sc_dmat, 915 NVME_DMA_MAP(q->q_ccb_prpls), 916 ccb->ccb_prpl_off, 917 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 918 BUS_DMASYNC_POSTWRITE); 919 } 920 921 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 922 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 923 bus_dmamap_unload(sc->sc_dmat, dmap); 924 } 925 926 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 927 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 928 } 929 930 static int 931 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 932 uint16_t nsid, struct lwp *l, bool is_adminq) 933 { 934 struct nvme_queue *q; 935 struct nvme_ccb *ccb; 936 void *buf = NULL; 937 int error; 938 939 /* limit command size to maximum data transfer size */ 940 if ((pt->buf == NULL && pt->len > 0) || 941 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 942 return EINVAL; 943 944 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 945 ccb = nvme_ccb_get(q); 946 if (ccb == NULL) 947 return EBUSY; 948 949 if (pt->buf != NULL) { 950 KASSERT(pt->len > 0); 951 buf = kmem_alloc(pt->len, KM_SLEEP); 952 if (buf == NULL) { 953 error = ENOMEM; 954 goto ccb_put; 955 } 956 if (!pt->is_read) { 957 error = copyin(pt->buf, buf, pt->len); 958 if (error) 959 goto kmem_free; 960 } 961 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 962 pt->len, NULL, 963 BUS_DMA_WAITOK | 964 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 965 if (error) 966 goto kmem_free; 967 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 968 0, ccb->ccb_dmamap->dm_mapsize, 969 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 970 } 971 972 ccb->ccb_done = nvme_pt_done; 973 ccb->ccb_cookie = pt; 974 975 pt->cmd.nsid = nsid; 976 if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) { 977 error = EIO; 978 goto out; 979 } 980 981 error = 0; 982 out: 983 if (buf != NULL) { 984 if (error == 0 && pt->is_read) 985 error = copyout(buf, pt->buf, pt->len); 986 kmem_free: 987 kmem_free(buf, pt->len); 988 } 989 ccb_put: 990 nvme_ccb_put(q, ccb); 991 return error; 992 } 993 994 static void 995 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 996 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 997 { 998 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 999 uint32_t tail; 1000 1001 mutex_enter(&q->q_sq_mtx); 1002 tail = q->q_sq_tail; 1003 if (++q->q_sq_tail >= q->q_entries) 1004 q->q_sq_tail = 0; 1005 1006 sqe += tail; 1007 1008 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1009 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 1010 memset(sqe, 0, sizeof(*sqe)); 1011 (*fill)(q, ccb, sqe); 1012 sqe->cid = ccb->ccb_id; 1013 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1014 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 1015 1016 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 1017 mutex_exit(&q->q_sq_mtx); 1018 } 1019 1020 struct nvme_poll_state { 1021 struct nvme_sqe s; 1022 struct nvme_cqe c; 1023 }; 1024 1025 static int 1026 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1027 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 1028 { 1029 struct nvme_poll_state state; 1030 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 1031 void *cookie; 1032 uint16_t flags; 1033 int step = 10; 1034 int maxloop = timo_sec * 1000000 / step; 1035 int error = 0; 1036 1037 memset(&state, 0, sizeof(state)); 1038 (*fill)(q, ccb, &state.s); 1039 1040 done = ccb->ccb_done; 1041 cookie = ccb->ccb_cookie; 1042 1043 ccb->ccb_done = nvme_poll_done; 1044 ccb->ccb_cookie = &state; 1045 1046 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1047 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1048 if (nvme_q_complete(sc, q) == 0) 1049 delay(step); 1050 1051 if (timo_sec >= 0 && --maxloop <= 0) { 1052 error = ETIMEDOUT; 1053 break; 1054 } 1055 } 1056 1057 ccb->ccb_cookie = cookie; 1058 done(q, ccb, &state.c); 1059 1060 if (error == 0) { 1061 flags = lemtoh16(&state.c.flags); 1062 return flags & ~NVME_CQE_PHASE; 1063 } else { 1064 return 1; 1065 } 1066 } 1067 1068 static void 1069 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1070 { 1071 struct nvme_sqe *sqe = slot; 1072 struct nvme_poll_state *state = ccb->ccb_cookie; 1073 1074 *sqe = state->s; 1075 } 1076 1077 static void 1078 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1079 struct nvme_cqe *cqe) 1080 { 1081 struct nvme_poll_state *state = ccb->ccb_cookie; 1082 1083 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1084 state->c = *cqe; 1085 } 1086 1087 static void 1088 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1089 { 1090 struct nvme_sqe *src = ccb->ccb_cookie; 1091 struct nvme_sqe *dst = slot; 1092 1093 *dst = *src; 1094 } 1095 1096 static void 1097 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1098 struct nvme_cqe *cqe) 1099 { 1100 } 1101 1102 static int 1103 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1104 { 1105 struct nvme_ccb *ccb; 1106 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1107 uint16_t flags; 1108 int rv = 0; 1109 1110 mutex_enter(&q->q_cq_mtx); 1111 1112 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1113 for (;;) { 1114 cqe = &ring[q->q_cq_head]; 1115 flags = lemtoh16(&cqe->flags); 1116 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1117 break; 1118 1119 ccb = &q->q_ccbs[cqe->cid]; 1120 1121 if (++q->q_cq_head >= q->q_entries) { 1122 q->q_cq_head = 0; 1123 q->q_cq_phase ^= NVME_CQE_PHASE; 1124 } 1125 1126 #ifdef DEBUG 1127 /* 1128 * If we get spurious completion notification, something 1129 * is seriously hosed up. Very likely DMA to some random 1130 * memory place happened, so just bail out. 1131 */ 1132 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1133 panic("%s: invalid ccb detected", 1134 device_xname(sc->sc_dev)); 1135 /* NOTREACHED */ 1136 } 1137 #endif 1138 1139 rv++; 1140 1141 /* 1142 * Unlock the mutex before calling the ccb_done callback 1143 * and re-lock afterwards. The callback triggers lddone() 1144 * which schedules another i/o, and also calls nvme_ccb_put(). 1145 * Unlock/relock avoids possibility of deadlock. 1146 */ 1147 mutex_exit(&q->q_cq_mtx); 1148 ccb->ccb_done(q, ccb, cqe); 1149 mutex_enter(&q->q_cq_mtx); 1150 } 1151 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1152 1153 if (rv) 1154 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1155 1156 mutex_exit(&q->q_cq_mtx); 1157 1158 if (rv) { 1159 mutex_enter(&q->q_ccb_mtx); 1160 q->q_nccbs_avail += rv; 1161 mutex_exit(&q->q_ccb_mtx); 1162 } 1163 1164 return rv; 1165 } 1166 1167 static int 1168 nvme_identify(struct nvme_softc *sc, u_int mps) 1169 { 1170 char sn[41], mn[81], fr[17]; 1171 struct nvm_identify_controller *identify; 1172 struct nvme_dmamem *mem; 1173 struct nvme_ccb *ccb; 1174 u_int mdts; 1175 int rv = 1; 1176 1177 ccb = nvme_ccb_get(sc->sc_admin_q); 1178 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1179 1180 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1181 if (mem == NULL) 1182 return 1; 1183 1184 ccb->ccb_done = nvme_empty_done; 1185 ccb->ccb_cookie = mem; 1186 1187 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1188 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1189 NVME_TIMO_IDENT); 1190 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1191 1192 nvme_ccb_put(sc->sc_admin_q, ccb); 1193 1194 if (rv != 0) 1195 goto done; 1196 1197 identify = NVME_DMA_KVA(mem); 1198 1199 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1200 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1201 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1202 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1203 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1204 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1205 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1206 sn); 1207 1208 if (identify->mdts > 0) { 1209 mdts = (1 << identify->mdts) * (1 << mps); 1210 if (mdts < sc->sc_mdts) 1211 sc->sc_mdts = mdts; 1212 } 1213 1214 sc->sc_nn = lemtoh32(&identify->nn); 1215 1216 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1217 1218 done: 1219 nvme_dmamem_free(sc, mem); 1220 1221 return rv; 1222 } 1223 1224 static int 1225 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1226 { 1227 struct nvme_sqe_q sqe; 1228 struct nvme_ccb *ccb; 1229 int rv; 1230 1231 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1232 return 1; 1233 1234 ccb = nvme_ccb_get(sc->sc_admin_q); 1235 KASSERT(ccb != NULL); 1236 1237 ccb->ccb_done = nvme_empty_done; 1238 ccb->ccb_cookie = &sqe; 1239 1240 memset(&sqe, 0, sizeof(sqe)); 1241 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1242 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1243 htolem16(&sqe.qsize, q->q_entries - 1); 1244 htolem16(&sqe.qid, q->q_id); 1245 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1246 if (sc->sc_use_mq) 1247 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1248 1249 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1250 if (rv != 0) 1251 goto fail; 1252 1253 ccb->ccb_done = nvme_empty_done; 1254 ccb->ccb_cookie = &sqe; 1255 1256 memset(&sqe, 0, sizeof(sqe)); 1257 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1258 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1259 htolem16(&sqe.qsize, q->q_entries - 1); 1260 htolem16(&sqe.qid, q->q_id); 1261 htolem16(&sqe.cqid, q->q_id); 1262 sqe.qflags = NVM_SQE_Q_PC; 1263 1264 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1265 if (rv != 0) 1266 goto fail; 1267 1268 fail: 1269 nvme_ccb_put(sc->sc_admin_q, ccb); 1270 return rv; 1271 } 1272 1273 static int 1274 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1275 { 1276 struct nvme_sqe_q sqe; 1277 struct nvme_ccb *ccb; 1278 int rv; 1279 1280 ccb = nvme_ccb_get(sc->sc_admin_q); 1281 KASSERT(ccb != NULL); 1282 1283 ccb->ccb_done = nvme_empty_done; 1284 ccb->ccb_cookie = &sqe; 1285 1286 memset(&sqe, 0, sizeof(sqe)); 1287 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1288 htolem16(&sqe.qid, q->q_id); 1289 1290 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1291 if (rv != 0) 1292 goto fail; 1293 1294 ccb->ccb_done = nvme_empty_done; 1295 ccb->ccb_cookie = &sqe; 1296 1297 memset(&sqe, 0, sizeof(sqe)); 1298 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1299 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1300 htolem16(&sqe.qid, q->q_id); 1301 1302 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1303 if (rv != 0) 1304 goto fail; 1305 1306 fail: 1307 nvme_ccb_put(sc->sc_admin_q, ccb); 1308 1309 if (rv == 0 && sc->sc_use_mq) { 1310 if (sc->sc_intr_disestablish(sc, q->q_id)) 1311 rv = 1; 1312 } 1313 1314 return rv; 1315 } 1316 1317 static void 1318 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1319 { 1320 struct nvme_sqe *sqe = slot; 1321 struct nvme_dmamem *mem = ccb->ccb_cookie; 1322 1323 sqe->opcode = NVM_ADMIN_IDENTIFY; 1324 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1325 htolem32(&sqe->cdw10, 1); 1326 } 1327 1328 static int 1329 nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap) 1330 { 1331 struct nvme_pt_command pt; 1332 struct nvme_ccb *ccb; 1333 uint16_t ncqa, nsqa; 1334 int rv; 1335 1336 ccb = nvme_ccb_get(sc->sc_admin_q); 1337 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1338 1339 memset(&pt, 0, sizeof(pt)); 1340 pt.cmd.opcode = NVM_ADMIN_GET_FEATURES; 1341 pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES; 1342 1343 ccb->ccb_done = nvme_pt_done; 1344 ccb->ccb_cookie = &pt; 1345 1346 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1347 1348 nvme_ccb_put(sc->sc_admin_q, ccb); 1349 1350 if (rv != 0) { 1351 *nqap = 0; 1352 return EIO; 1353 } 1354 1355 ncqa = pt.cpl.cdw0 >> 16; 1356 nsqa = pt.cpl.cdw0 & 0xffff; 1357 *nqap = MIN(ncqa, nsqa) + 1; 1358 1359 return 0; 1360 } 1361 1362 static int 1363 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1364 { 1365 struct nvme_softc *sc = q->q_sc; 1366 struct nvme_ccb *ccb; 1367 bus_addr_t off; 1368 uint64_t *prpl; 1369 u_int i; 1370 1371 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1372 SIMPLEQ_INIT(&q->q_ccb_list); 1373 1374 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1375 if (q->q_ccbs == NULL) 1376 return 1; 1377 1378 q->q_nccbs = nccbs; 1379 q->q_nccbs_avail = nccbs; 1380 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1381 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1382 1383 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1384 off = 0; 1385 1386 for (i = 0; i < nccbs; i++) { 1387 ccb = &q->q_ccbs[i]; 1388 1389 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1390 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1391 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1392 &ccb->ccb_dmamap) != 0) 1393 goto free_maps; 1394 1395 ccb->ccb_id = i; 1396 ccb->ccb_prpl = prpl; 1397 ccb->ccb_prpl_off = off; 1398 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1399 1400 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1401 1402 prpl += sc->sc_max_sgl; 1403 off += sizeof(*prpl) * sc->sc_max_sgl; 1404 } 1405 1406 return 0; 1407 1408 free_maps: 1409 nvme_ccbs_free(q); 1410 return 1; 1411 } 1412 1413 static struct nvme_ccb * 1414 nvme_ccb_get(struct nvme_queue *q) 1415 { 1416 struct nvme_ccb *ccb = NULL; 1417 1418 mutex_enter(&q->q_ccb_mtx); 1419 if (q->q_nccbs_avail > 0) { 1420 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1421 KASSERT(ccb != NULL); 1422 q->q_nccbs_avail--; 1423 1424 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1425 #ifdef DEBUG 1426 ccb->ccb_cookie = NULL; 1427 #endif 1428 } 1429 mutex_exit(&q->q_ccb_mtx); 1430 1431 return ccb; 1432 } 1433 1434 static void 1435 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1436 { 1437 1438 mutex_enter(&q->q_ccb_mtx); 1439 #ifdef DEBUG 1440 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1441 #endif 1442 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1443 mutex_exit(&q->q_ccb_mtx); 1444 } 1445 1446 static void 1447 nvme_ccbs_free(struct nvme_queue *q) 1448 { 1449 struct nvme_softc *sc = q->q_sc; 1450 struct nvme_ccb *ccb; 1451 1452 mutex_enter(&q->q_ccb_mtx); 1453 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1454 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1455 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1456 } 1457 mutex_exit(&q->q_ccb_mtx); 1458 1459 nvme_dmamem_free(sc, q->q_ccb_prpls); 1460 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1461 q->q_ccbs = NULL; 1462 mutex_destroy(&q->q_ccb_mtx); 1463 } 1464 1465 static struct nvme_queue * 1466 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1467 { 1468 struct nvme_queue *q; 1469 1470 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1471 if (q == NULL) 1472 return NULL; 1473 1474 q->q_sc = sc; 1475 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1476 sizeof(struct nvme_sqe) * entries); 1477 if (q->q_sq_dmamem == NULL) 1478 goto free; 1479 1480 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1481 sizeof(struct nvme_cqe) * entries); 1482 if (q->q_cq_dmamem == NULL) 1483 goto free_sq; 1484 1485 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1486 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1487 1488 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1489 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1490 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1491 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1492 q->q_id = id; 1493 q->q_entries = entries; 1494 q->q_sq_tail = 0; 1495 q->q_cq_head = 0; 1496 q->q_cq_phase = NVME_CQE_PHASE; 1497 1498 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1499 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1500 1501 /* 1502 * Due to definition of full and empty queue (queue is empty 1503 * when head == tail, full when tail is one less then head), 1504 * we can actually only have (entries - 1) in-flight commands. 1505 */ 1506 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1507 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1508 goto free_cq; 1509 } 1510 1511 return q; 1512 1513 free_cq: 1514 nvme_dmamem_free(sc, q->q_cq_dmamem); 1515 free_sq: 1516 nvme_dmamem_free(sc, q->q_sq_dmamem); 1517 free: 1518 kmem_free(q, sizeof(*q)); 1519 1520 return NULL; 1521 } 1522 1523 static void 1524 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1525 { 1526 nvme_ccbs_free(q); 1527 mutex_destroy(&q->q_sq_mtx); 1528 mutex_destroy(&q->q_cq_mtx); 1529 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1530 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1531 nvme_dmamem_free(sc, q->q_cq_dmamem); 1532 nvme_dmamem_free(sc, q->q_sq_dmamem); 1533 kmem_free(q, sizeof(*q)); 1534 } 1535 1536 int 1537 nvme_intr(void *xsc) 1538 { 1539 struct nvme_softc *sc = xsc; 1540 1541 /* 1542 * INTx is level triggered, controller deasserts the interrupt only 1543 * when we advance command queue head via write to the doorbell. 1544 * Tell the controller to block the interrupts while we process 1545 * the queue(s). 1546 */ 1547 nvme_write4(sc, NVME_INTMS, 1); 1548 1549 softint_schedule(sc->sc_softih[0]); 1550 1551 /* don't know, might not have been for us */ 1552 return 1; 1553 } 1554 1555 void 1556 nvme_softintr_intx(void *xq) 1557 { 1558 struct nvme_queue *q = xq; 1559 struct nvme_softc *sc = q->q_sc; 1560 1561 nvme_q_complete(sc, sc->sc_admin_q); 1562 if (sc->sc_q != NULL) 1563 nvme_q_complete(sc, sc->sc_q[0]); 1564 1565 /* 1566 * Processing done, tell controller to issue interrupts again. There 1567 * is no race, as NVMe spec requires the controller to maintain state, 1568 * and assert the interrupt whenever there are unacknowledged 1569 * completion queue entries. 1570 */ 1571 nvme_write4(sc, NVME_INTMC, 1); 1572 } 1573 1574 int 1575 nvme_intr_msi(void *xq) 1576 { 1577 struct nvme_queue *q = xq; 1578 1579 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1580 && q->q_sc->sc_softih[q->q_id]); 1581 1582 /* 1583 * MSI/MSI-X are edge triggered, so can handover processing to softint 1584 * without masking the interrupt. 1585 */ 1586 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1587 1588 return 1; 1589 } 1590 1591 void 1592 nvme_softintr_msi(void *xq) 1593 { 1594 struct nvme_queue *q = xq; 1595 struct nvme_softc *sc = q->q_sc; 1596 1597 nvme_q_complete(sc, q); 1598 } 1599 1600 static struct nvme_dmamem * 1601 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1602 { 1603 struct nvme_dmamem *ndm; 1604 int nsegs; 1605 1606 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1607 if (ndm == NULL) 1608 return NULL; 1609 1610 ndm->ndm_size = size; 1611 1612 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1613 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1614 goto ndmfree; 1615 1616 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1617 1, &nsegs, BUS_DMA_WAITOK) != 0) 1618 goto destroy; 1619 1620 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1621 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1622 goto free; 1623 memset(ndm->ndm_kva, 0, size); 1624 1625 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1626 NULL, BUS_DMA_WAITOK) != 0) 1627 goto unmap; 1628 1629 return ndm; 1630 1631 unmap: 1632 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1633 free: 1634 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1635 destroy: 1636 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1637 ndmfree: 1638 kmem_free(ndm, sizeof(*ndm)); 1639 return NULL; 1640 } 1641 1642 static void 1643 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1644 { 1645 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1646 0, NVME_DMA_LEN(mem), ops); 1647 } 1648 1649 void 1650 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1651 { 1652 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1653 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1654 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1655 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1656 kmem_free(ndm, sizeof(*ndm)); 1657 } 1658 1659 /* 1660 * ioctl 1661 */ 1662 1663 dev_type_open(nvmeopen); 1664 dev_type_close(nvmeclose); 1665 dev_type_ioctl(nvmeioctl); 1666 1667 const struct cdevsw nvme_cdevsw = { 1668 .d_open = nvmeopen, 1669 .d_close = nvmeclose, 1670 .d_read = noread, 1671 .d_write = nowrite, 1672 .d_ioctl = nvmeioctl, 1673 .d_stop = nostop, 1674 .d_tty = notty, 1675 .d_poll = nopoll, 1676 .d_mmap = nommap, 1677 .d_kqfilter = nokqfilter, 1678 .d_discard = nodiscard, 1679 .d_flag = D_OTHER, 1680 }; 1681 1682 extern struct cfdriver nvme_cd; 1683 1684 /* 1685 * Accept an open operation on the control device. 1686 */ 1687 int 1688 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1689 { 1690 struct nvme_softc *sc; 1691 int unit = minor(dev) / 0x10000; 1692 int nsid = minor(dev) & 0xffff; 1693 int nsidx; 1694 1695 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1696 return ENXIO; 1697 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1698 return ENXIO; 1699 1700 if (nsid == 0) { 1701 /* controller */ 1702 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1703 return EBUSY; 1704 SET(sc->sc_flags, NVME_F_OPEN); 1705 } else { 1706 /* namespace */ 1707 nsidx = nsid - 1; 1708 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1709 return ENXIO; 1710 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1711 return EBUSY; 1712 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1713 } 1714 return 0; 1715 } 1716 1717 /* 1718 * Accept the last close on the control device. 1719 */ 1720 int 1721 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1722 { 1723 struct nvme_softc *sc; 1724 int unit = minor(dev) / 0x10000; 1725 int nsid = minor(dev) & 0xffff; 1726 int nsidx; 1727 1728 sc = device_lookup_private(&nvme_cd, unit); 1729 if (sc == NULL) 1730 return ENXIO; 1731 1732 if (nsid == 0) { 1733 /* controller */ 1734 CLR(sc->sc_flags, NVME_F_OPEN); 1735 } else { 1736 /* namespace */ 1737 nsidx = nsid - 1; 1738 if (nsidx >= sc->sc_nn) 1739 return ENXIO; 1740 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1741 } 1742 1743 return 0; 1744 } 1745 1746 /* 1747 * Handle control operations. 1748 */ 1749 int 1750 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1751 { 1752 struct nvme_softc *sc; 1753 int unit = minor(dev) / 0x10000; 1754 int nsid = minor(dev) & 0xffff; 1755 struct nvme_pt_command *pt; 1756 1757 sc = device_lookup_private(&nvme_cd, unit); 1758 if (sc == NULL) 1759 return ENXIO; 1760 1761 switch (cmd) { 1762 case NVME_PASSTHROUGH_CMD: 1763 pt = data; 1764 return nvme_command_passthrough(sc, data, 1765 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1766 } 1767 1768 return ENOTTY; 1769 } 1770
Indexes created Mon Sep 29 03:10:08 GMT 2025