nvme.c revision 1.27
1 /* $NetBSD: nvme.c,v 1.27 2017/05/29 02:20:34 nonaka Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.27 2017/05/29 02:20:34 nonaka Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 int nvme_adminq_size = 32; 44 int nvme_ioq_size = 1024; 45 46 static int nvme_print(void *, const char *); 47 48 static int nvme_ready(struct nvme_softc *, uint32_t); 49 static int nvme_enable(struct nvme_softc *, u_int); 50 static int nvme_disable(struct nvme_softc *); 51 static int nvme_shutdown(struct nvme_softc *); 52 53 #ifdef NVME_DEBUG 54 static void nvme_dumpregs(struct nvme_softc *); 55 #endif 56 static int nvme_identify(struct nvme_softc *, u_int); 57 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 58 void *); 59 60 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 61 static void nvme_ccbs_free(struct nvme_queue *); 62 63 static struct nvme_ccb * 64 nvme_ccb_get(struct nvme_queue *); 65 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 66 67 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 68 struct nvme_ccb *, void (*)(struct nvme_queue *, 69 struct nvme_ccb *, void *), int); 70 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 71 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 72 struct nvme_cqe *); 73 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 74 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 75 struct nvme_cqe *); 76 77 static struct nvme_queue * 78 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 79 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 80 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 81 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 82 struct nvme_ccb *, void (*)(struct nvme_queue *, 83 struct nvme_ccb *, void *)); 84 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 85 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 86 87 static struct nvme_dmamem * 88 nvme_dmamem_alloc(struct nvme_softc *, size_t); 89 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 90 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 91 int); 92 93 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 94 void *); 95 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 96 struct nvme_cqe *); 97 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 98 void *); 99 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 100 struct nvme_cqe *); 101 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *, 102 void *); 103 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *, 104 struct nvme_cqe *); 105 106 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 107 void *); 108 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 109 struct nvme_cqe *); 110 static int nvme_command_passthrough(struct nvme_softc *, 111 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 112 113 static int nvme_get_number_of_queues(struct nvme_softc *, u_int *); 114 115 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 116 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 117 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 118 #define NVME_TIMO_SY 60 /* sync cache timeout */ 119 120 #define nvme_read4(_s, _r) \ 121 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 122 #define nvme_write4(_s, _r, _v) \ 123 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 124 #ifdef __LP64__ 125 #define nvme_read8(_s, _r) \ 126 bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r)) 127 #define nvme_write8(_s, _r, _v) \ 128 bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 129 #else /* __LP64__ */ 130 static inline uint64_t 131 nvme_read8(struct nvme_softc *sc, bus_size_t r) 132 { 133 uint64_t v; 134 uint32_t *a = (uint32_t *)&v; 135 136 #if _BYTE_ORDER == _LITTLE_ENDIAN 137 a[0] = nvme_read4(sc, r); 138 a[1] = nvme_read4(sc, r + 4); 139 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 140 a[1] = nvme_read4(sc, r); 141 a[0] = nvme_read4(sc, r + 4); 142 #endif 143 144 return v; 145 } 146 147 static inline void 148 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 149 { 150 uint32_t *a = (uint32_t *)&v; 151 152 #if _BYTE_ORDER == _LITTLE_ENDIAN 153 nvme_write4(sc, r, a[0]); 154 nvme_write4(sc, r + 4, a[1]); 155 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 156 nvme_write4(sc, r, a[1]); 157 nvme_write4(sc, r + 4, a[0]); 158 #endif 159 } 160 #endif /* __LP64__ */ 161 #define nvme_barrier(_s, _r, _l, _f) \ 162 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 163 164 #ifdef NVME_DEBUG 165 static __used void 166 nvme_dumpregs(struct nvme_softc *sc) 167 { 168 uint64_t r8; 169 uint32_t r4; 170 171 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 172 r8 = nvme_read8(sc, NVME_CAP); 173 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 174 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 175 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 176 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 177 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 178 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 179 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 180 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 181 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 182 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 183 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 184 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 185 186 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 187 188 r4 = nvme_read4(sc, NVME_CC); 189 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 190 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 191 (1 << NVME_CC_IOCQES_R(r4))); 192 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 193 (1 << NVME_CC_IOSQES_R(r4))); 194 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 195 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 196 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 197 (1 << NVME_CC_MPS_R(r4))); 198 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 199 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 200 201 r4 = nvme_read4(sc, NVME_CSTS); 202 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 203 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 204 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 205 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 206 207 r4 = nvme_read4(sc, NVME_AQA); 208 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 209 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 210 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 211 212 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 213 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 214 #undef DEVNAME 215 } 216 #endif /* NVME_DEBUG */ 217 218 static int 219 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 220 { 221 u_int i = 0; 222 uint32_t cc; 223 224 cc = nvme_read4(sc, NVME_CC); 225 if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) { 226 aprint_error_dev(sc->sc_dev, 227 "controller enabled status expected %d, found to be %d\n", 228 (rdy != 0), ((cc & NVME_CC_EN) != 0)); 229 return ENXIO; 230 } 231 232 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 233 if (i++ > sc->sc_rdy_to) 234 return ENXIO; 235 236 delay(1000); 237 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 238 } 239 240 return 0; 241 } 242 243 static int 244 nvme_enable(struct nvme_softc *sc, u_int mps) 245 { 246 uint32_t cc, csts; 247 248 cc = nvme_read4(sc, NVME_CC); 249 csts = nvme_read4(sc, NVME_CSTS); 250 251 if (ISSET(cc, NVME_CC_EN)) { 252 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 253 254 if (ISSET(csts, NVME_CSTS_RDY)) 255 return 1; 256 257 goto waitready; 258 } 259 260 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 261 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 262 delay(5000); 263 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 264 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 265 delay(5000); 266 267 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 268 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 269 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 270 delay(5000); 271 272 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 273 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 274 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 275 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 276 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 277 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 278 SET(cc, NVME_CC_MPS(mps)); 279 SET(cc, NVME_CC_EN); 280 281 nvme_write4(sc, NVME_CC, cc); 282 nvme_barrier(sc, 0, sc->sc_ios, 283 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 284 delay(5000); 285 286 waitready: 287 return nvme_ready(sc, NVME_CSTS_RDY); 288 } 289 290 static int 291 nvme_disable(struct nvme_softc *sc) 292 { 293 uint32_t cc, csts; 294 295 cc = nvme_read4(sc, NVME_CC); 296 csts = nvme_read4(sc, NVME_CSTS); 297 298 if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY)) 299 nvme_ready(sc, NVME_CSTS_RDY); 300 301 CLR(cc, NVME_CC_EN); 302 303 nvme_write4(sc, NVME_CC, cc); 304 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 305 306 delay(5000); 307 308 return nvme_ready(sc, 0); 309 } 310 311 int 312 nvme_attach(struct nvme_softc *sc) 313 { 314 uint64_t cap; 315 uint32_t reg; 316 u_int dstrd; 317 u_int mps = PAGE_SHIFT; 318 u_int ioq_allocated; 319 uint16_t adminq_entries = nvme_adminq_size; 320 uint16_t ioq_entries = nvme_ioq_size; 321 int i; 322 323 reg = nvme_read4(sc, NVME_VS); 324 if (reg == 0xffffffff) { 325 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 326 return 1; 327 } 328 329 if (NVME_VS_TER(reg) == 0) 330 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg), 331 NVME_VS_MNR(reg)); 332 else 333 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg), 334 NVME_VS_MNR(reg), NVME_VS_TER(reg)); 335 336 cap = nvme_read8(sc, NVME_CAP); 337 dstrd = NVME_CAP_DSTRD(cap); 338 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 339 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 340 "is greater than CPU page size %u\n", 341 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 342 return 1; 343 } 344 if (NVME_CAP_MPSMAX(cap) < mps) 345 mps = NVME_CAP_MPSMAX(cap); 346 if (ioq_entries > NVME_CAP_MQES(cap)) 347 ioq_entries = NVME_CAP_MQES(cap); 348 349 /* set initial values to be used for admin queue during probe */ 350 sc->sc_rdy_to = NVME_CAP_TO(cap); 351 sc->sc_mps = 1 << mps; 352 sc->sc_mdts = MAXPHYS; 353 sc->sc_max_sgl = 2; 354 355 if (nvme_disable(sc) != 0) { 356 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 357 return 1; 358 } 359 360 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 361 if (sc->sc_admin_q == NULL) { 362 aprint_error_dev(sc->sc_dev, 363 "unable to allocate admin queue\n"); 364 return 1; 365 } 366 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 367 goto free_admin_q; 368 369 if (nvme_enable(sc, mps) != 0) { 370 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 371 goto disestablish_admin_q; 372 } 373 374 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 375 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 376 goto disable; 377 } 378 379 /* we know how big things are now */ 380 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 381 382 /* reallocate ccbs of admin queue with new max sgl. */ 383 nvme_ccbs_free(sc->sc_admin_q); 384 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 385 386 if (sc->sc_use_mq) { 387 /* Limit the number of queues to the number allocated in HW */ 388 if (nvme_get_number_of_queues(sc, &ioq_allocated) != 0) { 389 aprint_error_dev(sc->sc_dev, 390 "unable to get number of queues\n"); 391 goto disable; 392 } 393 if (sc->sc_nq > ioq_allocated) 394 sc->sc_nq = ioq_allocated; 395 } 396 397 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 398 if (sc->sc_q == NULL) { 399 aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n"); 400 goto disable; 401 } 402 for (i = 0; i < sc->sc_nq; i++) { 403 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 404 if (sc->sc_q[i] == NULL) { 405 aprint_error_dev(sc->sc_dev, 406 "unable to allocate io queue\n"); 407 goto free_q; 408 } 409 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 410 aprint_error_dev(sc->sc_dev, 411 "unable to create io queue\n"); 412 nvme_q_free(sc, sc->sc_q[i]); 413 goto free_q; 414 } 415 } 416 417 if (!sc->sc_use_mq) 418 nvme_write4(sc, NVME_INTMC, 1); 419 420 /* probe subdevices */ 421 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 422 KM_SLEEP); 423 if (sc->sc_namespaces == NULL) 424 goto free_q; 425 nvme_rescan(sc->sc_dev, "nvme", &i); 426 427 return 0; 428 429 free_q: 430 while (--i >= 0) { 431 nvme_q_delete(sc, sc->sc_q[i]); 432 nvme_q_free(sc, sc->sc_q[i]); 433 } 434 disable: 435 nvme_disable(sc); 436 disestablish_admin_q: 437 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 438 free_admin_q: 439 nvme_q_free(sc, sc->sc_admin_q); 440 441 return 1; 442 } 443 444 int 445 nvme_rescan(device_t self, const char *attr, const int *flags) 446 { 447 struct nvme_softc *sc = device_private(self); 448 struct nvme_attach_args naa; 449 uint64_t cap; 450 int ioq_entries = nvme_ioq_size; 451 int i; 452 453 cap = nvme_read8(sc, NVME_CAP); 454 if (ioq_entries > NVME_CAP_MQES(cap)) 455 ioq_entries = NVME_CAP_MQES(cap); 456 457 for (i = 0; i < sc->sc_nn; i++) { 458 if (sc->sc_namespaces[i].dev) 459 continue; 460 memset(&naa, 0, sizeof(naa)); 461 naa.naa_nsid = i + 1; 462 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 463 naa.naa_maxphys = sc->sc_mdts; 464 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 465 nvme_print); 466 } 467 return 0; 468 } 469 470 static int 471 nvme_print(void *aux, const char *pnp) 472 { 473 struct nvme_attach_args *naa = aux; 474 475 if (pnp) 476 aprint_normal("at %s", pnp); 477 478 if (naa->naa_nsid > 0) 479 aprint_normal(" nsid %d", naa->naa_nsid); 480 481 return UNCONF; 482 } 483 484 int 485 nvme_detach(struct nvme_softc *sc, int flags) 486 { 487 int i, error; 488 489 error = config_detach_children(sc->sc_dev, flags); 490 if (error) 491 return error; 492 493 error = nvme_shutdown(sc); 494 if (error) 495 return error; 496 497 /* from now on we are committed to detach, following will never fail */ 498 for (i = 0; i < sc->sc_nq; i++) 499 nvme_q_free(sc, sc->sc_q[i]); 500 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 501 nvme_q_free(sc, sc->sc_admin_q); 502 503 return 0; 504 } 505 506 static int 507 nvme_shutdown(struct nvme_softc *sc) 508 { 509 uint32_t cc, csts; 510 bool disabled = false; 511 int i; 512 513 if (!sc->sc_use_mq) 514 nvme_write4(sc, NVME_INTMS, 1); 515 516 for (i = 0; i < sc->sc_nq; i++) { 517 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 518 aprint_error_dev(sc->sc_dev, 519 "unable to delete io queue %d, disabling\n", i + 1); 520 disabled = true; 521 } 522 } 523 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 524 if (disabled) 525 goto disable; 526 527 cc = nvme_read4(sc, NVME_CC); 528 CLR(cc, NVME_CC_SHN_MASK); 529 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 530 nvme_write4(sc, NVME_CC, cc); 531 532 for (i = 0; i < 4000; i++) { 533 nvme_barrier(sc, 0, sc->sc_ios, 534 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 535 csts = nvme_read4(sc, NVME_CSTS); 536 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 537 return 0; 538 539 delay(1000); 540 } 541 542 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 543 544 disable: 545 nvme_disable(sc); 546 return 0; 547 } 548 549 void 550 nvme_childdet(device_t self, device_t child) 551 { 552 struct nvme_softc *sc = device_private(self); 553 int i; 554 555 for (i = 0; i < sc->sc_nn; i++) { 556 if (sc->sc_namespaces[i].dev == child) { 557 /* Already freed ns->ident. */ 558 sc->sc_namespaces[i].dev = NULL; 559 break; 560 } 561 } 562 } 563 564 int 565 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 566 { 567 struct nvme_sqe sqe; 568 struct nvm_identify_namespace *identify; 569 struct nvme_dmamem *mem; 570 struct nvme_ccb *ccb; 571 struct nvme_namespace *ns; 572 int rv; 573 574 KASSERT(nsid > 0); 575 576 ccb = nvme_ccb_get(sc->sc_admin_q); 577 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 578 579 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 580 if (mem == NULL) 581 return ENOMEM; 582 583 memset(&sqe, 0, sizeof(sqe)); 584 sqe.opcode = NVM_ADMIN_IDENTIFY; 585 htolem32(&sqe.nsid, nsid); 586 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 587 htolem32(&sqe.cdw10, 0); 588 589 ccb->ccb_done = nvme_empty_done; 590 ccb->ccb_cookie = &sqe; 591 592 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 593 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 594 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 595 596 nvme_ccb_put(sc->sc_admin_q, ccb); 597 598 if (rv != 0) { 599 rv = EIO; 600 goto done; 601 } 602 603 /* commit */ 604 605 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 606 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 607 //memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 608 609 ns = nvme_ns_get(sc, nsid); 610 KASSERT(ns); 611 ns->ident = identify; 612 613 done: 614 nvme_dmamem_free(sc, mem); 615 616 return rv; 617 } 618 619 int 620 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 621 struct buf *bp, void *data, size_t datasize, 622 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 623 { 624 struct nvme_queue *q = nvme_get_q(sc); 625 struct nvme_ccb *ccb; 626 bus_dmamap_t dmap; 627 int i, error; 628 629 ccb = nvme_ccb_get(q); 630 if (ccb == NULL) 631 return EAGAIN; 632 633 ccb->ccb_done = nvme_ns_io_done; 634 ccb->ccb_cookie = cookie; 635 636 /* namespace context */ 637 ccb->nnc_nsid = nsid; 638 ccb->nnc_flags = flags; 639 ccb->nnc_buf = bp; 640 ccb->nnc_datasize = datasize; 641 ccb->nnc_secsize = secsize; 642 ccb->nnc_blkno = blkno; 643 ccb->nnc_done = nnc_done; 644 645 dmap = ccb->ccb_dmamap; 646 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 647 datasize, NULL, 648 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 649 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 650 (ISSET(flags, NVME_NS_CTX_F_READ) ? 651 BUS_DMA_READ : BUS_DMA_WRITE)); 652 if (error) { 653 nvme_ccb_put(q, ccb); 654 return error; 655 } 656 657 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 658 ISSET(flags, NVME_NS_CTX_F_READ) ? 659 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 660 661 if (dmap->dm_nsegs > 2) { 662 for (i = 1; i < dmap->dm_nsegs; i++) { 663 htolem64(&ccb->ccb_prpl[i - 1], 664 dmap->dm_segs[i].ds_addr); 665 } 666 bus_dmamap_sync(sc->sc_dmat, 667 NVME_DMA_MAP(q->q_ccb_prpls), 668 ccb->ccb_prpl_off, 669 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 670 BUS_DMASYNC_PREWRITE); 671 } 672 673 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 674 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 675 return EIO; 676 return 0; 677 } 678 679 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 680 return 0; 681 } 682 683 static void 684 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 685 { 686 struct nvme_sqe_io *sqe = slot; 687 bus_dmamap_t dmap = ccb->ccb_dmamap; 688 689 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 690 NVM_CMD_READ : NVM_CMD_WRITE; 691 htolem32(&sqe->nsid, ccb->nnc_nsid); 692 693 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 694 switch (dmap->dm_nsegs) { 695 case 1: 696 break; 697 case 2: 698 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 699 break; 700 default: 701 /* the prp list is already set up and synced */ 702 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 703 break; 704 } 705 706 htolem64(&sqe->slba, ccb->nnc_blkno); 707 708 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) 709 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA); 710 711 /* guaranteed by upper layers, but check just in case */ 712 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 713 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 714 } 715 716 static void 717 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 718 struct nvme_cqe *cqe) 719 { 720 struct nvme_softc *sc = q->q_sc; 721 bus_dmamap_t dmap = ccb->ccb_dmamap; 722 void *nnc_cookie = ccb->ccb_cookie; 723 nvme_nnc_done nnc_done = ccb->nnc_done; 724 struct buf *bp = ccb->nnc_buf; 725 726 if (dmap->dm_nsegs > 2) { 727 bus_dmamap_sync(sc->sc_dmat, 728 NVME_DMA_MAP(q->q_ccb_prpls), 729 ccb->ccb_prpl_off, 730 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 731 BUS_DMASYNC_POSTWRITE); 732 } 733 734 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 735 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 736 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 737 738 bus_dmamap_unload(sc->sc_dmat, dmap); 739 nvme_ccb_put(q, ccb); 740 741 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 742 } 743 744 /* 745 * If there is no volatile write cache, it makes no sense to issue 746 * flush commands or query for the status. 747 */ 748 bool 749 nvme_has_volatile_write_cache(struct nvme_softc *sc) 750 { 751 /* sc_identify is filled during attachment */ 752 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0); 753 } 754 755 int 756 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie, 757 int flags, nvme_nnc_done nnc_done) 758 { 759 struct nvme_queue *q = nvme_get_q(sc); 760 struct nvme_ccb *ccb; 761 762 ccb = nvme_ccb_get(q); 763 if (ccb == NULL) 764 return EAGAIN; 765 766 ccb->ccb_done = nvme_ns_sync_done; 767 ccb->ccb_cookie = cookie; 768 769 /* namespace context */ 770 ccb->nnc_nsid = nsid; 771 ccb->nnc_flags = flags; 772 ccb->nnc_done = nnc_done; 773 774 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 775 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 776 return EIO; 777 return 0; 778 } 779 780 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 781 return 0; 782 } 783 784 static void 785 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 786 { 787 struct nvme_sqe *sqe = slot; 788 789 sqe->opcode = NVM_CMD_FLUSH; 790 htolem32(&sqe->nsid, ccb->nnc_nsid); 791 } 792 793 static void 794 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 795 struct nvme_cqe *cqe) 796 { 797 void *cookie = ccb->ccb_cookie; 798 nvme_nnc_done nnc_done = ccb->nnc_done; 799 800 nvme_ccb_put(q, ccb); 801 802 nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 803 } 804 805 /* 806 * Get status of volatile write cache. Always asynchronous. 807 */ 808 int 809 nvme_admin_getcache(struct nvme_softc *sc, void *cookie, nvme_nnc_done nnc_done) 810 { 811 struct nvme_ccb *ccb; 812 struct nvme_queue *q = sc->sc_admin_q; 813 814 ccb = nvme_ccb_get(q); 815 if (ccb == NULL) 816 return EAGAIN; 817 818 ccb->ccb_done = nvme_getcache_done; 819 ccb->ccb_cookie = cookie; 820 821 /* namespace context */ 822 ccb->nnc_flags = 0; 823 ccb->nnc_done = nnc_done; 824 825 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); 826 return 0; 827 } 828 829 static void 830 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 831 { 832 struct nvme_sqe *sqe = slot; 833 834 sqe->opcode = NVM_ADMIN_GET_FEATURES; 835 sqe->cdw10 = NVM_FEATURE_VOLATILE_WRITE_CACHE; 836 } 837 838 static void 839 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 840 struct nvme_cqe *cqe) 841 { 842 void *cookie = ccb->ccb_cookie; 843 nvme_nnc_done nnc_done = ccb->nnc_done; 844 845 nvme_ccb_put(q, ccb); 846 847 nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 848 } 849 850 void 851 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 852 { 853 struct nvme_namespace *ns; 854 struct nvm_identify_namespace *identify; 855 856 ns = nvme_ns_get(sc, nsid); 857 KASSERT(ns); 858 859 identify = ns->ident; 860 ns->ident = NULL; 861 if (identify != NULL) 862 kmem_free(identify, sizeof(*identify)); 863 } 864 865 static void 866 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 867 { 868 struct nvme_softc *sc = q->q_sc; 869 struct nvme_sqe *sqe = slot; 870 struct nvme_pt_command *pt = ccb->ccb_cookie; 871 bus_dmamap_t dmap = ccb->ccb_dmamap; 872 int i; 873 874 sqe->opcode = pt->cmd.opcode; 875 htolem32(&sqe->nsid, pt->cmd.nsid); 876 877 if (pt->buf != NULL && pt->len > 0) { 878 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 879 switch (dmap->dm_nsegs) { 880 case 1: 881 break; 882 case 2: 883 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 884 break; 885 default: 886 for (i = 1; i < dmap->dm_nsegs; i++) { 887 htolem64(&ccb->ccb_prpl[i - 1], 888 dmap->dm_segs[i].ds_addr); 889 } 890 bus_dmamap_sync(sc->sc_dmat, 891 NVME_DMA_MAP(q->q_ccb_prpls), 892 ccb->ccb_prpl_off, 893 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 894 BUS_DMASYNC_PREWRITE); 895 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 896 break; 897 } 898 } 899 900 htolem32(&sqe->cdw10, pt->cmd.cdw10); 901 htolem32(&sqe->cdw11, pt->cmd.cdw11); 902 htolem32(&sqe->cdw12, pt->cmd.cdw12); 903 htolem32(&sqe->cdw13, pt->cmd.cdw13); 904 htolem32(&sqe->cdw14, pt->cmd.cdw14); 905 htolem32(&sqe->cdw15, pt->cmd.cdw15); 906 } 907 908 static void 909 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 910 { 911 struct nvme_softc *sc = q->q_sc; 912 struct nvme_pt_command *pt = ccb->ccb_cookie; 913 bus_dmamap_t dmap = ccb->ccb_dmamap; 914 915 if (pt->buf != NULL && pt->len > 0) { 916 if (dmap->dm_nsegs > 2) { 917 bus_dmamap_sync(sc->sc_dmat, 918 NVME_DMA_MAP(q->q_ccb_prpls), 919 ccb->ccb_prpl_off, 920 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 921 BUS_DMASYNC_POSTWRITE); 922 } 923 924 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 925 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 926 bus_dmamap_unload(sc->sc_dmat, dmap); 927 } 928 929 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 930 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 931 } 932 933 static int 934 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 935 uint16_t nsid, struct lwp *l, bool is_adminq) 936 { 937 struct nvme_queue *q; 938 struct nvme_ccb *ccb; 939 void *buf = NULL; 940 int error; 941 942 /* limit command size to maximum data transfer size */ 943 if ((pt->buf == NULL && pt->len > 0) || 944 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 945 return EINVAL; 946 947 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 948 ccb = nvme_ccb_get(q); 949 if (ccb == NULL) 950 return EBUSY; 951 952 if (pt->buf != NULL) { 953 KASSERT(pt->len > 0); 954 buf = kmem_alloc(pt->len, KM_SLEEP); 955 if (buf == NULL) { 956 error = ENOMEM; 957 goto ccb_put; 958 } 959 if (!pt->is_read) { 960 error = copyin(pt->buf, buf, pt->len); 961 if (error) 962 goto kmem_free; 963 } 964 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 965 pt->len, NULL, 966 BUS_DMA_WAITOK | 967 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 968 if (error) 969 goto kmem_free; 970 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 971 0, ccb->ccb_dmamap->dm_mapsize, 972 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 973 } 974 975 ccb->ccb_done = nvme_pt_done; 976 ccb->ccb_cookie = pt; 977 978 pt->cmd.nsid = nsid; 979 if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) { 980 error = EIO; 981 goto out; 982 } 983 984 error = 0; 985 out: 986 if (buf != NULL) { 987 if (error == 0 && pt->is_read) 988 error = copyout(buf, pt->buf, pt->len); 989 kmem_free: 990 kmem_free(buf, pt->len); 991 } 992 ccb_put: 993 nvme_ccb_put(q, ccb); 994 return error; 995 } 996 997 static void 998 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 999 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 1000 { 1001 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 1002 uint32_t tail; 1003 1004 mutex_enter(&q->q_sq_mtx); 1005 tail = q->q_sq_tail; 1006 if (++q->q_sq_tail >= q->q_entries) 1007 q->q_sq_tail = 0; 1008 1009 sqe += tail; 1010 1011 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1012 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 1013 memset(sqe, 0, sizeof(*sqe)); 1014 (*fill)(q, ccb, sqe); 1015 sqe->cid = ccb->ccb_id; 1016 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1017 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 1018 1019 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 1020 mutex_exit(&q->q_sq_mtx); 1021 } 1022 1023 struct nvme_poll_state { 1024 struct nvme_sqe s; 1025 struct nvme_cqe c; 1026 }; 1027 1028 static int 1029 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1030 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 1031 { 1032 struct nvme_poll_state state; 1033 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 1034 void *cookie; 1035 uint16_t flags; 1036 int step = 10; 1037 int maxloop = timo_sec * 1000000 / step; 1038 int error = 0; 1039 1040 memset(&state, 0, sizeof(state)); 1041 (*fill)(q, ccb, &state.s); 1042 1043 done = ccb->ccb_done; 1044 cookie = ccb->ccb_cookie; 1045 1046 ccb->ccb_done = nvme_poll_done; 1047 ccb->ccb_cookie = &state; 1048 1049 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1050 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1051 if (nvme_q_complete(sc, q) == 0) 1052 delay(step); 1053 1054 if (timo_sec >= 0 && --maxloop <= 0) { 1055 error = ETIMEDOUT; 1056 break; 1057 } 1058 } 1059 1060 ccb->ccb_cookie = cookie; 1061 done(q, ccb, &state.c); 1062 1063 if (error == 0) { 1064 flags = lemtoh16(&state.c.flags); 1065 return flags & ~NVME_CQE_PHASE; 1066 } else { 1067 return 1; 1068 } 1069 } 1070 1071 static void 1072 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1073 { 1074 struct nvme_sqe *sqe = slot; 1075 struct nvme_poll_state *state = ccb->ccb_cookie; 1076 1077 *sqe = state->s; 1078 } 1079 1080 static void 1081 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1082 struct nvme_cqe *cqe) 1083 { 1084 struct nvme_poll_state *state = ccb->ccb_cookie; 1085 1086 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1087 state->c = *cqe; 1088 } 1089 1090 static void 1091 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1092 { 1093 struct nvme_sqe *src = ccb->ccb_cookie; 1094 struct nvme_sqe *dst = slot; 1095 1096 *dst = *src; 1097 } 1098 1099 static void 1100 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1101 struct nvme_cqe *cqe) 1102 { 1103 } 1104 1105 static int 1106 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1107 { 1108 struct nvme_ccb *ccb; 1109 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1110 uint16_t flags; 1111 int rv = 0; 1112 1113 mutex_enter(&q->q_cq_mtx); 1114 1115 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1116 for (;;) { 1117 cqe = &ring[q->q_cq_head]; 1118 flags = lemtoh16(&cqe->flags); 1119 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1120 break; 1121 1122 ccb = &q->q_ccbs[cqe->cid]; 1123 1124 if (++q->q_cq_head >= q->q_entries) { 1125 q->q_cq_head = 0; 1126 q->q_cq_phase ^= NVME_CQE_PHASE; 1127 } 1128 1129 #ifdef DEBUG 1130 /* 1131 * If we get spurious completion notification, something 1132 * is seriously hosed up. Very likely DMA to some random 1133 * memory place happened, so just bail out. 1134 */ 1135 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1136 panic("%s: invalid ccb detected", 1137 device_xname(sc->sc_dev)); 1138 /* NOTREACHED */ 1139 } 1140 #endif 1141 1142 rv++; 1143 1144 /* 1145 * Unlock the mutex before calling the ccb_done callback 1146 * and re-lock afterwards. The callback triggers lddone() 1147 * which schedules another i/o, and also calls nvme_ccb_put(). 1148 * Unlock/relock avoids possibility of deadlock. 1149 */ 1150 mutex_exit(&q->q_cq_mtx); 1151 ccb->ccb_done(q, ccb, cqe); 1152 mutex_enter(&q->q_cq_mtx); 1153 } 1154 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1155 1156 if (rv) 1157 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1158 1159 mutex_exit(&q->q_cq_mtx); 1160 1161 if (rv) { 1162 mutex_enter(&q->q_ccb_mtx); 1163 q->q_nccbs_avail += rv; 1164 mutex_exit(&q->q_ccb_mtx); 1165 } 1166 1167 return rv; 1168 } 1169 1170 static int 1171 nvme_identify(struct nvme_softc *sc, u_int mps) 1172 { 1173 char sn[41], mn[81], fr[17]; 1174 struct nvm_identify_controller *identify; 1175 struct nvme_dmamem *mem; 1176 struct nvme_ccb *ccb; 1177 u_int mdts; 1178 int rv = 1; 1179 1180 ccb = nvme_ccb_get(sc->sc_admin_q); 1181 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1182 1183 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1184 if (mem == NULL) 1185 return 1; 1186 1187 ccb->ccb_done = nvme_empty_done; 1188 ccb->ccb_cookie = mem; 1189 1190 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1191 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1192 NVME_TIMO_IDENT); 1193 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1194 1195 nvme_ccb_put(sc->sc_admin_q, ccb); 1196 1197 if (rv != 0) 1198 goto done; 1199 1200 identify = NVME_DMA_KVA(mem); 1201 1202 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1203 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1204 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1205 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1206 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1207 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1208 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1209 sn); 1210 1211 if (identify->mdts > 0) { 1212 mdts = (1 << identify->mdts) * (1 << mps); 1213 if (mdts < sc->sc_mdts) 1214 sc->sc_mdts = mdts; 1215 } 1216 1217 sc->sc_nn = lemtoh32(&identify->nn); 1218 1219 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1220 1221 done: 1222 nvme_dmamem_free(sc, mem); 1223 1224 return rv; 1225 } 1226 1227 static int 1228 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1229 { 1230 struct nvme_sqe_q sqe; 1231 struct nvme_ccb *ccb; 1232 int rv; 1233 1234 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1235 return 1; 1236 1237 ccb = nvme_ccb_get(sc->sc_admin_q); 1238 KASSERT(ccb != NULL); 1239 1240 ccb->ccb_done = nvme_empty_done; 1241 ccb->ccb_cookie = &sqe; 1242 1243 memset(&sqe, 0, sizeof(sqe)); 1244 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1245 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1246 htolem16(&sqe.qsize, q->q_entries - 1); 1247 htolem16(&sqe.qid, q->q_id); 1248 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1249 if (sc->sc_use_mq) 1250 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1251 1252 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1253 if (rv != 0) 1254 goto fail; 1255 1256 ccb->ccb_done = nvme_empty_done; 1257 ccb->ccb_cookie = &sqe; 1258 1259 memset(&sqe, 0, sizeof(sqe)); 1260 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1261 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1262 htolem16(&sqe.qsize, q->q_entries - 1); 1263 htolem16(&sqe.qid, q->q_id); 1264 htolem16(&sqe.cqid, q->q_id); 1265 sqe.qflags = NVM_SQE_Q_PC; 1266 1267 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1268 if (rv != 0) 1269 goto fail; 1270 1271 fail: 1272 nvme_ccb_put(sc->sc_admin_q, ccb); 1273 return rv; 1274 } 1275 1276 static int 1277 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1278 { 1279 struct nvme_sqe_q sqe; 1280 struct nvme_ccb *ccb; 1281 int rv; 1282 1283 ccb = nvme_ccb_get(sc->sc_admin_q); 1284 KASSERT(ccb != NULL); 1285 1286 ccb->ccb_done = nvme_empty_done; 1287 ccb->ccb_cookie = &sqe; 1288 1289 memset(&sqe, 0, sizeof(sqe)); 1290 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1291 htolem16(&sqe.qid, q->q_id); 1292 1293 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1294 if (rv != 0) 1295 goto fail; 1296 1297 ccb->ccb_done = nvme_empty_done; 1298 ccb->ccb_cookie = &sqe; 1299 1300 memset(&sqe, 0, sizeof(sqe)); 1301 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1302 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1303 htolem16(&sqe.qid, q->q_id); 1304 1305 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1306 if (rv != 0) 1307 goto fail; 1308 1309 fail: 1310 nvme_ccb_put(sc->sc_admin_q, ccb); 1311 1312 if (rv == 0 && sc->sc_use_mq) { 1313 if (sc->sc_intr_disestablish(sc, q->q_id)) 1314 rv = 1; 1315 } 1316 1317 return rv; 1318 } 1319 1320 static void 1321 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1322 { 1323 struct nvme_sqe *sqe = slot; 1324 struct nvme_dmamem *mem = ccb->ccb_cookie; 1325 1326 sqe->opcode = NVM_ADMIN_IDENTIFY; 1327 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1328 htolem32(&sqe->cdw10, 1); 1329 } 1330 1331 static int 1332 nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap) 1333 { 1334 struct nvme_pt_command pt; 1335 struct nvme_ccb *ccb; 1336 uint16_t ncqa, nsqa; 1337 int rv; 1338 1339 ccb = nvme_ccb_get(sc->sc_admin_q); 1340 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1341 1342 memset(&pt, 0, sizeof(pt)); 1343 pt.cmd.opcode = NVM_ADMIN_GET_FEATURES; 1344 pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES; 1345 1346 ccb->ccb_done = nvme_pt_done; 1347 ccb->ccb_cookie = &pt; 1348 1349 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1350 1351 nvme_ccb_put(sc->sc_admin_q, ccb); 1352 1353 if (rv != 0) { 1354 *nqap = 0; 1355 return EIO; 1356 } 1357 1358 ncqa = pt.cpl.cdw0 >> 16; 1359 nsqa = pt.cpl.cdw0 & 0xffff; 1360 *nqap = MIN(ncqa, nsqa) + 1; 1361 1362 return 0; 1363 } 1364 1365 static int 1366 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1367 { 1368 struct nvme_softc *sc = q->q_sc; 1369 struct nvme_ccb *ccb; 1370 bus_addr_t off; 1371 uint64_t *prpl; 1372 u_int i; 1373 1374 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1375 SIMPLEQ_INIT(&q->q_ccb_list); 1376 1377 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1378 if (q->q_ccbs == NULL) 1379 return 1; 1380 1381 q->q_nccbs = nccbs; 1382 q->q_nccbs_avail = nccbs; 1383 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1384 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1385 1386 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1387 off = 0; 1388 1389 for (i = 0; i < nccbs; i++) { 1390 ccb = &q->q_ccbs[i]; 1391 1392 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1393 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1394 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1395 &ccb->ccb_dmamap) != 0) 1396 goto free_maps; 1397 1398 ccb->ccb_id = i; 1399 ccb->ccb_prpl = prpl; 1400 ccb->ccb_prpl_off = off; 1401 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1402 1403 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1404 1405 prpl += sc->sc_max_sgl; 1406 off += sizeof(*prpl) * sc->sc_max_sgl; 1407 } 1408 1409 return 0; 1410 1411 free_maps: 1412 nvme_ccbs_free(q); 1413 return 1; 1414 } 1415 1416 static struct nvme_ccb * 1417 nvme_ccb_get(struct nvme_queue *q) 1418 { 1419 struct nvme_ccb *ccb = NULL; 1420 1421 mutex_enter(&q->q_ccb_mtx); 1422 if (q->q_nccbs_avail > 0) { 1423 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1424 KASSERT(ccb != NULL); 1425 q->q_nccbs_avail--; 1426 1427 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1428 #ifdef DEBUG 1429 ccb->ccb_cookie = NULL; 1430 #endif 1431 } 1432 mutex_exit(&q->q_ccb_mtx); 1433 1434 return ccb; 1435 } 1436 1437 static void 1438 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1439 { 1440 1441 mutex_enter(&q->q_ccb_mtx); 1442 #ifdef DEBUG 1443 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1444 #endif 1445 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1446 mutex_exit(&q->q_ccb_mtx); 1447 } 1448 1449 static void 1450 nvme_ccbs_free(struct nvme_queue *q) 1451 { 1452 struct nvme_softc *sc = q->q_sc; 1453 struct nvme_ccb *ccb; 1454 1455 mutex_enter(&q->q_ccb_mtx); 1456 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1457 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1458 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1459 } 1460 mutex_exit(&q->q_ccb_mtx); 1461 1462 nvme_dmamem_free(sc, q->q_ccb_prpls); 1463 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1464 q->q_ccbs = NULL; 1465 mutex_destroy(&q->q_ccb_mtx); 1466 } 1467 1468 static struct nvme_queue * 1469 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1470 { 1471 struct nvme_queue *q; 1472 1473 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1474 if (q == NULL) 1475 return NULL; 1476 1477 q->q_sc = sc; 1478 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1479 sizeof(struct nvme_sqe) * entries); 1480 if (q->q_sq_dmamem == NULL) 1481 goto free; 1482 1483 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1484 sizeof(struct nvme_cqe) * entries); 1485 if (q->q_cq_dmamem == NULL) 1486 goto free_sq; 1487 1488 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1489 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1490 1491 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1492 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1493 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1494 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1495 q->q_id = id; 1496 q->q_entries = entries; 1497 q->q_sq_tail = 0; 1498 q->q_cq_head = 0; 1499 q->q_cq_phase = NVME_CQE_PHASE; 1500 1501 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1502 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1503 1504 /* 1505 * Due to definition of full and empty queue (queue is empty 1506 * when head == tail, full when tail is one less then head), 1507 * we can actually only have (entries - 1) in-flight commands. 1508 */ 1509 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1510 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1511 goto free_cq; 1512 } 1513 1514 return q; 1515 1516 free_cq: 1517 nvme_dmamem_free(sc, q->q_cq_dmamem); 1518 free_sq: 1519 nvme_dmamem_free(sc, q->q_sq_dmamem); 1520 free: 1521 kmem_free(q, sizeof(*q)); 1522 1523 return NULL; 1524 } 1525 1526 static void 1527 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1528 { 1529 nvme_ccbs_free(q); 1530 mutex_destroy(&q->q_sq_mtx); 1531 mutex_destroy(&q->q_cq_mtx); 1532 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1533 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1534 nvme_dmamem_free(sc, q->q_cq_dmamem); 1535 nvme_dmamem_free(sc, q->q_sq_dmamem); 1536 kmem_free(q, sizeof(*q)); 1537 } 1538 1539 int 1540 nvme_intr(void *xsc) 1541 { 1542 struct nvme_softc *sc = xsc; 1543 1544 /* 1545 * INTx is level triggered, controller deasserts the interrupt only 1546 * when we advance command queue head via write to the doorbell. 1547 * Tell the controller to block the interrupts while we process 1548 * the queue(s). 1549 */ 1550 nvme_write4(sc, NVME_INTMS, 1); 1551 1552 softint_schedule(sc->sc_softih[0]); 1553 1554 /* don't know, might not have been for us */ 1555 return 1; 1556 } 1557 1558 void 1559 nvme_softintr_intx(void *xq) 1560 { 1561 struct nvme_queue *q = xq; 1562 struct nvme_softc *sc = q->q_sc; 1563 1564 nvme_q_complete(sc, sc->sc_admin_q); 1565 if (sc->sc_q != NULL) 1566 nvme_q_complete(sc, sc->sc_q[0]); 1567 1568 /* 1569 * Processing done, tell controller to issue interrupts again. There 1570 * is no race, as NVMe spec requires the controller to maintain state, 1571 * and assert the interrupt whenever there are unacknowledged 1572 * completion queue entries. 1573 */ 1574 nvme_write4(sc, NVME_INTMC, 1); 1575 } 1576 1577 int 1578 nvme_intr_msi(void *xq) 1579 { 1580 struct nvme_queue *q = xq; 1581 1582 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1583 && q->q_sc->sc_softih[q->q_id]); 1584 1585 /* 1586 * MSI/MSI-X are edge triggered, so can handover processing to softint 1587 * without masking the interrupt. 1588 */ 1589 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1590 1591 return 1; 1592 } 1593 1594 void 1595 nvme_softintr_msi(void *xq) 1596 { 1597 struct nvme_queue *q = xq; 1598 struct nvme_softc *sc = q->q_sc; 1599 1600 nvme_q_complete(sc, q); 1601 } 1602 1603 static struct nvme_dmamem * 1604 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1605 { 1606 struct nvme_dmamem *ndm; 1607 int nsegs; 1608 1609 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1610 if (ndm == NULL) 1611 return NULL; 1612 1613 ndm->ndm_size = size; 1614 1615 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1616 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1617 goto ndmfree; 1618 1619 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1620 1, &nsegs, BUS_DMA_WAITOK) != 0) 1621 goto destroy; 1622 1623 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1624 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1625 goto free; 1626 memset(ndm->ndm_kva, 0, size); 1627 1628 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1629 NULL, BUS_DMA_WAITOK) != 0) 1630 goto unmap; 1631 1632 return ndm; 1633 1634 unmap: 1635 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1636 free: 1637 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1638 destroy: 1639 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1640 ndmfree: 1641 kmem_free(ndm, sizeof(*ndm)); 1642 return NULL; 1643 } 1644 1645 static void 1646 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1647 { 1648 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1649 0, NVME_DMA_LEN(mem), ops); 1650 } 1651 1652 void 1653 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1654 { 1655 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1656 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1657 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1658 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1659 kmem_free(ndm, sizeof(*ndm)); 1660 } 1661 1662 /* 1663 * ioctl 1664 */ 1665 1666 dev_type_open(nvmeopen); 1667 dev_type_close(nvmeclose); 1668 dev_type_ioctl(nvmeioctl); 1669 1670 const struct cdevsw nvme_cdevsw = { 1671 .d_open = nvmeopen, 1672 .d_close = nvmeclose, 1673 .d_read = noread, 1674 .d_write = nowrite, 1675 .d_ioctl = nvmeioctl, 1676 .d_stop = nostop, 1677 .d_tty = notty, 1678 .d_poll = nopoll, 1679 .d_mmap = nommap, 1680 .d_kqfilter = nokqfilter, 1681 .d_discard = nodiscard, 1682 .d_flag = D_OTHER, 1683 }; 1684 1685 extern struct cfdriver nvme_cd; 1686 1687 /* 1688 * Accept an open operation on the control device. 1689 */ 1690 int 1691 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1692 { 1693 struct nvme_softc *sc; 1694 int unit = minor(dev) / 0x10000; 1695 int nsid = minor(dev) & 0xffff; 1696 int nsidx; 1697 1698 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1699 return ENXIO; 1700 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1701 return ENXIO; 1702 1703 if (nsid == 0) { 1704 /* controller */ 1705 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1706 return EBUSY; 1707 SET(sc->sc_flags, NVME_F_OPEN); 1708 } else { 1709 /* namespace */ 1710 nsidx = nsid - 1; 1711 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1712 return ENXIO; 1713 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1714 return EBUSY; 1715 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1716 } 1717 return 0; 1718 } 1719 1720 /* 1721 * Accept the last close on the control device. 1722 */ 1723 int 1724 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1725 { 1726 struct nvme_softc *sc; 1727 int unit = minor(dev) / 0x10000; 1728 int nsid = minor(dev) & 0xffff; 1729 int nsidx; 1730 1731 sc = device_lookup_private(&nvme_cd, unit); 1732 if (sc == NULL) 1733 return ENXIO; 1734 1735 if (nsid == 0) { 1736 /* controller */ 1737 CLR(sc->sc_flags, NVME_F_OPEN); 1738 } else { 1739 /* namespace */ 1740 nsidx = nsid - 1; 1741 if (nsidx >= sc->sc_nn) 1742 return ENXIO; 1743 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1744 } 1745 1746 return 0; 1747 } 1748 1749 /* 1750 * Handle control operations. 1751 */ 1752 int 1753 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1754 { 1755 struct nvme_softc *sc; 1756 int unit = minor(dev) / 0x10000; 1757 int nsid = minor(dev) & 0xffff; 1758 struct nvme_pt_command *pt; 1759 1760 sc = device_lookup_private(&nvme_cd, unit); 1761 if (sc == NULL) 1762 return ENXIO; 1763 1764 switch (cmd) { 1765 case NVME_PASSTHROUGH_CMD: 1766 pt = data; 1767 return nvme_command_passthrough(sc, data, 1768 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1769 } 1770 1771 return ENOTTY; 1772 } 1773
Indexes created Wed Oct 22 13:09:56 GMT 2025