nvme.c revision 1.26.4.1
1 /* $NetBSD: nvme.c,v 1.26.4.1 2017/04/30 10:27:16 pgoyette Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.26.4.1 2017/04/30 10:27:16 pgoyette Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 int nvme_adminq_size = 32; 44 int nvme_ioq_size = 1024; 45 46 static int nvme_print(void *, const char *); 47 48 static int nvme_ready(struct nvme_softc *, uint32_t); 49 static int nvme_enable(struct nvme_softc *, u_int); 50 static int nvme_disable(struct nvme_softc *); 51 static int nvme_shutdown(struct nvme_softc *); 52 53 static void nvme_version(struct nvme_softc *, uint32_t); 54 #ifdef NVME_DEBUG 55 static void nvme_dumpregs(struct nvme_softc *); 56 #endif 57 static int nvme_identify(struct nvme_softc *, u_int); 58 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 59 void *); 60 61 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 62 static void nvme_ccbs_free(struct nvme_queue *); 63 64 static struct nvme_ccb * 65 nvme_ccb_get(struct nvme_queue *); 66 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 67 68 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 69 struct nvme_ccb *, void (*)(struct nvme_queue *, 70 struct nvme_ccb *, void *), int); 71 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 72 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 73 struct nvme_cqe *); 74 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 76 struct nvme_cqe *); 77 78 static struct nvme_queue * 79 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 80 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 81 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 82 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 83 struct nvme_ccb *, void (*)(struct nvme_queue *, 84 struct nvme_ccb *, void *)); 85 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 86 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 87 88 static struct nvme_dmamem * 89 nvme_dmamem_alloc(struct nvme_softc *, size_t); 90 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 91 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 92 int); 93 94 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 95 void *); 96 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 97 struct nvme_cqe *); 98 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 99 void *); 100 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 101 struct nvme_cqe *); 102 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *, 103 void *); 104 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *, 105 struct nvme_cqe *); 106 107 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 108 void *); 109 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 110 struct nvme_cqe *); 111 static int nvme_command_passthrough(struct nvme_softc *, 112 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 113 114 static int nvme_get_number_of_queues(struct nvme_softc *, u_int *); 115 116 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 117 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 118 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 119 #define NVME_TIMO_SY 60 /* sync cache timeout */ 120 121 #define nvme_read4(_s, _r) \ 122 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 123 #define nvme_write4(_s, _r, _v) \ 124 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 125 #ifdef __LP64__ 126 #define nvme_read8(_s, _r) \ 127 bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r)) 128 #define nvme_write8(_s, _r, _v) \ 129 bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 130 #else /* __LP64__ */ 131 static inline uint64_t 132 nvme_read8(struct nvme_softc *sc, bus_size_t r) 133 { 134 uint64_t v; 135 uint32_t *a = (uint32_t *)&v; 136 137 #if _BYTE_ORDER == _LITTLE_ENDIAN 138 a[0] = nvme_read4(sc, r); 139 a[1] = nvme_read4(sc, r + 4); 140 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 141 a[1] = nvme_read4(sc, r); 142 a[0] = nvme_read4(sc, r + 4); 143 #endif 144 145 return v; 146 } 147 148 static inline void 149 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 150 { 151 uint32_t *a = (uint32_t *)&v; 152 153 #if _BYTE_ORDER == _LITTLE_ENDIAN 154 nvme_write4(sc, r, a[0]); 155 nvme_write4(sc, r + 4, a[1]); 156 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 157 nvme_write4(sc, r, a[1]); 158 nvme_write4(sc, r + 4, a[0]); 159 #endif 160 } 161 #endif /* __LP64__ */ 162 #define nvme_barrier(_s, _r, _l, _f) \ 163 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 164 165 static void 166 nvme_version(struct nvme_softc *sc, uint32_t ver) 167 { 168 const char *v = NULL; 169 170 switch (ver) { 171 case NVME_VS_1_0: 172 v = "1.0"; 173 break; 174 case NVME_VS_1_1: 175 v = "1.1"; 176 break; 177 case NVME_VS_1_2: 178 v = "1.2"; 179 break; 180 case NVME_VS_1_2_1: 181 v = "1.2.1"; 182 break; 183 default: 184 aprint_error_dev(sc->sc_dev, "unknown version 0x%08x\n", ver); 185 return; 186 } 187 188 aprint_normal_dev(sc->sc_dev, "NVMe %s\n", v); 189 } 190 191 #ifdef NVME_DEBUG 192 static __used void 193 nvme_dumpregs(struct nvme_softc *sc) 194 { 195 uint64_t r8; 196 uint32_t r4; 197 198 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 199 r8 = nvme_read8(sc, NVME_CAP); 200 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 201 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 202 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 203 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 204 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 205 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 206 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 207 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 208 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 209 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 210 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 211 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 212 213 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 214 215 r4 = nvme_read4(sc, NVME_CC); 216 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 217 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 218 (1 << NVME_CC_IOCQES_R(r4))); 219 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 220 (1 << NVME_CC_IOSQES_R(r4))); 221 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 222 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 223 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 224 (1 << NVME_CC_MPS_R(r4))); 225 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 226 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 227 228 r4 = nvme_read4(sc, NVME_CSTS); 229 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 230 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 231 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 232 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 233 234 r4 = nvme_read4(sc, NVME_AQA); 235 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 236 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 237 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 238 239 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 240 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 241 #undef DEVNAME 242 } 243 #endif /* NVME_DEBUG */ 244 245 static int 246 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 247 { 248 u_int i = 0; 249 uint32_t cc; 250 251 cc = nvme_read4(sc, NVME_CC); 252 if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) { 253 aprint_error_dev(sc->sc_dev, 254 "controller enabled status expected %d, found to be %d\n", 255 (rdy != 0), ((cc & NVME_CC_EN) != 0)); 256 return ENXIO; 257 } 258 259 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 260 if (i++ > sc->sc_rdy_to) 261 return ENXIO; 262 263 delay(1000); 264 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 265 } 266 267 return 0; 268 } 269 270 static int 271 nvme_enable(struct nvme_softc *sc, u_int mps) 272 { 273 uint32_t cc, csts; 274 275 cc = nvme_read4(sc, NVME_CC); 276 csts = nvme_read4(sc, NVME_CSTS); 277 278 if (ISSET(cc, NVME_CC_EN)) { 279 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 280 281 if (ISSET(csts, NVME_CSTS_RDY)) 282 return 1; 283 284 goto waitready; 285 } 286 287 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 288 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 289 delay(5000); 290 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 291 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 292 delay(5000); 293 294 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 295 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 296 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 297 delay(5000); 298 299 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 300 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 301 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 302 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 303 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 304 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 305 SET(cc, NVME_CC_MPS(mps)); 306 SET(cc, NVME_CC_EN); 307 308 nvme_write4(sc, NVME_CC, cc); 309 nvme_barrier(sc, 0, sc->sc_ios, 310 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 311 delay(5000); 312 313 waitready: 314 return nvme_ready(sc, NVME_CSTS_RDY); 315 } 316 317 static int 318 nvme_disable(struct nvme_softc *sc) 319 { 320 uint32_t cc, csts; 321 322 cc = nvme_read4(sc, NVME_CC); 323 csts = nvme_read4(sc, NVME_CSTS); 324 325 if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY)) 326 nvme_ready(sc, NVME_CSTS_RDY); 327 328 CLR(cc, NVME_CC_EN); 329 330 nvme_write4(sc, NVME_CC, cc); 331 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 332 333 delay(5000); 334 335 return nvme_ready(sc, 0); 336 } 337 338 int 339 nvme_attach(struct nvme_softc *sc) 340 { 341 uint64_t cap; 342 uint32_t reg; 343 u_int dstrd; 344 u_int mps = PAGE_SHIFT; 345 u_int ioq_allocated; 346 uint16_t adminq_entries = nvme_adminq_size; 347 uint16_t ioq_entries = nvme_ioq_size; 348 int i; 349 350 reg = nvme_read4(sc, NVME_VS); 351 if (reg == 0xffffffff) { 352 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 353 return 1; 354 } 355 356 nvme_version(sc, reg); 357 358 cap = nvme_read8(sc, NVME_CAP); 359 dstrd = NVME_CAP_DSTRD(cap); 360 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 361 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 362 "is greater than CPU page size %u\n", 363 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 364 return 1; 365 } 366 if (NVME_CAP_MPSMAX(cap) < mps) 367 mps = NVME_CAP_MPSMAX(cap); 368 if (ioq_entries > NVME_CAP_MQES(cap)) 369 ioq_entries = NVME_CAP_MQES(cap); 370 371 /* set initial values to be used for admin queue during probe */ 372 sc->sc_rdy_to = NVME_CAP_TO(cap); 373 sc->sc_mps = 1 << mps; 374 sc->sc_mdts = MAXPHYS; 375 sc->sc_max_sgl = 2; 376 377 if (nvme_disable(sc) != 0) { 378 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 379 return 1; 380 } 381 382 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 383 if (sc->sc_admin_q == NULL) { 384 aprint_error_dev(sc->sc_dev, 385 "unable to allocate admin queue\n"); 386 return 1; 387 } 388 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 389 goto free_admin_q; 390 391 if (nvme_enable(sc, mps) != 0) { 392 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 393 goto disestablish_admin_q; 394 } 395 396 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 397 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 398 goto disable; 399 } 400 401 /* we know how big things are now */ 402 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 403 404 /* reallocate ccbs of admin queue with new max sgl. */ 405 nvme_ccbs_free(sc->sc_admin_q); 406 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 407 408 if (sc->sc_use_mq) { 409 /* Limit the number of queues to the number allocated in HW */ 410 if (nvme_get_number_of_queues(sc, &ioq_allocated) != 0) { 411 aprint_error_dev(sc->sc_dev, 412 "unable to get number of queues\n"); 413 goto disable; 414 } 415 if (sc->sc_nq > ioq_allocated) 416 sc->sc_nq = ioq_allocated; 417 } 418 419 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 420 if (sc->sc_q == NULL) { 421 aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n"); 422 goto disable; 423 } 424 for (i = 0; i < sc->sc_nq; i++) { 425 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 426 if (sc->sc_q[i] == NULL) { 427 aprint_error_dev(sc->sc_dev, 428 "unable to allocate io queue\n"); 429 goto free_q; 430 } 431 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 432 aprint_error_dev(sc->sc_dev, 433 "unable to create io queue\n"); 434 nvme_q_free(sc, sc->sc_q[i]); 435 goto free_q; 436 } 437 } 438 439 if (!sc->sc_use_mq) 440 nvme_write4(sc, NVME_INTMC, 1); 441 442 /* probe subdevices */ 443 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 444 KM_SLEEP); 445 if (sc->sc_namespaces == NULL) 446 goto free_q; 447 nvme_rescan(sc->sc_dev, "nvme", &i); 448 449 return 0; 450 451 free_q: 452 while (--i >= 0) { 453 nvme_q_delete(sc, sc->sc_q[i]); 454 nvme_q_free(sc, sc->sc_q[i]); 455 } 456 disable: 457 nvme_disable(sc); 458 disestablish_admin_q: 459 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 460 free_admin_q: 461 nvme_q_free(sc, sc->sc_admin_q); 462 463 return 1; 464 } 465 466 int 467 nvme_rescan(device_t self, const char *attr, const int *flags) 468 { 469 struct nvme_softc *sc = device_private(self); 470 struct nvme_attach_args naa; 471 uint64_t cap; 472 int ioq_entries = nvme_ioq_size; 473 int i; 474 475 cap = nvme_read8(sc, NVME_CAP); 476 if (ioq_entries > NVME_CAP_MQES(cap)) 477 ioq_entries = NVME_CAP_MQES(cap); 478 479 for (i = 0; i < sc->sc_nn; i++) { 480 if (sc->sc_namespaces[i].dev) 481 continue; 482 memset(&naa, 0, sizeof(naa)); 483 naa.naa_nsid = i + 1; 484 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 485 naa.naa_maxphys = sc->sc_mdts; 486 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 487 nvme_print); 488 } 489 return 0; 490 } 491 492 static int 493 nvme_print(void *aux, const char *pnp) 494 { 495 struct nvme_attach_args *naa = aux; 496 497 if (pnp) 498 aprint_normal("at %s", pnp); 499 500 if (naa->naa_nsid > 0) 501 aprint_normal(" nsid %d", naa->naa_nsid); 502 503 return UNCONF; 504 } 505 506 int 507 nvme_detach(struct nvme_softc *sc, int flags) 508 { 509 int i, error; 510 511 error = config_detach_children(sc->sc_dev, flags); 512 if (error) 513 return error; 514 515 error = nvme_shutdown(sc); 516 if (error) 517 return error; 518 519 /* from now on we are committed to detach, following will never fail */ 520 for (i = 0; i < sc->sc_nq; i++) 521 nvme_q_free(sc, sc->sc_q[i]); 522 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 523 nvme_q_free(sc, sc->sc_admin_q); 524 525 return 0; 526 } 527 528 static int 529 nvme_shutdown(struct nvme_softc *sc) 530 { 531 uint32_t cc, csts; 532 bool disabled = false; 533 int i; 534 535 if (!sc->sc_use_mq) 536 nvme_write4(sc, NVME_INTMS, 1); 537 538 for (i = 0; i < sc->sc_nq; i++) { 539 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 540 aprint_error_dev(sc->sc_dev, 541 "unable to delete io queue %d, disabling\n", i + 1); 542 disabled = true; 543 } 544 } 545 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 546 if (disabled) 547 goto disable; 548 549 cc = nvme_read4(sc, NVME_CC); 550 CLR(cc, NVME_CC_SHN_MASK); 551 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 552 nvme_write4(sc, NVME_CC, cc); 553 554 for (i = 0; i < 4000; i++) { 555 nvme_barrier(sc, 0, sc->sc_ios, 556 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 557 csts = nvme_read4(sc, NVME_CSTS); 558 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 559 return 0; 560 561 delay(1000); 562 } 563 564 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 565 566 disable: 567 nvme_disable(sc); 568 return 0; 569 } 570 571 void 572 nvme_childdet(device_t self, device_t child) 573 { 574 struct nvme_softc *sc = device_private(self); 575 int i; 576 577 for (i = 0; i < sc->sc_nn; i++) { 578 if (sc->sc_namespaces[i].dev == child) { 579 /* Already freed ns->ident. */ 580 sc->sc_namespaces[i].dev = NULL; 581 break; 582 } 583 } 584 } 585 586 int 587 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 588 { 589 struct nvme_sqe sqe; 590 struct nvm_identify_namespace *identify; 591 struct nvme_dmamem *mem; 592 struct nvme_ccb *ccb; 593 struct nvme_namespace *ns; 594 int rv; 595 596 KASSERT(nsid > 0); 597 598 ccb = nvme_ccb_get(sc->sc_admin_q); 599 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 600 601 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 602 if (mem == NULL) 603 return ENOMEM; 604 605 memset(&sqe, 0, sizeof(sqe)); 606 sqe.opcode = NVM_ADMIN_IDENTIFY; 607 htolem32(&sqe.nsid, nsid); 608 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 609 htolem32(&sqe.cdw10, 0); 610 611 ccb->ccb_done = nvme_empty_done; 612 ccb->ccb_cookie = &sqe; 613 614 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 615 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 616 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 617 618 nvme_ccb_put(sc->sc_admin_q, ccb); 619 620 if (rv != 0) { 621 rv = EIO; 622 goto done; 623 } 624 625 /* commit */ 626 627 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 628 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 629 //memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 630 631 ns = nvme_ns_get(sc, nsid); 632 KASSERT(ns); 633 ns->ident = identify; 634 635 done: 636 nvme_dmamem_free(sc, mem); 637 638 return rv; 639 } 640 641 int 642 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 643 struct buf *bp, void *data, size_t datasize, 644 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 645 { 646 struct nvme_queue *q = nvme_get_q(sc); 647 struct nvme_ccb *ccb; 648 bus_dmamap_t dmap; 649 int i, error; 650 651 ccb = nvme_ccb_get(q); 652 if (ccb == NULL) 653 return EAGAIN; 654 655 ccb->ccb_done = nvme_ns_io_done; 656 ccb->ccb_cookie = cookie; 657 658 /* namespace context */ 659 ccb->nnc_nsid = nsid; 660 ccb->nnc_flags = flags; 661 ccb->nnc_buf = bp; 662 ccb->nnc_datasize = datasize; 663 ccb->nnc_secsize = secsize; 664 ccb->nnc_blkno = blkno; 665 ccb->nnc_done = nnc_done; 666 667 dmap = ccb->ccb_dmamap; 668 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 669 datasize, NULL, 670 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 671 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 672 (ISSET(flags, NVME_NS_CTX_F_READ) ? 673 BUS_DMA_READ : BUS_DMA_WRITE)); 674 if (error) { 675 nvme_ccb_put(q, ccb); 676 return error; 677 } 678 679 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 680 ISSET(flags, NVME_NS_CTX_F_READ) ? 681 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 682 683 if (dmap->dm_nsegs > 2) { 684 for (i = 1; i < dmap->dm_nsegs; i++) { 685 htolem64(&ccb->ccb_prpl[i - 1], 686 dmap->dm_segs[i].ds_addr); 687 } 688 bus_dmamap_sync(sc->sc_dmat, 689 NVME_DMA_MAP(q->q_ccb_prpls), 690 ccb->ccb_prpl_off, 691 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 692 BUS_DMASYNC_PREWRITE); 693 } 694 695 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 696 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 697 return EIO; 698 return 0; 699 } 700 701 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 702 return 0; 703 } 704 705 static void 706 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 707 { 708 struct nvme_sqe_io *sqe = slot; 709 bus_dmamap_t dmap = ccb->ccb_dmamap; 710 711 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 712 NVM_CMD_READ : NVM_CMD_WRITE; 713 htolem32(&sqe->nsid, ccb->nnc_nsid); 714 715 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 716 switch (dmap->dm_nsegs) { 717 case 1: 718 break; 719 case 2: 720 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 721 break; 722 default: 723 /* the prp list is already set up and synced */ 724 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 725 break; 726 } 727 728 htolem64(&sqe->slba, ccb->nnc_blkno); 729 730 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) 731 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA); 732 733 /* guaranteed by upper layers, but check just in case */ 734 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 735 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 736 } 737 738 static void 739 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 740 struct nvme_cqe *cqe) 741 { 742 struct nvme_softc *sc = q->q_sc; 743 bus_dmamap_t dmap = ccb->ccb_dmamap; 744 void *nnc_cookie = ccb->ccb_cookie; 745 nvme_nnc_done nnc_done = ccb->nnc_done; 746 struct buf *bp = ccb->nnc_buf; 747 748 if (dmap->dm_nsegs > 2) { 749 bus_dmamap_sync(sc->sc_dmat, 750 NVME_DMA_MAP(q->q_ccb_prpls), 751 ccb->ccb_prpl_off, 752 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 753 BUS_DMASYNC_POSTWRITE); 754 } 755 756 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 757 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 758 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 759 760 bus_dmamap_unload(sc->sc_dmat, dmap); 761 nvme_ccb_put(q, ccb); 762 763 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 764 } 765 766 /* 767 * If there is no volatile write cache, it makes no sense to issue 768 * flush commands or query for the status. 769 */ 770 bool 771 nvme_has_volatile_write_cache(struct nvme_softc *sc) 772 { 773 /* sc_identify is filled during attachment */ 774 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0); 775 } 776 777 int 778 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie, 779 int flags, nvme_nnc_done nnc_done) 780 { 781 struct nvme_queue *q = nvme_get_q(sc); 782 struct nvme_ccb *ccb; 783 784 ccb = nvme_ccb_get(q); 785 if (ccb == NULL) 786 return EAGAIN; 787 788 ccb->ccb_done = nvme_ns_sync_done; 789 ccb->ccb_cookie = cookie; 790 791 /* namespace context */ 792 ccb->nnc_nsid = nsid; 793 ccb->nnc_flags = flags; 794 ccb->nnc_done = nnc_done; 795 796 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 797 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 798 return EIO; 799 return 0; 800 } 801 802 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 803 return 0; 804 } 805 806 static void 807 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 808 { 809 struct nvme_sqe *sqe = slot; 810 811 sqe->opcode = NVM_CMD_FLUSH; 812 htolem32(&sqe->nsid, ccb->nnc_nsid); 813 } 814 815 static void 816 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 817 struct nvme_cqe *cqe) 818 { 819 void *cookie = ccb->ccb_cookie; 820 nvme_nnc_done nnc_done = ccb->nnc_done; 821 822 nvme_ccb_put(q, ccb); 823 824 nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 825 } 826 827 /* 828 * Get status of volatile write cache. Always asynchronous. 829 */ 830 int 831 nvme_admin_getcache(struct nvme_softc *sc, void *cookie, nvme_nnc_done nnc_done) 832 { 833 struct nvme_ccb *ccb; 834 struct nvme_queue *q = sc->sc_admin_q; 835 836 ccb = nvme_ccb_get(q); 837 if (ccb == NULL) 838 return EAGAIN; 839 840 ccb->ccb_done = nvme_getcache_done; 841 ccb->ccb_cookie = cookie; 842 843 /* namespace context */ 844 ccb->nnc_flags = 0; 845 ccb->nnc_done = nnc_done; 846 847 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); 848 return 0; 849 } 850 851 static void 852 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 853 { 854 struct nvme_sqe *sqe = slot; 855 856 sqe->opcode = NVM_ADMIN_GET_FEATURES; 857 sqe->cdw10 = NVM_FEATURE_VOLATILE_WRITE_CACHE; 858 } 859 860 static void 861 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 862 struct nvme_cqe *cqe) 863 { 864 void *cookie = ccb->ccb_cookie; 865 nvme_nnc_done nnc_done = ccb->nnc_done; 866 867 nvme_ccb_put(q, ccb); 868 869 nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 870 } 871 872 void 873 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 874 { 875 struct nvme_namespace *ns; 876 struct nvm_identify_namespace *identify; 877 878 ns = nvme_ns_get(sc, nsid); 879 KASSERT(ns); 880 881 identify = ns->ident; 882 ns->ident = NULL; 883 if (identify != NULL) 884 kmem_free(identify, sizeof(*identify)); 885 } 886 887 static void 888 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 889 { 890 struct nvme_softc *sc = q->q_sc; 891 struct nvme_sqe *sqe = slot; 892 struct nvme_pt_command *pt = ccb->ccb_cookie; 893 bus_dmamap_t dmap = ccb->ccb_dmamap; 894 int i; 895 896 sqe->opcode = pt->cmd.opcode; 897 htolem32(&sqe->nsid, pt->cmd.nsid); 898 899 if (pt->buf != NULL && pt->len > 0) { 900 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 901 switch (dmap->dm_nsegs) { 902 case 1: 903 break; 904 case 2: 905 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 906 break; 907 default: 908 for (i = 1; i < dmap->dm_nsegs; i++) { 909 htolem64(&ccb->ccb_prpl[i - 1], 910 dmap->dm_segs[i].ds_addr); 911 } 912 bus_dmamap_sync(sc->sc_dmat, 913 NVME_DMA_MAP(q->q_ccb_prpls), 914 ccb->ccb_prpl_off, 915 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 916 BUS_DMASYNC_PREWRITE); 917 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 918 break; 919 } 920 } 921 922 htolem32(&sqe->cdw10, pt->cmd.cdw10); 923 htolem32(&sqe->cdw11, pt->cmd.cdw11); 924 htolem32(&sqe->cdw12, pt->cmd.cdw12); 925 htolem32(&sqe->cdw13, pt->cmd.cdw13); 926 htolem32(&sqe->cdw14, pt->cmd.cdw14); 927 htolem32(&sqe->cdw15, pt->cmd.cdw15); 928 } 929 930 static void 931 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 932 { 933 struct nvme_softc *sc = q->q_sc; 934 struct nvme_pt_command *pt = ccb->ccb_cookie; 935 bus_dmamap_t dmap = ccb->ccb_dmamap; 936 937 if (pt->buf != NULL && pt->len > 0) { 938 if (dmap->dm_nsegs > 2) { 939 bus_dmamap_sync(sc->sc_dmat, 940 NVME_DMA_MAP(q->q_ccb_prpls), 941 ccb->ccb_prpl_off, 942 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 943 BUS_DMASYNC_POSTWRITE); 944 } 945 946 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 947 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 948 bus_dmamap_unload(sc->sc_dmat, dmap); 949 } 950 951 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 952 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 953 } 954 955 static int 956 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 957 uint16_t nsid, struct lwp *l, bool is_adminq) 958 { 959 struct nvme_queue *q; 960 struct nvme_ccb *ccb; 961 void *buf = NULL; 962 int error; 963 964 /* limit command size to maximum data transfer size */ 965 if ((pt->buf == NULL && pt->len > 0) || 966 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 967 return EINVAL; 968 969 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 970 ccb = nvme_ccb_get(q); 971 if (ccb == NULL) 972 return EBUSY; 973 974 if (pt->buf != NULL) { 975 KASSERT(pt->len > 0); 976 buf = kmem_alloc(pt->len, KM_SLEEP); 977 if (buf == NULL) { 978 error = ENOMEM; 979 goto ccb_put; 980 } 981 if (!pt->is_read) { 982 error = copyin(pt->buf, buf, pt->len); 983 if (error) 984 goto kmem_free; 985 } 986 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 987 pt->len, NULL, 988 BUS_DMA_WAITOK | 989 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 990 if (error) 991 goto kmem_free; 992 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 993 0, ccb->ccb_dmamap->dm_mapsize, 994 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 995 } 996 997 ccb->ccb_done = nvme_pt_done; 998 ccb->ccb_cookie = pt; 999 1000 pt->cmd.nsid = nsid; 1001 if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) { 1002 error = EIO; 1003 goto out; 1004 } 1005 1006 error = 0; 1007 out: 1008 if (buf != NULL) { 1009 if (error == 0 && pt->is_read) 1010 error = copyout(buf, pt->buf, pt->len); 1011 kmem_free: 1012 kmem_free(buf, pt->len); 1013 } 1014 ccb_put: 1015 nvme_ccb_put(q, ccb); 1016 return error; 1017 } 1018 1019 static void 1020 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1021 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 1022 { 1023 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 1024 uint32_t tail; 1025 1026 mutex_enter(&q->q_sq_mtx); 1027 tail = q->q_sq_tail; 1028 if (++q->q_sq_tail >= q->q_entries) 1029 q->q_sq_tail = 0; 1030 1031 sqe += tail; 1032 1033 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1034 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 1035 memset(sqe, 0, sizeof(*sqe)); 1036 (*fill)(q, ccb, sqe); 1037 sqe->cid = ccb->ccb_id; 1038 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1039 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 1040 1041 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 1042 mutex_exit(&q->q_sq_mtx); 1043 } 1044 1045 struct nvme_poll_state { 1046 struct nvme_sqe s; 1047 struct nvme_cqe c; 1048 }; 1049 1050 static int 1051 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1052 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 1053 { 1054 struct nvme_poll_state state; 1055 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 1056 void *cookie; 1057 uint16_t flags; 1058 int step = 10; 1059 int maxloop = timo_sec * 1000000 / step; 1060 int error = 0; 1061 1062 memset(&state, 0, sizeof(state)); 1063 (*fill)(q, ccb, &state.s); 1064 1065 done = ccb->ccb_done; 1066 cookie = ccb->ccb_cookie; 1067 1068 ccb->ccb_done = nvme_poll_done; 1069 ccb->ccb_cookie = &state; 1070 1071 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1072 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1073 if (nvme_q_complete(sc, q) == 0) 1074 delay(step); 1075 1076 if (timo_sec >= 0 && --maxloop <= 0) { 1077 error = ETIMEDOUT; 1078 break; 1079 } 1080 } 1081 1082 ccb->ccb_cookie = cookie; 1083 done(q, ccb, &state.c); 1084 1085 if (error == 0) { 1086 flags = lemtoh16(&state.c.flags); 1087 return flags & ~NVME_CQE_PHASE; 1088 } else { 1089 return 1; 1090 } 1091 } 1092 1093 static void 1094 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1095 { 1096 struct nvme_sqe *sqe = slot; 1097 struct nvme_poll_state *state = ccb->ccb_cookie; 1098 1099 *sqe = state->s; 1100 } 1101 1102 static void 1103 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1104 struct nvme_cqe *cqe) 1105 { 1106 struct nvme_poll_state *state = ccb->ccb_cookie; 1107 1108 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1109 state->c = *cqe; 1110 } 1111 1112 static void 1113 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1114 { 1115 struct nvme_sqe *src = ccb->ccb_cookie; 1116 struct nvme_sqe *dst = slot; 1117 1118 *dst = *src; 1119 } 1120 1121 static void 1122 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1123 struct nvme_cqe *cqe) 1124 { 1125 } 1126 1127 static int 1128 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1129 { 1130 struct nvme_ccb *ccb; 1131 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1132 uint16_t flags; 1133 int rv = 0; 1134 1135 mutex_enter(&q->q_cq_mtx); 1136 1137 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1138 for (;;) { 1139 cqe = &ring[q->q_cq_head]; 1140 flags = lemtoh16(&cqe->flags); 1141 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1142 break; 1143 1144 ccb = &q->q_ccbs[cqe->cid]; 1145 1146 if (++q->q_cq_head >= q->q_entries) { 1147 q->q_cq_head = 0; 1148 q->q_cq_phase ^= NVME_CQE_PHASE; 1149 } 1150 1151 #ifdef DEBUG 1152 /* 1153 * If we get spurious completion notification, something 1154 * is seriously hosed up. Very likely DMA to some random 1155 * memory place happened, so just bail out. 1156 */ 1157 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1158 panic("%s: invalid ccb detected", 1159 device_xname(sc->sc_dev)); 1160 /* NOTREACHED */ 1161 } 1162 #endif 1163 1164 rv++; 1165 1166 /* 1167 * Unlock the mutex before calling the ccb_done callback 1168 * and re-lock afterwards. The callback triggers lddone() 1169 * which schedules another i/o, and also calls nvme_ccb_put(). 1170 * Unlock/relock avoids possibility of deadlock. 1171 */ 1172 mutex_exit(&q->q_cq_mtx); 1173 ccb->ccb_done(q, ccb, cqe); 1174 mutex_enter(&q->q_cq_mtx); 1175 } 1176 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1177 1178 if (rv) 1179 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1180 1181 mutex_exit(&q->q_cq_mtx); 1182 1183 if (rv) { 1184 mutex_enter(&q->q_ccb_mtx); 1185 q->q_nccbs_avail += rv; 1186 mutex_exit(&q->q_ccb_mtx); 1187 } 1188 1189 return rv; 1190 } 1191 1192 static int 1193 nvme_identify(struct nvme_softc *sc, u_int mps) 1194 { 1195 char sn[41], mn[81], fr[17]; 1196 struct nvm_identify_controller *identify; 1197 struct nvme_dmamem *mem; 1198 struct nvme_ccb *ccb; 1199 u_int mdts; 1200 int rv = 1; 1201 1202 ccb = nvme_ccb_get(sc->sc_admin_q); 1203 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1204 1205 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1206 if (mem == NULL) 1207 return 1; 1208 1209 ccb->ccb_done = nvme_empty_done; 1210 ccb->ccb_cookie = mem; 1211 1212 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1213 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1214 NVME_TIMO_IDENT); 1215 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1216 1217 nvme_ccb_put(sc->sc_admin_q, ccb); 1218 1219 if (rv != 0) 1220 goto done; 1221 1222 identify = NVME_DMA_KVA(mem); 1223 1224 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1225 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1226 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1227 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1228 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1229 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1230 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1231 sn); 1232 1233 if (identify->mdts > 0) { 1234 mdts = (1 << identify->mdts) * (1 << mps); 1235 if (mdts < sc->sc_mdts) 1236 sc->sc_mdts = mdts; 1237 } 1238 1239 sc->sc_nn = lemtoh32(&identify->nn); 1240 1241 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1242 1243 done: 1244 nvme_dmamem_free(sc, mem); 1245 1246 return rv; 1247 } 1248 1249 static int 1250 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1251 { 1252 struct nvme_sqe_q sqe; 1253 struct nvme_ccb *ccb; 1254 int rv; 1255 1256 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1257 return 1; 1258 1259 ccb = nvme_ccb_get(sc->sc_admin_q); 1260 KASSERT(ccb != NULL); 1261 1262 ccb->ccb_done = nvme_empty_done; 1263 ccb->ccb_cookie = &sqe; 1264 1265 memset(&sqe, 0, sizeof(sqe)); 1266 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1267 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1268 htolem16(&sqe.qsize, q->q_entries - 1); 1269 htolem16(&sqe.qid, q->q_id); 1270 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1271 if (sc->sc_use_mq) 1272 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1273 1274 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1275 if (rv != 0) 1276 goto fail; 1277 1278 ccb->ccb_done = nvme_empty_done; 1279 ccb->ccb_cookie = &sqe; 1280 1281 memset(&sqe, 0, sizeof(sqe)); 1282 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1283 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1284 htolem16(&sqe.qsize, q->q_entries - 1); 1285 htolem16(&sqe.qid, q->q_id); 1286 htolem16(&sqe.cqid, q->q_id); 1287 sqe.qflags = NVM_SQE_Q_PC; 1288 1289 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1290 if (rv != 0) 1291 goto fail; 1292 1293 fail: 1294 nvme_ccb_put(sc->sc_admin_q, ccb); 1295 return rv; 1296 } 1297 1298 static int 1299 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1300 { 1301 struct nvme_sqe_q sqe; 1302 struct nvme_ccb *ccb; 1303 int rv; 1304 1305 ccb = nvme_ccb_get(sc->sc_admin_q); 1306 KASSERT(ccb != NULL); 1307 1308 ccb->ccb_done = nvme_empty_done; 1309 ccb->ccb_cookie = &sqe; 1310 1311 memset(&sqe, 0, sizeof(sqe)); 1312 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1313 htolem16(&sqe.qid, q->q_id); 1314 1315 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1316 if (rv != 0) 1317 goto fail; 1318 1319 ccb->ccb_done = nvme_empty_done; 1320 ccb->ccb_cookie = &sqe; 1321 1322 memset(&sqe, 0, sizeof(sqe)); 1323 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1324 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1325 htolem16(&sqe.qid, q->q_id); 1326 1327 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1328 if (rv != 0) 1329 goto fail; 1330 1331 fail: 1332 nvme_ccb_put(sc->sc_admin_q, ccb); 1333 1334 if (rv == 0 && sc->sc_use_mq) { 1335 if (sc->sc_intr_disestablish(sc, q->q_id)) 1336 rv = 1; 1337 } 1338 1339 return rv; 1340 } 1341 1342 static void 1343 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1344 { 1345 struct nvme_sqe *sqe = slot; 1346 struct nvme_dmamem *mem = ccb->ccb_cookie; 1347 1348 sqe->opcode = NVM_ADMIN_IDENTIFY; 1349 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1350 htolem32(&sqe->cdw10, 1); 1351 } 1352 1353 static int 1354 nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap) 1355 { 1356 struct nvme_pt_command pt; 1357 struct nvme_ccb *ccb; 1358 uint16_t ncqa, nsqa; 1359 int rv; 1360 1361 ccb = nvme_ccb_get(sc->sc_admin_q); 1362 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1363 1364 memset(&pt, 0, sizeof(pt)); 1365 pt.cmd.opcode = NVM_ADMIN_GET_FEATURES; 1366 pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES; 1367 1368 ccb->ccb_done = nvme_pt_done; 1369 ccb->ccb_cookie = &pt; 1370 1371 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1372 1373 nvme_ccb_put(sc->sc_admin_q, ccb); 1374 1375 if (rv != 0) { 1376 *nqap = 0; 1377 return EIO; 1378 } 1379 1380 ncqa = pt.cpl.cdw0 >> 16; 1381 nsqa = pt.cpl.cdw0 & 0xffff; 1382 *nqap = MIN(ncqa, nsqa) + 1; 1383 1384 return 0; 1385 } 1386 1387 static int 1388 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1389 { 1390 struct nvme_softc *sc = q->q_sc; 1391 struct nvme_ccb *ccb; 1392 bus_addr_t off; 1393 uint64_t *prpl; 1394 u_int i; 1395 1396 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1397 SIMPLEQ_INIT(&q->q_ccb_list); 1398 1399 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1400 if (q->q_ccbs == NULL) 1401 return 1; 1402 1403 q->q_nccbs = nccbs; 1404 q->q_nccbs_avail = nccbs; 1405 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1406 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1407 1408 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1409 off = 0; 1410 1411 for (i = 0; i < nccbs; i++) { 1412 ccb = &q->q_ccbs[i]; 1413 1414 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1415 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1416 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1417 &ccb->ccb_dmamap) != 0) 1418 goto free_maps; 1419 1420 ccb->ccb_id = i; 1421 ccb->ccb_prpl = prpl; 1422 ccb->ccb_prpl_off = off; 1423 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1424 1425 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1426 1427 prpl += sc->sc_max_sgl; 1428 off += sizeof(*prpl) * sc->sc_max_sgl; 1429 } 1430 1431 return 0; 1432 1433 free_maps: 1434 nvme_ccbs_free(q); 1435 return 1; 1436 } 1437 1438 static struct nvme_ccb * 1439 nvme_ccb_get(struct nvme_queue *q) 1440 { 1441 struct nvme_ccb *ccb = NULL; 1442 1443 mutex_enter(&q->q_ccb_mtx); 1444 if (q->q_nccbs_avail > 0) { 1445 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1446 KASSERT(ccb != NULL); 1447 q->q_nccbs_avail--; 1448 1449 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1450 #ifdef DEBUG 1451 ccb->ccb_cookie = NULL; 1452 #endif 1453 } 1454 mutex_exit(&q->q_ccb_mtx); 1455 1456 return ccb; 1457 } 1458 1459 static void 1460 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1461 { 1462 1463 mutex_enter(&q->q_ccb_mtx); 1464 #ifdef DEBUG 1465 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1466 #endif 1467 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1468 mutex_exit(&q->q_ccb_mtx); 1469 } 1470 1471 static void 1472 nvme_ccbs_free(struct nvme_queue *q) 1473 { 1474 struct nvme_softc *sc = q->q_sc; 1475 struct nvme_ccb *ccb; 1476 1477 mutex_enter(&q->q_ccb_mtx); 1478 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1479 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1480 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1481 } 1482 mutex_exit(&q->q_ccb_mtx); 1483 1484 nvme_dmamem_free(sc, q->q_ccb_prpls); 1485 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1486 q->q_ccbs = NULL; 1487 mutex_destroy(&q->q_ccb_mtx); 1488 } 1489 1490 static struct nvme_queue * 1491 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1492 { 1493 struct nvme_queue *q; 1494 1495 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1496 if (q == NULL) 1497 return NULL; 1498 1499 q->q_sc = sc; 1500 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1501 sizeof(struct nvme_sqe) * entries); 1502 if (q->q_sq_dmamem == NULL) 1503 goto free; 1504 1505 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1506 sizeof(struct nvme_cqe) * entries); 1507 if (q->q_cq_dmamem == NULL) 1508 goto free_sq; 1509 1510 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1511 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1512 1513 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1514 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1515 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1516 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1517 q->q_id = id; 1518 q->q_entries = entries; 1519 q->q_sq_tail = 0; 1520 q->q_cq_head = 0; 1521 q->q_cq_phase = NVME_CQE_PHASE; 1522 1523 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1524 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1525 1526 /* 1527 * Due to definition of full and empty queue (queue is empty 1528 * when head == tail, full when tail is one less then head), 1529 * we can actually only have (entries - 1) in-flight commands. 1530 */ 1531 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1532 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1533 goto free_cq; 1534 } 1535 1536 return q; 1537 1538 free_cq: 1539 nvme_dmamem_free(sc, q->q_cq_dmamem); 1540 free_sq: 1541 nvme_dmamem_free(sc, q->q_sq_dmamem); 1542 free: 1543 kmem_free(q, sizeof(*q)); 1544 1545 return NULL; 1546 } 1547 1548 static void 1549 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1550 { 1551 nvme_ccbs_free(q); 1552 mutex_destroy(&q->q_sq_mtx); 1553 mutex_destroy(&q->q_cq_mtx); 1554 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1555 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1556 nvme_dmamem_free(sc, q->q_cq_dmamem); 1557 nvme_dmamem_free(sc, q->q_sq_dmamem); 1558 kmem_free(q, sizeof(*q)); 1559 } 1560 1561 int 1562 nvme_intr(void *xsc) 1563 { 1564 struct nvme_softc *sc = xsc; 1565 1566 /* 1567 * INTx is level triggered, controller deasserts the interrupt only 1568 * when we advance command queue head via write to the doorbell. 1569 * Tell the controller to block the interrupts while we process 1570 * the queue(s). 1571 */ 1572 nvme_write4(sc, NVME_INTMS, 1); 1573 1574 softint_schedule(sc->sc_softih[0]); 1575 1576 /* don't know, might not have been for us */ 1577 return 1; 1578 } 1579 1580 void 1581 nvme_softintr_intx(void *xq) 1582 { 1583 struct nvme_queue *q = xq; 1584 struct nvme_softc *sc = q->q_sc; 1585 1586 nvme_q_complete(sc, sc->sc_admin_q); 1587 if (sc->sc_q != NULL) 1588 nvme_q_complete(sc, sc->sc_q[0]); 1589 1590 /* 1591 * Processing done, tell controller to issue interrupts again. There 1592 * is no race, as NVMe spec requires the controller to maintain state, 1593 * and assert the interrupt whenever there are unacknowledged 1594 * completion queue entries. 1595 */ 1596 nvme_write4(sc, NVME_INTMC, 1); 1597 } 1598 1599 int 1600 nvme_intr_msi(void *xq) 1601 { 1602 struct nvme_queue *q = xq; 1603 1604 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1605 && q->q_sc->sc_softih[q->q_id]); 1606 1607 /* 1608 * MSI/MSI-X are edge triggered, so can handover processing to softint 1609 * without masking the interrupt. 1610 */ 1611 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1612 1613 return 1; 1614 } 1615 1616 void 1617 nvme_softintr_msi(void *xq) 1618 { 1619 struct nvme_queue *q = xq; 1620 struct nvme_softc *sc = q->q_sc; 1621 1622 nvme_q_complete(sc, q); 1623 } 1624 1625 static struct nvme_dmamem * 1626 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1627 { 1628 struct nvme_dmamem *ndm; 1629 int nsegs; 1630 1631 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1632 if (ndm == NULL) 1633 return NULL; 1634 1635 ndm->ndm_size = size; 1636 1637 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1638 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1639 goto ndmfree; 1640 1641 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1642 1, &nsegs, BUS_DMA_WAITOK) != 0) 1643 goto destroy; 1644 1645 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1646 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1647 goto free; 1648 memset(ndm->ndm_kva, 0, size); 1649 1650 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1651 NULL, BUS_DMA_WAITOK) != 0) 1652 goto unmap; 1653 1654 return ndm; 1655 1656 unmap: 1657 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1658 free: 1659 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1660 destroy: 1661 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1662 ndmfree: 1663 kmem_free(ndm, sizeof(*ndm)); 1664 return NULL; 1665 } 1666 1667 static void 1668 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1669 { 1670 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1671 0, NVME_DMA_LEN(mem), ops); 1672 } 1673 1674 void 1675 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1676 { 1677 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1678 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1679 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1680 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1681 kmem_free(ndm, sizeof(*ndm)); 1682 } 1683 1684 /* 1685 * ioctl 1686 */ 1687 1688 dev_type_open(nvmeopen); 1689 dev_type_close(nvmeclose); 1690 dev_type_ioctl(nvmeioctl); 1691 1692 const struct cdevsw nvme_cdevsw = { 1693 DEVSW_MODULE_INIT 1694 .d_open = nvmeopen, 1695 .d_close = nvmeclose, 1696 .d_read = noread, 1697 .d_write = nowrite, 1698 .d_ioctl = nvmeioctl, 1699 .d_stop = nostop, 1700 .d_tty = notty, 1701 .d_poll = nopoll, 1702 .d_mmap = nommap, 1703 .d_kqfilter = nokqfilter, 1704 .d_discard = nodiscard, 1705 .d_flag = D_OTHER, 1706 }; 1707 1708 extern struct cfdriver nvme_cd; 1709 1710 /* 1711 * Accept an open operation on the control device. 1712 */ 1713 int 1714 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1715 { 1716 struct nvme_softc *sc; 1717 int unit = minor(dev) / 0x10000; 1718 int nsid = minor(dev) & 0xffff; 1719 int nsidx; 1720 1721 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1722 return ENXIO; 1723 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1724 return ENXIO; 1725 1726 if (nsid == 0) { 1727 /* controller */ 1728 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1729 return EBUSY; 1730 SET(sc->sc_flags, NVME_F_OPEN); 1731 } else { 1732 /* namespace */ 1733 nsidx = nsid - 1; 1734 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1735 return ENXIO; 1736 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1737 return EBUSY; 1738 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1739 } 1740 return 0; 1741 } 1742 1743 /* 1744 * Accept the last close on the control device. 1745 */ 1746 int 1747 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1748 { 1749 struct nvme_softc *sc; 1750 int unit = minor(dev) / 0x10000; 1751 int nsid = minor(dev) & 0xffff; 1752 int nsidx; 1753 1754 sc = device_lookup_private(&nvme_cd, unit); 1755 if (sc == NULL) 1756 return ENXIO; 1757 1758 if (nsid == 0) { 1759 /* controller */ 1760 CLR(sc->sc_flags, NVME_F_OPEN); 1761 } else { 1762 /* namespace */ 1763 nsidx = nsid - 1; 1764 if (nsidx >= sc->sc_nn) 1765 return ENXIO; 1766 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1767 } 1768 1769 return 0; 1770 } 1771 1772 /* 1773 * Handle control operations. 1774 */ 1775 int 1776 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1777 { 1778 struct nvme_softc *sc; 1779 int unit = minor(dev) / 0x10000; 1780 int nsid = minor(dev) & 0xffff; 1781 struct nvme_pt_command *pt; 1782 1783 sc = device_lookup_private(&nvme_cd, unit); 1784 if (sc == NULL) 1785 return ENXIO; 1786 1787 switch (cmd) { 1788 case NVME_PASSTHROUGH_CMD: 1789 pt = data; 1790 return nvme_command_passthrough(sc, data, 1791 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1792 } 1793 1794 return ENOTTY; 1795 } 1796
Indexes created Mon Oct 20 20:10:13 GMT 2025