nvme.c revision 1.35
1 /* $NetBSD: nvme.c,v 1.35 2018/03/17 00:28:03 jdolecek Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.35 2018/03/17 00:28:03 jdolecek Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 #include "ioconf.h" 44 45 int nvme_adminq_size = 32; 46 int nvme_ioq_size = 1024; 47 48 static int nvme_print(void *, const char *); 49 50 static int nvme_ready(struct nvme_softc *, uint32_t); 51 static int nvme_enable(struct nvme_softc *, u_int); 52 static int nvme_disable(struct nvme_softc *); 53 static int nvme_shutdown(struct nvme_softc *); 54 55 #ifdef NVME_DEBUG 56 static void nvme_dumpregs(struct nvme_softc *); 57 #endif 58 static int nvme_identify(struct nvme_softc *, u_int); 59 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 60 void *); 61 62 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 63 static void nvme_ccbs_free(struct nvme_queue *); 64 65 static struct nvme_ccb * 66 nvme_ccb_get(struct nvme_queue *, bool); 67 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 68 69 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 70 struct nvme_ccb *, void (*)(struct nvme_queue *, 71 struct nvme_ccb *, void *), int); 72 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 73 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 74 struct nvme_cqe *); 75 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 76 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 77 struct nvme_cqe *); 78 79 static struct nvme_queue * 80 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 81 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 82 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 83 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 84 struct nvme_ccb *, void (*)(struct nvme_queue *, 85 struct nvme_ccb *, void *)); 86 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 87 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 88 static void nvme_q_wait_complete(struct nvme_softc *, struct nvme_queue *, 89 bool (*)(void *), void *); 90 91 static struct nvme_dmamem * 92 nvme_dmamem_alloc(struct nvme_softc *, size_t); 93 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 94 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 95 int); 96 97 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 98 void *); 99 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 100 struct nvme_cqe *); 101 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 102 void *); 103 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 104 struct nvme_cqe *); 105 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *, 106 void *); 107 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *, 108 struct nvme_cqe *); 109 110 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 111 void *); 112 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 113 struct nvme_cqe *); 114 static int nvme_command_passthrough(struct nvme_softc *, 115 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 116 117 static int nvme_get_number_of_queues(struct nvme_softc *, u_int *); 118 119 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 120 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 121 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 122 #define NVME_TIMO_SY 60 /* sync cache timeout */ 123 124 #define nvme_read4(_s, _r) \ 125 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 126 #define nvme_write4(_s, _r, _v) \ 127 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 128 /* 129 * Some controllers, at least Apple NVMe, always require split 130 * transfers, so don't use bus_space_{read,write}_8() on LP64. 131 */ 132 static inline uint64_t 133 nvme_read8(struct nvme_softc *sc, bus_size_t r) 134 { 135 uint64_t v; 136 uint32_t *a = (uint32_t *)&v; 137 138 #if _BYTE_ORDER == _LITTLE_ENDIAN 139 a[0] = nvme_read4(sc, r); 140 a[1] = nvme_read4(sc, r + 4); 141 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 142 a[1] = nvme_read4(sc, r); 143 a[0] = nvme_read4(sc, r + 4); 144 #endif 145 146 return v; 147 } 148 149 static inline void 150 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 151 { 152 uint32_t *a = (uint32_t *)&v; 153 154 #if _BYTE_ORDER == _LITTLE_ENDIAN 155 nvme_write4(sc, r, a[0]); 156 nvme_write4(sc, r + 4, a[1]); 157 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 158 nvme_write4(sc, r, a[1]); 159 nvme_write4(sc, r + 4, a[0]); 160 #endif 161 } 162 #define nvme_barrier(_s, _r, _l, _f) \ 163 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 164 165 #ifdef NVME_DEBUG 166 static __used void 167 nvme_dumpregs(struct nvme_softc *sc) 168 { 169 uint64_t r8; 170 uint32_t r4; 171 172 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 173 r8 = nvme_read8(sc, NVME_CAP); 174 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 175 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 176 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 177 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 178 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 179 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 180 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 181 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 182 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 183 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 184 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 185 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 186 187 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 188 189 r4 = nvme_read4(sc, NVME_CC); 190 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 191 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 192 (1 << NVME_CC_IOCQES_R(r4))); 193 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 194 (1 << NVME_CC_IOSQES_R(r4))); 195 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 196 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 197 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 198 (1 << NVME_CC_MPS_R(r4))); 199 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 200 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 201 202 r4 = nvme_read4(sc, NVME_CSTS); 203 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 204 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 205 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 206 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 207 208 r4 = nvme_read4(sc, NVME_AQA); 209 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 210 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 211 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 212 213 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 214 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 215 #undef DEVNAME 216 } 217 #endif /* NVME_DEBUG */ 218 219 static int 220 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 221 { 222 u_int i = 0; 223 uint32_t cc; 224 225 cc = nvme_read4(sc, NVME_CC); 226 if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) { 227 aprint_error_dev(sc->sc_dev, 228 "controller enabled status expected %d, found to be %d\n", 229 (rdy != 0), ((cc & NVME_CC_EN) != 0)); 230 return ENXIO; 231 } 232 233 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 234 if (i++ > sc->sc_rdy_to) 235 return ENXIO; 236 237 delay(1000); 238 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 239 } 240 241 return 0; 242 } 243 244 static int 245 nvme_enable(struct nvme_softc *sc, u_int mps) 246 { 247 uint32_t cc, csts; 248 249 cc = nvme_read4(sc, NVME_CC); 250 csts = nvme_read4(sc, NVME_CSTS); 251 252 if (ISSET(cc, NVME_CC_EN)) { 253 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 254 255 if (ISSET(csts, NVME_CSTS_RDY)) 256 return 1; 257 258 goto waitready; 259 } 260 261 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 262 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 263 delay(5000); 264 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 265 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 266 delay(5000); 267 268 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 269 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 270 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 271 delay(5000); 272 273 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 274 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 275 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 276 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 277 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 278 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 279 SET(cc, NVME_CC_MPS(mps)); 280 SET(cc, NVME_CC_EN); 281 282 nvme_write4(sc, NVME_CC, cc); 283 nvme_barrier(sc, 0, sc->sc_ios, 284 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 285 delay(5000); 286 287 waitready: 288 return nvme_ready(sc, NVME_CSTS_RDY); 289 } 290 291 static int 292 nvme_disable(struct nvme_softc *sc) 293 { 294 uint32_t cc, csts; 295 296 cc = nvme_read4(sc, NVME_CC); 297 csts = nvme_read4(sc, NVME_CSTS); 298 299 if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY)) 300 nvme_ready(sc, NVME_CSTS_RDY); 301 302 CLR(cc, NVME_CC_EN); 303 304 nvme_write4(sc, NVME_CC, cc); 305 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 306 307 delay(5000); 308 309 return nvme_ready(sc, 0); 310 } 311 312 int 313 nvme_attach(struct nvme_softc *sc) 314 { 315 uint64_t cap; 316 uint32_t reg; 317 u_int dstrd; 318 u_int mps = PAGE_SHIFT; 319 u_int ioq_allocated; 320 uint16_t adminq_entries = nvme_adminq_size; 321 uint16_t ioq_entries = nvme_ioq_size; 322 int i; 323 324 reg = nvme_read4(sc, NVME_VS); 325 if (reg == 0xffffffff) { 326 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 327 return 1; 328 } 329 330 if (NVME_VS_TER(reg) == 0) 331 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg), 332 NVME_VS_MNR(reg)); 333 else 334 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg), 335 NVME_VS_MNR(reg), NVME_VS_TER(reg)); 336 337 cap = nvme_read8(sc, NVME_CAP); 338 dstrd = NVME_CAP_DSTRD(cap); 339 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 340 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 341 "is greater than CPU page size %u\n", 342 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 343 return 1; 344 } 345 if (NVME_CAP_MPSMAX(cap) < mps) 346 mps = NVME_CAP_MPSMAX(cap); 347 if (ioq_entries > NVME_CAP_MQES(cap)) 348 ioq_entries = NVME_CAP_MQES(cap); 349 350 /* set initial values to be used for admin queue during probe */ 351 sc->sc_rdy_to = NVME_CAP_TO(cap); 352 sc->sc_mps = 1 << mps; 353 sc->sc_mdts = MAXPHYS; 354 sc->sc_max_sgl = 2; 355 356 if (nvme_disable(sc) != 0) { 357 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 358 return 1; 359 } 360 361 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 362 if (sc->sc_admin_q == NULL) { 363 aprint_error_dev(sc->sc_dev, 364 "unable to allocate admin queue\n"); 365 return 1; 366 } 367 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 368 goto free_admin_q; 369 370 if (nvme_enable(sc, mps) != 0) { 371 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 372 goto disestablish_admin_q; 373 } 374 375 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 376 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 377 goto disable; 378 } 379 380 /* we know how big things are now */ 381 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 382 383 /* reallocate ccbs of admin queue with new max sgl. */ 384 nvme_ccbs_free(sc->sc_admin_q); 385 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 386 387 if (sc->sc_use_mq) { 388 /* Limit the number of queues to the number allocated in HW */ 389 if (nvme_get_number_of_queues(sc, &ioq_allocated) != 0) { 390 aprint_error_dev(sc->sc_dev, 391 "unable to get number of queues\n"); 392 goto disable; 393 } 394 if (sc->sc_nq > ioq_allocated) 395 sc->sc_nq = ioq_allocated; 396 } 397 398 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 399 for (i = 0; i < sc->sc_nq; i++) { 400 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 401 if (sc->sc_q[i] == NULL) { 402 aprint_error_dev(sc->sc_dev, 403 "unable to allocate io queue\n"); 404 goto free_q; 405 } 406 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 407 aprint_error_dev(sc->sc_dev, 408 "unable to create io queue\n"); 409 nvme_q_free(sc, sc->sc_q[i]); 410 goto free_q; 411 } 412 } 413 414 if (!sc->sc_use_mq) 415 nvme_write4(sc, NVME_INTMC, 1); 416 417 /* probe subdevices */ 418 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 419 KM_SLEEP); 420 nvme_rescan(sc->sc_dev, "nvme", &i); 421 422 return 0; 423 424 free_q: 425 while (--i >= 0) { 426 nvme_q_delete(sc, sc->sc_q[i]); 427 nvme_q_free(sc, sc->sc_q[i]); 428 } 429 disable: 430 nvme_disable(sc); 431 disestablish_admin_q: 432 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 433 free_admin_q: 434 nvme_q_free(sc, sc->sc_admin_q); 435 436 return 1; 437 } 438 439 int 440 nvme_rescan(device_t self, const char *attr, const int *flags) 441 { 442 struct nvme_softc *sc = device_private(self); 443 struct nvme_attach_args naa; 444 uint64_t cap; 445 int ioq_entries = nvme_ioq_size; 446 int i; 447 448 cap = nvme_read8(sc, NVME_CAP); 449 if (ioq_entries > NVME_CAP_MQES(cap)) 450 ioq_entries = NVME_CAP_MQES(cap); 451 452 for (i = 0; i < sc->sc_nn; i++) { 453 if (sc->sc_namespaces[i].dev) 454 continue; 455 memset(&naa, 0, sizeof(naa)); 456 naa.naa_nsid = i + 1; 457 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 458 naa.naa_maxphys = sc->sc_mdts; 459 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 460 nvme_print); 461 } 462 return 0; 463 } 464 465 static int 466 nvme_print(void *aux, const char *pnp) 467 { 468 struct nvme_attach_args *naa = aux; 469 470 if (pnp) 471 aprint_normal("at %s", pnp); 472 473 if (naa->naa_nsid > 0) 474 aprint_normal(" nsid %d", naa->naa_nsid); 475 476 return UNCONF; 477 } 478 479 int 480 nvme_detach(struct nvme_softc *sc, int flags) 481 { 482 int i, error; 483 484 error = config_detach_children(sc->sc_dev, flags); 485 if (error) 486 return error; 487 488 error = nvme_shutdown(sc); 489 if (error) 490 return error; 491 492 /* from now on we are committed to detach, following will never fail */ 493 for (i = 0; i < sc->sc_nq; i++) 494 nvme_q_free(sc, sc->sc_q[i]); 495 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 496 nvme_q_free(sc, sc->sc_admin_q); 497 498 return 0; 499 } 500 501 static int 502 nvme_shutdown(struct nvme_softc *sc) 503 { 504 uint32_t cc, csts; 505 bool disabled = false; 506 int i; 507 508 if (!sc->sc_use_mq) 509 nvme_write4(sc, NVME_INTMS, 1); 510 511 for (i = 0; i < sc->sc_nq; i++) { 512 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 513 aprint_error_dev(sc->sc_dev, 514 "unable to delete io queue %d, disabling\n", i + 1); 515 disabled = true; 516 } 517 } 518 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 519 if (disabled) 520 goto disable; 521 522 cc = nvme_read4(sc, NVME_CC); 523 CLR(cc, NVME_CC_SHN_MASK); 524 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 525 nvme_write4(sc, NVME_CC, cc); 526 527 for (i = 0; i < 4000; i++) { 528 nvme_barrier(sc, 0, sc->sc_ios, 529 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 530 csts = nvme_read4(sc, NVME_CSTS); 531 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 532 return 0; 533 534 delay(1000); 535 } 536 537 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 538 539 disable: 540 nvme_disable(sc); 541 return 0; 542 } 543 544 void 545 nvme_childdet(device_t self, device_t child) 546 { 547 struct nvme_softc *sc = device_private(self); 548 int i; 549 550 for (i = 0; i < sc->sc_nn; i++) { 551 if (sc->sc_namespaces[i].dev == child) { 552 /* Already freed ns->ident. */ 553 sc->sc_namespaces[i].dev = NULL; 554 break; 555 } 556 } 557 } 558 559 int 560 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 561 { 562 struct nvme_sqe sqe; 563 struct nvm_identify_namespace *identify; 564 struct nvme_dmamem *mem; 565 struct nvme_ccb *ccb; 566 struct nvme_namespace *ns; 567 int rv; 568 569 KASSERT(nsid > 0); 570 571 ccb = nvme_ccb_get(sc->sc_admin_q, false); 572 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 573 574 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 575 if (mem == NULL) { 576 nvme_ccb_put(sc->sc_admin_q, ccb); 577 return ENOMEM; 578 } 579 580 memset(&sqe, 0, sizeof(sqe)); 581 sqe.opcode = NVM_ADMIN_IDENTIFY; 582 htolem32(&sqe.nsid, nsid); 583 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 584 htolem32(&sqe.cdw10, 0); 585 586 ccb->ccb_done = nvme_empty_done; 587 ccb->ccb_cookie = &sqe; 588 589 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 590 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 591 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 592 593 nvme_ccb_put(sc->sc_admin_q, ccb); 594 595 if (rv != 0) { 596 rv = EIO; 597 goto done; 598 } 599 600 /* commit */ 601 602 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 603 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 604 //memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 605 606 ns = nvme_ns_get(sc, nsid); 607 KASSERT(ns); 608 KASSERT(ns->ident == NULL); 609 ns->ident = identify; 610 611 done: 612 nvme_dmamem_free(sc, mem); 613 614 return rv; 615 } 616 617 int 618 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 619 struct buf *bp, void *data, size_t datasize, 620 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 621 { 622 struct nvme_queue *q = nvme_get_q(sc); 623 struct nvme_ccb *ccb; 624 bus_dmamap_t dmap; 625 int i, error; 626 627 ccb = nvme_ccb_get(q, false); 628 if (ccb == NULL) 629 return EAGAIN; 630 631 ccb->ccb_done = nvme_ns_io_done; 632 ccb->ccb_cookie = cookie; 633 634 /* namespace context */ 635 ccb->nnc_nsid = nsid; 636 ccb->nnc_flags = flags; 637 ccb->nnc_buf = bp; 638 ccb->nnc_datasize = datasize; 639 ccb->nnc_secsize = secsize; 640 ccb->nnc_blkno = blkno; 641 ccb->nnc_done = nnc_done; 642 643 dmap = ccb->ccb_dmamap; 644 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 645 datasize, NULL, 646 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 647 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 648 (ISSET(flags, NVME_NS_CTX_F_READ) ? 649 BUS_DMA_READ : BUS_DMA_WRITE)); 650 if (error) { 651 nvme_ccb_put(q, ccb); 652 return error; 653 } 654 655 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 656 ISSET(flags, NVME_NS_CTX_F_READ) ? 657 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 658 659 if (dmap->dm_nsegs > 2) { 660 for (i = 1; i < dmap->dm_nsegs; i++) { 661 htolem64(&ccb->ccb_prpl[i - 1], 662 dmap->dm_segs[i].ds_addr); 663 } 664 bus_dmamap_sync(sc->sc_dmat, 665 NVME_DMA_MAP(q->q_ccb_prpls), 666 ccb->ccb_prpl_off, 667 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 668 BUS_DMASYNC_PREWRITE); 669 } 670 671 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 672 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 673 return EIO; 674 return 0; 675 } 676 677 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 678 return 0; 679 } 680 681 static void 682 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 683 { 684 struct nvme_sqe_io *sqe = slot; 685 bus_dmamap_t dmap = ccb->ccb_dmamap; 686 687 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 688 NVM_CMD_READ : NVM_CMD_WRITE; 689 htolem32(&sqe->nsid, ccb->nnc_nsid); 690 691 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 692 switch (dmap->dm_nsegs) { 693 case 1: 694 break; 695 case 2: 696 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 697 break; 698 default: 699 /* the prp list is already set up and synced */ 700 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 701 break; 702 } 703 704 htolem64(&sqe->slba, ccb->nnc_blkno); 705 706 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) 707 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA); 708 709 /* guaranteed by upper layers, but check just in case */ 710 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 711 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 712 } 713 714 static void 715 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 716 struct nvme_cqe *cqe) 717 { 718 struct nvme_softc *sc = q->q_sc; 719 bus_dmamap_t dmap = ccb->ccb_dmamap; 720 void *nnc_cookie = ccb->ccb_cookie; 721 nvme_nnc_done nnc_done = ccb->nnc_done; 722 struct buf *bp = ccb->nnc_buf; 723 724 if (dmap->dm_nsegs > 2) { 725 bus_dmamap_sync(sc->sc_dmat, 726 NVME_DMA_MAP(q->q_ccb_prpls), 727 ccb->ccb_prpl_off, 728 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 729 BUS_DMASYNC_POSTWRITE); 730 } 731 732 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 733 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 734 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 735 736 bus_dmamap_unload(sc->sc_dmat, dmap); 737 nvme_ccb_put(q, ccb); 738 739 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 740 } 741 742 /* 743 * If there is no volatile write cache, it makes no sense to issue 744 * flush commands or query for the status. 745 */ 746 static bool 747 nvme_has_volatile_write_cache(struct nvme_softc *sc) 748 { 749 /* sc_identify is filled during attachment */ 750 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0); 751 } 752 753 static bool 754 nvme_ns_sync_finished(void *cookie) 755 { 756 int *result = cookie; 757 758 return (*result != 0); 759 } 760 761 int 762 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, int flags) 763 { 764 struct nvme_queue *q = nvme_get_q(sc); 765 struct nvme_ccb *ccb; 766 int result = 0; 767 768 if (!nvme_has_volatile_write_cache(sc)) { 769 /* cache not present, no value in trying to flush it */ 770 return 0; 771 } 772 773 ccb = nvme_ccb_get(q, true); 774 if (ccb == NULL) 775 return EAGAIN; 776 777 ccb->ccb_done = nvme_ns_sync_done; 778 ccb->ccb_cookie = &result; 779 780 /* namespace context */ 781 ccb->nnc_nsid = nsid; 782 ccb->nnc_flags = flags; 783 ccb->nnc_done = NULL; 784 785 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 786 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 787 return EIO; 788 return 0; 789 } 790 791 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 792 793 /* wait for completion */ 794 nvme_q_wait_complete(sc, q, nvme_ns_sync_finished, &result); 795 KASSERT(result != 0); 796 797 return (result > 0) ? 0 : EIO; 798 } 799 800 static void 801 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 802 { 803 struct nvme_sqe *sqe = slot; 804 805 sqe->opcode = NVM_CMD_FLUSH; 806 htolem32(&sqe->nsid, ccb->nnc_nsid); 807 } 808 809 static void 810 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 811 struct nvme_cqe *cqe) 812 { 813 int *result = ccb->ccb_cookie; 814 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 815 816 if (status == NVME_CQE_SC_SUCCESS) 817 *result = 1; 818 else 819 *result = -1; 820 821 nvme_ccb_put(q, ccb); 822 } 823 824 static bool 825 nvme_getcache_finished(void *xc) 826 { 827 int *addr = xc; 828 829 return (*addr != 0); 830 } 831 832 /* 833 * Get status of volatile write cache. Always asynchronous. 834 */ 835 int 836 nvme_admin_getcache(struct nvme_softc *sc, int *addr) 837 { 838 struct nvme_ccb *ccb; 839 struct nvme_queue *q = sc->sc_admin_q; 840 int result = 0, error; 841 842 if (!nvme_has_volatile_write_cache(sc)) { 843 /* cache simply not present */ 844 *addr = 0; 845 return 0; 846 } 847 848 ccb = nvme_ccb_get(q, true); 849 KASSERT(ccb != NULL); 850 851 ccb->ccb_done = nvme_getcache_done; 852 ccb->ccb_cookie = &result; 853 854 /* namespace context */ 855 ccb->nnc_flags = 0; 856 ccb->nnc_done = NULL; 857 858 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); 859 860 /* wait for completion */ 861 nvme_q_wait_complete(sc, q, nvme_getcache_finished, &result); 862 KASSERT(result != 0); 863 864 if (result > 0) { 865 *addr = result; 866 error = 0; 867 } else 868 error = EINVAL; 869 870 return error; 871 } 872 873 static void 874 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 875 { 876 struct nvme_sqe *sqe = slot; 877 878 sqe->opcode = NVM_ADMIN_GET_FEATURES; 879 sqe->cdw10 = NVM_FEATURE_VOLATILE_WRITE_CACHE; 880 } 881 882 static void 883 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 884 struct nvme_cqe *cqe) 885 { 886 int *addr = ccb->ccb_cookie; 887 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 888 uint32_t cdw0 = lemtoh32(&cqe->cdw0); 889 int result; 890 891 if (status == NVME_CQE_SC_SUCCESS) { 892 result = 0; 893 894 /* 895 * DPO not supported, Dataset Management (DSM) field doesn't 896 * specify the same semantics. FUA is always supported. 897 */ 898 result = DKCACHE_FUA; 899 900 if (cdw0 & NVME_CQE_CDW0_VWC_WCE) 901 result |= DKCACHE_WRITE; 902 903 /* 904 * If volatile write cache is present, the flag shall also be 905 * settable. 906 */ 907 result |= DKCACHE_WCHANGE; 908 } else { 909 result = -1; 910 } 911 912 *addr = result; 913 914 nvme_ccb_put(q, ccb); 915 } 916 917 void 918 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 919 { 920 struct nvme_namespace *ns; 921 struct nvm_identify_namespace *identify; 922 923 ns = nvme_ns_get(sc, nsid); 924 KASSERT(ns); 925 926 identify = ns->ident; 927 ns->ident = NULL; 928 if (identify != NULL) 929 kmem_free(identify, sizeof(*identify)); 930 } 931 932 struct nvme_pt_state { 933 struct nvme_pt_command *pt; 934 bool finished; 935 }; 936 937 static void 938 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 939 { 940 struct nvme_softc *sc = q->q_sc; 941 struct nvme_sqe *sqe = slot; 942 struct nvme_pt_state *state = ccb->ccb_cookie; 943 struct nvme_pt_command *pt = state->pt; 944 bus_dmamap_t dmap = ccb->ccb_dmamap; 945 int i; 946 947 sqe->opcode = pt->cmd.opcode; 948 htolem32(&sqe->nsid, pt->cmd.nsid); 949 950 if (pt->buf != NULL && pt->len > 0) { 951 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 952 switch (dmap->dm_nsegs) { 953 case 1: 954 break; 955 case 2: 956 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 957 break; 958 default: 959 for (i = 1; i < dmap->dm_nsegs; i++) { 960 htolem64(&ccb->ccb_prpl[i - 1], 961 dmap->dm_segs[i].ds_addr); 962 } 963 bus_dmamap_sync(sc->sc_dmat, 964 NVME_DMA_MAP(q->q_ccb_prpls), 965 ccb->ccb_prpl_off, 966 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 967 BUS_DMASYNC_PREWRITE); 968 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 969 break; 970 } 971 } 972 973 htolem32(&sqe->cdw10, pt->cmd.cdw10); 974 htolem32(&sqe->cdw11, pt->cmd.cdw11); 975 htolem32(&sqe->cdw12, pt->cmd.cdw12); 976 htolem32(&sqe->cdw13, pt->cmd.cdw13); 977 htolem32(&sqe->cdw14, pt->cmd.cdw14); 978 htolem32(&sqe->cdw15, pt->cmd.cdw15); 979 } 980 981 static void 982 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 983 { 984 struct nvme_softc *sc = q->q_sc; 985 struct nvme_pt_state *state = ccb->ccb_cookie; 986 struct nvme_pt_command *pt = state->pt; 987 bus_dmamap_t dmap = ccb->ccb_dmamap; 988 989 if (pt->buf != NULL && pt->len > 0) { 990 if (dmap->dm_nsegs > 2) { 991 bus_dmamap_sync(sc->sc_dmat, 992 NVME_DMA_MAP(q->q_ccb_prpls), 993 ccb->ccb_prpl_off, 994 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 995 BUS_DMASYNC_POSTWRITE); 996 } 997 998 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 999 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 1000 bus_dmamap_unload(sc->sc_dmat, dmap); 1001 } 1002 1003 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 1004 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 1005 1006 state->finished = true; 1007 1008 nvme_ccb_put(q, ccb); 1009 } 1010 1011 static bool 1012 nvme_pt_finished(void *cookie) 1013 { 1014 struct nvme_pt_state *state = cookie; 1015 1016 return state->finished; 1017 } 1018 1019 static int 1020 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 1021 uint16_t nsid, struct lwp *l, bool is_adminq) 1022 { 1023 struct nvme_queue *q; 1024 struct nvme_ccb *ccb; 1025 void *buf = NULL; 1026 struct nvme_pt_state state; 1027 int error; 1028 1029 /* limit command size to maximum data transfer size */ 1030 if ((pt->buf == NULL && pt->len > 0) || 1031 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 1032 return EINVAL; 1033 1034 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 1035 ccb = nvme_ccb_get(q, true); 1036 KASSERT(ccb != NULL); 1037 1038 if (pt->buf != NULL) { 1039 KASSERT(pt->len > 0); 1040 buf = kmem_alloc(pt->len, KM_SLEEP); 1041 if (!pt->is_read) { 1042 error = copyin(pt->buf, buf, pt->len); 1043 if (error) 1044 goto kmem_free; 1045 } 1046 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 1047 pt->len, NULL, 1048 BUS_DMA_WAITOK | 1049 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 1050 if (error) 1051 goto kmem_free; 1052 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 1053 0, ccb->ccb_dmamap->dm_mapsize, 1054 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 1055 } 1056 1057 memset(&state, 0, sizeof(state)); 1058 state.pt = pt; 1059 state.finished = false; 1060 1061 ccb->ccb_done = nvme_pt_done; 1062 ccb->ccb_cookie = &state; 1063 1064 pt->cmd.nsid = nsid; 1065 1066 nvme_q_submit(sc, q, ccb, nvme_pt_fill); 1067 1068 /* wait for completion */ 1069 nvme_q_wait_complete(sc, q, nvme_pt_finished, &state); 1070 KASSERT(state.finished); 1071 1072 error = 0; 1073 1074 if (buf != NULL) { 1075 if (error == 0 && pt->is_read) 1076 error = copyout(buf, pt->buf, pt->len); 1077 kmem_free: 1078 kmem_free(buf, pt->len); 1079 } 1080 1081 return error; 1082 } 1083 1084 static void 1085 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1086 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 1087 { 1088 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 1089 uint32_t tail; 1090 1091 mutex_enter(&q->q_sq_mtx); 1092 tail = q->q_sq_tail; 1093 if (++q->q_sq_tail >= q->q_entries) 1094 q->q_sq_tail = 0; 1095 1096 sqe += tail; 1097 1098 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1099 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 1100 memset(sqe, 0, sizeof(*sqe)); 1101 (*fill)(q, ccb, sqe); 1102 sqe->cid = ccb->ccb_id; 1103 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1104 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 1105 1106 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 1107 mutex_exit(&q->q_sq_mtx); 1108 } 1109 1110 struct nvme_poll_state { 1111 struct nvme_sqe s; 1112 struct nvme_cqe c; 1113 void *cookie; 1114 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 1115 }; 1116 1117 static int 1118 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1119 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 1120 { 1121 struct nvme_poll_state state; 1122 uint16_t flags; 1123 int step = 10; 1124 int maxloop = timo_sec * 1000000 / step; 1125 int error = 0; 1126 1127 memset(&state, 0, sizeof(state)); 1128 (*fill)(q, ccb, &state.s); 1129 1130 state.done = ccb->ccb_done; 1131 state.cookie = ccb->ccb_cookie; 1132 1133 ccb->ccb_done = nvme_poll_done; 1134 ccb->ccb_cookie = &state; 1135 1136 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1137 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1138 if (nvme_q_complete(sc, q) == 0) 1139 delay(step); 1140 1141 if (timo_sec >= 0 && --maxloop <= 0) { 1142 error = ETIMEDOUT; 1143 break; 1144 } 1145 } 1146 1147 if (error == 0) { 1148 flags = lemtoh16(&state.c.flags); 1149 return flags & ~NVME_CQE_PHASE; 1150 } else { 1151 /* 1152 * If it succeds later, it would hit ccb which will have been 1153 * already reused for something else. Not good. Cross 1154 * fingers and hope for best. XXX do controller reset? 1155 */ 1156 aprint_error_dev(sc->sc_dev, "polled command timed out\n"); 1157 1158 /* Invoke the callback to clean state anyway */ 1159 struct nvme_cqe cqe; 1160 memset(&cqe, 0, sizeof(cqe)); 1161 ccb->ccb_done(q, ccb, &cqe); 1162 1163 return 1; 1164 } 1165 } 1166 1167 static void 1168 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1169 { 1170 struct nvme_sqe *sqe = slot; 1171 struct nvme_poll_state *state = ccb->ccb_cookie; 1172 1173 *sqe = state->s; 1174 } 1175 1176 static void 1177 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1178 struct nvme_cqe *cqe) 1179 { 1180 struct nvme_poll_state *state = ccb->ccb_cookie; 1181 1182 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1183 state->c = *cqe; 1184 1185 ccb->ccb_cookie = state->cookie; 1186 state->done(q, ccb, &state->c); 1187 } 1188 1189 static void 1190 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1191 { 1192 struct nvme_sqe *src = ccb->ccb_cookie; 1193 struct nvme_sqe *dst = slot; 1194 1195 *dst = *src; 1196 } 1197 1198 static void 1199 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1200 struct nvme_cqe *cqe) 1201 { 1202 } 1203 1204 static int 1205 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1206 { 1207 struct nvme_ccb *ccb; 1208 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1209 uint16_t flags; 1210 int rv = 0; 1211 1212 mutex_enter(&q->q_cq_mtx); 1213 1214 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1215 for (;;) { 1216 cqe = &ring[q->q_cq_head]; 1217 flags = lemtoh16(&cqe->flags); 1218 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1219 break; 1220 1221 ccb = &q->q_ccbs[cqe->cid]; 1222 1223 if (++q->q_cq_head >= q->q_entries) { 1224 q->q_cq_head = 0; 1225 q->q_cq_phase ^= NVME_CQE_PHASE; 1226 } 1227 1228 #ifdef DEBUG 1229 /* 1230 * If we get spurious completion notification, something 1231 * is seriously hosed up. Very likely DMA to some random 1232 * memory place happened, so just bail out. 1233 */ 1234 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1235 panic("%s: invalid ccb detected", 1236 device_xname(sc->sc_dev)); 1237 /* NOTREACHED */ 1238 } 1239 #endif 1240 1241 rv++; 1242 1243 /* 1244 * Unlock the mutex before calling the ccb_done callback 1245 * and re-lock afterwards. The callback triggers lddone() 1246 * which schedules another i/o, and also calls nvme_ccb_put(). 1247 * Unlock/relock avoids possibility of deadlock. 1248 */ 1249 mutex_exit(&q->q_cq_mtx); 1250 ccb->ccb_done(q, ccb, cqe); 1251 mutex_enter(&q->q_cq_mtx); 1252 } 1253 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1254 1255 if (rv) 1256 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1257 1258 mutex_exit(&q->q_cq_mtx); 1259 1260 return rv; 1261 } 1262 1263 static void 1264 nvme_q_wait_complete(struct nvme_softc *sc, 1265 struct nvme_queue *q, bool (*finished)(void *), void *cookie) 1266 { 1267 mutex_enter(&q->q_ccb_mtx); 1268 if (finished(cookie)) 1269 goto out; 1270 1271 for(;;) { 1272 q->q_ccb_waiting = true; 1273 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1274 1275 if (finished(cookie)) 1276 break; 1277 } 1278 1279 out: 1280 mutex_exit(&q->q_ccb_mtx); 1281 } 1282 1283 static int 1284 nvme_identify(struct nvme_softc *sc, u_int mps) 1285 { 1286 char sn[41], mn[81], fr[17]; 1287 struct nvm_identify_controller *identify; 1288 struct nvme_dmamem *mem; 1289 struct nvme_ccb *ccb; 1290 u_int mdts; 1291 int rv = 1; 1292 1293 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1294 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1295 1296 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1297 if (mem == NULL) 1298 return 1; 1299 1300 ccb->ccb_done = nvme_empty_done; 1301 ccb->ccb_cookie = mem; 1302 1303 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1304 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1305 NVME_TIMO_IDENT); 1306 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1307 1308 nvme_ccb_put(sc->sc_admin_q, ccb); 1309 1310 if (rv != 0) 1311 goto done; 1312 1313 identify = NVME_DMA_KVA(mem); 1314 1315 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1316 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1317 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1318 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1319 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1320 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1321 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1322 sn); 1323 1324 if (identify->mdts > 0) { 1325 mdts = (1 << identify->mdts) * (1 << mps); 1326 if (mdts < sc->sc_mdts) 1327 sc->sc_mdts = mdts; 1328 } 1329 1330 sc->sc_nn = lemtoh32(&identify->nn); 1331 1332 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1333 1334 done: 1335 nvme_dmamem_free(sc, mem); 1336 1337 return rv; 1338 } 1339 1340 static int 1341 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1342 { 1343 struct nvme_sqe_q sqe; 1344 struct nvme_ccb *ccb; 1345 int rv; 1346 1347 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1348 return 1; 1349 1350 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1351 KASSERT(ccb != NULL); 1352 1353 ccb->ccb_done = nvme_empty_done; 1354 ccb->ccb_cookie = &sqe; 1355 1356 memset(&sqe, 0, sizeof(sqe)); 1357 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1358 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1359 htolem16(&sqe.qsize, q->q_entries - 1); 1360 htolem16(&sqe.qid, q->q_id); 1361 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1362 if (sc->sc_use_mq) 1363 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1364 1365 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1366 if (rv != 0) 1367 goto fail; 1368 1369 ccb->ccb_done = nvme_empty_done; 1370 ccb->ccb_cookie = &sqe; 1371 1372 memset(&sqe, 0, sizeof(sqe)); 1373 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1374 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1375 htolem16(&sqe.qsize, q->q_entries - 1); 1376 htolem16(&sqe.qid, q->q_id); 1377 htolem16(&sqe.cqid, q->q_id); 1378 sqe.qflags = NVM_SQE_Q_PC; 1379 1380 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1381 if (rv != 0) 1382 goto fail; 1383 1384 fail: 1385 nvme_ccb_put(sc->sc_admin_q, ccb); 1386 return rv; 1387 } 1388 1389 static int 1390 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1391 { 1392 struct nvme_sqe_q sqe; 1393 struct nvme_ccb *ccb; 1394 int rv; 1395 1396 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1397 KASSERT(ccb != NULL); 1398 1399 ccb->ccb_done = nvme_empty_done; 1400 ccb->ccb_cookie = &sqe; 1401 1402 memset(&sqe, 0, sizeof(sqe)); 1403 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1404 htolem16(&sqe.qid, q->q_id); 1405 1406 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1407 if (rv != 0) 1408 goto fail; 1409 1410 ccb->ccb_done = nvme_empty_done; 1411 ccb->ccb_cookie = &sqe; 1412 1413 memset(&sqe, 0, sizeof(sqe)); 1414 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1415 htolem16(&sqe.qid, q->q_id); 1416 1417 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1418 if (rv != 0) 1419 goto fail; 1420 1421 fail: 1422 nvme_ccb_put(sc->sc_admin_q, ccb); 1423 1424 if (rv == 0 && sc->sc_use_mq) { 1425 if (sc->sc_intr_disestablish(sc, q->q_id)) 1426 rv = 1; 1427 } 1428 1429 return rv; 1430 } 1431 1432 static void 1433 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1434 { 1435 struct nvme_sqe *sqe = slot; 1436 struct nvme_dmamem *mem = ccb->ccb_cookie; 1437 1438 sqe->opcode = NVM_ADMIN_IDENTIFY; 1439 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1440 htolem32(&sqe->cdw10, 1); 1441 } 1442 1443 static int 1444 nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap) 1445 { 1446 struct nvme_pt_command pt; 1447 struct nvme_ccb *ccb; 1448 uint16_t ncqa, nsqa; 1449 int rv; 1450 1451 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1452 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1453 1454 memset(&pt, 0, sizeof(pt)); 1455 pt.cmd.opcode = NVM_ADMIN_GET_FEATURES; 1456 pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES; 1457 1458 ccb->ccb_done = nvme_pt_done; 1459 ccb->ccb_cookie = &pt; 1460 1461 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1462 1463 nvme_ccb_put(sc->sc_admin_q, ccb); 1464 1465 if (rv != 0) { 1466 *nqap = 0; 1467 return EIO; 1468 } 1469 1470 ncqa = pt.cpl.cdw0 >> 16; 1471 nsqa = pt.cpl.cdw0 & 0xffff; 1472 *nqap = MIN(ncqa, nsqa) + 1; 1473 1474 return 0; 1475 } 1476 1477 static int 1478 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1479 { 1480 struct nvme_softc *sc = q->q_sc; 1481 struct nvme_ccb *ccb; 1482 bus_addr_t off; 1483 uint64_t *prpl; 1484 u_int i; 1485 1486 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1487 cv_init(&q->q_ccb_wait, "nvmeqw"); 1488 q->q_ccb_waiting = false; 1489 SIMPLEQ_INIT(&q->q_ccb_list); 1490 1491 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1492 1493 q->q_nccbs = nccbs; 1494 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1495 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1496 1497 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1498 off = 0; 1499 1500 for (i = 0; i < nccbs; i++) { 1501 ccb = &q->q_ccbs[i]; 1502 1503 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1504 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1505 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1506 &ccb->ccb_dmamap) != 0) 1507 goto free_maps; 1508 1509 ccb->ccb_id = i; 1510 ccb->ccb_prpl = prpl; 1511 ccb->ccb_prpl_off = off; 1512 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1513 1514 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1515 1516 prpl += sc->sc_max_sgl; 1517 off += sizeof(*prpl) * sc->sc_max_sgl; 1518 } 1519 1520 return 0; 1521 1522 free_maps: 1523 nvme_ccbs_free(q); 1524 return 1; 1525 } 1526 1527 static struct nvme_ccb * 1528 nvme_ccb_get(struct nvme_queue *q, bool wait) 1529 { 1530 struct nvme_ccb *ccb = NULL; 1531 1532 mutex_enter(&q->q_ccb_mtx); 1533 again: 1534 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1535 if (ccb != NULL) { 1536 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1537 #ifdef DEBUG 1538 ccb->ccb_cookie = NULL; 1539 #endif 1540 } else { 1541 if (__predict_false(wait)) { 1542 q->q_ccb_waiting = true; 1543 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1544 goto again; 1545 } 1546 } 1547 mutex_exit(&q->q_ccb_mtx); 1548 1549 return ccb; 1550 } 1551 1552 static void 1553 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1554 { 1555 1556 mutex_enter(&q->q_ccb_mtx); 1557 #ifdef DEBUG 1558 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1559 #endif 1560 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1561 1562 /* It's unlikely there are any waiters, it's not used for regular I/O */ 1563 if (__predict_false(q->q_ccb_waiting)) { 1564 q->q_ccb_waiting = false; 1565 cv_broadcast(&q->q_ccb_wait); 1566 } 1567 1568 mutex_exit(&q->q_ccb_mtx); 1569 } 1570 1571 static void 1572 nvme_ccbs_free(struct nvme_queue *q) 1573 { 1574 struct nvme_softc *sc = q->q_sc; 1575 struct nvme_ccb *ccb; 1576 1577 mutex_enter(&q->q_ccb_mtx); 1578 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1579 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1580 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1581 } 1582 mutex_exit(&q->q_ccb_mtx); 1583 1584 nvme_dmamem_free(sc, q->q_ccb_prpls); 1585 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1586 q->q_ccbs = NULL; 1587 cv_destroy(&q->q_ccb_wait); 1588 mutex_destroy(&q->q_ccb_mtx); 1589 } 1590 1591 static struct nvme_queue * 1592 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1593 { 1594 struct nvme_queue *q; 1595 1596 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1597 q->q_sc = sc; 1598 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1599 sizeof(struct nvme_sqe) * entries); 1600 if (q->q_sq_dmamem == NULL) 1601 goto free; 1602 1603 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1604 sizeof(struct nvme_cqe) * entries); 1605 if (q->q_cq_dmamem == NULL) 1606 goto free_sq; 1607 1608 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1609 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1610 1611 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1612 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1613 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1614 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1615 q->q_id = id; 1616 q->q_entries = entries; 1617 q->q_sq_tail = 0; 1618 q->q_cq_head = 0; 1619 q->q_cq_phase = NVME_CQE_PHASE; 1620 1621 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1622 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1623 1624 /* 1625 * Due to definition of full and empty queue (queue is empty 1626 * when head == tail, full when tail is one less then head), 1627 * we can actually only have (entries - 1) in-flight commands. 1628 */ 1629 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1630 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1631 goto free_cq; 1632 } 1633 1634 return q; 1635 1636 free_cq: 1637 nvme_dmamem_free(sc, q->q_cq_dmamem); 1638 free_sq: 1639 nvme_dmamem_free(sc, q->q_sq_dmamem); 1640 free: 1641 kmem_free(q, sizeof(*q)); 1642 1643 return NULL; 1644 } 1645 1646 static void 1647 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1648 { 1649 nvme_ccbs_free(q); 1650 mutex_destroy(&q->q_sq_mtx); 1651 mutex_destroy(&q->q_cq_mtx); 1652 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1653 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1654 nvme_dmamem_free(sc, q->q_cq_dmamem); 1655 nvme_dmamem_free(sc, q->q_sq_dmamem); 1656 kmem_free(q, sizeof(*q)); 1657 } 1658 1659 int 1660 nvme_intr(void *xsc) 1661 { 1662 struct nvme_softc *sc = xsc; 1663 1664 /* 1665 * INTx is level triggered, controller deasserts the interrupt only 1666 * when we advance command queue head via write to the doorbell. 1667 * Tell the controller to block the interrupts while we process 1668 * the queue(s). 1669 */ 1670 nvme_write4(sc, NVME_INTMS, 1); 1671 1672 softint_schedule(sc->sc_softih[0]); 1673 1674 /* don't know, might not have been for us */ 1675 return 1; 1676 } 1677 1678 void 1679 nvme_softintr_intx(void *xq) 1680 { 1681 struct nvme_queue *q = xq; 1682 struct nvme_softc *sc = q->q_sc; 1683 1684 nvme_q_complete(sc, sc->sc_admin_q); 1685 if (sc->sc_q != NULL) 1686 nvme_q_complete(sc, sc->sc_q[0]); 1687 1688 /* 1689 * Processing done, tell controller to issue interrupts again. There 1690 * is no race, as NVMe spec requires the controller to maintain state, 1691 * and assert the interrupt whenever there are unacknowledged 1692 * completion queue entries. 1693 */ 1694 nvme_write4(sc, NVME_INTMC, 1); 1695 } 1696 1697 int 1698 nvme_intr_msi(void *xq) 1699 { 1700 struct nvme_queue *q = xq; 1701 1702 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1703 && q->q_sc->sc_softih[q->q_id]); 1704 1705 /* 1706 * MSI/MSI-X are edge triggered, so can handover processing to softint 1707 * without masking the interrupt. 1708 */ 1709 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1710 1711 return 1; 1712 } 1713 1714 void 1715 nvme_softintr_msi(void *xq) 1716 { 1717 struct nvme_queue *q = xq; 1718 struct nvme_softc *sc = q->q_sc; 1719 1720 nvme_q_complete(sc, q); 1721 } 1722 1723 static struct nvme_dmamem * 1724 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1725 { 1726 struct nvme_dmamem *ndm; 1727 int nsegs; 1728 1729 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1730 if (ndm == NULL) 1731 return NULL; 1732 1733 ndm->ndm_size = size; 1734 1735 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1736 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1737 goto ndmfree; 1738 1739 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1740 1, &nsegs, BUS_DMA_WAITOK) != 0) 1741 goto destroy; 1742 1743 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1744 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1745 goto free; 1746 memset(ndm->ndm_kva, 0, size); 1747 1748 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1749 NULL, BUS_DMA_WAITOK) != 0) 1750 goto unmap; 1751 1752 return ndm; 1753 1754 unmap: 1755 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1756 free: 1757 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1758 destroy: 1759 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1760 ndmfree: 1761 kmem_free(ndm, sizeof(*ndm)); 1762 return NULL; 1763 } 1764 1765 static void 1766 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1767 { 1768 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1769 0, NVME_DMA_LEN(mem), ops); 1770 } 1771 1772 void 1773 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1774 { 1775 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1776 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1777 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1778 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1779 kmem_free(ndm, sizeof(*ndm)); 1780 } 1781 1782 /* 1783 * ioctl 1784 */ 1785 1786 dev_type_open(nvmeopen); 1787 dev_type_close(nvmeclose); 1788 dev_type_ioctl(nvmeioctl); 1789 1790 const struct cdevsw nvme_cdevsw = { 1791 .d_open = nvmeopen, 1792 .d_close = nvmeclose, 1793 .d_read = noread, 1794 .d_write = nowrite, 1795 .d_ioctl = nvmeioctl, 1796 .d_stop = nostop, 1797 .d_tty = notty, 1798 .d_poll = nopoll, 1799 .d_mmap = nommap, 1800 .d_kqfilter = nokqfilter, 1801 .d_discard = nodiscard, 1802 .d_flag = D_OTHER, 1803 }; 1804 1805 /* 1806 * Accept an open operation on the control device. 1807 */ 1808 int 1809 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1810 { 1811 struct nvme_softc *sc; 1812 int unit = minor(dev) / 0x10000; 1813 int nsid = minor(dev) & 0xffff; 1814 int nsidx; 1815 1816 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1817 return ENXIO; 1818 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1819 return ENXIO; 1820 1821 if (nsid == 0) { 1822 /* controller */ 1823 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1824 return EBUSY; 1825 SET(sc->sc_flags, NVME_F_OPEN); 1826 } else { 1827 /* namespace */ 1828 nsidx = nsid - 1; 1829 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1830 return ENXIO; 1831 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1832 return EBUSY; 1833 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1834 } 1835 return 0; 1836 } 1837 1838 /* 1839 * Accept the last close on the control device. 1840 */ 1841 int 1842 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1843 { 1844 struct nvme_softc *sc; 1845 int unit = minor(dev) / 0x10000; 1846 int nsid = minor(dev) & 0xffff; 1847 int nsidx; 1848 1849 sc = device_lookup_private(&nvme_cd, unit); 1850 if (sc == NULL) 1851 return ENXIO; 1852 1853 if (nsid == 0) { 1854 /* controller */ 1855 CLR(sc->sc_flags, NVME_F_OPEN); 1856 } else { 1857 /* namespace */ 1858 nsidx = nsid - 1; 1859 if (nsidx >= sc->sc_nn) 1860 return ENXIO; 1861 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1862 } 1863 1864 return 0; 1865 } 1866 1867 /* 1868 * Handle control operations. 1869 */ 1870 int 1871 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1872 { 1873 struct nvme_softc *sc; 1874 int unit = minor(dev) / 0x10000; 1875 int nsid = minor(dev) & 0xffff; 1876 struct nvme_pt_command *pt; 1877 1878 sc = device_lookup_private(&nvme_cd, unit); 1879 if (sc == NULL) 1880 return ENXIO; 1881 1882 switch (cmd) { 1883 case NVME_PASSTHROUGH_CMD: 1884 pt = data; 1885 return nvme_command_passthrough(sc, data, 1886 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1887 } 1888 1889 return ENOTTY; 1890 } 1891
Indexes created Sat Sep 27 22:09:54 GMT 2025