nvme.c revision 1.38
1 /* $NetBSD: nvme.c,v 1.38 2018/04/18 10:05:59 nonaka Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.38 2018/04/18 10:05:59 nonaka Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 #include "ioconf.h" 44 45 #define B4_CHK_RDY_DELAY_MS 2300 /* workaround controller bug */ 46 47 int nvme_adminq_size = 32; 48 int nvme_ioq_size = 1024; 49 50 static int nvme_print(void *, const char *); 51 52 static int nvme_ready(struct nvme_softc *, uint32_t); 53 static int nvme_enable(struct nvme_softc *, u_int); 54 static int nvme_disable(struct nvme_softc *); 55 static int nvme_shutdown(struct nvme_softc *); 56 57 #ifdef NVME_DEBUG 58 static void nvme_dumpregs(struct nvme_softc *); 59 #endif 60 static int nvme_identify(struct nvme_softc *, u_int); 61 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 62 void *); 63 64 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 65 static void nvme_ccbs_free(struct nvme_queue *); 66 67 static struct nvme_ccb * 68 nvme_ccb_get(struct nvme_queue *, bool); 69 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 70 71 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 72 struct nvme_ccb *, void (*)(struct nvme_queue *, 73 struct nvme_ccb *, void *), int); 74 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 75 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 76 struct nvme_cqe *); 77 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 78 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 79 struct nvme_cqe *); 80 81 static struct nvme_queue * 82 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 83 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 84 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 85 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 86 struct nvme_ccb *, void (*)(struct nvme_queue *, 87 struct nvme_ccb *, void *)); 88 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 89 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 90 static void nvme_q_wait_complete(struct nvme_softc *, struct nvme_queue *, 91 bool (*)(void *), void *); 92 93 static struct nvme_dmamem * 94 nvme_dmamem_alloc(struct nvme_softc *, size_t); 95 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 96 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 97 int); 98 99 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 100 void *); 101 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 102 struct nvme_cqe *); 103 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 104 void *); 105 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 106 struct nvme_cqe *); 107 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *, 108 void *); 109 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *, 110 struct nvme_cqe *); 111 112 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 113 void *); 114 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 115 struct nvme_cqe *); 116 static int nvme_command_passthrough(struct nvme_softc *, 117 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 118 119 static int nvme_get_number_of_queues(struct nvme_softc *, u_int *); 120 121 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 122 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 123 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 124 #define NVME_TIMO_SY 60 /* sync cache timeout */ 125 126 #define nvme_read4(_s, _r) \ 127 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 128 #define nvme_write4(_s, _r, _v) \ 129 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 130 /* 131 * Some controllers, at least Apple NVMe, always require split 132 * transfers, so don't use bus_space_{read,write}_8() on LP64. 133 */ 134 static inline uint64_t 135 nvme_read8(struct nvme_softc *sc, bus_size_t r) 136 { 137 uint64_t v; 138 uint32_t *a = (uint32_t *)&v; 139 140 #if _BYTE_ORDER == _LITTLE_ENDIAN 141 a[0] = nvme_read4(sc, r); 142 a[1] = nvme_read4(sc, r + 4); 143 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 144 a[1] = nvme_read4(sc, r); 145 a[0] = nvme_read4(sc, r + 4); 146 #endif 147 148 return v; 149 } 150 151 static inline void 152 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 153 { 154 uint32_t *a = (uint32_t *)&v; 155 156 #if _BYTE_ORDER == _LITTLE_ENDIAN 157 nvme_write4(sc, r, a[0]); 158 nvme_write4(sc, r + 4, a[1]); 159 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 160 nvme_write4(sc, r, a[1]); 161 nvme_write4(sc, r + 4, a[0]); 162 #endif 163 } 164 #define nvme_barrier(_s, _r, _l, _f) \ 165 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 166 167 #ifdef NVME_DEBUG 168 static __used void 169 nvme_dumpregs(struct nvme_softc *sc) 170 { 171 uint64_t r8; 172 uint32_t r4; 173 174 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 175 r8 = nvme_read8(sc, NVME_CAP); 176 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 177 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 178 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 179 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 180 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 181 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 182 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 183 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 184 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 185 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 186 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 187 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 188 189 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 190 191 r4 = nvme_read4(sc, NVME_CC); 192 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 193 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 194 (1 << NVME_CC_IOCQES_R(r4))); 195 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 196 (1 << NVME_CC_IOSQES_R(r4))); 197 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 198 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 199 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 200 (1 << NVME_CC_MPS_R(r4))); 201 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 202 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 203 204 r4 = nvme_read4(sc, NVME_CSTS); 205 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 206 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 207 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 208 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 209 210 r4 = nvme_read4(sc, NVME_AQA); 211 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 212 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 213 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 214 215 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 216 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 217 #undef DEVNAME 218 } 219 #endif /* NVME_DEBUG */ 220 221 static int 222 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 223 { 224 u_int i = 0; 225 226 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 227 if (i++ > sc->sc_rdy_to) 228 return ENXIO; 229 230 delay(1000); 231 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 232 } 233 234 return 0; 235 } 236 237 static int 238 nvme_enable(struct nvme_softc *sc, u_int mps) 239 { 240 uint32_t cc, csts; 241 int error; 242 243 cc = nvme_read4(sc, NVME_CC); 244 csts = nvme_read4(sc, NVME_CSTS); 245 246 /* 247 * See note in nvme_disable. Short circuit if we're already enabled. 248 */ 249 if (ISSET(cc, NVME_CC_EN)) { 250 if (ISSET(csts, NVME_CSTS_RDY)) 251 return 0; 252 253 goto waitready; 254 } else { 255 /* EN == 0 already wait for RDY == 0 or fail */ 256 error = nvme_ready(sc, 0); 257 if (error) 258 return error; 259 } 260 261 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 262 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 263 delay(5000); 264 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 265 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 266 delay(5000); 267 268 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 269 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 270 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 271 delay(5000); 272 273 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 274 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 275 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 276 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 277 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 278 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 279 SET(cc, NVME_CC_MPS(mps)); 280 SET(cc, NVME_CC_EN); 281 282 nvme_write4(sc, NVME_CC, cc); 283 nvme_barrier(sc, 0, sc->sc_ios, 284 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 285 286 waitready: 287 return nvme_ready(sc, NVME_CSTS_RDY); 288 } 289 290 static int 291 nvme_disable(struct nvme_softc *sc) 292 { 293 uint32_t cc, csts; 294 int error; 295 296 cc = nvme_read4(sc, NVME_CC); 297 csts = nvme_read4(sc, NVME_CSTS); 298 299 /* 300 * Per 3.1.5 in NVME 1.3 spec, transitioning CC.EN from 0 to 1 301 * when CSTS.RDY is 1 or transitioning CC.EN from 1 to 0 when 302 * CSTS.RDY is 0 "has undefined results" So make sure that CSTS.RDY 303 * isn't the desired value. Short circuit if we're already disabled. 304 */ 305 if (ISSET(cc, NVME_CC_EN)) { 306 if (!ISSET(csts, NVME_CSTS_RDY)) { 307 /* EN == 1, wait for RDY == 1 or fail */ 308 error = nvme_ready(sc, NVME_CSTS_RDY); 309 if (error) 310 return error; 311 } 312 } else { 313 /* EN == 0 already wait for RDY == 0 */ 314 if (!ISSET(csts, NVME_CSTS_RDY)) 315 return 0; 316 317 goto waitready; 318 } 319 320 CLR(cc, NVME_CC_EN); 321 nvme_write4(sc, NVME_CC, cc); 322 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 323 324 /* 325 * Some drives have issues with accessing the mmio after we disable, 326 * so delay for a bit after we write the bit to cope with these issues. 327 */ 328 if (ISSET(sc->sc_quirks, NVME_QUIRK_DELAY_B4_CHK_RDY)) 329 delay(B4_CHK_RDY_DELAY_MS); 330 331 waitready: 332 return nvme_ready(sc, 0); 333 } 334 335 int 336 nvme_attach(struct nvme_softc *sc) 337 { 338 uint64_t cap; 339 uint32_t reg; 340 u_int dstrd; 341 u_int mps = PAGE_SHIFT; 342 u_int ioq_allocated; 343 uint16_t adminq_entries = nvme_adminq_size; 344 uint16_t ioq_entries = nvme_ioq_size; 345 int i; 346 347 reg = nvme_read4(sc, NVME_VS); 348 if (reg == 0xffffffff) { 349 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 350 return 1; 351 } 352 353 if (NVME_VS_TER(reg) == 0) 354 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg), 355 NVME_VS_MNR(reg)); 356 else 357 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg), 358 NVME_VS_MNR(reg), NVME_VS_TER(reg)); 359 360 cap = nvme_read8(sc, NVME_CAP); 361 dstrd = NVME_CAP_DSTRD(cap); 362 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 363 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 364 "is greater than CPU page size %u\n", 365 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 366 return 1; 367 } 368 if (NVME_CAP_MPSMAX(cap) < mps) 369 mps = NVME_CAP_MPSMAX(cap); 370 if (ioq_entries > NVME_CAP_MQES(cap)) 371 ioq_entries = NVME_CAP_MQES(cap); 372 373 /* set initial values to be used for admin queue during probe */ 374 sc->sc_rdy_to = NVME_CAP_TO(cap); 375 sc->sc_mps = 1 << mps; 376 sc->sc_mdts = MAXPHYS; 377 sc->sc_max_sgl = 2; 378 379 if (nvme_disable(sc) != 0) { 380 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 381 return 1; 382 } 383 384 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 385 if (sc->sc_admin_q == NULL) { 386 aprint_error_dev(sc->sc_dev, 387 "unable to allocate admin queue\n"); 388 return 1; 389 } 390 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 391 goto free_admin_q; 392 393 if (nvme_enable(sc, mps) != 0) { 394 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 395 goto disestablish_admin_q; 396 } 397 398 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 399 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 400 goto disable; 401 } 402 403 /* we know how big things are now */ 404 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 405 406 /* reallocate ccbs of admin queue with new max sgl. */ 407 nvme_ccbs_free(sc->sc_admin_q); 408 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 409 410 if (sc->sc_use_mq) { 411 /* Limit the number of queues to the number allocated in HW */ 412 if (nvme_get_number_of_queues(sc, &ioq_allocated) != 0) { 413 aprint_error_dev(sc->sc_dev, 414 "unable to get number of queues\n"); 415 goto disable; 416 } 417 if (sc->sc_nq > ioq_allocated) 418 sc->sc_nq = ioq_allocated; 419 } 420 421 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 422 for (i = 0; i < sc->sc_nq; i++) { 423 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 424 if (sc->sc_q[i] == NULL) { 425 aprint_error_dev(sc->sc_dev, 426 "unable to allocate io queue\n"); 427 goto free_q; 428 } 429 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 430 aprint_error_dev(sc->sc_dev, 431 "unable to create io queue\n"); 432 nvme_q_free(sc, sc->sc_q[i]); 433 goto free_q; 434 } 435 } 436 437 if (!sc->sc_use_mq) 438 nvme_write4(sc, NVME_INTMC, 1); 439 440 /* probe subdevices */ 441 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 442 KM_SLEEP); 443 nvme_rescan(sc->sc_dev, "nvme", &i); 444 445 return 0; 446 447 free_q: 448 while (--i >= 0) { 449 nvme_q_delete(sc, sc->sc_q[i]); 450 nvme_q_free(sc, sc->sc_q[i]); 451 } 452 disable: 453 nvme_disable(sc); 454 disestablish_admin_q: 455 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 456 free_admin_q: 457 nvme_q_free(sc, sc->sc_admin_q); 458 459 return 1; 460 } 461 462 int 463 nvme_rescan(device_t self, const char *attr, const int *flags) 464 { 465 struct nvme_softc *sc = device_private(self); 466 struct nvme_attach_args naa; 467 uint64_t cap; 468 int ioq_entries = nvme_ioq_size; 469 int i; 470 471 cap = nvme_read8(sc, NVME_CAP); 472 if (ioq_entries > NVME_CAP_MQES(cap)) 473 ioq_entries = NVME_CAP_MQES(cap); 474 475 for (i = 0; i < sc->sc_nn; i++) { 476 if (sc->sc_namespaces[i].dev) 477 continue; 478 memset(&naa, 0, sizeof(naa)); 479 naa.naa_nsid = i + 1; 480 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 481 naa.naa_maxphys = sc->sc_mdts; 482 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 483 nvme_print); 484 } 485 return 0; 486 } 487 488 static int 489 nvme_print(void *aux, const char *pnp) 490 { 491 struct nvme_attach_args *naa = aux; 492 493 if (pnp) 494 aprint_normal("at %s", pnp); 495 496 if (naa->naa_nsid > 0) 497 aprint_normal(" nsid %d", naa->naa_nsid); 498 499 return UNCONF; 500 } 501 502 int 503 nvme_detach(struct nvme_softc *sc, int flags) 504 { 505 int i, error; 506 507 error = config_detach_children(sc->sc_dev, flags); 508 if (error) 509 return error; 510 511 error = nvme_shutdown(sc); 512 if (error) 513 return error; 514 515 /* from now on we are committed to detach, following will never fail */ 516 for (i = 0; i < sc->sc_nq; i++) 517 nvme_q_free(sc, sc->sc_q[i]); 518 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 519 nvme_q_free(sc, sc->sc_admin_q); 520 521 return 0; 522 } 523 524 static int 525 nvme_shutdown(struct nvme_softc *sc) 526 { 527 uint32_t cc, csts; 528 bool disabled = false; 529 int i; 530 531 if (!sc->sc_use_mq) 532 nvme_write4(sc, NVME_INTMS, 1); 533 534 for (i = 0; i < sc->sc_nq; i++) { 535 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 536 aprint_error_dev(sc->sc_dev, 537 "unable to delete io queue %d, disabling\n", i + 1); 538 disabled = true; 539 } 540 } 541 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 542 if (disabled) 543 goto disable; 544 545 cc = nvme_read4(sc, NVME_CC); 546 CLR(cc, NVME_CC_SHN_MASK); 547 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 548 nvme_write4(sc, NVME_CC, cc); 549 550 for (i = 0; i < 4000; i++) { 551 nvme_barrier(sc, 0, sc->sc_ios, 552 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 553 csts = nvme_read4(sc, NVME_CSTS); 554 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 555 return 0; 556 557 delay(1000); 558 } 559 560 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 561 562 disable: 563 nvme_disable(sc); 564 return 0; 565 } 566 567 void 568 nvme_childdet(device_t self, device_t child) 569 { 570 struct nvme_softc *sc = device_private(self); 571 int i; 572 573 for (i = 0; i < sc->sc_nn; i++) { 574 if (sc->sc_namespaces[i].dev == child) { 575 /* Already freed ns->ident. */ 576 sc->sc_namespaces[i].dev = NULL; 577 break; 578 } 579 } 580 } 581 582 int 583 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 584 { 585 struct nvme_sqe sqe; 586 struct nvm_identify_namespace *identify; 587 struct nvme_dmamem *mem; 588 struct nvme_ccb *ccb; 589 struct nvme_namespace *ns; 590 int rv; 591 592 KASSERT(nsid > 0); 593 594 ccb = nvme_ccb_get(sc->sc_admin_q, false); 595 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 596 597 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 598 if (mem == NULL) { 599 nvme_ccb_put(sc->sc_admin_q, ccb); 600 return ENOMEM; 601 } 602 603 memset(&sqe, 0, sizeof(sqe)); 604 sqe.opcode = NVM_ADMIN_IDENTIFY; 605 htolem32(&sqe.nsid, nsid); 606 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 607 htolem32(&sqe.cdw10, 0); 608 609 ccb->ccb_done = nvme_empty_done; 610 ccb->ccb_cookie = &sqe; 611 612 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 613 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 614 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 615 616 nvme_ccb_put(sc->sc_admin_q, ccb); 617 618 if (rv != 0) { 619 rv = EIO; 620 goto done; 621 } 622 623 /* commit */ 624 625 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 626 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 627 //memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 628 629 ns = nvme_ns_get(sc, nsid); 630 KASSERT(ns); 631 KASSERT(ns->ident == NULL); 632 ns->ident = identify; 633 634 done: 635 nvme_dmamem_free(sc, mem); 636 637 return rv; 638 } 639 640 int 641 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 642 struct buf *bp, void *data, size_t datasize, 643 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 644 { 645 struct nvme_queue *q = nvme_get_q(sc); 646 struct nvme_ccb *ccb; 647 bus_dmamap_t dmap; 648 int i, error; 649 650 ccb = nvme_ccb_get(q, false); 651 if (ccb == NULL) 652 return EAGAIN; 653 654 ccb->ccb_done = nvme_ns_io_done; 655 ccb->ccb_cookie = cookie; 656 657 /* namespace context */ 658 ccb->nnc_nsid = nsid; 659 ccb->nnc_flags = flags; 660 ccb->nnc_buf = bp; 661 ccb->nnc_datasize = datasize; 662 ccb->nnc_secsize = secsize; 663 ccb->nnc_blkno = blkno; 664 ccb->nnc_done = nnc_done; 665 666 dmap = ccb->ccb_dmamap; 667 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 668 datasize, NULL, 669 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 670 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 671 (ISSET(flags, NVME_NS_CTX_F_READ) ? 672 BUS_DMA_READ : BUS_DMA_WRITE)); 673 if (error) { 674 nvme_ccb_put(q, ccb); 675 return error; 676 } 677 678 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 679 ISSET(flags, NVME_NS_CTX_F_READ) ? 680 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 681 682 if (dmap->dm_nsegs > 2) { 683 for (i = 1; i < dmap->dm_nsegs; i++) { 684 htolem64(&ccb->ccb_prpl[i - 1], 685 dmap->dm_segs[i].ds_addr); 686 } 687 bus_dmamap_sync(sc->sc_dmat, 688 NVME_DMA_MAP(q->q_ccb_prpls), 689 ccb->ccb_prpl_off, 690 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 691 BUS_DMASYNC_PREWRITE); 692 } 693 694 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 695 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 696 return EIO; 697 return 0; 698 } 699 700 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 701 return 0; 702 } 703 704 static void 705 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 706 { 707 struct nvme_sqe_io *sqe = slot; 708 bus_dmamap_t dmap = ccb->ccb_dmamap; 709 710 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 711 NVM_CMD_READ : NVM_CMD_WRITE; 712 htolem32(&sqe->nsid, ccb->nnc_nsid); 713 714 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 715 switch (dmap->dm_nsegs) { 716 case 1: 717 break; 718 case 2: 719 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 720 break; 721 default: 722 /* the prp list is already set up and synced */ 723 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 724 break; 725 } 726 727 htolem64(&sqe->slba, ccb->nnc_blkno); 728 729 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) 730 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA); 731 732 /* guaranteed by upper layers, but check just in case */ 733 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 734 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 735 } 736 737 static void 738 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 739 struct nvme_cqe *cqe) 740 { 741 struct nvme_softc *sc = q->q_sc; 742 bus_dmamap_t dmap = ccb->ccb_dmamap; 743 void *nnc_cookie = ccb->ccb_cookie; 744 nvme_nnc_done nnc_done = ccb->nnc_done; 745 struct buf *bp = ccb->nnc_buf; 746 747 if (dmap->dm_nsegs > 2) { 748 bus_dmamap_sync(sc->sc_dmat, 749 NVME_DMA_MAP(q->q_ccb_prpls), 750 ccb->ccb_prpl_off, 751 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 752 BUS_DMASYNC_POSTWRITE); 753 } 754 755 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 756 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 757 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 758 759 bus_dmamap_unload(sc->sc_dmat, dmap); 760 nvme_ccb_put(q, ccb); 761 762 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 763 } 764 765 /* 766 * If there is no volatile write cache, it makes no sense to issue 767 * flush commands or query for the status. 768 */ 769 static bool 770 nvme_has_volatile_write_cache(struct nvme_softc *sc) 771 { 772 /* sc_identify is filled during attachment */ 773 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0); 774 } 775 776 static bool 777 nvme_ns_sync_finished(void *cookie) 778 { 779 int *result = cookie; 780 781 return (*result != 0); 782 } 783 784 int 785 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, int flags) 786 { 787 struct nvme_queue *q = nvme_get_q(sc); 788 struct nvme_ccb *ccb; 789 int result = 0; 790 791 if (!nvme_has_volatile_write_cache(sc)) { 792 /* cache not present, no value in trying to flush it */ 793 return 0; 794 } 795 796 ccb = nvme_ccb_get(q, true); 797 if (ccb == NULL) 798 return EAGAIN; 799 800 ccb->ccb_done = nvme_ns_sync_done; 801 ccb->ccb_cookie = &result; 802 803 /* namespace context */ 804 ccb->nnc_nsid = nsid; 805 ccb->nnc_flags = flags; 806 ccb->nnc_done = NULL; 807 808 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 809 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 810 return EIO; 811 return 0; 812 } 813 814 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 815 816 /* wait for completion */ 817 nvme_q_wait_complete(sc, q, nvme_ns_sync_finished, &result); 818 KASSERT(result != 0); 819 820 return (result > 0) ? 0 : EIO; 821 } 822 823 static void 824 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 825 { 826 struct nvme_sqe *sqe = slot; 827 828 sqe->opcode = NVM_CMD_FLUSH; 829 htolem32(&sqe->nsid, ccb->nnc_nsid); 830 } 831 832 static void 833 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 834 struct nvme_cqe *cqe) 835 { 836 int *result = ccb->ccb_cookie; 837 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 838 839 if (status == NVME_CQE_SC_SUCCESS) 840 *result = 1; 841 else 842 *result = -1; 843 844 nvme_ccb_put(q, ccb); 845 } 846 847 static bool 848 nvme_getcache_finished(void *xc) 849 { 850 int *addr = xc; 851 852 return (*addr != 0); 853 } 854 855 /* 856 * Get status of volatile write cache. Always asynchronous. 857 */ 858 int 859 nvme_admin_getcache(struct nvme_softc *sc, int *addr) 860 { 861 struct nvme_ccb *ccb; 862 struct nvme_queue *q = sc->sc_admin_q; 863 int result = 0, error; 864 865 if (!nvme_has_volatile_write_cache(sc)) { 866 /* cache simply not present */ 867 *addr = 0; 868 return 0; 869 } 870 871 ccb = nvme_ccb_get(q, true); 872 KASSERT(ccb != NULL); 873 874 ccb->ccb_done = nvme_getcache_done; 875 ccb->ccb_cookie = &result; 876 877 /* namespace context */ 878 ccb->nnc_flags = 0; 879 ccb->nnc_done = NULL; 880 881 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); 882 883 /* wait for completion */ 884 nvme_q_wait_complete(sc, q, nvme_getcache_finished, &result); 885 KASSERT(result != 0); 886 887 if (result > 0) { 888 *addr = result; 889 error = 0; 890 } else 891 error = EINVAL; 892 893 return error; 894 } 895 896 static void 897 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 898 { 899 struct nvme_sqe *sqe = slot; 900 901 sqe->opcode = NVM_ADMIN_GET_FEATURES; 902 sqe->cdw10 = NVM_FEATURE_VOLATILE_WRITE_CACHE; 903 } 904 905 static void 906 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 907 struct nvme_cqe *cqe) 908 { 909 int *addr = ccb->ccb_cookie; 910 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags)); 911 uint32_t cdw0 = lemtoh32(&cqe->cdw0); 912 int result; 913 914 if (status == NVME_CQE_SC_SUCCESS) { 915 result = 0; 916 917 /* 918 * DPO not supported, Dataset Management (DSM) field doesn't 919 * specify the same semantics. FUA is always supported. 920 */ 921 result = DKCACHE_FUA; 922 923 if (cdw0 & NVME_CQE_CDW0_VWC_WCE) 924 result |= DKCACHE_WRITE; 925 926 /* 927 * If volatile write cache is present, the flag shall also be 928 * settable. 929 */ 930 result |= DKCACHE_WCHANGE; 931 } else { 932 result = -1; 933 } 934 935 *addr = result; 936 937 nvme_ccb_put(q, ccb); 938 } 939 940 void 941 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 942 { 943 struct nvme_namespace *ns; 944 struct nvm_identify_namespace *identify; 945 946 ns = nvme_ns_get(sc, nsid); 947 KASSERT(ns); 948 949 identify = ns->ident; 950 ns->ident = NULL; 951 if (identify != NULL) 952 kmem_free(identify, sizeof(*identify)); 953 } 954 955 struct nvme_pt_state { 956 struct nvme_pt_command *pt; 957 bool finished; 958 }; 959 960 static void 961 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 962 { 963 struct nvme_softc *sc = q->q_sc; 964 struct nvme_sqe *sqe = slot; 965 struct nvme_pt_state *state = ccb->ccb_cookie; 966 struct nvme_pt_command *pt = state->pt; 967 bus_dmamap_t dmap = ccb->ccb_dmamap; 968 int i; 969 970 sqe->opcode = pt->cmd.opcode; 971 htolem32(&sqe->nsid, pt->cmd.nsid); 972 973 if (pt->buf != NULL && pt->len > 0) { 974 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 975 switch (dmap->dm_nsegs) { 976 case 1: 977 break; 978 case 2: 979 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 980 break; 981 default: 982 for (i = 1; i < dmap->dm_nsegs; i++) { 983 htolem64(&ccb->ccb_prpl[i - 1], 984 dmap->dm_segs[i].ds_addr); 985 } 986 bus_dmamap_sync(sc->sc_dmat, 987 NVME_DMA_MAP(q->q_ccb_prpls), 988 ccb->ccb_prpl_off, 989 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 990 BUS_DMASYNC_PREWRITE); 991 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 992 break; 993 } 994 } 995 996 htolem32(&sqe->cdw10, pt->cmd.cdw10); 997 htolem32(&sqe->cdw11, pt->cmd.cdw11); 998 htolem32(&sqe->cdw12, pt->cmd.cdw12); 999 htolem32(&sqe->cdw13, pt->cmd.cdw13); 1000 htolem32(&sqe->cdw14, pt->cmd.cdw14); 1001 htolem32(&sqe->cdw15, pt->cmd.cdw15); 1002 } 1003 1004 static void 1005 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 1006 { 1007 struct nvme_softc *sc = q->q_sc; 1008 struct nvme_pt_state *state = ccb->ccb_cookie; 1009 struct nvme_pt_command *pt = state->pt; 1010 bus_dmamap_t dmap = ccb->ccb_dmamap; 1011 1012 if (pt->buf != NULL && pt->len > 0) { 1013 if (dmap->dm_nsegs > 2) { 1014 bus_dmamap_sync(sc->sc_dmat, 1015 NVME_DMA_MAP(q->q_ccb_prpls), 1016 ccb->ccb_prpl_off, 1017 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 1018 BUS_DMASYNC_POSTWRITE); 1019 } 1020 1021 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 1022 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 1023 bus_dmamap_unload(sc->sc_dmat, dmap); 1024 } 1025 1026 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 1027 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 1028 1029 state->finished = true; 1030 1031 nvme_ccb_put(q, ccb); 1032 } 1033 1034 static bool 1035 nvme_pt_finished(void *cookie) 1036 { 1037 struct nvme_pt_state *state = cookie; 1038 1039 return state->finished; 1040 } 1041 1042 static int 1043 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 1044 uint16_t nsid, struct lwp *l, bool is_adminq) 1045 { 1046 struct nvme_queue *q; 1047 struct nvme_ccb *ccb; 1048 void *buf = NULL; 1049 struct nvme_pt_state state; 1050 int error; 1051 1052 /* limit command size to maximum data transfer size */ 1053 if ((pt->buf == NULL && pt->len > 0) || 1054 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 1055 return EINVAL; 1056 1057 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 1058 ccb = nvme_ccb_get(q, true); 1059 KASSERT(ccb != NULL); 1060 1061 if (pt->buf != NULL) { 1062 KASSERT(pt->len > 0); 1063 buf = kmem_alloc(pt->len, KM_SLEEP); 1064 if (!pt->is_read) { 1065 error = copyin(pt->buf, buf, pt->len); 1066 if (error) 1067 goto kmem_free; 1068 } 1069 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 1070 pt->len, NULL, 1071 BUS_DMA_WAITOK | 1072 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 1073 if (error) 1074 goto kmem_free; 1075 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 1076 0, ccb->ccb_dmamap->dm_mapsize, 1077 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 1078 } 1079 1080 memset(&state, 0, sizeof(state)); 1081 state.pt = pt; 1082 state.finished = false; 1083 1084 ccb->ccb_done = nvme_pt_done; 1085 ccb->ccb_cookie = &state; 1086 1087 pt->cmd.nsid = nsid; 1088 1089 nvme_q_submit(sc, q, ccb, nvme_pt_fill); 1090 1091 /* wait for completion */ 1092 nvme_q_wait_complete(sc, q, nvme_pt_finished, &state); 1093 KASSERT(state.finished); 1094 1095 error = 0; 1096 1097 if (buf != NULL) { 1098 if (error == 0 && pt->is_read) 1099 error = copyout(buf, pt->buf, pt->len); 1100 kmem_free: 1101 kmem_free(buf, pt->len); 1102 } 1103 1104 return error; 1105 } 1106 1107 static void 1108 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1109 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 1110 { 1111 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 1112 uint32_t tail; 1113 1114 mutex_enter(&q->q_sq_mtx); 1115 tail = q->q_sq_tail; 1116 if (++q->q_sq_tail >= q->q_entries) 1117 q->q_sq_tail = 0; 1118 1119 sqe += tail; 1120 1121 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1122 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 1123 memset(sqe, 0, sizeof(*sqe)); 1124 (*fill)(q, ccb, sqe); 1125 sqe->cid = ccb->ccb_id; 1126 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1127 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 1128 1129 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 1130 mutex_exit(&q->q_sq_mtx); 1131 } 1132 1133 struct nvme_poll_state { 1134 struct nvme_sqe s; 1135 struct nvme_cqe c; 1136 void *cookie; 1137 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 1138 }; 1139 1140 static int 1141 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1142 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 1143 { 1144 struct nvme_poll_state state; 1145 uint16_t flags; 1146 int step = 10; 1147 int maxloop = timo_sec * 1000000 / step; 1148 int error = 0; 1149 1150 memset(&state, 0, sizeof(state)); 1151 (*fill)(q, ccb, &state.s); 1152 1153 state.done = ccb->ccb_done; 1154 state.cookie = ccb->ccb_cookie; 1155 1156 ccb->ccb_done = nvme_poll_done; 1157 ccb->ccb_cookie = &state; 1158 1159 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1160 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1161 if (nvme_q_complete(sc, q) == 0) 1162 delay(step); 1163 1164 if (timo_sec >= 0 && --maxloop <= 0) { 1165 error = ETIMEDOUT; 1166 break; 1167 } 1168 } 1169 1170 if (error == 0) { 1171 flags = lemtoh16(&state.c.flags); 1172 return flags & ~NVME_CQE_PHASE; 1173 } else { 1174 /* 1175 * If it succeds later, it would hit ccb which will have been 1176 * already reused for something else. Not good. Cross 1177 * fingers and hope for best. XXX do controller reset? 1178 */ 1179 aprint_error_dev(sc->sc_dev, "polled command timed out\n"); 1180 1181 /* Invoke the callback to clean state anyway */ 1182 struct nvme_cqe cqe; 1183 memset(&cqe, 0, sizeof(cqe)); 1184 ccb->ccb_done(q, ccb, &cqe); 1185 1186 return 1; 1187 } 1188 } 1189 1190 static void 1191 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1192 { 1193 struct nvme_sqe *sqe = slot; 1194 struct nvme_poll_state *state = ccb->ccb_cookie; 1195 1196 *sqe = state->s; 1197 } 1198 1199 static void 1200 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1201 struct nvme_cqe *cqe) 1202 { 1203 struct nvme_poll_state *state = ccb->ccb_cookie; 1204 1205 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1206 state->c = *cqe; 1207 1208 ccb->ccb_cookie = state->cookie; 1209 state->done(q, ccb, &state->c); 1210 } 1211 1212 static void 1213 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1214 { 1215 struct nvme_sqe *src = ccb->ccb_cookie; 1216 struct nvme_sqe *dst = slot; 1217 1218 *dst = *src; 1219 } 1220 1221 static void 1222 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1223 struct nvme_cqe *cqe) 1224 { 1225 } 1226 1227 static int 1228 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1229 { 1230 struct nvme_ccb *ccb; 1231 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1232 uint16_t flags; 1233 int rv = 0; 1234 1235 mutex_enter(&q->q_cq_mtx); 1236 1237 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1238 for (;;) { 1239 cqe = &ring[q->q_cq_head]; 1240 flags = lemtoh16(&cqe->flags); 1241 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1242 break; 1243 1244 ccb = &q->q_ccbs[cqe->cid]; 1245 1246 if (++q->q_cq_head >= q->q_entries) { 1247 q->q_cq_head = 0; 1248 q->q_cq_phase ^= NVME_CQE_PHASE; 1249 } 1250 1251 #ifdef DEBUG 1252 /* 1253 * If we get spurious completion notification, something 1254 * is seriously hosed up. Very likely DMA to some random 1255 * memory place happened, so just bail out. 1256 */ 1257 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1258 panic("%s: invalid ccb detected", 1259 device_xname(sc->sc_dev)); 1260 /* NOTREACHED */ 1261 } 1262 #endif 1263 1264 rv++; 1265 1266 /* 1267 * Unlock the mutex before calling the ccb_done callback 1268 * and re-lock afterwards. The callback triggers lddone() 1269 * which schedules another i/o, and also calls nvme_ccb_put(). 1270 * Unlock/relock avoids possibility of deadlock. 1271 */ 1272 mutex_exit(&q->q_cq_mtx); 1273 ccb->ccb_done(q, ccb, cqe); 1274 mutex_enter(&q->q_cq_mtx); 1275 } 1276 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1277 1278 if (rv) 1279 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1280 1281 mutex_exit(&q->q_cq_mtx); 1282 1283 return rv; 1284 } 1285 1286 static void 1287 nvme_q_wait_complete(struct nvme_softc *sc, 1288 struct nvme_queue *q, bool (*finished)(void *), void *cookie) 1289 { 1290 mutex_enter(&q->q_ccb_mtx); 1291 if (finished(cookie)) 1292 goto out; 1293 1294 for(;;) { 1295 q->q_ccb_waiting = true; 1296 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1297 1298 if (finished(cookie)) 1299 break; 1300 } 1301 1302 out: 1303 mutex_exit(&q->q_ccb_mtx); 1304 } 1305 1306 static int 1307 nvme_identify(struct nvme_softc *sc, u_int mps) 1308 { 1309 char sn[41], mn[81], fr[17]; 1310 struct nvm_identify_controller *identify; 1311 struct nvme_dmamem *mem; 1312 struct nvme_ccb *ccb; 1313 u_int mdts; 1314 int rv = 1; 1315 1316 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1317 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1318 1319 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1320 if (mem == NULL) 1321 return 1; 1322 1323 ccb->ccb_done = nvme_empty_done; 1324 ccb->ccb_cookie = mem; 1325 1326 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1327 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1328 NVME_TIMO_IDENT); 1329 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1330 1331 nvme_ccb_put(sc->sc_admin_q, ccb); 1332 1333 if (rv != 0) 1334 goto done; 1335 1336 identify = NVME_DMA_KVA(mem); 1337 1338 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1339 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1340 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1341 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1342 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1343 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1344 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1345 sn); 1346 1347 if (identify->mdts > 0) { 1348 mdts = (1 << identify->mdts) * (1 << mps); 1349 if (mdts < sc->sc_mdts) 1350 sc->sc_mdts = mdts; 1351 } 1352 1353 sc->sc_nn = lemtoh32(&identify->nn); 1354 1355 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1356 1357 done: 1358 nvme_dmamem_free(sc, mem); 1359 1360 return rv; 1361 } 1362 1363 static int 1364 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1365 { 1366 struct nvme_sqe_q sqe; 1367 struct nvme_ccb *ccb; 1368 int rv; 1369 1370 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1371 return 1; 1372 1373 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1374 KASSERT(ccb != NULL); 1375 1376 ccb->ccb_done = nvme_empty_done; 1377 ccb->ccb_cookie = &sqe; 1378 1379 memset(&sqe, 0, sizeof(sqe)); 1380 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1381 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1382 htolem16(&sqe.qsize, q->q_entries - 1); 1383 htolem16(&sqe.qid, q->q_id); 1384 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1385 if (sc->sc_use_mq) 1386 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1387 1388 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1389 if (rv != 0) 1390 goto fail; 1391 1392 ccb->ccb_done = nvme_empty_done; 1393 ccb->ccb_cookie = &sqe; 1394 1395 memset(&sqe, 0, sizeof(sqe)); 1396 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1397 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1398 htolem16(&sqe.qsize, q->q_entries - 1); 1399 htolem16(&sqe.qid, q->q_id); 1400 htolem16(&sqe.cqid, q->q_id); 1401 sqe.qflags = NVM_SQE_Q_PC; 1402 1403 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1404 if (rv != 0) 1405 goto fail; 1406 1407 fail: 1408 nvme_ccb_put(sc->sc_admin_q, ccb); 1409 return rv; 1410 } 1411 1412 static int 1413 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1414 { 1415 struct nvme_sqe_q sqe; 1416 struct nvme_ccb *ccb; 1417 int rv; 1418 1419 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1420 KASSERT(ccb != NULL); 1421 1422 ccb->ccb_done = nvme_empty_done; 1423 ccb->ccb_cookie = &sqe; 1424 1425 memset(&sqe, 0, sizeof(sqe)); 1426 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1427 htolem16(&sqe.qid, q->q_id); 1428 1429 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1430 if (rv != 0) 1431 goto fail; 1432 1433 ccb->ccb_done = nvme_empty_done; 1434 ccb->ccb_cookie = &sqe; 1435 1436 memset(&sqe, 0, sizeof(sqe)); 1437 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1438 htolem16(&sqe.qid, q->q_id); 1439 1440 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1441 if (rv != 0) 1442 goto fail; 1443 1444 fail: 1445 nvme_ccb_put(sc->sc_admin_q, ccb); 1446 1447 if (rv == 0 && sc->sc_use_mq) { 1448 if (sc->sc_intr_disestablish(sc, q->q_id)) 1449 rv = 1; 1450 } 1451 1452 return rv; 1453 } 1454 1455 static void 1456 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1457 { 1458 struct nvme_sqe *sqe = slot; 1459 struct nvme_dmamem *mem = ccb->ccb_cookie; 1460 1461 sqe->opcode = NVM_ADMIN_IDENTIFY; 1462 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1463 htolem32(&sqe->cdw10, 1); 1464 } 1465 1466 static int 1467 nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap) 1468 { 1469 struct nvme_pt_state state; 1470 struct nvme_pt_command pt; 1471 struct nvme_ccb *ccb; 1472 uint16_t ncqa, nsqa; 1473 int rv; 1474 1475 ccb = nvme_ccb_get(sc->sc_admin_q, false); 1476 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1477 1478 memset(&pt, 0, sizeof(pt)); 1479 pt.cmd.opcode = NVM_ADMIN_GET_FEATURES; 1480 pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES; 1481 1482 memset(&state, 0, sizeof(state)); 1483 state.pt = &pt; 1484 state.finished = false; 1485 1486 ccb->ccb_done = nvme_pt_done; 1487 ccb->ccb_cookie = &state; 1488 1489 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1490 1491 if (rv != 0) { 1492 *nqap = 0; 1493 return EIO; 1494 } 1495 1496 ncqa = pt.cpl.cdw0 >> 16; 1497 nsqa = pt.cpl.cdw0 & 0xffff; 1498 *nqap = MIN(ncqa, nsqa) + 1; 1499 1500 return 0; 1501 } 1502 1503 static int 1504 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1505 { 1506 struct nvme_softc *sc = q->q_sc; 1507 struct nvme_ccb *ccb; 1508 bus_addr_t off; 1509 uint64_t *prpl; 1510 u_int i; 1511 1512 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1513 cv_init(&q->q_ccb_wait, "nvmeqw"); 1514 q->q_ccb_waiting = false; 1515 SIMPLEQ_INIT(&q->q_ccb_list); 1516 1517 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1518 1519 q->q_nccbs = nccbs; 1520 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1521 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1522 1523 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1524 off = 0; 1525 1526 for (i = 0; i < nccbs; i++) { 1527 ccb = &q->q_ccbs[i]; 1528 1529 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1530 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1531 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1532 &ccb->ccb_dmamap) != 0) 1533 goto free_maps; 1534 1535 ccb->ccb_id = i; 1536 ccb->ccb_prpl = prpl; 1537 ccb->ccb_prpl_off = off; 1538 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1539 1540 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1541 1542 prpl += sc->sc_max_sgl; 1543 off += sizeof(*prpl) * sc->sc_max_sgl; 1544 } 1545 1546 return 0; 1547 1548 free_maps: 1549 nvme_ccbs_free(q); 1550 return 1; 1551 } 1552 1553 static struct nvme_ccb * 1554 nvme_ccb_get(struct nvme_queue *q, bool wait) 1555 { 1556 struct nvme_ccb *ccb = NULL; 1557 1558 mutex_enter(&q->q_ccb_mtx); 1559 again: 1560 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1561 if (ccb != NULL) { 1562 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1563 #ifdef DEBUG 1564 ccb->ccb_cookie = NULL; 1565 #endif 1566 } else { 1567 if (__predict_false(wait)) { 1568 q->q_ccb_waiting = true; 1569 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx); 1570 goto again; 1571 } 1572 } 1573 mutex_exit(&q->q_ccb_mtx); 1574 1575 return ccb; 1576 } 1577 1578 static void 1579 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1580 { 1581 1582 mutex_enter(&q->q_ccb_mtx); 1583 #ifdef DEBUG 1584 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1585 #endif 1586 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1587 1588 /* It's unlikely there are any waiters, it's not used for regular I/O */ 1589 if (__predict_false(q->q_ccb_waiting)) { 1590 q->q_ccb_waiting = false; 1591 cv_broadcast(&q->q_ccb_wait); 1592 } 1593 1594 mutex_exit(&q->q_ccb_mtx); 1595 } 1596 1597 static void 1598 nvme_ccbs_free(struct nvme_queue *q) 1599 { 1600 struct nvme_softc *sc = q->q_sc; 1601 struct nvme_ccb *ccb; 1602 1603 mutex_enter(&q->q_ccb_mtx); 1604 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1605 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1606 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1607 } 1608 mutex_exit(&q->q_ccb_mtx); 1609 1610 nvme_dmamem_free(sc, q->q_ccb_prpls); 1611 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1612 q->q_ccbs = NULL; 1613 cv_destroy(&q->q_ccb_wait); 1614 mutex_destroy(&q->q_ccb_mtx); 1615 } 1616 1617 static struct nvme_queue * 1618 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1619 { 1620 struct nvme_queue *q; 1621 1622 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1623 q->q_sc = sc; 1624 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1625 sizeof(struct nvme_sqe) * entries); 1626 if (q->q_sq_dmamem == NULL) 1627 goto free; 1628 1629 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1630 sizeof(struct nvme_cqe) * entries); 1631 if (q->q_cq_dmamem == NULL) 1632 goto free_sq; 1633 1634 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1635 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1636 1637 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1638 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1639 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1640 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1641 q->q_id = id; 1642 q->q_entries = entries; 1643 q->q_sq_tail = 0; 1644 q->q_cq_head = 0; 1645 q->q_cq_phase = NVME_CQE_PHASE; 1646 1647 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1648 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1649 1650 /* 1651 * Due to definition of full and empty queue (queue is empty 1652 * when head == tail, full when tail is one less then head), 1653 * we can actually only have (entries - 1) in-flight commands. 1654 */ 1655 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1656 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1657 goto free_cq; 1658 } 1659 1660 return q; 1661 1662 free_cq: 1663 nvme_dmamem_free(sc, q->q_cq_dmamem); 1664 free_sq: 1665 nvme_dmamem_free(sc, q->q_sq_dmamem); 1666 free: 1667 kmem_free(q, sizeof(*q)); 1668 1669 return NULL; 1670 } 1671 1672 static void 1673 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1674 { 1675 nvme_ccbs_free(q); 1676 mutex_destroy(&q->q_sq_mtx); 1677 mutex_destroy(&q->q_cq_mtx); 1678 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1679 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1680 nvme_dmamem_free(sc, q->q_cq_dmamem); 1681 nvme_dmamem_free(sc, q->q_sq_dmamem); 1682 kmem_free(q, sizeof(*q)); 1683 } 1684 1685 int 1686 nvme_intr(void *xsc) 1687 { 1688 struct nvme_softc *sc = xsc; 1689 1690 /* 1691 * INTx is level triggered, controller deasserts the interrupt only 1692 * when we advance command queue head via write to the doorbell. 1693 * Tell the controller to block the interrupts while we process 1694 * the queue(s). 1695 */ 1696 nvme_write4(sc, NVME_INTMS, 1); 1697 1698 softint_schedule(sc->sc_softih[0]); 1699 1700 /* don't know, might not have been for us */ 1701 return 1; 1702 } 1703 1704 void 1705 nvme_softintr_intx(void *xq) 1706 { 1707 struct nvme_queue *q = xq; 1708 struct nvme_softc *sc = q->q_sc; 1709 1710 nvme_q_complete(sc, sc->sc_admin_q); 1711 if (sc->sc_q != NULL) 1712 nvme_q_complete(sc, sc->sc_q[0]); 1713 1714 /* 1715 * Processing done, tell controller to issue interrupts again. There 1716 * is no race, as NVMe spec requires the controller to maintain state, 1717 * and assert the interrupt whenever there are unacknowledged 1718 * completion queue entries. 1719 */ 1720 nvme_write4(sc, NVME_INTMC, 1); 1721 } 1722 1723 int 1724 nvme_intr_msi(void *xq) 1725 { 1726 struct nvme_queue *q = xq; 1727 1728 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1729 && q->q_sc->sc_softih[q->q_id]); 1730 1731 /* 1732 * MSI/MSI-X are edge triggered, so can handover processing to softint 1733 * without masking the interrupt. 1734 */ 1735 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1736 1737 return 1; 1738 } 1739 1740 void 1741 nvme_softintr_msi(void *xq) 1742 { 1743 struct nvme_queue *q = xq; 1744 struct nvme_softc *sc = q->q_sc; 1745 1746 nvme_q_complete(sc, q); 1747 } 1748 1749 static struct nvme_dmamem * 1750 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1751 { 1752 struct nvme_dmamem *ndm; 1753 int nsegs; 1754 1755 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1756 if (ndm == NULL) 1757 return NULL; 1758 1759 ndm->ndm_size = size; 1760 1761 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1762 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1763 goto ndmfree; 1764 1765 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1766 1, &nsegs, BUS_DMA_WAITOK) != 0) 1767 goto destroy; 1768 1769 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1770 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1771 goto free; 1772 memset(ndm->ndm_kva, 0, size); 1773 1774 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1775 NULL, BUS_DMA_WAITOK) != 0) 1776 goto unmap; 1777 1778 return ndm; 1779 1780 unmap: 1781 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1782 free: 1783 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1784 destroy: 1785 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1786 ndmfree: 1787 kmem_free(ndm, sizeof(*ndm)); 1788 return NULL; 1789 } 1790 1791 static void 1792 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1793 { 1794 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1795 0, NVME_DMA_LEN(mem), ops); 1796 } 1797 1798 void 1799 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1800 { 1801 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1802 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1803 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1804 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1805 kmem_free(ndm, sizeof(*ndm)); 1806 } 1807 1808 /* 1809 * ioctl 1810 */ 1811 1812 dev_type_open(nvmeopen); 1813 dev_type_close(nvmeclose); 1814 dev_type_ioctl(nvmeioctl); 1815 1816 const struct cdevsw nvme_cdevsw = { 1817 .d_open = nvmeopen, 1818 .d_close = nvmeclose, 1819 .d_read = noread, 1820 .d_write = nowrite, 1821 .d_ioctl = nvmeioctl, 1822 .d_stop = nostop, 1823 .d_tty = notty, 1824 .d_poll = nopoll, 1825 .d_mmap = nommap, 1826 .d_kqfilter = nokqfilter, 1827 .d_discard = nodiscard, 1828 .d_flag = D_OTHER, 1829 }; 1830 1831 /* 1832 * Accept an open operation on the control device. 1833 */ 1834 int 1835 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1836 { 1837 struct nvme_softc *sc; 1838 int unit = minor(dev) / 0x10000; 1839 int nsid = minor(dev) & 0xffff; 1840 int nsidx; 1841 1842 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1843 return ENXIO; 1844 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1845 return ENXIO; 1846 1847 if (nsid == 0) { 1848 /* controller */ 1849 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1850 return EBUSY; 1851 SET(sc->sc_flags, NVME_F_OPEN); 1852 } else { 1853 /* namespace */ 1854 nsidx = nsid - 1; 1855 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1856 return ENXIO; 1857 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1858 return EBUSY; 1859 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1860 } 1861 return 0; 1862 } 1863 1864 /* 1865 * Accept the last close on the control device. 1866 */ 1867 int 1868 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1869 { 1870 struct nvme_softc *sc; 1871 int unit = minor(dev) / 0x10000; 1872 int nsid = minor(dev) & 0xffff; 1873 int nsidx; 1874 1875 sc = device_lookup_private(&nvme_cd, unit); 1876 if (sc == NULL) 1877 return ENXIO; 1878 1879 if (nsid == 0) { 1880 /* controller */ 1881 CLR(sc->sc_flags, NVME_F_OPEN); 1882 } else { 1883 /* namespace */ 1884 nsidx = nsid - 1; 1885 if (nsidx >= sc->sc_nn) 1886 return ENXIO; 1887 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1888 } 1889 1890 return 0; 1891 } 1892 1893 /* 1894 * Handle control operations. 1895 */ 1896 int 1897 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1898 { 1899 struct nvme_softc *sc; 1900 int unit = minor(dev) / 0x10000; 1901 int nsid = minor(dev) & 0xffff; 1902 struct nvme_pt_command *pt; 1903 1904 sc = device_lookup_private(&nvme_cd, unit); 1905 if (sc == NULL) 1906 return ENXIO; 1907 1908 switch (cmd) { 1909 case NVME_PASSTHROUGH_CMD: 1910 pt = data; 1911 return nvme_command_passthrough(sc, data, 1912 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1913 } 1914 1915 return ENOTTY; 1916 } 1917
Indexes created Mon Sep 29 21:09:56 GMT 2025