nvme.c revision 1.32
1 /* $NetBSD: nvme.c,v 1.32 2018/02/27 12:59:53 christos Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.32 2018/02/27 12:59:53 christos Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 #include "ioconf.h" 44 45 int nvme_adminq_size = 32; 46 int nvme_ioq_size = 1024; 47 48 static int nvme_print(void *, const char *); 49 50 static int nvme_ready(struct nvme_softc *, uint32_t); 51 static int nvme_enable(struct nvme_softc *, u_int); 52 static int nvme_disable(struct nvme_softc *); 53 static int nvme_shutdown(struct nvme_softc *); 54 55 #ifdef NVME_DEBUG 56 static void nvme_dumpregs(struct nvme_softc *); 57 #endif 58 static int nvme_identify(struct nvme_softc *, u_int); 59 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 60 void *); 61 62 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 63 static void nvme_ccbs_free(struct nvme_queue *); 64 65 static struct nvme_ccb * 66 nvme_ccb_get(struct nvme_queue *); 67 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 68 69 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 70 struct nvme_ccb *, void (*)(struct nvme_queue *, 71 struct nvme_ccb *, void *), int); 72 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 73 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 74 struct nvme_cqe *); 75 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 76 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 77 struct nvme_cqe *); 78 79 static struct nvme_queue * 80 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 81 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 82 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 83 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 84 struct nvme_ccb *, void (*)(struct nvme_queue *, 85 struct nvme_ccb *, void *)); 86 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 87 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 88 89 static struct nvme_dmamem * 90 nvme_dmamem_alloc(struct nvme_softc *, size_t); 91 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 92 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 93 int); 94 95 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 96 void *); 97 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 98 struct nvme_cqe *); 99 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 100 void *); 101 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 102 struct nvme_cqe *); 103 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *, 104 void *); 105 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *, 106 struct nvme_cqe *); 107 108 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 109 void *); 110 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 111 struct nvme_cqe *); 112 static int nvme_command_passthrough(struct nvme_softc *, 113 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 114 115 static int nvme_get_number_of_queues(struct nvme_softc *, u_int *); 116 117 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 118 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 119 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 120 #define NVME_TIMO_SY 60 /* sync cache timeout */ 121 122 #define nvme_read4(_s, _r) \ 123 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 124 #define nvme_write4(_s, _r, _v) \ 125 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 126 /* 127 * Some controllers, at least Apple NVMe, always require split 128 * transfers, so don't use bus_space_{read,write}_8() on LP64. 129 */ 130 static inline uint64_t 131 nvme_read8(struct nvme_softc *sc, bus_size_t r) 132 { 133 uint64_t v; 134 uint32_t *a = (uint32_t *)&v; 135 136 #if _BYTE_ORDER == _LITTLE_ENDIAN 137 a[0] = nvme_read4(sc, r); 138 a[1] = nvme_read4(sc, r + 4); 139 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 140 a[1] = nvme_read4(sc, r); 141 a[0] = nvme_read4(sc, r + 4); 142 #endif 143 144 return v; 145 } 146 147 static inline void 148 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 149 { 150 uint32_t *a = (uint32_t *)&v; 151 152 #if _BYTE_ORDER == _LITTLE_ENDIAN 153 nvme_write4(sc, r, a[0]); 154 nvme_write4(sc, r + 4, a[1]); 155 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 156 nvme_write4(sc, r, a[1]); 157 nvme_write4(sc, r + 4, a[0]); 158 #endif 159 } 160 #define nvme_barrier(_s, _r, _l, _f) \ 161 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 162 163 #ifdef NVME_DEBUG 164 static __used void 165 nvme_dumpregs(struct nvme_softc *sc) 166 { 167 uint64_t r8; 168 uint32_t r4; 169 170 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 171 r8 = nvme_read8(sc, NVME_CAP); 172 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 173 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 174 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 175 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 176 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 177 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 178 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 179 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 180 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 181 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 182 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 183 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 184 185 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 186 187 r4 = nvme_read4(sc, NVME_CC); 188 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 189 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 190 (1 << NVME_CC_IOCQES_R(r4))); 191 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 192 (1 << NVME_CC_IOSQES_R(r4))); 193 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 194 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 195 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 196 (1 << NVME_CC_MPS_R(r4))); 197 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 198 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 199 200 r4 = nvme_read4(sc, NVME_CSTS); 201 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 202 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 203 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 204 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 205 206 r4 = nvme_read4(sc, NVME_AQA); 207 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 208 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 209 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 210 211 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 212 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 213 #undef DEVNAME 214 } 215 #endif /* NVME_DEBUG */ 216 217 static int 218 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 219 { 220 u_int i = 0; 221 uint32_t cc; 222 223 cc = nvme_read4(sc, NVME_CC); 224 if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) { 225 aprint_error_dev(sc->sc_dev, 226 "controller enabled status expected %d, found to be %d\n", 227 (rdy != 0), ((cc & NVME_CC_EN) != 0)); 228 return ENXIO; 229 } 230 231 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 232 if (i++ > sc->sc_rdy_to) 233 return ENXIO; 234 235 delay(1000); 236 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 237 } 238 239 return 0; 240 } 241 242 static int 243 nvme_enable(struct nvme_softc *sc, u_int mps) 244 { 245 uint32_t cc, csts; 246 247 cc = nvme_read4(sc, NVME_CC); 248 csts = nvme_read4(sc, NVME_CSTS); 249 250 if (ISSET(cc, NVME_CC_EN)) { 251 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 252 253 if (ISSET(csts, NVME_CSTS_RDY)) 254 return 1; 255 256 goto waitready; 257 } 258 259 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 260 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 261 delay(5000); 262 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 263 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 264 delay(5000); 265 266 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 267 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 268 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 269 delay(5000); 270 271 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 272 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 273 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 274 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 275 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 276 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 277 SET(cc, NVME_CC_MPS(mps)); 278 SET(cc, NVME_CC_EN); 279 280 nvme_write4(sc, NVME_CC, cc); 281 nvme_barrier(sc, 0, sc->sc_ios, 282 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 283 delay(5000); 284 285 waitready: 286 return nvme_ready(sc, NVME_CSTS_RDY); 287 } 288 289 static int 290 nvme_disable(struct nvme_softc *sc) 291 { 292 uint32_t cc, csts; 293 294 cc = nvme_read4(sc, NVME_CC); 295 csts = nvme_read4(sc, NVME_CSTS); 296 297 if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY)) 298 nvme_ready(sc, NVME_CSTS_RDY); 299 300 CLR(cc, NVME_CC_EN); 301 302 nvme_write4(sc, NVME_CC, cc); 303 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 304 305 delay(5000); 306 307 return nvme_ready(sc, 0); 308 } 309 310 int 311 nvme_attach(struct nvme_softc *sc) 312 { 313 uint64_t cap; 314 uint32_t reg; 315 u_int dstrd; 316 u_int mps = PAGE_SHIFT; 317 u_int ioq_allocated; 318 uint16_t adminq_entries = nvme_adminq_size; 319 uint16_t ioq_entries = nvme_ioq_size; 320 int i; 321 322 reg = nvme_read4(sc, NVME_VS); 323 if (reg == 0xffffffff) { 324 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 325 return 1; 326 } 327 328 if (NVME_VS_TER(reg) == 0) 329 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg), 330 NVME_VS_MNR(reg)); 331 else 332 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg), 333 NVME_VS_MNR(reg), NVME_VS_TER(reg)); 334 335 cap = nvme_read8(sc, NVME_CAP); 336 dstrd = NVME_CAP_DSTRD(cap); 337 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 338 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 339 "is greater than CPU page size %u\n", 340 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 341 return 1; 342 } 343 if (NVME_CAP_MPSMAX(cap) < mps) 344 mps = NVME_CAP_MPSMAX(cap); 345 if (ioq_entries > NVME_CAP_MQES(cap)) 346 ioq_entries = NVME_CAP_MQES(cap); 347 348 /* set initial values to be used for admin queue during probe */ 349 sc->sc_rdy_to = NVME_CAP_TO(cap); 350 sc->sc_mps = 1 << mps; 351 sc->sc_mdts = MAXPHYS; 352 sc->sc_max_sgl = 2; 353 354 if (nvme_disable(sc) != 0) { 355 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 356 return 1; 357 } 358 359 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 360 if (sc->sc_admin_q == NULL) { 361 aprint_error_dev(sc->sc_dev, 362 "unable to allocate admin queue\n"); 363 return 1; 364 } 365 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 366 goto free_admin_q; 367 368 if (nvme_enable(sc, mps) != 0) { 369 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 370 goto disestablish_admin_q; 371 } 372 373 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 374 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 375 goto disable; 376 } 377 378 /* we know how big things are now */ 379 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 380 381 /* reallocate ccbs of admin queue with new max sgl. */ 382 nvme_ccbs_free(sc->sc_admin_q); 383 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 384 385 if (sc->sc_use_mq) { 386 /* Limit the number of queues to the number allocated in HW */ 387 if (nvme_get_number_of_queues(sc, &ioq_allocated) != 0) { 388 aprint_error_dev(sc->sc_dev, 389 "unable to get number of queues\n"); 390 goto disable; 391 } 392 if (sc->sc_nq > ioq_allocated) 393 sc->sc_nq = ioq_allocated; 394 } 395 396 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 397 for (i = 0; i < sc->sc_nq; i++) { 398 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 399 if (sc->sc_q[i] == NULL) { 400 aprint_error_dev(sc->sc_dev, 401 "unable to allocate io queue\n"); 402 goto free_q; 403 } 404 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 405 aprint_error_dev(sc->sc_dev, 406 "unable to create io queue\n"); 407 nvme_q_free(sc, sc->sc_q[i]); 408 goto free_q; 409 } 410 } 411 412 if (!sc->sc_use_mq) 413 nvme_write4(sc, NVME_INTMC, 1); 414 415 /* probe subdevices */ 416 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 417 KM_SLEEP); 418 nvme_rescan(sc->sc_dev, "nvme", &i); 419 420 return 0; 421 422 free_q: 423 while (--i >= 0) { 424 nvme_q_delete(sc, sc->sc_q[i]); 425 nvme_q_free(sc, sc->sc_q[i]); 426 } 427 disable: 428 nvme_disable(sc); 429 disestablish_admin_q: 430 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 431 free_admin_q: 432 nvme_q_free(sc, sc->sc_admin_q); 433 434 return 1; 435 } 436 437 int 438 nvme_rescan(device_t self, const char *attr, const int *flags) 439 { 440 struct nvme_softc *sc = device_private(self); 441 struct nvme_attach_args naa; 442 uint64_t cap; 443 int ioq_entries = nvme_ioq_size; 444 int i; 445 446 cap = nvme_read8(sc, NVME_CAP); 447 if (ioq_entries > NVME_CAP_MQES(cap)) 448 ioq_entries = NVME_CAP_MQES(cap); 449 450 for (i = 0; i < sc->sc_nn; i++) { 451 if (sc->sc_namespaces[i].dev) 452 continue; 453 memset(&naa, 0, sizeof(naa)); 454 naa.naa_nsid = i + 1; 455 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 456 naa.naa_maxphys = sc->sc_mdts; 457 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 458 nvme_print); 459 } 460 return 0; 461 } 462 463 static int 464 nvme_print(void *aux, const char *pnp) 465 { 466 struct nvme_attach_args *naa = aux; 467 468 if (pnp) 469 aprint_normal("at %s", pnp); 470 471 if (naa->naa_nsid > 0) 472 aprint_normal(" nsid %d", naa->naa_nsid); 473 474 return UNCONF; 475 } 476 477 int 478 nvme_detach(struct nvme_softc *sc, int flags) 479 { 480 int i, error; 481 482 error = config_detach_children(sc->sc_dev, flags); 483 if (error) 484 return error; 485 486 error = nvme_shutdown(sc); 487 if (error) 488 return error; 489 490 /* from now on we are committed to detach, following will never fail */ 491 for (i = 0; i < sc->sc_nq; i++) 492 nvme_q_free(sc, sc->sc_q[i]); 493 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 494 nvme_q_free(sc, sc->sc_admin_q); 495 496 return 0; 497 } 498 499 static int 500 nvme_shutdown(struct nvme_softc *sc) 501 { 502 uint32_t cc, csts; 503 bool disabled = false; 504 int i; 505 506 if (!sc->sc_use_mq) 507 nvme_write4(sc, NVME_INTMS, 1); 508 509 for (i = 0; i < sc->sc_nq; i++) { 510 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 511 aprint_error_dev(sc->sc_dev, 512 "unable to delete io queue %d, disabling\n", i + 1); 513 disabled = true; 514 } 515 } 516 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 517 if (disabled) 518 goto disable; 519 520 cc = nvme_read4(sc, NVME_CC); 521 CLR(cc, NVME_CC_SHN_MASK); 522 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 523 nvme_write4(sc, NVME_CC, cc); 524 525 for (i = 0; i < 4000; i++) { 526 nvme_barrier(sc, 0, sc->sc_ios, 527 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 528 csts = nvme_read4(sc, NVME_CSTS); 529 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 530 return 0; 531 532 delay(1000); 533 } 534 535 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 536 537 disable: 538 nvme_disable(sc); 539 return 0; 540 } 541 542 void 543 nvme_childdet(device_t self, device_t child) 544 { 545 struct nvme_softc *sc = device_private(self); 546 int i; 547 548 for (i = 0; i < sc->sc_nn; i++) { 549 if (sc->sc_namespaces[i].dev == child) { 550 /* Already freed ns->ident. */ 551 sc->sc_namespaces[i].dev = NULL; 552 break; 553 } 554 } 555 } 556 557 int 558 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 559 { 560 struct nvme_sqe sqe; 561 struct nvm_identify_namespace *identify; 562 struct nvme_dmamem *mem; 563 struct nvme_ccb *ccb; 564 struct nvme_namespace *ns; 565 int rv; 566 567 KASSERT(nsid > 0); 568 569 ccb = nvme_ccb_get(sc->sc_admin_q); 570 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 571 572 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 573 if (mem == NULL) { 574 nvme_ccb_put(sc->sc_admin_q, ccb); 575 return ENOMEM; 576 } 577 578 memset(&sqe, 0, sizeof(sqe)); 579 sqe.opcode = NVM_ADMIN_IDENTIFY; 580 htolem32(&sqe.nsid, nsid); 581 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 582 htolem32(&sqe.cdw10, 0); 583 584 ccb->ccb_done = nvme_empty_done; 585 ccb->ccb_cookie = &sqe; 586 587 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 588 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 589 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 590 591 nvme_ccb_put(sc->sc_admin_q, ccb); 592 593 if (rv != 0) { 594 rv = EIO; 595 goto done; 596 } 597 598 /* commit */ 599 600 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 601 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 602 //memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 603 604 ns = nvme_ns_get(sc, nsid); 605 KASSERT(ns); 606 KASSERT(ns->ident == NULL); 607 ns->ident = identify; 608 609 done: 610 nvme_dmamem_free(sc, mem); 611 612 return rv; 613 } 614 615 int 616 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 617 struct buf *bp, void *data, size_t datasize, 618 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 619 { 620 struct nvme_queue *q = nvme_get_q(sc); 621 struct nvme_ccb *ccb; 622 bus_dmamap_t dmap; 623 int i, error; 624 625 ccb = nvme_ccb_get(q); 626 if (ccb == NULL) 627 return EAGAIN; 628 629 ccb->ccb_done = nvme_ns_io_done; 630 ccb->ccb_cookie = cookie; 631 632 /* namespace context */ 633 ccb->nnc_nsid = nsid; 634 ccb->nnc_flags = flags; 635 ccb->nnc_buf = bp; 636 ccb->nnc_datasize = datasize; 637 ccb->nnc_secsize = secsize; 638 ccb->nnc_blkno = blkno; 639 ccb->nnc_done = nnc_done; 640 641 dmap = ccb->ccb_dmamap; 642 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 643 datasize, NULL, 644 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 645 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 646 (ISSET(flags, NVME_NS_CTX_F_READ) ? 647 BUS_DMA_READ : BUS_DMA_WRITE)); 648 if (error) { 649 nvme_ccb_put(q, ccb); 650 return error; 651 } 652 653 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 654 ISSET(flags, NVME_NS_CTX_F_READ) ? 655 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 656 657 if (dmap->dm_nsegs > 2) { 658 for (i = 1; i < dmap->dm_nsegs; i++) { 659 htolem64(&ccb->ccb_prpl[i - 1], 660 dmap->dm_segs[i].ds_addr); 661 } 662 bus_dmamap_sync(sc->sc_dmat, 663 NVME_DMA_MAP(q->q_ccb_prpls), 664 ccb->ccb_prpl_off, 665 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 666 BUS_DMASYNC_PREWRITE); 667 } 668 669 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 670 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 671 return EIO; 672 return 0; 673 } 674 675 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 676 return 0; 677 } 678 679 static void 680 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 681 { 682 struct nvme_sqe_io *sqe = slot; 683 bus_dmamap_t dmap = ccb->ccb_dmamap; 684 685 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 686 NVM_CMD_READ : NVM_CMD_WRITE; 687 htolem32(&sqe->nsid, ccb->nnc_nsid); 688 689 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 690 switch (dmap->dm_nsegs) { 691 case 1: 692 break; 693 case 2: 694 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 695 break; 696 default: 697 /* the prp list is already set up and synced */ 698 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 699 break; 700 } 701 702 htolem64(&sqe->slba, ccb->nnc_blkno); 703 704 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) 705 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA); 706 707 /* guaranteed by upper layers, but check just in case */ 708 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 709 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 710 } 711 712 static void 713 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 714 struct nvme_cqe *cqe) 715 { 716 struct nvme_softc *sc = q->q_sc; 717 bus_dmamap_t dmap = ccb->ccb_dmamap; 718 void *nnc_cookie = ccb->ccb_cookie; 719 nvme_nnc_done nnc_done = ccb->nnc_done; 720 struct buf *bp = ccb->nnc_buf; 721 722 if (dmap->dm_nsegs > 2) { 723 bus_dmamap_sync(sc->sc_dmat, 724 NVME_DMA_MAP(q->q_ccb_prpls), 725 ccb->ccb_prpl_off, 726 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 727 BUS_DMASYNC_POSTWRITE); 728 } 729 730 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 731 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 732 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 733 734 bus_dmamap_unload(sc->sc_dmat, dmap); 735 nvme_ccb_put(q, ccb); 736 737 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 738 } 739 740 /* 741 * If there is no volatile write cache, it makes no sense to issue 742 * flush commands or query for the status. 743 */ 744 bool 745 nvme_has_volatile_write_cache(struct nvme_softc *sc) 746 { 747 /* sc_identify is filled during attachment */ 748 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0); 749 } 750 751 int 752 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie, 753 int flags, nvme_nnc_done nnc_done) 754 { 755 struct nvme_queue *q = nvme_get_q(sc); 756 struct nvme_ccb *ccb; 757 758 ccb = nvme_ccb_get(q); 759 if (ccb == NULL) 760 return EAGAIN; 761 762 ccb->ccb_done = nvme_ns_sync_done; 763 ccb->ccb_cookie = cookie; 764 765 /* namespace context */ 766 ccb->nnc_nsid = nsid; 767 ccb->nnc_flags = flags; 768 ccb->nnc_done = nnc_done; 769 770 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 771 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 772 return EIO; 773 return 0; 774 } 775 776 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 777 return 0; 778 } 779 780 static void 781 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 782 { 783 struct nvme_sqe *sqe = slot; 784 785 sqe->opcode = NVM_CMD_FLUSH; 786 htolem32(&sqe->nsid, ccb->nnc_nsid); 787 } 788 789 static void 790 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 791 struct nvme_cqe *cqe) 792 { 793 void *cookie = ccb->ccb_cookie; 794 nvme_nnc_done nnc_done = ccb->nnc_done; 795 796 nvme_ccb_put(q, ccb); 797 798 nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 799 } 800 801 /* 802 * Get status of volatile write cache. Always asynchronous. 803 */ 804 int 805 nvme_admin_getcache(struct nvme_softc *sc, void *cookie, nvme_nnc_done nnc_done) 806 { 807 struct nvme_ccb *ccb; 808 struct nvme_queue *q = sc->sc_admin_q; 809 810 ccb = nvme_ccb_get(q); 811 if (ccb == NULL) 812 return EAGAIN; 813 814 ccb->ccb_done = nvme_getcache_done; 815 ccb->ccb_cookie = cookie; 816 817 /* namespace context */ 818 ccb->nnc_flags = 0; 819 ccb->nnc_done = nnc_done; 820 821 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); 822 return 0; 823 } 824 825 static void 826 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 827 { 828 struct nvme_sqe *sqe = slot; 829 830 sqe->opcode = NVM_ADMIN_GET_FEATURES; 831 sqe->cdw10 = NVM_FEATURE_VOLATILE_WRITE_CACHE; 832 } 833 834 static void 835 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 836 struct nvme_cqe *cqe) 837 { 838 void *cookie = ccb->ccb_cookie; 839 nvme_nnc_done nnc_done = ccb->nnc_done; 840 841 nvme_ccb_put(q, ccb); 842 843 nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 844 } 845 846 void 847 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 848 { 849 struct nvme_namespace *ns; 850 struct nvm_identify_namespace *identify; 851 852 ns = nvme_ns_get(sc, nsid); 853 KASSERT(ns); 854 855 identify = ns->ident; 856 ns->ident = NULL; 857 if (identify != NULL) 858 kmem_free(identify, sizeof(*identify)); 859 } 860 861 static void 862 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 863 { 864 struct nvme_softc *sc = q->q_sc; 865 struct nvme_sqe *sqe = slot; 866 struct nvme_pt_command *pt = ccb->ccb_cookie; 867 bus_dmamap_t dmap = ccb->ccb_dmamap; 868 int i; 869 870 sqe->opcode = pt->cmd.opcode; 871 htolem32(&sqe->nsid, pt->cmd.nsid); 872 873 if (pt->buf != NULL && pt->len > 0) { 874 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 875 switch (dmap->dm_nsegs) { 876 case 1: 877 break; 878 case 2: 879 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 880 break; 881 default: 882 for (i = 1; i < dmap->dm_nsegs; i++) { 883 htolem64(&ccb->ccb_prpl[i - 1], 884 dmap->dm_segs[i].ds_addr); 885 } 886 bus_dmamap_sync(sc->sc_dmat, 887 NVME_DMA_MAP(q->q_ccb_prpls), 888 ccb->ccb_prpl_off, 889 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 890 BUS_DMASYNC_PREWRITE); 891 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 892 break; 893 } 894 } 895 896 htolem32(&sqe->cdw10, pt->cmd.cdw10); 897 htolem32(&sqe->cdw11, pt->cmd.cdw11); 898 htolem32(&sqe->cdw12, pt->cmd.cdw12); 899 htolem32(&sqe->cdw13, pt->cmd.cdw13); 900 htolem32(&sqe->cdw14, pt->cmd.cdw14); 901 htolem32(&sqe->cdw15, pt->cmd.cdw15); 902 } 903 904 static void 905 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 906 { 907 struct nvme_softc *sc = q->q_sc; 908 struct nvme_pt_command *pt = ccb->ccb_cookie; 909 bus_dmamap_t dmap = ccb->ccb_dmamap; 910 911 if (pt->buf != NULL && pt->len > 0) { 912 if (dmap->dm_nsegs > 2) { 913 bus_dmamap_sync(sc->sc_dmat, 914 NVME_DMA_MAP(q->q_ccb_prpls), 915 ccb->ccb_prpl_off, 916 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 917 BUS_DMASYNC_POSTWRITE); 918 } 919 920 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 921 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 922 bus_dmamap_unload(sc->sc_dmat, dmap); 923 } 924 925 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 926 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 927 } 928 929 static int 930 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 931 uint16_t nsid, struct lwp *l, bool is_adminq) 932 { 933 struct nvme_queue *q; 934 struct nvme_ccb *ccb; 935 void *buf = NULL; 936 int error; 937 938 /* limit command size to maximum data transfer size */ 939 if ((pt->buf == NULL && pt->len > 0) || 940 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 941 return EINVAL; 942 943 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 944 ccb = nvme_ccb_get(q); 945 if (ccb == NULL) 946 return EBUSY; 947 948 if (pt->buf != NULL) { 949 KASSERT(pt->len > 0); 950 buf = kmem_alloc(pt->len, KM_SLEEP); 951 if (!pt->is_read) { 952 error = copyin(pt->buf, buf, pt->len); 953 if (error) 954 goto kmem_free; 955 } 956 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 957 pt->len, NULL, 958 BUS_DMA_WAITOK | 959 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 960 if (error) 961 goto kmem_free; 962 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 963 0, ccb->ccb_dmamap->dm_mapsize, 964 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 965 } 966 967 ccb->ccb_done = nvme_pt_done; 968 ccb->ccb_cookie = pt; 969 970 pt->cmd.nsid = nsid; 971 if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) { 972 error = EIO; 973 goto out; 974 } 975 976 error = 0; 977 out: 978 if (buf != NULL) { 979 if (error == 0 && pt->is_read) 980 error = copyout(buf, pt->buf, pt->len); 981 kmem_free: 982 kmem_free(buf, pt->len); 983 } 984 nvme_ccb_put(q, ccb); 985 return error; 986 } 987 988 static void 989 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 990 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 991 { 992 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 993 uint32_t tail; 994 995 mutex_enter(&q->q_sq_mtx); 996 tail = q->q_sq_tail; 997 if (++q->q_sq_tail >= q->q_entries) 998 q->q_sq_tail = 0; 999 1000 sqe += tail; 1001 1002 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1003 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 1004 memset(sqe, 0, sizeof(*sqe)); 1005 (*fill)(q, ccb, sqe); 1006 sqe->cid = ccb->ccb_id; 1007 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1008 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 1009 1010 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 1011 mutex_exit(&q->q_sq_mtx); 1012 } 1013 1014 struct nvme_poll_state { 1015 struct nvme_sqe s; 1016 struct nvme_cqe c; 1017 }; 1018 1019 static int 1020 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1021 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 1022 { 1023 struct nvme_poll_state state; 1024 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 1025 void *cookie; 1026 uint16_t flags; 1027 int step = 10; 1028 int maxloop = timo_sec * 1000000 / step; 1029 int error = 0; 1030 1031 memset(&state, 0, sizeof(state)); 1032 (*fill)(q, ccb, &state.s); 1033 1034 done = ccb->ccb_done; 1035 cookie = ccb->ccb_cookie; 1036 1037 ccb->ccb_done = nvme_poll_done; 1038 ccb->ccb_cookie = &state; 1039 1040 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1041 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1042 if (nvme_q_complete(sc, q) == 0) 1043 delay(step); 1044 1045 if (timo_sec >= 0 && --maxloop <= 0) { 1046 error = ETIMEDOUT; 1047 break; 1048 } 1049 } 1050 1051 ccb->ccb_cookie = cookie; 1052 done(q, ccb, &state.c); 1053 1054 if (error == 0) { 1055 flags = lemtoh16(&state.c.flags); 1056 return flags & ~NVME_CQE_PHASE; 1057 } else { 1058 return 1; 1059 } 1060 } 1061 1062 static void 1063 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1064 { 1065 struct nvme_sqe *sqe = slot; 1066 struct nvme_poll_state *state = ccb->ccb_cookie; 1067 1068 *sqe = state->s; 1069 } 1070 1071 static void 1072 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1073 struct nvme_cqe *cqe) 1074 { 1075 struct nvme_poll_state *state = ccb->ccb_cookie; 1076 1077 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1078 state->c = *cqe; 1079 } 1080 1081 static void 1082 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1083 { 1084 struct nvme_sqe *src = ccb->ccb_cookie; 1085 struct nvme_sqe *dst = slot; 1086 1087 *dst = *src; 1088 } 1089 1090 static void 1091 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1092 struct nvme_cqe *cqe) 1093 { 1094 } 1095 1096 static int 1097 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1098 { 1099 struct nvme_ccb *ccb; 1100 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1101 uint16_t flags; 1102 int rv = 0; 1103 1104 mutex_enter(&q->q_cq_mtx); 1105 1106 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1107 for (;;) { 1108 cqe = &ring[q->q_cq_head]; 1109 flags = lemtoh16(&cqe->flags); 1110 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1111 break; 1112 1113 ccb = &q->q_ccbs[cqe->cid]; 1114 1115 if (++q->q_cq_head >= q->q_entries) { 1116 q->q_cq_head = 0; 1117 q->q_cq_phase ^= NVME_CQE_PHASE; 1118 } 1119 1120 #ifdef DEBUG 1121 /* 1122 * If we get spurious completion notification, something 1123 * is seriously hosed up. Very likely DMA to some random 1124 * memory place happened, so just bail out. 1125 */ 1126 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1127 panic("%s: invalid ccb detected", 1128 device_xname(sc->sc_dev)); 1129 /* NOTREACHED */ 1130 } 1131 #endif 1132 1133 rv++; 1134 1135 /* 1136 * Unlock the mutex before calling the ccb_done callback 1137 * and re-lock afterwards. The callback triggers lddone() 1138 * which schedules another i/o, and also calls nvme_ccb_put(). 1139 * Unlock/relock avoids possibility of deadlock. 1140 */ 1141 mutex_exit(&q->q_cq_mtx); 1142 ccb->ccb_done(q, ccb, cqe); 1143 mutex_enter(&q->q_cq_mtx); 1144 } 1145 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1146 1147 if (rv) 1148 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1149 1150 mutex_exit(&q->q_cq_mtx); 1151 1152 if (rv) { 1153 mutex_enter(&q->q_ccb_mtx); 1154 q->q_nccbs_avail += rv; 1155 mutex_exit(&q->q_ccb_mtx); 1156 } 1157 1158 return rv; 1159 } 1160 1161 static int 1162 nvme_identify(struct nvme_softc *sc, u_int mps) 1163 { 1164 char sn[41], mn[81], fr[17]; 1165 struct nvm_identify_controller *identify; 1166 struct nvme_dmamem *mem; 1167 struct nvme_ccb *ccb; 1168 u_int mdts; 1169 int rv = 1; 1170 1171 ccb = nvme_ccb_get(sc->sc_admin_q); 1172 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1173 1174 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1175 if (mem == NULL) 1176 return 1; 1177 1178 ccb->ccb_done = nvme_empty_done; 1179 ccb->ccb_cookie = mem; 1180 1181 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1182 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1183 NVME_TIMO_IDENT); 1184 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1185 1186 nvme_ccb_put(sc->sc_admin_q, ccb); 1187 1188 if (rv != 0) 1189 goto done; 1190 1191 identify = NVME_DMA_KVA(mem); 1192 1193 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1194 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1195 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1196 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1197 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1198 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1199 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1200 sn); 1201 1202 if (identify->mdts > 0) { 1203 mdts = (1 << identify->mdts) * (1 << mps); 1204 if (mdts < sc->sc_mdts) 1205 sc->sc_mdts = mdts; 1206 } 1207 1208 sc->sc_nn = lemtoh32(&identify->nn); 1209 1210 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1211 1212 done: 1213 nvme_dmamem_free(sc, mem); 1214 1215 return rv; 1216 } 1217 1218 static int 1219 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1220 { 1221 struct nvme_sqe_q sqe; 1222 struct nvme_ccb *ccb; 1223 int rv; 1224 1225 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1226 return 1; 1227 1228 ccb = nvme_ccb_get(sc->sc_admin_q); 1229 KASSERT(ccb != NULL); 1230 1231 ccb->ccb_done = nvme_empty_done; 1232 ccb->ccb_cookie = &sqe; 1233 1234 memset(&sqe, 0, sizeof(sqe)); 1235 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1236 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1237 htolem16(&sqe.qsize, q->q_entries - 1); 1238 htolem16(&sqe.qid, q->q_id); 1239 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1240 if (sc->sc_use_mq) 1241 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1242 1243 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1244 if (rv != 0) 1245 goto fail; 1246 1247 ccb->ccb_done = nvme_empty_done; 1248 ccb->ccb_cookie = &sqe; 1249 1250 memset(&sqe, 0, sizeof(sqe)); 1251 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1252 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1253 htolem16(&sqe.qsize, q->q_entries - 1); 1254 htolem16(&sqe.qid, q->q_id); 1255 htolem16(&sqe.cqid, q->q_id); 1256 sqe.qflags = NVM_SQE_Q_PC; 1257 1258 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1259 if (rv != 0) 1260 goto fail; 1261 1262 fail: 1263 nvme_ccb_put(sc->sc_admin_q, ccb); 1264 return rv; 1265 } 1266 1267 static int 1268 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1269 { 1270 struct nvme_sqe_q sqe; 1271 struct nvme_ccb *ccb; 1272 int rv; 1273 1274 ccb = nvme_ccb_get(sc->sc_admin_q); 1275 KASSERT(ccb != NULL); 1276 1277 ccb->ccb_done = nvme_empty_done; 1278 ccb->ccb_cookie = &sqe; 1279 1280 memset(&sqe, 0, sizeof(sqe)); 1281 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1282 htolem16(&sqe.qid, q->q_id); 1283 1284 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1285 if (rv != 0) 1286 goto fail; 1287 1288 ccb->ccb_done = nvme_empty_done; 1289 ccb->ccb_cookie = &sqe; 1290 1291 memset(&sqe, 0, sizeof(sqe)); 1292 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1293 htolem16(&sqe.qid, q->q_id); 1294 1295 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1296 if (rv != 0) 1297 goto fail; 1298 1299 fail: 1300 nvme_ccb_put(sc->sc_admin_q, ccb); 1301 1302 if (rv == 0 && sc->sc_use_mq) { 1303 if (sc->sc_intr_disestablish(sc, q->q_id)) 1304 rv = 1; 1305 } 1306 1307 return rv; 1308 } 1309 1310 static void 1311 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1312 { 1313 struct nvme_sqe *sqe = slot; 1314 struct nvme_dmamem *mem = ccb->ccb_cookie; 1315 1316 sqe->opcode = NVM_ADMIN_IDENTIFY; 1317 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1318 htolem32(&sqe->cdw10, 1); 1319 } 1320 1321 static int 1322 nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap) 1323 { 1324 struct nvme_pt_command pt; 1325 struct nvme_ccb *ccb; 1326 uint16_t ncqa, nsqa; 1327 int rv; 1328 1329 ccb = nvme_ccb_get(sc->sc_admin_q); 1330 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1331 1332 memset(&pt, 0, sizeof(pt)); 1333 pt.cmd.opcode = NVM_ADMIN_GET_FEATURES; 1334 pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES; 1335 1336 ccb->ccb_done = nvme_pt_done; 1337 ccb->ccb_cookie = &pt; 1338 1339 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1340 1341 nvme_ccb_put(sc->sc_admin_q, ccb); 1342 1343 if (rv != 0) { 1344 *nqap = 0; 1345 return EIO; 1346 } 1347 1348 ncqa = pt.cpl.cdw0 >> 16; 1349 nsqa = pt.cpl.cdw0 & 0xffff; 1350 *nqap = MIN(ncqa, nsqa) + 1; 1351 1352 return 0; 1353 } 1354 1355 static int 1356 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1357 { 1358 struct nvme_softc *sc = q->q_sc; 1359 struct nvme_ccb *ccb; 1360 bus_addr_t off; 1361 uint64_t *prpl; 1362 u_int i; 1363 1364 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1365 SIMPLEQ_INIT(&q->q_ccb_list); 1366 1367 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1368 1369 q->q_nccbs = nccbs; 1370 q->q_nccbs_avail = nccbs; 1371 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1372 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1373 1374 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1375 off = 0; 1376 1377 for (i = 0; i < nccbs; i++) { 1378 ccb = &q->q_ccbs[i]; 1379 1380 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1381 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1382 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1383 &ccb->ccb_dmamap) != 0) 1384 goto free_maps; 1385 1386 ccb->ccb_id = i; 1387 ccb->ccb_prpl = prpl; 1388 ccb->ccb_prpl_off = off; 1389 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1390 1391 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1392 1393 prpl += sc->sc_max_sgl; 1394 off += sizeof(*prpl) * sc->sc_max_sgl; 1395 } 1396 1397 return 0; 1398 1399 free_maps: 1400 nvme_ccbs_free(q); 1401 return 1; 1402 } 1403 1404 static struct nvme_ccb * 1405 nvme_ccb_get(struct nvme_queue *q) 1406 { 1407 struct nvme_ccb *ccb = NULL; 1408 1409 mutex_enter(&q->q_ccb_mtx); 1410 if (q->q_nccbs_avail > 0) { 1411 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1412 KASSERT(ccb != NULL); 1413 q->q_nccbs_avail--; 1414 1415 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1416 #ifdef DEBUG 1417 ccb->ccb_cookie = NULL; 1418 #endif 1419 } 1420 mutex_exit(&q->q_ccb_mtx); 1421 1422 return ccb; 1423 } 1424 1425 static void 1426 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1427 { 1428 1429 mutex_enter(&q->q_ccb_mtx); 1430 #ifdef DEBUG 1431 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1432 #endif 1433 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1434 mutex_exit(&q->q_ccb_mtx); 1435 } 1436 1437 static void 1438 nvme_ccbs_free(struct nvme_queue *q) 1439 { 1440 struct nvme_softc *sc = q->q_sc; 1441 struct nvme_ccb *ccb; 1442 1443 mutex_enter(&q->q_ccb_mtx); 1444 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1445 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1446 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1447 } 1448 mutex_exit(&q->q_ccb_mtx); 1449 1450 nvme_dmamem_free(sc, q->q_ccb_prpls); 1451 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1452 q->q_ccbs = NULL; 1453 mutex_destroy(&q->q_ccb_mtx); 1454 } 1455 1456 static struct nvme_queue * 1457 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1458 { 1459 struct nvme_queue *q; 1460 1461 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1462 q->q_sc = sc; 1463 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1464 sizeof(struct nvme_sqe) * entries); 1465 if (q->q_sq_dmamem == NULL) 1466 goto free; 1467 1468 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1469 sizeof(struct nvme_cqe) * entries); 1470 if (q->q_cq_dmamem == NULL) 1471 goto free_sq; 1472 1473 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1474 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1475 1476 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1477 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1478 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1479 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1480 q->q_id = id; 1481 q->q_entries = entries; 1482 q->q_sq_tail = 0; 1483 q->q_cq_head = 0; 1484 q->q_cq_phase = NVME_CQE_PHASE; 1485 1486 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1487 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1488 1489 /* 1490 * Due to definition of full and empty queue (queue is empty 1491 * when head == tail, full when tail is one less then head), 1492 * we can actually only have (entries - 1) in-flight commands. 1493 */ 1494 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1495 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1496 goto free_cq; 1497 } 1498 1499 return q; 1500 1501 free_cq: 1502 nvme_dmamem_free(sc, q->q_cq_dmamem); 1503 free_sq: 1504 nvme_dmamem_free(sc, q->q_sq_dmamem); 1505 free: 1506 kmem_free(q, sizeof(*q)); 1507 1508 return NULL; 1509 } 1510 1511 static void 1512 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1513 { 1514 nvme_ccbs_free(q); 1515 mutex_destroy(&q->q_sq_mtx); 1516 mutex_destroy(&q->q_cq_mtx); 1517 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1518 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1519 nvme_dmamem_free(sc, q->q_cq_dmamem); 1520 nvme_dmamem_free(sc, q->q_sq_dmamem); 1521 kmem_free(q, sizeof(*q)); 1522 } 1523 1524 int 1525 nvme_intr(void *xsc) 1526 { 1527 struct nvme_softc *sc = xsc; 1528 1529 /* 1530 * INTx is level triggered, controller deasserts the interrupt only 1531 * when we advance command queue head via write to the doorbell. 1532 * Tell the controller to block the interrupts while we process 1533 * the queue(s). 1534 */ 1535 nvme_write4(sc, NVME_INTMS, 1); 1536 1537 softint_schedule(sc->sc_softih[0]); 1538 1539 /* don't know, might not have been for us */ 1540 return 1; 1541 } 1542 1543 void 1544 nvme_softintr_intx(void *xq) 1545 { 1546 struct nvme_queue *q = xq; 1547 struct nvme_softc *sc = q->q_sc; 1548 1549 nvme_q_complete(sc, sc->sc_admin_q); 1550 if (sc->sc_q != NULL) 1551 nvme_q_complete(sc, sc->sc_q[0]); 1552 1553 /* 1554 * Processing done, tell controller to issue interrupts again. There 1555 * is no race, as NVMe spec requires the controller to maintain state, 1556 * and assert the interrupt whenever there are unacknowledged 1557 * completion queue entries. 1558 */ 1559 nvme_write4(sc, NVME_INTMC, 1); 1560 } 1561 1562 int 1563 nvme_intr_msi(void *xq) 1564 { 1565 struct nvme_queue *q = xq; 1566 1567 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1568 && q->q_sc->sc_softih[q->q_id]); 1569 1570 /* 1571 * MSI/MSI-X are edge triggered, so can handover processing to softint 1572 * without masking the interrupt. 1573 */ 1574 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1575 1576 return 1; 1577 } 1578 1579 void 1580 nvme_softintr_msi(void *xq) 1581 { 1582 struct nvme_queue *q = xq; 1583 struct nvme_softc *sc = q->q_sc; 1584 1585 nvme_q_complete(sc, q); 1586 } 1587 1588 static struct nvme_dmamem * 1589 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1590 { 1591 struct nvme_dmamem *ndm; 1592 int nsegs; 1593 1594 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1595 if (ndm == NULL) 1596 return NULL; 1597 1598 ndm->ndm_size = size; 1599 1600 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1601 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1602 goto ndmfree; 1603 1604 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1605 1, &nsegs, BUS_DMA_WAITOK) != 0) 1606 goto destroy; 1607 1608 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1609 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1610 goto free; 1611 memset(ndm->ndm_kva, 0, size); 1612 1613 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1614 NULL, BUS_DMA_WAITOK) != 0) 1615 goto unmap; 1616 1617 return ndm; 1618 1619 unmap: 1620 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1621 free: 1622 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1623 destroy: 1624 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1625 ndmfree: 1626 kmem_free(ndm, sizeof(*ndm)); 1627 return NULL; 1628 } 1629 1630 static void 1631 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1632 { 1633 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1634 0, NVME_DMA_LEN(mem), ops); 1635 } 1636 1637 void 1638 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1639 { 1640 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1641 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1642 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1643 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1644 kmem_free(ndm, sizeof(*ndm)); 1645 } 1646 1647 /* 1648 * ioctl 1649 */ 1650 1651 dev_type_open(nvmeopen); 1652 dev_type_close(nvmeclose); 1653 dev_type_ioctl(nvmeioctl); 1654 1655 const struct cdevsw nvme_cdevsw = { 1656 .d_open = nvmeopen, 1657 .d_close = nvmeclose, 1658 .d_read = noread, 1659 .d_write = nowrite, 1660 .d_ioctl = nvmeioctl, 1661 .d_stop = nostop, 1662 .d_tty = notty, 1663 .d_poll = nopoll, 1664 .d_mmap = nommap, 1665 .d_kqfilter = nokqfilter, 1666 .d_discard = nodiscard, 1667 .d_flag = D_OTHER, 1668 }; 1669 1670 /* 1671 * Accept an open operation on the control device. 1672 */ 1673 int 1674 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1675 { 1676 struct nvme_softc *sc; 1677 int unit = minor(dev) / 0x10000; 1678 int nsid = minor(dev) & 0xffff; 1679 int nsidx; 1680 1681 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1682 return ENXIO; 1683 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1684 return ENXIO; 1685 1686 if (nsid == 0) { 1687 /* controller */ 1688 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1689 return EBUSY; 1690 SET(sc->sc_flags, NVME_F_OPEN); 1691 } else { 1692 /* namespace */ 1693 nsidx = nsid - 1; 1694 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1695 return ENXIO; 1696 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1697 return EBUSY; 1698 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1699 } 1700 return 0; 1701 } 1702 1703 /* 1704 * Accept the last close on the control device. 1705 */ 1706 int 1707 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1708 { 1709 struct nvme_softc *sc; 1710 int unit = minor(dev) / 0x10000; 1711 int nsid = minor(dev) & 0xffff; 1712 int nsidx; 1713 1714 sc = device_lookup_private(&nvme_cd, unit); 1715 if (sc == NULL) 1716 return ENXIO; 1717 1718 if (nsid == 0) { 1719 /* controller */ 1720 CLR(sc->sc_flags, NVME_F_OPEN); 1721 } else { 1722 /* namespace */ 1723 nsidx = nsid - 1; 1724 if (nsidx >= sc->sc_nn) 1725 return ENXIO; 1726 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1727 } 1728 1729 return 0; 1730 } 1731 1732 /* 1733 * Handle control operations. 1734 */ 1735 int 1736 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1737 { 1738 struct nvme_softc *sc; 1739 int unit = minor(dev) / 0x10000; 1740 int nsid = minor(dev) & 0xffff; 1741 struct nvme_pt_command *pt; 1742 1743 sc = device_lookup_private(&nvme_cd, unit); 1744 if (sc == NULL) 1745 return ENXIO; 1746 1747 switch (cmd) { 1748 case NVME_PASSTHROUGH_CMD: 1749 pt = data; 1750 return nvme_command_passthrough(sc, data, 1751 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1752 } 1753 1754 return ENOTTY; 1755 } 1756
Indexes created Sat Sep 27 14:09:57 GMT 2025