nvme.c revision 1.31
1 /* $NetBSD: nvme.c,v 1.31 2017/10/28 04:53:55 riastradh Exp $ */ 2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */ 3 4 /* 5 * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/cdefs.h> 21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.31 2017/10/28 04:53:55 riastradh Exp $"); 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/kernel.h> 26 #include <sys/atomic.h> 27 #include <sys/bus.h> 28 #include <sys/buf.h> 29 #include <sys/conf.h> 30 #include <sys/device.h> 31 #include <sys/kmem.h> 32 #include <sys/once.h> 33 #include <sys/proc.h> 34 #include <sys/queue.h> 35 #include <sys/mutex.h> 36 37 #include <uvm/uvm_extern.h> 38 39 #include <dev/ic/nvmereg.h> 40 #include <dev/ic/nvmevar.h> 41 #include <dev/ic/nvmeio.h> 42 43 #include "ioconf.h" 44 45 int nvme_adminq_size = 32; 46 int nvme_ioq_size = 1024; 47 48 static int nvme_print(void *, const char *); 49 50 static int nvme_ready(struct nvme_softc *, uint32_t); 51 static int nvme_enable(struct nvme_softc *, u_int); 52 static int nvme_disable(struct nvme_softc *); 53 static int nvme_shutdown(struct nvme_softc *); 54 55 #ifdef NVME_DEBUG 56 static void nvme_dumpregs(struct nvme_softc *); 57 #endif 58 static int nvme_identify(struct nvme_softc *, u_int); 59 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *, 60 void *); 61 62 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t); 63 static void nvme_ccbs_free(struct nvme_queue *); 64 65 static struct nvme_ccb * 66 nvme_ccb_get(struct nvme_queue *); 67 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *); 68 69 static int nvme_poll(struct nvme_softc *, struct nvme_queue *, 70 struct nvme_ccb *, void (*)(struct nvme_queue *, 71 struct nvme_ccb *, void *), int); 72 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *); 73 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *, 74 struct nvme_cqe *); 75 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *); 76 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *, 77 struct nvme_cqe *); 78 79 static struct nvme_queue * 80 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int); 81 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 82 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 83 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *, 84 struct nvme_ccb *, void (*)(struct nvme_queue *, 85 struct nvme_ccb *, void *)); 86 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q); 87 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 88 89 static struct nvme_dmamem * 90 nvme_dmamem_alloc(struct nvme_softc *, size_t); 91 static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 92 static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, 93 int); 94 95 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *, 96 void *); 97 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *, 98 struct nvme_cqe *); 99 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *, 100 void *); 101 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *, 102 struct nvme_cqe *); 103 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *, 104 void *); 105 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *, 106 struct nvme_cqe *); 107 108 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *, 109 void *); 110 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *, 111 struct nvme_cqe *); 112 static int nvme_command_passthrough(struct nvme_softc *, 113 struct nvme_pt_command *, uint16_t, struct lwp *, bool); 114 115 static int nvme_get_number_of_queues(struct nvme_softc *, u_int *); 116 117 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */ 118 #define NVME_TIMO_IDENT 10 /* probe identify timeout */ 119 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */ 120 #define NVME_TIMO_SY 60 /* sync cache timeout */ 121 122 #define nvme_read4(_s, _r) \ 123 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 124 #define nvme_write4(_s, _r, _v) \ 125 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 126 /* 127 * Some controllers, at least Apple NVMe, always require split 128 * transfers, so don't use bus_space_{read,write}_8() on LP64. 129 */ 130 static inline uint64_t 131 nvme_read8(struct nvme_softc *sc, bus_size_t r) 132 { 133 uint64_t v; 134 uint32_t *a = (uint32_t *)&v; 135 136 #if _BYTE_ORDER == _LITTLE_ENDIAN 137 a[0] = nvme_read4(sc, r); 138 a[1] = nvme_read4(sc, r + 4); 139 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 140 a[1] = nvme_read4(sc, r); 141 a[0] = nvme_read4(sc, r + 4); 142 #endif 143 144 return v; 145 } 146 147 static inline void 148 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v) 149 { 150 uint32_t *a = (uint32_t *)&v; 151 152 #if _BYTE_ORDER == _LITTLE_ENDIAN 153 nvme_write4(sc, r, a[0]); 154 nvme_write4(sc, r + 4, a[1]); 155 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 156 nvme_write4(sc, r, a[1]); 157 nvme_write4(sc, r + 4, a[0]); 158 #endif 159 } 160 #define nvme_barrier(_s, _r, _l, _f) \ 161 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 162 163 #ifdef NVME_DEBUG 164 static __used void 165 nvme_dumpregs(struct nvme_softc *sc) 166 { 167 uint64_t r8; 168 uint32_t r4; 169 170 #define DEVNAME(_sc) device_xname((_sc)->sc_dev) 171 r8 = nvme_read8(sc, NVME_CAP); 172 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 173 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 174 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 175 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 176 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 177 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 178 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 179 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 180 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 181 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 182 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 183 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 184 185 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 186 187 r4 = nvme_read4(sc, NVME_CC); 188 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 189 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4), 190 (1 << NVME_CC_IOCQES_R(r4))); 191 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4), 192 (1 << NVME_CC_IOSQES_R(r4))); 193 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 194 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 195 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4), 196 (1 << NVME_CC_MPS_R(r4))); 197 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 198 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0); 199 200 r4 = nvme_read4(sc, NVME_CSTS); 201 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4); 202 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY); 203 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS); 204 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK); 205 206 r4 = nvme_read4(sc, NVME_AQA); 207 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4); 208 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4)); 209 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4)); 210 211 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 212 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 213 #undef DEVNAME 214 } 215 #endif /* NVME_DEBUG */ 216 217 static int 218 nvme_ready(struct nvme_softc *sc, uint32_t rdy) 219 { 220 u_int i = 0; 221 uint32_t cc; 222 223 cc = nvme_read4(sc, NVME_CC); 224 if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) { 225 aprint_error_dev(sc->sc_dev, 226 "controller enabled status expected %d, found to be %d\n", 227 (rdy != 0), ((cc & NVME_CC_EN) != 0)); 228 return ENXIO; 229 } 230 231 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 232 if (i++ > sc->sc_rdy_to) 233 return ENXIO; 234 235 delay(1000); 236 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 237 } 238 239 return 0; 240 } 241 242 static int 243 nvme_enable(struct nvme_softc *sc, u_int mps) 244 { 245 uint32_t cc, csts; 246 247 cc = nvme_read4(sc, NVME_CC); 248 csts = nvme_read4(sc, NVME_CSTS); 249 250 if (ISSET(cc, NVME_CC_EN)) { 251 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n"); 252 253 if (ISSET(csts, NVME_CSTS_RDY)) 254 return 1; 255 256 goto waitready; 257 } 258 259 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 260 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 261 delay(5000); 262 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 263 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 264 delay(5000); 265 266 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 267 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 268 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 269 delay(5000); 270 271 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 272 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 273 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 274 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 275 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 276 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 277 SET(cc, NVME_CC_MPS(mps)); 278 SET(cc, NVME_CC_EN); 279 280 nvme_write4(sc, NVME_CC, cc); 281 nvme_barrier(sc, 0, sc->sc_ios, 282 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 283 delay(5000); 284 285 waitready: 286 return nvme_ready(sc, NVME_CSTS_RDY); 287 } 288 289 static int 290 nvme_disable(struct nvme_softc *sc) 291 { 292 uint32_t cc, csts; 293 294 cc = nvme_read4(sc, NVME_CC); 295 csts = nvme_read4(sc, NVME_CSTS); 296 297 if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY)) 298 nvme_ready(sc, NVME_CSTS_RDY); 299 300 CLR(cc, NVME_CC_EN); 301 302 nvme_write4(sc, NVME_CC, cc); 303 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ); 304 305 delay(5000); 306 307 return nvme_ready(sc, 0); 308 } 309 310 int 311 nvme_attach(struct nvme_softc *sc) 312 { 313 uint64_t cap; 314 uint32_t reg; 315 u_int dstrd; 316 u_int mps = PAGE_SHIFT; 317 u_int ioq_allocated; 318 uint16_t adminq_entries = nvme_adminq_size; 319 uint16_t ioq_entries = nvme_ioq_size; 320 int i; 321 322 reg = nvme_read4(sc, NVME_VS); 323 if (reg == 0xffffffff) { 324 aprint_error_dev(sc->sc_dev, "invalid mapping\n"); 325 return 1; 326 } 327 328 if (NVME_VS_TER(reg) == 0) 329 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg), 330 NVME_VS_MNR(reg)); 331 else 332 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg), 333 NVME_VS_MNR(reg), NVME_VS_TER(reg)); 334 335 cap = nvme_read8(sc, NVME_CAP); 336 dstrd = NVME_CAP_DSTRD(cap); 337 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 338 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u " 339 "is greater than CPU page size %u\n", 340 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 341 return 1; 342 } 343 if (NVME_CAP_MPSMAX(cap) < mps) 344 mps = NVME_CAP_MPSMAX(cap); 345 if (ioq_entries > NVME_CAP_MQES(cap)) 346 ioq_entries = NVME_CAP_MQES(cap); 347 348 /* set initial values to be used for admin queue during probe */ 349 sc->sc_rdy_to = NVME_CAP_TO(cap); 350 sc->sc_mps = 1 << mps; 351 sc->sc_mdts = MAXPHYS; 352 sc->sc_max_sgl = 2; 353 354 if (nvme_disable(sc) != 0) { 355 aprint_error_dev(sc->sc_dev, "unable to disable controller\n"); 356 return 1; 357 } 358 359 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd); 360 if (sc->sc_admin_q == NULL) { 361 aprint_error_dev(sc->sc_dev, 362 "unable to allocate admin queue\n"); 363 return 1; 364 } 365 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q)) 366 goto free_admin_q; 367 368 if (nvme_enable(sc, mps) != 0) { 369 aprint_error_dev(sc->sc_dev, "unable to enable controller\n"); 370 goto disestablish_admin_q; 371 } 372 373 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 374 aprint_error_dev(sc->sc_dev, "unable to identify controller\n"); 375 goto disable; 376 } 377 378 /* we know how big things are now */ 379 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 380 381 /* reallocate ccbs of admin queue with new max sgl. */ 382 nvme_ccbs_free(sc->sc_admin_q); 383 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries); 384 385 if (sc->sc_use_mq) { 386 /* Limit the number of queues to the number allocated in HW */ 387 if (nvme_get_number_of_queues(sc, &ioq_allocated) != 0) { 388 aprint_error_dev(sc->sc_dev, 389 "unable to get number of queues\n"); 390 goto disable; 391 } 392 if (sc->sc_nq > ioq_allocated) 393 sc->sc_nq = ioq_allocated; 394 } 395 396 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP); 397 for (i = 0; i < sc->sc_nq; i++) { 398 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd); 399 if (sc->sc_q[i] == NULL) { 400 aprint_error_dev(sc->sc_dev, 401 "unable to allocate io queue\n"); 402 goto free_q; 403 } 404 if (nvme_q_create(sc, sc->sc_q[i]) != 0) { 405 aprint_error_dev(sc->sc_dev, 406 "unable to create io queue\n"); 407 nvme_q_free(sc, sc->sc_q[i]); 408 goto free_q; 409 } 410 } 411 412 if (!sc->sc_use_mq) 413 nvme_write4(sc, NVME_INTMC, 1); 414 415 /* probe subdevices */ 416 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn, 417 KM_SLEEP); 418 nvme_rescan(sc->sc_dev, "nvme", &i); 419 420 return 0; 421 422 free_q: 423 while (--i >= 0) { 424 nvme_q_delete(sc, sc->sc_q[i]); 425 nvme_q_free(sc, sc->sc_q[i]); 426 } 427 disable: 428 nvme_disable(sc); 429 disestablish_admin_q: 430 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 431 free_admin_q: 432 nvme_q_free(sc, sc->sc_admin_q); 433 434 return 1; 435 } 436 437 int 438 nvme_rescan(device_t self, const char *attr, const int *flags) 439 { 440 struct nvme_softc *sc = device_private(self); 441 struct nvme_attach_args naa; 442 uint64_t cap; 443 int ioq_entries = nvme_ioq_size; 444 int i; 445 446 cap = nvme_read8(sc, NVME_CAP); 447 if (ioq_entries > NVME_CAP_MQES(cap)) 448 ioq_entries = NVME_CAP_MQES(cap); 449 450 for (i = 0; i < sc->sc_nn; i++) { 451 if (sc->sc_namespaces[i].dev) 452 continue; 453 memset(&naa, 0, sizeof(naa)); 454 naa.naa_nsid = i + 1; 455 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq; 456 naa.naa_maxphys = sc->sc_mdts; 457 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa, 458 nvme_print); 459 } 460 return 0; 461 } 462 463 static int 464 nvme_print(void *aux, const char *pnp) 465 { 466 struct nvme_attach_args *naa = aux; 467 468 if (pnp) 469 aprint_normal("at %s", pnp); 470 471 if (naa->naa_nsid > 0) 472 aprint_normal(" nsid %d", naa->naa_nsid); 473 474 return UNCONF; 475 } 476 477 int 478 nvme_detach(struct nvme_softc *sc, int flags) 479 { 480 int i, error; 481 482 error = config_detach_children(sc->sc_dev, flags); 483 if (error) 484 return error; 485 486 error = nvme_shutdown(sc); 487 if (error) 488 return error; 489 490 /* from now on we are committed to detach, following will never fail */ 491 for (i = 0; i < sc->sc_nq; i++) 492 nvme_q_free(sc, sc->sc_q[i]); 493 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq); 494 nvme_q_free(sc, sc->sc_admin_q); 495 496 return 0; 497 } 498 499 static int 500 nvme_shutdown(struct nvme_softc *sc) 501 { 502 uint32_t cc, csts; 503 bool disabled = false; 504 int i; 505 506 if (!sc->sc_use_mq) 507 nvme_write4(sc, NVME_INTMS, 1); 508 509 for (i = 0; i < sc->sc_nq; i++) { 510 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) { 511 aprint_error_dev(sc->sc_dev, 512 "unable to delete io queue %d, disabling\n", i + 1); 513 disabled = true; 514 } 515 } 516 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q); 517 if (disabled) 518 goto disable; 519 520 cc = nvme_read4(sc, NVME_CC); 521 CLR(cc, NVME_CC_SHN_MASK); 522 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 523 nvme_write4(sc, NVME_CC, cc); 524 525 for (i = 0; i < 4000; i++) { 526 nvme_barrier(sc, 0, sc->sc_ios, 527 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 528 csts = nvme_read4(sc, NVME_CSTS); 529 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 530 return 0; 531 532 delay(1000); 533 } 534 535 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n"); 536 537 disable: 538 nvme_disable(sc); 539 return 0; 540 } 541 542 void 543 nvme_childdet(device_t self, device_t child) 544 { 545 struct nvme_softc *sc = device_private(self); 546 int i; 547 548 for (i = 0; i < sc->sc_nn; i++) { 549 if (sc->sc_namespaces[i].dev == child) { 550 /* Already freed ns->ident. */ 551 sc->sc_namespaces[i].dev = NULL; 552 break; 553 } 554 } 555 } 556 557 int 558 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid) 559 { 560 struct nvme_sqe sqe; 561 struct nvm_identify_namespace *identify; 562 struct nvme_dmamem *mem; 563 struct nvme_ccb *ccb; 564 struct nvme_namespace *ns; 565 int rv; 566 567 KASSERT(nsid > 0); 568 569 ccb = nvme_ccb_get(sc->sc_admin_q); 570 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 571 572 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 573 if (mem == NULL) 574 return ENOMEM; 575 576 memset(&sqe, 0, sizeof(sqe)); 577 sqe.opcode = NVM_ADMIN_IDENTIFY; 578 htolem32(&sqe.nsid, nsid); 579 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 580 htolem32(&sqe.cdw10, 0); 581 582 ccb->ccb_done = nvme_empty_done; 583 ccb->ccb_cookie = &sqe; 584 585 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 586 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); 587 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 588 589 nvme_ccb_put(sc->sc_admin_q, ccb); 590 591 if (rv != 0) { 592 rv = EIO; 593 goto done; 594 } 595 596 /* commit */ 597 598 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP); 599 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem)); 600 //memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 601 602 ns = nvme_ns_get(sc, nsid); 603 KASSERT(ns); 604 ns->ident = identify; 605 606 done: 607 nvme_dmamem_free(sc, mem); 608 609 return rv; 610 } 611 612 int 613 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie, 614 struct buf *bp, void *data, size_t datasize, 615 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done) 616 { 617 struct nvme_queue *q = nvme_get_q(sc); 618 struct nvme_ccb *ccb; 619 bus_dmamap_t dmap; 620 int i, error; 621 622 ccb = nvme_ccb_get(q); 623 if (ccb == NULL) 624 return EAGAIN; 625 626 ccb->ccb_done = nvme_ns_io_done; 627 ccb->ccb_cookie = cookie; 628 629 /* namespace context */ 630 ccb->nnc_nsid = nsid; 631 ccb->nnc_flags = flags; 632 ccb->nnc_buf = bp; 633 ccb->nnc_datasize = datasize; 634 ccb->nnc_secsize = secsize; 635 ccb->nnc_blkno = blkno; 636 ccb->nnc_done = nnc_done; 637 638 dmap = ccb->ccb_dmamap; 639 error = bus_dmamap_load(sc->sc_dmat, dmap, data, 640 datasize, NULL, 641 (ISSET(flags, NVME_NS_CTX_F_POLL) ? 642 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) | 643 (ISSET(flags, NVME_NS_CTX_F_READ) ? 644 BUS_DMA_READ : BUS_DMA_WRITE)); 645 if (error) { 646 nvme_ccb_put(q, ccb); 647 return error; 648 } 649 650 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 651 ISSET(flags, NVME_NS_CTX_F_READ) ? 652 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 653 654 if (dmap->dm_nsegs > 2) { 655 for (i = 1; i < dmap->dm_nsegs; i++) { 656 htolem64(&ccb->ccb_prpl[i - 1], 657 dmap->dm_segs[i].ds_addr); 658 } 659 bus_dmamap_sync(sc->sc_dmat, 660 NVME_DMA_MAP(q->q_ccb_prpls), 661 ccb->ccb_prpl_off, 662 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 663 BUS_DMASYNC_PREWRITE); 664 } 665 666 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 667 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) 668 return EIO; 669 return 0; 670 } 671 672 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); 673 return 0; 674 } 675 676 static void 677 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 678 { 679 struct nvme_sqe_io *sqe = slot; 680 bus_dmamap_t dmap = ccb->ccb_dmamap; 681 682 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 683 NVM_CMD_READ : NVM_CMD_WRITE; 684 htolem32(&sqe->nsid, ccb->nnc_nsid); 685 686 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 687 switch (dmap->dm_nsegs) { 688 case 1: 689 break; 690 case 2: 691 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 692 break; 693 default: 694 /* the prp list is already set up and synced */ 695 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 696 break; 697 } 698 699 htolem64(&sqe->slba, ccb->nnc_blkno); 700 701 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) 702 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA); 703 704 /* guaranteed by upper layers, but check just in case */ 705 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); 706 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); 707 } 708 709 static void 710 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, 711 struct nvme_cqe *cqe) 712 { 713 struct nvme_softc *sc = q->q_sc; 714 bus_dmamap_t dmap = ccb->ccb_dmamap; 715 void *nnc_cookie = ccb->ccb_cookie; 716 nvme_nnc_done nnc_done = ccb->nnc_done; 717 struct buf *bp = ccb->nnc_buf; 718 719 if (dmap->dm_nsegs > 2) { 720 bus_dmamap_sync(sc->sc_dmat, 721 NVME_DMA_MAP(q->q_ccb_prpls), 722 ccb->ccb_prpl_off, 723 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 724 BUS_DMASYNC_POSTWRITE); 725 } 726 727 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 728 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? 729 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 730 731 bus_dmamap_unload(sc->sc_dmat, dmap); 732 nvme_ccb_put(q, ccb); 733 734 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 735 } 736 737 /* 738 * If there is no volatile write cache, it makes no sense to issue 739 * flush commands or query for the status. 740 */ 741 bool 742 nvme_has_volatile_write_cache(struct nvme_softc *sc) 743 { 744 /* sc_identify is filled during attachment */ 745 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0); 746 } 747 748 int 749 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie, 750 int flags, nvme_nnc_done nnc_done) 751 { 752 struct nvme_queue *q = nvme_get_q(sc); 753 struct nvme_ccb *ccb; 754 755 ccb = nvme_ccb_get(q); 756 if (ccb == NULL) 757 return EAGAIN; 758 759 ccb->ccb_done = nvme_ns_sync_done; 760 ccb->ccb_cookie = cookie; 761 762 /* namespace context */ 763 ccb->nnc_nsid = nsid; 764 ccb->nnc_flags = flags; 765 ccb->nnc_done = nnc_done; 766 767 if (ISSET(flags, NVME_NS_CTX_F_POLL)) { 768 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) 769 return EIO; 770 return 0; 771 } 772 773 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); 774 return 0; 775 } 776 777 static void 778 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 779 { 780 struct nvme_sqe *sqe = slot; 781 782 sqe->opcode = NVM_CMD_FLUSH; 783 htolem32(&sqe->nsid, ccb->nnc_nsid); 784 } 785 786 static void 787 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, 788 struct nvme_cqe *cqe) 789 { 790 void *cookie = ccb->ccb_cookie; 791 nvme_nnc_done nnc_done = ccb->nnc_done; 792 793 nvme_ccb_put(q, ccb); 794 795 nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 796 } 797 798 /* 799 * Get status of volatile write cache. Always asynchronous. 800 */ 801 int 802 nvme_admin_getcache(struct nvme_softc *sc, void *cookie, nvme_nnc_done nnc_done) 803 { 804 struct nvme_ccb *ccb; 805 struct nvme_queue *q = sc->sc_admin_q; 806 807 ccb = nvme_ccb_get(q); 808 if (ccb == NULL) 809 return EAGAIN; 810 811 ccb->ccb_done = nvme_getcache_done; 812 ccb->ccb_cookie = cookie; 813 814 /* namespace context */ 815 ccb->nnc_flags = 0; 816 ccb->nnc_done = nnc_done; 817 818 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); 819 return 0; 820 } 821 822 static void 823 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 824 { 825 struct nvme_sqe *sqe = slot; 826 827 sqe->opcode = NVM_ADMIN_GET_FEATURES; 828 sqe->cdw10 = NVM_FEATURE_VOLATILE_WRITE_CACHE; 829 } 830 831 static void 832 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, 833 struct nvme_cqe *cqe) 834 { 835 void *cookie = ccb->ccb_cookie; 836 nvme_nnc_done nnc_done = ccb->nnc_done; 837 838 nvme_ccb_put(q, ccb); 839 840 nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0)); 841 } 842 843 void 844 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid) 845 { 846 struct nvme_namespace *ns; 847 struct nvm_identify_namespace *identify; 848 849 ns = nvme_ns_get(sc, nsid); 850 KASSERT(ns); 851 852 identify = ns->ident; 853 ns->ident = NULL; 854 if (identify != NULL) 855 kmem_free(identify, sizeof(*identify)); 856 } 857 858 static void 859 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 860 { 861 struct nvme_softc *sc = q->q_sc; 862 struct nvme_sqe *sqe = slot; 863 struct nvme_pt_command *pt = ccb->ccb_cookie; 864 bus_dmamap_t dmap = ccb->ccb_dmamap; 865 int i; 866 867 sqe->opcode = pt->cmd.opcode; 868 htolem32(&sqe->nsid, pt->cmd.nsid); 869 870 if (pt->buf != NULL && pt->len > 0) { 871 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 872 switch (dmap->dm_nsegs) { 873 case 1: 874 break; 875 case 2: 876 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 877 break; 878 default: 879 for (i = 1; i < dmap->dm_nsegs; i++) { 880 htolem64(&ccb->ccb_prpl[i - 1], 881 dmap->dm_segs[i].ds_addr); 882 } 883 bus_dmamap_sync(sc->sc_dmat, 884 NVME_DMA_MAP(q->q_ccb_prpls), 885 ccb->ccb_prpl_off, 886 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 887 BUS_DMASYNC_PREWRITE); 888 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 889 break; 890 } 891 } 892 893 htolem32(&sqe->cdw10, pt->cmd.cdw10); 894 htolem32(&sqe->cdw11, pt->cmd.cdw11); 895 htolem32(&sqe->cdw12, pt->cmd.cdw12); 896 htolem32(&sqe->cdw13, pt->cmd.cdw13); 897 htolem32(&sqe->cdw14, pt->cmd.cdw14); 898 htolem32(&sqe->cdw15, pt->cmd.cdw15); 899 } 900 901 static void 902 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) 903 { 904 struct nvme_softc *sc = q->q_sc; 905 struct nvme_pt_command *pt = ccb->ccb_cookie; 906 bus_dmamap_t dmap = ccb->ccb_dmamap; 907 908 if (pt->buf != NULL && pt->len > 0) { 909 if (dmap->dm_nsegs > 2) { 910 bus_dmamap_sync(sc->sc_dmat, 911 NVME_DMA_MAP(q->q_ccb_prpls), 912 ccb->ccb_prpl_off, 913 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), 914 BUS_DMASYNC_POSTWRITE); 915 } 916 917 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 918 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 919 bus_dmamap_unload(sc->sc_dmat, dmap); 920 } 921 922 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0); 923 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE; 924 } 925 926 static int 927 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt, 928 uint16_t nsid, struct lwp *l, bool is_adminq) 929 { 930 struct nvme_queue *q; 931 struct nvme_ccb *ccb; 932 void *buf = NULL; 933 int error; 934 935 /* limit command size to maximum data transfer size */ 936 if ((pt->buf == NULL && pt->len > 0) || 937 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts))) 938 return EINVAL; 939 940 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc); 941 ccb = nvme_ccb_get(q); 942 if (ccb == NULL) 943 return EBUSY; 944 945 if (pt->buf != NULL) { 946 KASSERT(pt->len > 0); 947 buf = kmem_alloc(pt->len, KM_SLEEP); 948 if (!pt->is_read) { 949 error = copyin(pt->buf, buf, pt->len); 950 if (error) 951 goto kmem_free; 952 } 953 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, 954 pt->len, NULL, 955 BUS_DMA_WAITOK | 956 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE)); 957 if (error) 958 goto kmem_free; 959 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 960 0, ccb->ccb_dmamap->dm_mapsize, 961 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 962 } 963 964 ccb->ccb_done = nvme_pt_done; 965 ccb->ccb_cookie = pt; 966 967 pt->cmd.nsid = nsid; 968 if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) { 969 error = EIO; 970 goto out; 971 } 972 973 error = 0; 974 out: 975 if (buf != NULL) { 976 if (error == 0 && pt->is_read) 977 error = copyout(buf, pt->buf, pt->len); 978 kmem_free: 979 kmem_free(buf, pt->len); 980 } 981 nvme_ccb_put(q, ccb); 982 return error; 983 } 984 985 static void 986 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 987 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *)) 988 { 989 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 990 uint32_t tail; 991 992 mutex_enter(&q->q_sq_mtx); 993 tail = q->q_sq_tail; 994 if (++q->q_sq_tail >= q->q_entries) 995 q->q_sq_tail = 0; 996 997 sqe += tail; 998 999 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1000 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 1001 memset(sqe, 0, sizeof(*sqe)); 1002 (*fill)(q, ccb, sqe); 1003 sqe->cid = ccb->ccb_id; 1004 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 1005 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 1006 1007 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 1008 mutex_exit(&q->q_sq_mtx); 1009 } 1010 1011 struct nvme_poll_state { 1012 struct nvme_sqe s; 1013 struct nvme_cqe c; 1014 }; 1015 1016 static int 1017 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 1018 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec) 1019 { 1020 struct nvme_poll_state state; 1021 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *); 1022 void *cookie; 1023 uint16_t flags; 1024 int step = 10; 1025 int maxloop = timo_sec * 1000000 / step; 1026 int error = 0; 1027 1028 memset(&state, 0, sizeof(state)); 1029 (*fill)(q, ccb, &state.s); 1030 1031 done = ccb->ccb_done; 1032 cookie = ccb->ccb_cookie; 1033 1034 ccb->ccb_done = nvme_poll_done; 1035 ccb->ccb_cookie = &state; 1036 1037 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 1038 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 1039 if (nvme_q_complete(sc, q) == 0) 1040 delay(step); 1041 1042 if (timo_sec >= 0 && --maxloop <= 0) { 1043 error = ETIMEDOUT; 1044 break; 1045 } 1046 } 1047 1048 ccb->ccb_cookie = cookie; 1049 done(q, ccb, &state.c); 1050 1051 if (error == 0) { 1052 flags = lemtoh16(&state.c.flags); 1053 return flags & ~NVME_CQE_PHASE; 1054 } else { 1055 return 1; 1056 } 1057 } 1058 1059 static void 1060 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1061 { 1062 struct nvme_sqe *sqe = slot; 1063 struct nvme_poll_state *state = ccb->ccb_cookie; 1064 1065 *sqe = state->s; 1066 } 1067 1068 static void 1069 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1070 struct nvme_cqe *cqe) 1071 { 1072 struct nvme_poll_state *state = ccb->ccb_cookie; 1073 1074 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 1075 state->c = *cqe; 1076 } 1077 1078 static void 1079 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1080 { 1081 struct nvme_sqe *src = ccb->ccb_cookie; 1082 struct nvme_sqe *dst = slot; 1083 1084 *dst = *src; 1085 } 1086 1087 static void 1088 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, 1089 struct nvme_cqe *cqe) 1090 { 1091 } 1092 1093 static int 1094 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 1095 { 1096 struct nvme_ccb *ccb; 1097 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 1098 uint16_t flags; 1099 int rv = 0; 1100 1101 mutex_enter(&q->q_cq_mtx); 1102 1103 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1104 for (;;) { 1105 cqe = &ring[q->q_cq_head]; 1106 flags = lemtoh16(&cqe->flags); 1107 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 1108 break; 1109 1110 ccb = &q->q_ccbs[cqe->cid]; 1111 1112 if (++q->q_cq_head >= q->q_entries) { 1113 q->q_cq_head = 0; 1114 q->q_cq_phase ^= NVME_CQE_PHASE; 1115 } 1116 1117 #ifdef DEBUG 1118 /* 1119 * If we get spurious completion notification, something 1120 * is seriously hosed up. Very likely DMA to some random 1121 * memory place happened, so just bail out. 1122 */ 1123 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { 1124 panic("%s: invalid ccb detected", 1125 device_xname(sc->sc_dev)); 1126 /* NOTREACHED */ 1127 } 1128 #endif 1129 1130 rv++; 1131 1132 /* 1133 * Unlock the mutex before calling the ccb_done callback 1134 * and re-lock afterwards. The callback triggers lddone() 1135 * which schedules another i/o, and also calls nvme_ccb_put(). 1136 * Unlock/relock avoids possibility of deadlock. 1137 */ 1138 mutex_exit(&q->q_cq_mtx); 1139 ccb->ccb_done(q, ccb, cqe); 1140 mutex_enter(&q->q_cq_mtx); 1141 } 1142 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1143 1144 if (rv) 1145 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 1146 1147 mutex_exit(&q->q_cq_mtx); 1148 1149 if (rv) { 1150 mutex_enter(&q->q_ccb_mtx); 1151 q->q_nccbs_avail += rv; 1152 mutex_exit(&q->q_ccb_mtx); 1153 } 1154 1155 return rv; 1156 } 1157 1158 static int 1159 nvme_identify(struct nvme_softc *sc, u_int mps) 1160 { 1161 char sn[41], mn[81], fr[17]; 1162 struct nvm_identify_controller *identify; 1163 struct nvme_dmamem *mem; 1164 struct nvme_ccb *ccb; 1165 u_int mdts; 1166 int rv = 1; 1167 1168 ccb = nvme_ccb_get(sc->sc_admin_q); 1169 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1170 1171 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1172 if (mem == NULL) 1173 return 1; 1174 1175 ccb->ccb_done = nvme_empty_done; 1176 ccb->ccb_cookie = mem; 1177 1178 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1179 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, 1180 NVME_TIMO_IDENT); 1181 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1182 1183 nvme_ccb_put(sc->sc_admin_q, ccb); 1184 1185 if (rv != 0) 1186 goto done; 1187 1188 identify = NVME_DMA_KVA(mem); 1189 1190 strnvisx(sn, sizeof(sn), (const char *)identify->sn, 1191 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1192 strnvisx(mn, sizeof(mn), (const char *)identify->mn, 1193 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1194 strnvisx(fr, sizeof(fr), (const char *)identify->fr, 1195 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL); 1196 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr, 1197 sn); 1198 1199 if (identify->mdts > 0) { 1200 mdts = (1 << identify->mdts) * (1 << mps); 1201 if (mdts < sc->sc_mdts) 1202 sc->sc_mdts = mdts; 1203 } 1204 1205 sc->sc_nn = lemtoh32(&identify->nn); 1206 1207 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1208 1209 done: 1210 nvme_dmamem_free(sc, mem); 1211 1212 return rv; 1213 } 1214 1215 static int 1216 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1217 { 1218 struct nvme_sqe_q sqe; 1219 struct nvme_ccb *ccb; 1220 int rv; 1221 1222 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0) 1223 return 1; 1224 1225 ccb = nvme_ccb_get(sc->sc_admin_q); 1226 KASSERT(ccb != NULL); 1227 1228 ccb->ccb_done = nvme_empty_done; 1229 ccb->ccb_cookie = &sqe; 1230 1231 memset(&sqe, 0, sizeof(sqe)); 1232 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1233 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1234 htolem16(&sqe.qsize, q->q_entries - 1); 1235 htolem16(&sqe.qid, q->q_id); 1236 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1237 if (sc->sc_use_mq) 1238 htolem16(&sqe.cqid, q->q_id); /* qid == vector */ 1239 1240 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1241 if (rv != 0) 1242 goto fail; 1243 1244 ccb->ccb_done = nvme_empty_done; 1245 ccb->ccb_cookie = &sqe; 1246 1247 memset(&sqe, 0, sizeof(sqe)); 1248 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1249 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1250 htolem16(&sqe.qsize, q->q_entries - 1); 1251 htolem16(&sqe.qid, q->q_id); 1252 htolem16(&sqe.cqid, q->q_id); 1253 sqe.qflags = NVM_SQE_Q_PC; 1254 1255 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1256 if (rv != 0) 1257 goto fail; 1258 1259 fail: 1260 nvme_ccb_put(sc->sc_admin_q, ccb); 1261 return rv; 1262 } 1263 1264 static int 1265 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1266 { 1267 struct nvme_sqe_q sqe; 1268 struct nvme_ccb *ccb; 1269 int rv; 1270 1271 ccb = nvme_ccb_get(sc->sc_admin_q); 1272 KASSERT(ccb != NULL); 1273 1274 ccb->ccb_done = nvme_empty_done; 1275 ccb->ccb_cookie = &sqe; 1276 1277 memset(&sqe, 0, sizeof(sqe)); 1278 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1279 htolem16(&sqe.qid, q->q_id); 1280 1281 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1282 if (rv != 0) 1283 goto fail; 1284 1285 ccb->ccb_done = nvme_empty_done; 1286 ccb->ccb_cookie = &sqe; 1287 1288 memset(&sqe, 0, sizeof(sqe)); 1289 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1290 htolem16(&sqe.qid, q->q_id); 1291 1292 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); 1293 if (rv != 0) 1294 goto fail; 1295 1296 fail: 1297 nvme_ccb_put(sc->sc_admin_q, ccb); 1298 1299 if (rv == 0 && sc->sc_use_mq) { 1300 if (sc->sc_intr_disestablish(sc, q->q_id)) 1301 rv = 1; 1302 } 1303 1304 return rv; 1305 } 1306 1307 static void 1308 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) 1309 { 1310 struct nvme_sqe *sqe = slot; 1311 struct nvme_dmamem *mem = ccb->ccb_cookie; 1312 1313 sqe->opcode = NVM_ADMIN_IDENTIFY; 1314 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1315 htolem32(&sqe->cdw10, 1); 1316 } 1317 1318 static int 1319 nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap) 1320 { 1321 struct nvme_pt_command pt; 1322 struct nvme_ccb *ccb; 1323 uint16_t ncqa, nsqa; 1324 int rv; 1325 1326 ccb = nvme_ccb_get(sc->sc_admin_q); 1327 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ 1328 1329 memset(&pt, 0, sizeof(pt)); 1330 pt.cmd.opcode = NVM_ADMIN_GET_FEATURES; 1331 pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES; 1332 1333 ccb->ccb_done = nvme_pt_done; 1334 ccb->ccb_cookie = &pt; 1335 1336 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); 1337 1338 nvme_ccb_put(sc->sc_admin_q, ccb); 1339 1340 if (rv != 0) { 1341 *nqap = 0; 1342 return EIO; 1343 } 1344 1345 ncqa = pt.cpl.cdw0 >> 16; 1346 nsqa = pt.cpl.cdw0 & 0xffff; 1347 *nqap = MIN(ncqa, nsqa) + 1; 1348 1349 return 0; 1350 } 1351 1352 static int 1353 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs) 1354 { 1355 struct nvme_softc *sc = q->q_sc; 1356 struct nvme_ccb *ccb; 1357 bus_addr_t off; 1358 uint64_t *prpl; 1359 u_int i; 1360 1361 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO); 1362 SIMPLEQ_INIT(&q->q_ccb_list); 1363 1364 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); 1365 1366 q->q_nccbs = nccbs; 1367 q->q_nccbs_avail = nccbs; 1368 q->q_ccb_prpls = nvme_dmamem_alloc(sc, 1369 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1370 1371 prpl = NVME_DMA_KVA(q->q_ccb_prpls); 1372 off = 0; 1373 1374 for (i = 0; i < nccbs; i++) { 1375 ccb = &q->q_ccbs[i]; 1376 1377 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1378 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1379 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1380 &ccb->ccb_dmamap) != 0) 1381 goto free_maps; 1382 1383 ccb->ccb_id = i; 1384 ccb->ccb_prpl = prpl; 1385 ccb->ccb_prpl_off = off; 1386 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; 1387 1388 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); 1389 1390 prpl += sc->sc_max_sgl; 1391 off += sizeof(*prpl) * sc->sc_max_sgl; 1392 } 1393 1394 return 0; 1395 1396 free_maps: 1397 nvme_ccbs_free(q); 1398 return 1; 1399 } 1400 1401 static struct nvme_ccb * 1402 nvme_ccb_get(struct nvme_queue *q) 1403 { 1404 struct nvme_ccb *ccb = NULL; 1405 1406 mutex_enter(&q->q_ccb_mtx); 1407 if (q->q_nccbs_avail > 0) { 1408 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); 1409 KASSERT(ccb != NULL); 1410 q->q_nccbs_avail--; 1411 1412 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1413 #ifdef DEBUG 1414 ccb->ccb_cookie = NULL; 1415 #endif 1416 } 1417 mutex_exit(&q->q_ccb_mtx); 1418 1419 return ccb; 1420 } 1421 1422 static void 1423 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) 1424 { 1425 1426 mutex_enter(&q->q_ccb_mtx); 1427 #ifdef DEBUG 1428 ccb->ccb_cookie = (void *)NVME_CCB_FREE; 1429 #endif 1430 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); 1431 mutex_exit(&q->q_ccb_mtx); 1432 } 1433 1434 static void 1435 nvme_ccbs_free(struct nvme_queue *q) 1436 { 1437 struct nvme_softc *sc = q->q_sc; 1438 struct nvme_ccb *ccb; 1439 1440 mutex_enter(&q->q_ccb_mtx); 1441 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { 1442 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry); 1443 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1444 } 1445 mutex_exit(&q->q_ccb_mtx); 1446 1447 nvme_dmamem_free(sc, q->q_ccb_prpls); 1448 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); 1449 q->q_ccbs = NULL; 1450 mutex_destroy(&q->q_ccb_mtx); 1451 } 1452 1453 static struct nvme_queue * 1454 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd) 1455 { 1456 struct nvme_queue *q; 1457 1458 q = kmem_alloc(sizeof(*q), KM_SLEEP); 1459 q->q_sc = sc; 1460 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1461 sizeof(struct nvme_sqe) * entries); 1462 if (q->q_sq_dmamem == NULL) 1463 goto free; 1464 1465 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1466 sizeof(struct nvme_cqe) * entries); 1467 if (q->q_cq_dmamem == NULL) 1468 goto free_sq; 1469 1470 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1471 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1472 1473 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO); 1474 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO); 1475 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1476 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1477 q->q_id = id; 1478 q->q_entries = entries; 1479 q->q_sq_tail = 0; 1480 q->q_cq_head = 0; 1481 q->q_cq_phase = NVME_CQE_PHASE; 1482 1483 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1484 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1485 1486 /* 1487 * Due to definition of full and empty queue (queue is empty 1488 * when head == tail, full when tail is one less then head), 1489 * we can actually only have (entries - 1) in-flight commands. 1490 */ 1491 if (nvme_ccbs_alloc(q, entries - 1) != 0) { 1492 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n"); 1493 goto free_cq; 1494 } 1495 1496 return q; 1497 1498 free_cq: 1499 nvme_dmamem_free(sc, q->q_cq_dmamem); 1500 free_sq: 1501 nvme_dmamem_free(sc, q->q_sq_dmamem); 1502 free: 1503 kmem_free(q, sizeof(*q)); 1504 1505 return NULL; 1506 } 1507 1508 static void 1509 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1510 { 1511 nvme_ccbs_free(q); 1512 mutex_destroy(&q->q_sq_mtx); 1513 mutex_destroy(&q->q_cq_mtx); 1514 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1515 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1516 nvme_dmamem_free(sc, q->q_cq_dmamem); 1517 nvme_dmamem_free(sc, q->q_sq_dmamem); 1518 kmem_free(q, sizeof(*q)); 1519 } 1520 1521 int 1522 nvme_intr(void *xsc) 1523 { 1524 struct nvme_softc *sc = xsc; 1525 1526 /* 1527 * INTx is level triggered, controller deasserts the interrupt only 1528 * when we advance command queue head via write to the doorbell. 1529 * Tell the controller to block the interrupts while we process 1530 * the queue(s). 1531 */ 1532 nvme_write4(sc, NVME_INTMS, 1); 1533 1534 softint_schedule(sc->sc_softih[0]); 1535 1536 /* don't know, might not have been for us */ 1537 return 1; 1538 } 1539 1540 void 1541 nvme_softintr_intx(void *xq) 1542 { 1543 struct nvme_queue *q = xq; 1544 struct nvme_softc *sc = q->q_sc; 1545 1546 nvme_q_complete(sc, sc->sc_admin_q); 1547 if (sc->sc_q != NULL) 1548 nvme_q_complete(sc, sc->sc_q[0]); 1549 1550 /* 1551 * Processing done, tell controller to issue interrupts again. There 1552 * is no race, as NVMe spec requires the controller to maintain state, 1553 * and assert the interrupt whenever there are unacknowledged 1554 * completion queue entries. 1555 */ 1556 nvme_write4(sc, NVME_INTMC, 1); 1557 } 1558 1559 int 1560 nvme_intr_msi(void *xq) 1561 { 1562 struct nvme_queue *q = xq; 1563 1564 KASSERT(q && q->q_sc && q->q_sc->sc_softih 1565 && q->q_sc->sc_softih[q->q_id]); 1566 1567 /* 1568 * MSI/MSI-X are edge triggered, so can handover processing to softint 1569 * without masking the interrupt. 1570 */ 1571 softint_schedule(q->q_sc->sc_softih[q->q_id]); 1572 1573 return 1; 1574 } 1575 1576 void 1577 nvme_softintr_msi(void *xq) 1578 { 1579 struct nvme_queue *q = xq; 1580 struct nvme_softc *sc = q->q_sc; 1581 1582 nvme_q_complete(sc, q); 1583 } 1584 1585 static struct nvme_dmamem * 1586 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1587 { 1588 struct nvme_dmamem *ndm; 1589 int nsegs; 1590 1591 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP); 1592 if (ndm == NULL) 1593 return NULL; 1594 1595 ndm->ndm_size = size; 1596 1597 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1598 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1599 goto ndmfree; 1600 1601 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1602 1, &nsegs, BUS_DMA_WAITOK) != 0) 1603 goto destroy; 1604 1605 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1606 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1607 goto free; 1608 memset(ndm->ndm_kva, 0, size); 1609 1610 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1611 NULL, BUS_DMA_WAITOK) != 0) 1612 goto unmap; 1613 1614 return ndm; 1615 1616 unmap: 1617 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1618 free: 1619 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1620 destroy: 1621 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1622 ndmfree: 1623 kmem_free(ndm, sizeof(*ndm)); 1624 return NULL; 1625 } 1626 1627 static void 1628 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1629 { 1630 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1631 0, NVME_DMA_LEN(mem), ops); 1632 } 1633 1634 void 1635 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1636 { 1637 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1638 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1639 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1640 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1641 kmem_free(ndm, sizeof(*ndm)); 1642 } 1643 1644 /* 1645 * ioctl 1646 */ 1647 1648 dev_type_open(nvmeopen); 1649 dev_type_close(nvmeclose); 1650 dev_type_ioctl(nvmeioctl); 1651 1652 const struct cdevsw nvme_cdevsw = { 1653 .d_open = nvmeopen, 1654 .d_close = nvmeclose, 1655 .d_read = noread, 1656 .d_write = nowrite, 1657 .d_ioctl = nvmeioctl, 1658 .d_stop = nostop, 1659 .d_tty = notty, 1660 .d_poll = nopoll, 1661 .d_mmap = nommap, 1662 .d_kqfilter = nokqfilter, 1663 .d_discard = nodiscard, 1664 .d_flag = D_OTHER, 1665 }; 1666 1667 /* 1668 * Accept an open operation on the control device. 1669 */ 1670 int 1671 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l) 1672 { 1673 struct nvme_softc *sc; 1674 int unit = minor(dev) / 0x10000; 1675 int nsid = minor(dev) & 0xffff; 1676 int nsidx; 1677 1678 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL) 1679 return ENXIO; 1680 if ((sc->sc_flags & NVME_F_ATTACHED) == 0) 1681 return ENXIO; 1682 1683 if (nsid == 0) { 1684 /* controller */ 1685 if (ISSET(sc->sc_flags, NVME_F_OPEN)) 1686 return EBUSY; 1687 SET(sc->sc_flags, NVME_F_OPEN); 1688 } else { 1689 /* namespace */ 1690 nsidx = nsid - 1; 1691 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL) 1692 return ENXIO; 1693 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN)) 1694 return EBUSY; 1695 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1696 } 1697 return 0; 1698 } 1699 1700 /* 1701 * Accept the last close on the control device. 1702 */ 1703 int 1704 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l) 1705 { 1706 struct nvme_softc *sc; 1707 int unit = minor(dev) / 0x10000; 1708 int nsid = minor(dev) & 0xffff; 1709 int nsidx; 1710 1711 sc = device_lookup_private(&nvme_cd, unit); 1712 if (sc == NULL) 1713 return ENXIO; 1714 1715 if (nsid == 0) { 1716 /* controller */ 1717 CLR(sc->sc_flags, NVME_F_OPEN); 1718 } else { 1719 /* namespace */ 1720 nsidx = nsid - 1; 1721 if (nsidx >= sc->sc_nn) 1722 return ENXIO; 1723 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN); 1724 } 1725 1726 return 0; 1727 } 1728 1729 /* 1730 * Handle control operations. 1731 */ 1732 int 1733 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1734 { 1735 struct nvme_softc *sc; 1736 int unit = minor(dev) / 0x10000; 1737 int nsid = minor(dev) & 0xffff; 1738 struct nvme_pt_command *pt; 1739 1740 sc = device_lookup_private(&nvme_cd, unit); 1741 if (sc == NULL) 1742 return ENXIO; 1743 1744 switch (cmd) { 1745 case NVME_PASSTHROUGH_CMD: 1746 pt = data; 1747 return nvme_command_passthrough(sc, data, 1748 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0); 1749 } 1750 1751 return ENOTTY; 1752 } 1753
Indexes created Mon Sep 29 03:10:08 GMT 2025