1 /* $NetBSD: apple_nvme.c,v 1.1 2022/05/07 08:20:03 skrll Exp $ */ 2 /* $OpenBSD: aplns.c,v 1.5 2021/08/29 11:23:29 kettenis Exp $ */ 3 4 /*- 5 * Copyright (c) 2022 The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Nick Hudson 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 2014, 2021 David Gwynne <dlg (at) openbsd.org> 35 * 36 * Permission to use, copy, modify, and distribute this software for any 37 * purpose with or without fee is hereby granted, provided that the above 38 * copyright notice and this permission notice appear in all copies. 39 * 40 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 41 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 42 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 43 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 44 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 45 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 46 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 47 */ 48 49 #include <sys/cdefs.h> 50 __KERNEL_RCSID(0, "$NetBSD: apple_nvme.c,v 1.1 2022/05/07 08:20:03 skrll Exp $"); 51 52 #include <sys/param.h> 53 54 #include <sys/bus.h> 55 #include <sys/device.h> 56 #include <sys/kmem.h> 57 #include <sys/intr.h> 58 59 #include <dev/ic/nvmereg.h> 60 #include <dev/ic/nvmevar.h> 61 62 #include <dev/fdt/fdtvar.h> 63 64 #include <arm/apple/apple_rtkit.h> 65 66 int apple_nvme_mpsafe = 1; 67 68 #define NVME_IO_Q 1 69 70 #define ANS_CPU_CTRL 0x0044 71 #define ANS_CPU_CTRL_RUN __BIT(4) 72 73 #define ANS_MAX_PEND_CMDS_CTRL 0x01210 74 #define ANS_MAX_QUEUE_DEPTH 64 75 #define ANS_BOOT_STATUS 0x01300 76 #define ANS_BOOT_STATUS_OK 0xde71ce55 77 78 #define ANS_MODESEL_REG 0x01304 79 #define ANS_UNKNOWN_CTRL 0x24008 80 #define ANS_PRP_NULL_CHECK __BIT(11) 81 #define ANS_LINEAR_SQ_CTRL 0x24908 82 #define ANS_LINEAR_SQ_CTRL_EN __BIT(0) 83 #define ANS_LINEAR_ASQ_DB 0x2490c 84 #define ANS_LINEAR_IOSQ_DB 0x24910 85 86 #define ANS_NVMMU_NUM 0x28100 87 #define ANS_NVMMU_BASE_ASQ 0x28108 88 #define ANS_NVMMU_BASE_IOSQ 0x28110 89 #define ANS_NVMMU_TCB_INVAL 0x28118 90 #define ANS_NVMMU_TCB_STAT 0x28120 91 92 #define ANS_NVMMU_TCB_SIZE 0x4000 93 #define ANS_NVMMU_TCB_PITCH 0x80 94 95 struct ans_nvmmu_tcb { 96 uint8_t tcb_opcode; 97 uint8_t tcb_flags; 98 #define ANS_NVMMU_TCB_WRITE __BIT(0) 99 #define ANS_NVMMU_TCB_READ __BIT(1) 100 uint8_t tcb_cid; 101 uint8_t tcb_pad0[1]; 102 103 uint32_t tcb_prpl_len; 104 uint8_t tcb_pad1[16]; 105 106 uint64_t tcb_prp[2]; 107 }; 108 109 void nvme_ans_enable(struct nvme_softc *); 110 111 int nvme_ans_q_alloc(struct nvme_softc *, struct nvme_queue *); 112 void nvme_ans_q_free(struct nvme_softc *, struct nvme_queue *); 113 114 uint32_t 115 nvme_ans_sq_enter(struct nvme_softc *, struct nvme_queue *, 116 struct nvme_ccb *); 117 void nvme_ans_sq_leave(struct nvme_softc *, 118 struct nvme_queue *, struct nvme_ccb *); 119 120 void nvme_ans_cq_done(struct nvme_softc *, 121 struct nvme_queue *, struct nvme_ccb *); 122 123 static const struct nvme_ops nvme_ans_ops = { 124 .op_enable = nvme_ans_enable, 125 126 .op_q_alloc = nvme_ans_q_alloc, 127 .op_q_free = nvme_ans_q_free, 128 129 .op_sq_enter = nvme_ans_sq_enter, 130 .op_sq_leave = nvme_ans_sq_leave, 131 .op_sq_enter_locked = nvme_ans_sq_enter, 132 .op_sq_leave_locked = nvme_ans_sq_leave, 133 134 .op_cq_done = nvme_ans_cq_done, 135 }; 136 137 static const struct device_compatible_entry compat_data[] = { 138 { .compat = "apple,nvme-m1" }, 139 { .compat = "apple,nvme-ans2" }, 140 DEVICE_COMPAT_EOL 141 }; 142 143 struct apple_nvme_softc { 144 struct nvme_softc asc_nvme; 145 int asc_phandle; 146 147 bus_space_tag_t asc_iot; 148 bus_space_handle_t asc_ioh; 149 bus_size_t asc_size; 150 151 struct rtkit_state *asc_rtkit; 152 153 size_t asc_nintrs; 154 void *asc_ihs; 155 }; 156 157 void 158 nvme_ans_enable(struct nvme_softc *sc) 159 { 160 nvme_write4(sc, ANS_NVMMU_NUM, 161 (ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH) - 1); 162 nvme_write4(sc, ANS_MODESEL_REG, 0); 163 } 164 165 166 int 167 nvme_ans_q_alloc(struct nvme_softc *sc, struct nvme_queue *q) 168 { 169 bus_size_t db, base; 170 171 KASSERT(q->q_entries <= (ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH)); 172 173 q->q_nvmmu_dmamem = nvme_dmamem_alloc(sc, ANS_NVMMU_TCB_SIZE); 174 if (q->q_nvmmu_dmamem == NULL) 175 return -1; 176 177 memset(NVME_DMA_KVA(q->q_nvmmu_dmamem), 0, 178 NVME_DMA_LEN(q->q_nvmmu_dmamem)); 179 180 switch (q->q_id) { 181 case NVME_IO_Q: 182 db = ANS_LINEAR_IOSQ_DB; 183 base = ANS_NVMMU_BASE_IOSQ; 184 break; 185 case NVME_ADMIN_Q: 186 db = ANS_LINEAR_ASQ_DB; 187 base = ANS_NVMMU_BASE_ASQ; 188 break; 189 default: 190 panic("unsupported queue id %u", q->q_id); 191 /* NOTREACHED */ 192 } 193 194 q->q_sqtdbl = db; 195 196 nvme_dmamem_sync(sc, q->q_nvmmu_dmamem, BUS_DMASYNC_PREWRITE); 197 nvme_write8(sc, base, NVME_DMA_DVA(q->q_nvmmu_dmamem)); 198 199 return 0; 200 } 201 202 void 203 nvme_ans_q_free(struct nvme_softc *sc, 204 struct nvme_queue *q) 205 { 206 nvme_dmamem_sync(sc, q->q_nvmmu_dmamem, BUS_DMASYNC_POSTWRITE); 207 nvme_dmamem_free(sc, q->q_nvmmu_dmamem); 208 } 209 210 uint32_t 211 nvme_ans_sq_enter(struct nvme_softc *sc, 212 struct nvme_queue *q, struct nvme_ccb *ccb) 213 { 214 return ccb->ccb_id; 215 } 216 217 static inline struct ans_nvmmu_tcb * 218 nvme_ans_tcb(struct nvme_queue *q, unsigned int qid) 219 { 220 uint8_t *ptr = NVME_DMA_KVA(q->q_nvmmu_dmamem); 221 ptr += qid * ANS_NVMMU_TCB_PITCH; 222 223 return (struct ans_nvmmu_tcb *)ptr; 224 } 225 226 void 227 nvme_ans_sq_leave(struct nvme_softc *sc, 228 struct nvme_queue *q, struct nvme_ccb *ccb) 229 { 230 unsigned int id = ccb->ccb_id; 231 struct ans_nvmmu_tcb *tcb = nvme_ans_tcb(q, id); 232 struct nvme_sqe_io *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 233 sqe += id; 234 235 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem), 236 ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_POSTWRITE); 237 238 memset(tcb, 0, sizeof(*tcb)); 239 tcb->tcb_opcode = sqe->opcode; 240 tcb->tcb_flags = ANS_NVMMU_TCB_WRITE | ANS_NVMMU_TCB_READ; 241 tcb->tcb_cid = id; 242 tcb->tcb_prpl_len = sqe->nlb; 243 tcb->tcb_prp[0] = sqe->entry.prp[0]; 244 tcb->tcb_prp[1] = sqe->entry.prp[1]; 245 246 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem), 247 ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_PREWRITE); 248 nvme_write4(sc, q->q_sqtdbl, id); 249 } 250 251 void 252 nvme_ans_cq_done(struct nvme_softc *sc, 253 struct nvme_queue *q, struct nvme_ccb *ccb) 254 { 255 unsigned int id = ccb->ccb_id; 256 struct ans_nvmmu_tcb *tcb = nvme_ans_tcb(q, id); 257 258 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem), 259 ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_POSTWRITE); 260 memset(tcb, 0, sizeof(*tcb)); 261 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem), 262 ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_PREWRITE); 263 264 nvme_write4(sc, ANS_NVMMU_TCB_INVAL, id); 265 uint32_t stat = nvme_read4(sc, ANS_NVMMU_TCB_STAT); 266 if (stat != 0) { 267 printf("%s: nvmmu tcp stat is non-zero: 0x%08x\n", 268 device_xname(sc->sc_dev), stat); 269 } 270 } 271 272 273 static int 274 apple_nvme_intr_establish(struct nvme_softc *sc, uint16_t qid, 275 struct nvme_queue *q) 276 { 277 struct apple_nvme_softc * const asc = 278 container_of(sc, struct apple_nvme_softc, asc_nvme); 279 const int phandle = asc->asc_phandle; 280 char intr_xname[INTRDEVNAMEBUF]; 281 char intrstr[128]; 282 const device_t self = sc->sc_dev; 283 284 KASSERT(sc->sc_use_mq || qid == NVME_ADMIN_Q); 285 KASSERT(sc->sc_ih[qid] == NULL); 286 287 if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) { 288 aprint_error(": couldn't decode interrupt\n"); 289 return 1; 290 } 291 sc->sc_ih[qid] = fdtbus_intr_establish_xname(phandle, 0, IPL_BIO, 292 FDT_INTR_MPSAFE, nvme_intr, sc, device_xname(sc->sc_dev)); 293 if (sc->sc_ih[qid] == NULL) { 294 aprint_error_dev(self, "couldn't establish interrupt on %s\n", 295 intrstr); 296 return 1; 297 } 298 299 /* establish also the software interrupt */ 300 sc->sc_softih[qid] = softint_establish( 301 SOFTINT_BIO|(apple_nvme_mpsafe ? SOFTINT_MPSAFE : 0), 302 nvme_softintr_intx, q); 303 if (sc->sc_softih[qid] == NULL) { 304 fdtbus_intr_disestablish(phandle, sc->sc_ih[qid]); 305 sc->sc_ih[qid] = NULL; 306 307 aprint_error_dev(sc->sc_dev, 308 "unable to establish %s soft interrupt\n", 309 intr_xname); 310 return 1; 311 } 312 313 if (!sc->sc_use_mq) { 314 aprint_normal_dev(sc->sc_dev, "interrupting on %s\n", intrstr); 315 } else if (qid == NVME_ADMIN_Q) { 316 aprint_normal_dev(sc->sc_dev, 317 "for admin queue interrupting on %s\n", intrstr); 318 } else { 319 aprint_normal_dev(sc->sc_dev, 320 "for io queue %d interrupting on %s\n", qid, intrstr); 321 } 322 return 0; 323 } 324 325 static int 326 apple_nvme_intr_disestablish(struct nvme_softc *sc, uint16_t qid) 327 { 328 struct apple_nvme_softc * const asc = 329 container_of(sc, struct apple_nvme_softc, asc_nvme); 330 331 KASSERT(sc->sc_use_mq || qid == NVME_ADMIN_Q); 332 KASSERT(sc->sc_ih[qid] != NULL); 333 334 if (sc->sc_softih) { 335 softint_disestablish(sc->sc_softih[qid]); 336 sc->sc_softih[qid] = NULL; 337 } 338 339 fdtbus_intr_disestablish(asc->asc_phandle, sc->sc_ih[qid]); 340 sc->sc_ih[qid] = NULL; 341 342 return 0; 343 } 344 345 346 static int 347 apple_nvme_setup_intr(struct fdt_attach_args * const faa, 348 struct apple_nvme_softc * const asc) 349 { 350 struct nvme_softc * const sc = &asc->asc_nvme; 351 352 asc->asc_nintrs = 1; 353 asc->asc_phandle = faa->faa_phandle; 354 355 sc->sc_use_mq = asc->asc_nintrs > 1; 356 sc->sc_nq = 1; /* sc_use_mq */ 357 358 return 0; 359 } 360 361 362 static int 363 apple_nvme_match(device_t parent, cfdata_t cf, void *aux) 364 { 365 struct fdt_attach_args * const faa = aux; 366 367 return of_compatible_match(faa->faa_phandle, compat_data); 368 } 369 370 static void 371 apple_nvme_attach(device_t parent, device_t self, void *aux) 372 { 373 struct apple_nvme_softc * const asc = device_private(self); 374 struct nvme_softc *sc = &asc->asc_nvme; 375 struct fdt_attach_args * const faa = aux; 376 const int phandle = faa->faa_phandle; 377 bus_addr_t addr, ans_addr; 378 bus_size_t size, ans_size; 379 uint32_t ctrl, status; 380 381 382 if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) { 383 aprint_error(": couldn't get NVME registers\n"); 384 return; 385 } 386 387 if (fdtbus_get_reg(phandle, 1, &ans_addr, &ans_size) != 0) { 388 aprint_error(": couldn't get ANS registers\n"); 389 return; 390 } 391 392 sc->sc_dev = self; 393 sc->sc_iot = asc->asc_iot = faa->faa_bst; 394 sc->sc_ios = size; 395 396 if (bus_space_map(sc->sc_iot, addr, size, 0, &sc->sc_ioh) != 0) { 397 aprint_error(": couldn't map NVME registers\n"); 398 return; 399 } 400 401 if (bus_space_map(asc->asc_iot, ans_addr, ans_size, 0, &asc->asc_ioh) != 0) { 402 aprint_error(": couldn't map ANS registers\n"); 403 goto fail_ansmap; 404 } 405 406 sc->sc_dmat = faa->faa_dmat; 407 sc->sc_ops = &nvme_ans_ops; 408 aprint_naive("\n"); 409 aprint_normal(": Apple NVME\n"); 410 411 apple_nvme_setup_intr(faa, asc); 412 413 sc->sc_intr_establish = apple_nvme_intr_establish; 414 sc->sc_intr_disestablish = apple_nvme_intr_disestablish; 415 416 sc->sc_ih = kmem_zalloc(sizeof(*sc->sc_ih) * asc->asc_nintrs, KM_SLEEP); 417 sc->sc_softih = kmem_zalloc(sizeof(*sc->sc_softih) * asc->asc_nintrs, 418 KM_SLEEP); 419 420 asc->asc_rtkit = rtkit_init(phandle, NULL); 421 if (asc->asc_rtkit == NULL) { 422 aprint_error("can't map mailbox channel\n"); 423 goto fail_rtkit; 424 } 425 426 ctrl = bus_space_read_4(asc->asc_iot, asc->asc_ioh, ANS_CPU_CTRL); 427 bus_space_write_4(asc->asc_iot, asc->asc_ioh, ANS_CPU_CTRL, 428 ctrl | ANS_CPU_CTRL_RUN); 429 430 status = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ANS_BOOT_STATUS); 431 if (status != ANS_BOOT_STATUS_OK) 432 rtkit_boot(asc->asc_rtkit); 433 434 status = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ANS_BOOT_STATUS); 435 if (status != ANS_BOOT_STATUS_OK) { 436 aprint_error("firmware not ready\n"); 437 goto fail_ansnotready; 438 } 439 440 bus_space_write_4(sc->sc_iot, sc->sc_ioh, ANS_LINEAR_SQ_CTRL, 441 ANS_LINEAR_SQ_CTRL_EN); 442 bus_space_write_4(sc->sc_iot, sc->sc_ioh, ANS_MAX_PEND_CMDS_CTRL, 443 (ANS_MAX_QUEUE_DEPTH << 16) | ANS_MAX_QUEUE_DEPTH); 444 445 ctrl = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ANS_UNKNOWN_CTRL); 446 bus_space_write_4(sc->sc_iot, sc->sc_ioh, ANS_UNKNOWN_CTRL, 447 ctrl & ~ANS_PRP_NULL_CHECK); 448 449 if (nvme_attach(sc) != 0) { 450 /* error printed by nvme_attach() */ 451 return; 452 } 453 454 SET(sc->sc_flags, NVME_F_ATTACHED); 455 return; 456 fail_ansnotready: 457 458 fail_rtkit: 459 460 fail_ansmap: 461 bus_space_unmap(sc->sc_iot, sc->sc_ioh, size); 462 } 463 464 CFATTACH_DECL_NEW(apple_nvme, sizeof(struct apple_nvme_softc), 465 apple_nvme_match, apple_nvme_attach, NULL, NULL); 466
Indexes created Sat Sep 20 22:09:52 GMT 2025