1 /* $NetBSD: apple_pcie.c,v 1.7 2026/09/20 11:00:31 skrll Exp $ */ 2 /* $OpenBSD: aplpcie.c,v 1.13 2022/04/06 18:59:26 naddy Exp $ */ 3 4 /*- 5 * Copyright (c) 2021 Jared McNeill <jmcneill (at) invisible.ca> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 22 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 24 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /* 31 * Copyright (c) 2021 Mark Kettenis <kettenis (at) openbsd.org> 32 * 33 * Permission to use, copy, modify, and distribute this software for any 34 * purpose with or without fee is hereby granted, provided that the above 35 * copyright notice and this permission notice appear in all copies. 36 * 37 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 38 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 39 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 40 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 41 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 42 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 43 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 44 */ 45 46 47 #include <sys/cdefs.h> 48 __KERNEL_RCSID(0, "$NetBSD: apple_pcie.c,v 1.7 2026/09/20 11:00:31 skrll Exp $"); 49 50 #include <sys/param.h> 51 #include <sys/device.h> 52 #include <sys/kernel.h> 53 #include <sys/systm.h> 54 #include <sys/bus.h> 55 #include <sys/kmem.h> 56 #include <sys/bitops.h> 57 58 #include <dev/pci/pcireg.h> 59 #include <dev/pci/pcivar.h> 60 #include <dev/pci/pciconf.h> 61 62 #include <dev/fdt/fdtvar.h> 63 #include <dev/fdt/pcihost_fdtvar.h> 64 65 #define PCIE_CORE_LANE_CONF(port) (0x84000 + (port) * 0x4000) 66 #define PCIE_CORE_LANE_CONF_REFCLK0REQ __BIT(0) 67 #define PCIE_CORE_LANE_CONF_REFCLK1REQ __BIT(1) 68 #define PCIE_CORE_LANE_CONF_REFCLK0ACK __BIT(2) 69 #define PCIE_CORE_LANE_CONF_REFCLK1ACK __BIT(3) 70 #define PCIE_CORE_LANE_CONF_REFCLK0EN __BIT(9) 71 #define PCIE_CORE_LANE_CONF_REFCLK1EN __BIT(10) 72 #define PCIE_CORE_LANE_CTRL(port) (0x84004 + (port) * 0x4000) 73 #define PCIE_CORE_LANE_CTRL_CFGACC __BIT(15) 74 75 #define PCIE_PORT_LTSSM_CTRL 0x0080 76 #define PCIE_PORT_LTSSM_CTRL_START __BIT(0) 77 #define PCIE_PORT_MSI_CTRL 0x0124 78 #define PCIE_PORT_MSI_CTRL_EN __BIT(0) 79 #define PCIE_PORT_MSI_CTRL_32 __SHIFTIN(5U, __BITS(7,4)) 80 #define PCIE_PORT_MSI_REMAP 0x0128 81 #define PCIE_PORT_MSI_DOORBELL 0x0168 82 #define PCIE_PORT_LINK_STAT 0x0208 83 #define PCIE_PORT_LINK_STAT_UP __BIT(0) 84 #define PCIE_PORT_APPCLK 0x0800 85 #define PCIE_PORT_APPCLK_EN __BIT(0) 86 #define PCIE_PORT_APPCLK_CGDIS __BIT(8) 87 #define PCIE_PORT_STAT 0x0804 88 #define PCIE_PORT_STAT_READY __BIT(0) 89 #define PCIE_PORT_REFCLK 0x0810 90 #define PCIE_PORT_REFCLK_EN __BIT(0) 91 #define PCIE_PORT_REFCLK_CGDIS __BIT(8) 92 #define PCIE_PORT_PERST 0x0814 93 #define PCIE_PORT_PERST_DIS __BIT(0) 94 95 extern struct bus_space arm_generic_bs_tag; 96 97 struct apple_pcie_softc { 98 struct pcihost_softc sc_pcihost; 99 100 bus_space_tag_t sc_rc_bst; 101 bus_space_handle_t sc_rc_bsh; 102 103 int sc_phandle; 104 struct md_pci_msi sc_msi; 105 u_int sc_msi_start; 106 u_int sc_nmsi; 107 struct pci_attach_args **sc_msi_pa; 108 void **sc_msi_ih; 109 uint64_t sc_msi_addr; 110 }; 111 112 static int apple_pcie_match(device_t, cfdata_t, void *); 113 static void apple_pcie_attach(device_t, device_t, void *); 114 115 static void apple_pcie_attach_hook(device_t, device_t, 116 struct pcibus_attach_args *); 117 static int apple_pcie_msi_init(struct apple_pcie_softc *); 118 119 CFATTACH_DECL_NEW(apple_pcie, sizeof(struct apple_pcie_softc), 120 apple_pcie_match, apple_pcie_attach, NULL, NULL); 121 122 static const struct device_compatible_entry compat_data[] = { 123 { .compat = "apple,pcie" }, 124 DEVICE_COMPAT_EOL 125 }; 126 127 #define RREAD4(sc, reg) \ 128 (bus_space_read_4((sc)->sc_rc_bst, (sc)->sc_rc_bsh, (reg))) 129 #define RWRITE4(sc, reg, val) \ 130 bus_space_write_4((sc)->sc_rc_bst, (sc)->sc_rc_bsh, (reg), (val)) 131 #define RSET4(sc, reg, bits) \ 132 RWRITE4((sc), (reg), RREAD4((sc), (reg)) | (bits)) 133 #define RCLR4(sc, reg, bits) \ 134 RWRITE4((sc), (reg), RREAD4((sc), (reg)) & ~(bits)) 135 136 137 static void 138 apple_pcie_setup_port(struct apple_pcie_softc *sc, int phandle) 139 { 140 const bus_space_tag_t bst = sc->sc_pcihost.sc_bst; 141 const device_t dev = sc->sc_pcihost.sc_dev; 142 const int parent = sc->sc_pcihost.sc_phandle; 143 char regname[sizeof("portX")]; 144 bus_space_handle_t bsh; 145 bus_addr_t addr; 146 bus_size_t size; 147 int error; 148 int timo; 149 int len; 150 151 const u_int *reg = fdtbus_get_prop(phandle, "reg", &len); 152 if (len != 5 * sizeof(uint32_t)) { 153 aprint_error(": couldn't get port number\n"); 154 } 155 156 u_int portno = __SHIFTOUT(be32toh(reg[0]), __BITS(13,11)); 157 snprintf(regname, sizeof(regname), "port%u", portno); 158 159 if (fdtbus_get_reg_byname(parent, regname, &addr, &size) != 0) { 160 aprint_error(": couldn't get %s regs\n", regname); 161 return; 162 } 163 error = bus_space_map(bst, addr, size, 0, &bsh); 164 if (error != 0) { 165 aprint_error(": couldn't map %s regs\n", regname); 166 return; 167 } 168 169 #define PREAD4(bst, bsh, reg) \ 170 bus_space_read_4((bst), (bsh), (reg)) 171 #define PWRITE4(bst, bsh, reg, val) \ 172 bus_space_write_4((bst), (bsh), (reg), (val)) 173 #define PSET4(bst, bsh, reg, bits) \ 174 PWRITE4((bst), (bsh), (reg), PREAD4((bst), (bsh), (reg)) | (bits)) 175 #define PCLR4(bst, bsh, reg, bits) \ 176 PWRITE4((bst), (bsh), (reg), PREAD4((bst), (bsh), (reg)) & ~(bits)) 177 178 /* Doorbell address must be below 4GB */ 179 KASSERT((sc->sc_msi_addr & ~0xffffffffUL) == 0); 180 181 int pwren_gpiolen, reset_gpiolen; 182 183 pwren_gpiolen = OF_getproplen(phandle, "pwren-gpios"); 184 reset_gpiolen = OF_getproplen(phandle, "reset-gpios"); 185 if (reset_gpiolen <= 0) 186 return; 187 188 /* 189 * Set things up such that we can share the 32 available MSIs 190 * across all ports. 191 */ 192 PWRITE4(bst, bsh, PCIE_PORT_MSI_CTRL, 193 PCIE_PORT_MSI_CTRL_32 | PCIE_PORT_MSI_CTRL_EN); 194 PWRITE4(bst, bsh, PCIE_PORT_MSI_REMAP, 0); 195 PWRITE4(bst, bsh, PCIE_PORT_MSI_DOORBELL, 196 __SHIFTOUT(sc->sc_msi_addr, __BITS(31, 0))); 197 198 /* Check if the link is already up. */ 199 uint32_t stat = PREAD4(bst, bsh, PCIE_PORT_LINK_STAT); 200 if (stat & PCIE_PORT_LINK_STAT_UP) { 201 aprint_debug_dev(dev, "link already up\n"); 202 return; 203 } 204 aprint_debug_dev(dev, "bringing link up\n"); 205 206 PSET4(bst, bsh, PCIE_PORT_APPCLK, PCIE_PORT_APPCLK_EN); 207 208 struct fdtbus_gpio_pin *gpio_reset = fdtbus_gpio_acquire(phandle, 209 "reset-gpios", GPIO_PIN_OUTPUT); 210 211 if (gpio_reset == NULL) { 212 aprint_debug_dev(dev, "failed to get reset-gpios\n"); 213 return; 214 } 215 216 fdtbus_gpio_write(gpio_reset, 1); 217 218 /* Power up the device if necessary. */ 219 if (pwren_gpiolen > 0) { 220 struct fdtbus_gpio_pin *gpio_pwren = fdtbus_gpio_acquire(phandle, 221 "pwren-gpios", GPIO_PIN_OUTPUT); 222 223 if (gpio_pwren == NULL) { 224 aprint_debug_dev(dev, "failed to get pwren-gpios\n"); 225 return; 226 } 227 228 fdtbus_gpio_write(gpio_pwren, 1); 229 } 230 231 /* Setup Refclk. */ 232 RSET4(sc, PCIE_CORE_LANE_CTRL(portno), PCIE_CORE_LANE_CTRL_CFGACC); 233 RSET4(sc, PCIE_CORE_LANE_CONF(portno), PCIE_CORE_LANE_CONF_REFCLK0REQ); 234 for (timo = 500; timo > 0; timo--) { 235 stat = RREAD4(sc, PCIE_CORE_LANE_CONF(portno)); 236 if (stat & PCIE_CORE_LANE_CONF_REFCLK0ACK) 237 break; 238 delay(100); 239 } 240 RSET4(sc, PCIE_CORE_LANE_CONF(portno), PCIE_CORE_LANE_CONF_REFCLK1REQ); 241 for (timo = 500; timo > 0; timo--) { 242 stat = RREAD4(sc, PCIE_CORE_LANE_CONF(portno)); 243 if (stat & PCIE_CORE_LANE_CONF_REFCLK1ACK) 244 break; 245 delay(100); 246 } 247 RCLR4(sc, PCIE_CORE_LANE_CTRL(portno), PCIE_CORE_LANE_CTRL_CFGACC); 248 RSET4(sc, PCIE_CORE_LANE_CONF(portno), 249 PCIE_CORE_LANE_CONF_REFCLK0EN | PCIE_CORE_LANE_CONF_REFCLK1EN); 250 PSET4(bst, bsh, PCIE_PORT_REFCLK, PCIE_PORT_REFCLK_EN); 251 252 /* 253 * PERST# must remain asserted for at least 100us after the 254 * reference clock becomes stable. But also has to remain 255 * active at least 100ms after power up. 256 */ 257 if (pwren_gpiolen > 0) 258 delay(100000); 259 else 260 delay(100); 261 262 /* Deassert PERST#. */ 263 PSET4(bst, bsh, PCIE_PORT_PERST, PCIE_PORT_PERST_DIS); 264 fdtbus_gpio_write(gpio_reset, 0); 265 266 for (timo = 2500; timo > 0; timo--) { 267 stat = PREAD4(bst, bsh, PCIE_PORT_STAT); 268 if (stat & PCIE_PORT_STAT_READY) 269 break; 270 delay(100); 271 } 272 if ((stat & PCIE_PORT_STAT_READY) == 0) { 273 aprint_debug_dev(dev, "link up\n"); 274 return; 275 } 276 277 PCLR4(bst, bsh, PCIE_PORT_REFCLK, PCIE_PORT_REFCLK_CGDIS); 278 PCLR4(bst, bsh, PCIE_PORT_APPCLK, PCIE_PORT_APPCLK_CGDIS); 279 280 /* Bring up the link. */ 281 PWRITE4(bst, bsh, PCIE_PORT_LTSSM_CTRL, PCIE_PORT_LTSSM_CTRL_START); 282 for (timo = 1000; timo > 0; timo--) { 283 stat = PREAD4(bst, bsh, PCIE_PORT_LINK_STAT); 284 if (stat & PCIE_PORT_LINK_STAT_UP) 285 break; 286 delay(100); 287 } 288 289 #undef PREAD4 290 #undef PWRITE4 291 #undef PCLR4 292 #undef PSET4 293 294 bus_space_unmap(bst, bsh, size); 295 } 296 297 static int 298 apple_pcie_match(device_t parent, cfdata_t cf, void *aux) 299 { 300 struct fdt_attach_args * const faa = aux; 301 302 return of_compatible_match(faa->faa_phandle, compat_data); 303 } 304 305 static void 306 apple_pcie_attach(device_t parent, device_t self, void *aux) 307 { 308 struct apple_pcie_softc * const asc = device_private(self); 309 struct pcihost_softc * const sc = &asc->sc_pcihost; 310 struct fdt_attach_args * const faa = aux; 311 const int phandle = faa->faa_phandle; 312 bus_addr_t cs_addr, rc_addr; 313 bus_size_t cs_size, rc_size; 314 int error; 315 316 if (fdtbus_get_reg_byname(phandle, "config", &cs_addr, &cs_size) != 0) { 317 aprint_error(": couldn't get registers (%s)\n", "config"); 318 return; 319 } 320 321 if (fdtbus_get_reg_byname(phandle, "rc", &rc_addr, &rc_size) != 0) { 322 aprint_error(": couldn't get registers (%s)\n", "rc"); 323 return; 324 } 325 326 sc->sc_dev = self; 327 sc->sc_dmat = faa->faa_dmat; 328 sc->sc_bst = asc->sc_rc_bst = faa->faa_bst; 329 /* 330 * Create a new bus tag for PCIe devices that does not inherit the 331 * nonposted MMIO flag from the host controller. 332 */ 333 sc->sc_pci_bst = &arm_generic_bs_tag; 334 sc->sc_phandle = phandle; 335 error = bus_space_map(faa->faa_bst, cs_addr, cs_size, 0, &sc->sc_bsh); 336 if (error) { 337 aprint_error(": couldn't map registers (%s): %d\n", "config", 338 error); 339 return; 340 } 341 error = bus_space_map(asc->sc_rc_bst, rc_addr, rc_size, 0, 342 &asc->sc_rc_bsh); 343 if (error) { 344 aprint_error(": couldn't map registers (%s): %d\n", "rc", 345 error); 346 return; 347 } 348 sc->sc_type = PCIHOST_ECAM; 349 350 if (apple_pcie_msi_init(asc) == 0) { 351 sc->sc_pci_flags |= PCI_FLAGS_MSI_OKAY; 352 #if notyet 353 sc->sc_pci_flags |= PCI_FLAGS_MSIX_OKAY; 354 #endif 355 } 356 357 aprint_naive("\n"); 358 aprint_normal(": Apple PCIe host controller\n"); 359 360 for (int node = OF_child(phandle); node; node = OF_peer(node)) 361 apple_pcie_setup_port(asc, node); 362 363 /* 364 * Must wait at least 100ms after link training completes 365 * before sending a configuration request to a device 366 * immediately below a port. 367 */ 368 delay(100000); 369 370 pcihost_init(&sc->sc_pc, sc); 371 372 sc->sc_pc.pc_attach_hook = apple_pcie_attach_hook; 373 pcihost_init2(sc); 374 } 375 376 377 378 static void 379 apple_pcie_attach_hook(device_t parent, device_t self, 380 struct pcibus_attach_args *pba) 381 { 382 struct apple_pcie_softc *sc = pba->pba_pc->pc_conf_v; 383 const int phandle = sc->sc_pcihost.sc_phandle; 384 bus_dma_tag_t dmat; 385 386 KASSERT(device_is_a(sc->sc_pcihost.sc_dev, "applepcie")); 387 388 /* XXX this should be per-device, not per-bus */ 389 const uint32_t rid = pba->pba_bus << 8; 390 391 dmat = fdtbus_iommu_map_pci(phandle, rid, sc->sc_pcihost.sc_dmat); 392 393 pba->pba_dmat = pba->pba_dmat64 = dmat; 394 } 395 396 static int 397 apple_pcie_msi_alloc_msi(struct apple_pcie_softc *sc, int count, 398 const struct pci_attach_args *pa) 399 { 400 struct pci_attach_args *new_pa; 401 int msi, n; 402 403 for (msi = 0; msi < sc->sc_nmsi; msi += n) { 404 /* Look for first empty slot */ 405 if (sc->sc_msi_pa[msi] != NULL) { 406 /* skip the used entry */ 407 n = 1; 408 continue; 409 } 410 411 /* Now check that 'count' entries are also empty */ 412 for (n = 1; n < count && msi + n < sc->sc_nmsi; n++) { 413 if (sc->sc_msi_pa[msi + n] != NULL) { 414 break; 415 } 416 } 417 /* 418 * If 'count' empty entries weren't found then the search 419 * continues. 420 */ 421 if (n != count) 422 continue; 423 for (n = 0; n < count; n++) { 424 new_pa = kmem_alloc(sizeof(*new_pa), KM_SLEEP); 425 memcpy(new_pa, pa, sizeof(*new_pa)); 426 sc->sc_msi_pa[msi + n] = new_pa; 427 } 428 429 return msi; 430 } 431 432 return -1; 433 } 434 435 static void 436 apple_pcie_msi_free_msi(struct apple_pcie_softc *sc, int msi) 437 { 438 struct pci_attach_args *pa; 439 440 pa = sc->sc_msi_pa[msi]; 441 sc->sc_msi_pa[msi] = NULL; 442 443 if (pa != NULL) { 444 kmem_free(pa, sizeof(*pa)); 445 } 446 } 447 448 static int 449 apple_pcie_msi_available_msi(struct apple_pcie_softc *sc) 450 { 451 int msi, n; 452 453 for (n = 0, msi = 0; msi < sc->sc_nmsi; msi++) { 454 if (sc->sc_msi_pa[msi] == NULL) { 455 n++; 456 } 457 } 458 459 return n; 460 } 461 462 static void 463 apple_pcie_msi_msi_enable(struct apple_pcie_softc *sc, int msi, int count) 464 { 465 const struct pci_attach_args *pa = sc->sc_msi_pa[msi]; 466 pci_chipset_tag_t pc = pa->pa_pc; 467 pcitag_t tag = pa->pa_tag; 468 pcireg_t ctl; 469 int off; 470 471 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) 472 panic("apple_pcie_msi_msi_enable: device is not MSI-capable"); 473 474 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 475 ctl &= ~PCI_MSI_CTL_MSI_ENABLE; 476 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 477 478 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 479 ctl &= ~PCI_MSI_CTL_MME_MASK; 480 ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK); 481 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 482 483 const uint64_t addr = sc->sc_msi_addr; 484 const uint32_t data = msi; 485 486 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 487 if (ctl & PCI_MSI_CTL_64BIT_ADDR) { 488 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, 489 __SHIFTOUT(addr, __BITS(31, 0))); 490 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, 491 __SHIFTOUT(addr, __BITS(63, 32))); 492 pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, data); 493 } else { 494 pci_conf_write(pc, tag, off + PCI_MSI_MADDR, 495 __SHIFTOUT(addr, __BITS(31, 0))); 496 pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data); 497 } 498 ctl |= PCI_MSI_CTL_MSI_ENABLE; 499 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 500 } 501 502 static void 503 apple_pcie_msi_msi_disable(struct apple_pcie_softc *sc, int msi) 504 { 505 const struct pci_attach_args *pa = sc->sc_msi_pa[msi]; 506 pci_chipset_tag_t pc = pa->pa_pc; 507 pcitag_t tag = pa->pa_tag; 508 pcireg_t ctl; 509 int off; 510 511 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) 512 panic("apple_pcie_msi_msi_disable: device is not MSI-capable"); 513 514 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 515 ctl &= ~PCI_MSI_CTL_MSI_ENABLE; 516 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 517 } 518 519 static void 520 apple_pcie_msi_msix_enable(struct apple_pcie_softc *sc, int msi, int msix_vec, 521 bus_space_tag_t bst, bus_space_handle_t bsh) 522 { 523 const struct pci_attach_args *pa = sc->sc_msi_pa[msi]; 524 pci_chipset_tag_t pc = pa->pa_pc; 525 pcitag_t tag = pa->pa_tag; 526 pcireg_t ctl; 527 uint32_t val; 528 int off; 529 530 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) 531 panic("apple_pcie_msi_msix_enable: device is not MSI-X-capable"); 532 533 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 534 ctl &= ~PCI_MSIX_CTL_ENABLE; 535 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 536 537 const uint64_t addr = sc->sc_msi_addr; 538 const uint32_t data = msi; 539 const uint64_t entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec; 540 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, 541 __SHIFTOUT(addr, __BITS(31, 0))); 542 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, 543 __SHIFTOUT(addr, __BITS(63, 32))); 544 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_DATA, 545 data); 546 val = bus_space_read_4(bst, bsh, 547 entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL); 548 val &= ~PCI_MSIX_VECTCTL_MASK; 549 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, 550 val); 551 552 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 553 ctl |= PCI_MSIX_CTL_ENABLE; 554 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 555 } 556 557 static void 558 apple_pcie_msi_msix_disable(struct apple_pcie_softc *sc, int msi) 559 { 560 const struct pci_attach_args *pa = sc->sc_msi_pa[msi]; 561 pci_chipset_tag_t pc = pa->pa_pc; 562 pcitag_t tag = pa->pa_tag; 563 pcireg_t ctl; 564 int off; 565 566 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) 567 panic("apple_pcie_msi_msix_disable: device is not MSI-X-capable"); 568 569 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 570 ctl &= ~PCI_MSIX_CTL_ENABLE; 571 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 572 } 573 574 static pci_intr_handle_t * 575 apple_pcie_msi_msi_alloc(struct md_pci_msi *msi, int *count, 576 const struct pci_attach_args *pa, bool exact) 577 { 578 struct apple_pcie_softc * const sc = msi->msi_priv; 579 pci_intr_handle_t *vectors; 580 int n, off; 581 582 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSI, &off, NULL)) 583 return NULL; 584 585 const int avail = apple_pcie_msi_available_msi(sc); 586 if (avail == 0) 587 return NULL; 588 589 if (exact && *count > avail) 590 return NULL; 591 592 while (*count > avail) { 593 (*count) >>= 1; 594 } 595 if (*count == 0) 596 return NULL; 597 598 const int msi_base = apple_pcie_msi_alloc_msi(sc, *count, pa); 599 if (msi_base == -1) 600 return NULL; 601 602 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); 603 for (n = 0; n < *count; n++) { 604 const int msino = msi_base + n; 605 vectors[n] = MD_PCI_INTR_MSI | 606 __SHIFTIN(msino, MD_PCI_INTR_IRQ) | 607 __SHIFTIN(n, MD_PCI_INTR_MSI_VEC) | 608 __SHIFTIN(msi->msi_id, MD_PCI_INTR_FRAME); 609 } 610 611 apple_pcie_msi_msi_enable(sc, msi_base, *count); 612 613 return vectors; 614 } 615 616 static pci_intr_handle_t * 617 apple_pcie_msi_msix_alloc(struct md_pci_msi *msi, u_int *table_indexes, 618 int *count, const struct pci_attach_args *pa, bool exact) 619 { 620 struct apple_pcie_softc * const sc = msi->msi_priv; 621 pci_intr_handle_t *vectors; 622 bus_space_tag_t bst; 623 bus_space_handle_t bsh; 624 bus_size_t bsz; 625 uint32_t table_offset, table_size; 626 int n, off, bar, error; 627 pcireg_t tbl; 628 629 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &off, NULL)) 630 return NULL; 631 632 const int avail = apple_pcie_msi_available_msi(sc); 633 if (exact && *count > avail) 634 return NULL; 635 636 while (*count > avail) { 637 (*count) >>= 1; 638 } 639 if (*count == 0) 640 return NULL; 641 642 tbl = pci_conf_read(pa->pa_pc, pa->pa_tag, off + PCI_MSIX_TBLOFFSET); 643 bar = PCI_BAR0 + (4 * (tbl & PCI_MSIX_TBLBIR_MASK)); 644 table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK; 645 table_size = pci_msix_count(pa->pa_pc, pa->pa_tag) * PCI_MSIX_TABLE_ENTRY_SIZE; 646 if (table_size == 0) 647 return NULL; 648 649 error = pci_mapreg_submap(pa, bar, pci_mapreg_type(pa->pa_pc, pa->pa_tag, bar), 650 BUS_SPACE_MAP_LINEAR, roundup(table_size, PAGE_SIZE), table_offset, 651 &bst, &bsh, NULL, &bsz); 652 if (error) 653 return NULL; 654 655 const int msi_base = apple_pcie_msi_alloc_msi(sc, *count, pa); 656 if (msi_base == -1) { 657 bus_space_unmap(bst, bsh, bsz); 658 return NULL; 659 } 660 661 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); 662 for (n = 0; n < *count; n++) { 663 const int msino = msi_base + n; 664 const int msix_vec = table_indexes ? table_indexes[n] : n; 665 vectors[msix_vec] = MD_PCI_INTR_MSIX | 666 __SHIFTIN(msino, MD_PCI_INTR_IRQ) | 667 __SHIFTIN(msix_vec, MD_PCI_INTR_MSI_VEC) | 668 __SHIFTIN(msi->msi_id, MD_PCI_INTR_FRAME); 669 670 apple_pcie_msi_msix_enable(sc, msino, msix_vec, bst, bsh); 671 } 672 673 bus_space_unmap(bst, bsh, bsz); 674 675 return vectors; 676 } 677 678 static void * 679 apple_pcie_msi_intr_establish(struct md_pci_msi *msi, 680 pci_intr_handle_t ih, int ipl, int (*func)(void *), void *arg, const char *xname) 681 { 682 struct apple_pcie_softc * const sc = msi->msi_priv; 683 684 const int msino = __SHIFTOUT(ih, MD_PCI_INTR_IRQ); 685 const int mpsafe = (ih & MD_PCI_INTR_MPSAFE) ? FDT_INTR_MPSAFE : 0; 686 687 KASSERT(sc->sc_msi_ih[msino] == NULL); 688 sc->sc_msi_ih[msino] = intr_establish_xname(sc->sc_msi_start + msino, 689 ipl, IST_LEVEL | (mpsafe ? IST_MPSAFE : 0), func, arg, xname); 690 691 return sc->sc_msi_ih[msino]; 692 } 693 694 static void 695 apple_pcie_msi_intr_release(struct md_pci_msi *msi, pci_intr_handle_t *pih, 696 int count) 697 { 698 struct apple_pcie_softc * const sc = msi->msi_priv; 699 int n; 700 701 for (n = 0; n < count; n++) { 702 const int msino = __SHIFTOUT(pih[n], MD_PCI_INTR_IRQ); 703 if (pih[n] & MD_PCI_INTR_MSIX) 704 apple_pcie_msi_msix_disable(sc, msino); 705 if (pih[n] & MD_PCI_INTR_MSI) 706 apple_pcie_msi_msi_disable(sc, msino); 707 apple_pcie_msi_free_msi(sc, msino); 708 if (sc->sc_msi_ih[msino] != NULL) { 709 intr_disestablish(sc->sc_msi_ih[msino]); 710 sc->sc_msi_ih[msino] = NULL; 711 } 712 } 713 } 714 715 static int 716 apple_pcie_msi_init(struct apple_pcie_softc *sc) 717 { 718 struct md_pci_msi *msi = &sc->sc_msi; 719 const int phandle = sc->sc_pcihost.sc_phandle; 720 int len; 721 722 const u_int *data = fdtbus_get_prop(phandle, "msi-ranges", &len); 723 switch (len) { 724 case 8: 725 /* two cells: start and count */ 726 sc->sc_msi_start = be32toh(data[0]); 727 sc->sc_nmsi = be32toh(data[1]); 728 break; 729 case 20: 730 /* 5 cells: xref, specifier (3 cells), and count */ 731 sc->sc_msi_start = be32toh(data[2]); 732 sc->sc_nmsi = be32toh(data[4]); 733 break; 734 default: 735 aprint_error_dev(sc->sc_pcihost.sc_dev, 736 "WARNING: bad msi-ranges property, MSI not enabled!\n"); 737 return ENXIO; 738 } 739 sc->sc_msi_pa = kmem_zalloc(sizeof(*sc->sc_msi_pa) * sc->sc_nmsi, 740 KM_SLEEP); 741 sc->sc_msi_ih = kmem_zalloc(sizeof(*sc->sc_msi_ih) * sc->sc_nmsi, 742 KM_SLEEP); 743 744 if (of_getprop_uint64(phandle, "msi-doorbell", &sc->sc_msi_addr)) { 745 sc->sc_msi_addr = 0xffff000ULL; 746 } 747 748 msi->msi_dev = sc->sc_pcihost.sc_dev; 749 msi->msi_priv = sc; 750 msi->msi_alloc = apple_pcie_msi_msi_alloc; 751 msi->msix_alloc = apple_pcie_msi_msix_alloc; 752 msi->msi_intr_establish = apple_pcie_msi_intr_establish; 753 msi->msi_intr_release = apple_pcie_msi_intr_release; 754 755 return md_pci_msi_add(msi); 756 } 757
Indexes created Sun Oct 25 00:26:45 UTC 2026