1 /* $NetBSD: apple_pcie.c,v 1.6 2022/04/27 08:03:06 skrll Exp $ */ 2 /* $OpenBSD: aplpcie.c,v 1.13 2022/04/06 18:59:26 naddy Exp $ */ 3 4 /*- 5 * Copyright (c) 2021 Jared McNeill <jmcneill (at) invisible.ca> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 22 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 24 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /* 31 * Copyright (c) 2021 Mark Kettenis <kettenis (at) openbsd.org> 32 * 33 * Permission to use, copy, modify, and distribute this software for any 34 * purpose with or without fee is hereby granted, provided that the above 35 * copyright notice and this permission notice appear in all copies. 36 * 37 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 38 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 39 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 40 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 41 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 42 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 43 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 44 */ 45 46 47 #include <sys/cdefs.h> 48 __KERNEL_RCSID(0, "$NetBSD: apple_pcie.c,v 1.6 2022/04/27 08:03:06 skrll Exp $"); 49 50 #include <sys/param.h> 51 #include <sys/device.h> 52 #include <sys/kernel.h> 53 #include <sys/systm.h> 54 #include <sys/bus.h> 55 #include <sys/kmem.h> 56 #include <sys/bitops.h> 57 58 #include <dev/pci/pcireg.h> 59 #include <dev/pci/pcivar.h> 60 #include <dev/pci/pciconf.h> 61 62 #include <dev/fdt/fdtvar.h> 63 64 #include <arm/pci/pci_msi_machdep.h> 65 #include <arm/fdt/pcihost_fdtvar.h> 66 67 #define PCIE_CORE_LANE_CONF(port) (0x84000 + (port) * 0x4000) 68 #define PCIE_CORE_LANE_CONF_REFCLK0REQ __BIT(0) 69 #define PCIE_CORE_LANE_CONF_REFCLK1REQ __BIT(1) 70 #define PCIE_CORE_LANE_CONF_REFCLK0ACK __BIT(2) 71 #define PCIE_CORE_LANE_CONF_REFCLK1ACK __BIT(3) 72 #define PCIE_CORE_LANE_CONF_REFCLK0EN __BIT(9) 73 #define PCIE_CORE_LANE_CONF_REFCLK1EN __BIT(10) 74 #define PCIE_CORE_LANE_CTRL(port) (0x84004 + (port) * 0x4000) 75 #define PCIE_CORE_LANE_CTRL_CFGACC __BIT(15) 76 77 #define PCIE_PORT_LTSSM_CTRL 0x0080 78 #define PCIE_PORT_LTSSM_CTRL_START __BIT(0) 79 #define PCIE_PORT_MSI_CTRL 0x0124 80 #define PCIE_PORT_MSI_CTRL_EN __BIT(0) 81 #define PCIE_PORT_MSI_CTRL_32 __SHIFTIN(5U, __BITS(7,4)) 82 #define PCIE_PORT_MSI_REMAP 0x0128 83 #define PCIE_PORT_MSI_DOORBELL 0x0168 84 #define PCIE_PORT_LINK_STAT 0x0208 85 #define PCIE_PORT_LINK_STAT_UP __BIT(0) 86 #define PCIE_PORT_APPCLK 0x0800 87 #define PCIE_PORT_APPCLK_EN __BIT(0) 88 #define PCIE_PORT_APPCLK_CGDIS __BIT(8) 89 #define PCIE_PORT_STAT 0x0804 90 #define PCIE_PORT_STAT_READY __BIT(0) 91 #define PCIE_PORT_REFCLK 0x0810 92 #define PCIE_PORT_REFCLK_EN __BIT(0) 93 #define PCIE_PORT_REFCLK_CGDIS __BIT(8) 94 #define PCIE_PORT_PERST 0x0814 95 #define PCIE_PORT_PERST_DIS __BIT(0) 96 97 extern struct bus_space arm_generic_bs_tag; 98 99 struct apple_pcie_softc { 100 struct pcihost_softc sc_pcihost; 101 102 bus_space_tag_t sc_rc_bst; 103 bus_space_handle_t sc_rc_bsh; 104 105 int sc_phandle; 106 struct arm_pci_msi sc_msi; 107 u_int sc_msi_start; 108 u_int sc_nmsi; 109 struct pci_attach_args **sc_msi_pa; 110 void **sc_msi_ih; 111 uint64_t sc_msi_addr; 112 }; 113 114 static int apple_pcie_match(device_t, cfdata_t, void *); 115 static void apple_pcie_attach(device_t, device_t, void *); 116 117 static void apple_pcie_attach_hook(device_t, device_t, 118 struct pcibus_attach_args *); 119 static int apple_pcie_msi_init(struct apple_pcie_softc *); 120 121 CFATTACH_DECL_NEW(apple_pcie, sizeof(struct apple_pcie_softc), 122 apple_pcie_match, apple_pcie_attach, NULL, NULL); 123 124 static const struct device_compatible_entry compat_data[] = { 125 { .compat = "apple,pcie" }, 126 DEVICE_COMPAT_EOL 127 }; 128 129 #define RREAD4(sc, reg) \ 130 (bus_space_read_4((sc)->sc_rc_bst, (sc)->sc_rc_bsh, (reg))) 131 #define RWRITE4(sc, reg, val) \ 132 bus_space_write_4((sc)->sc_rc_bst, (sc)->sc_rc_bsh, (reg), (val)) 133 #define RSET4(sc, reg, bits) \ 134 RWRITE4((sc), (reg), RREAD4((sc), (reg)) | (bits)) 135 #define RCLR4(sc, reg, bits) \ 136 RWRITE4((sc), (reg), RREAD4((sc), (reg)) & ~(bits)) 137 138 139 static void 140 apple_pcie_setup_port(struct apple_pcie_softc *sc, int phandle) 141 { 142 const bus_space_tag_t bst = sc->sc_pcihost.sc_bst; 143 const device_t dev = sc->sc_pcihost.sc_dev; 144 const int parent = sc->sc_pcihost.sc_phandle; 145 char regname[sizeof("portX")]; 146 bus_space_handle_t bsh; 147 bus_addr_t addr; 148 bus_size_t size; 149 int error; 150 int timo; 151 int len; 152 153 const u_int *reg = fdtbus_get_prop(phandle, "reg", &len); 154 if (len != 5 * sizeof(uint32_t)) { 155 aprint_error(": couldn't get port number\n"); 156 } 157 158 u_int portno = __SHIFTOUT(be32toh(reg[0]), __BITS(13,11)); 159 snprintf(regname, sizeof(regname), "port%u", portno); 160 161 if (fdtbus_get_reg_byname(parent, regname, &addr, &size) != 0) { 162 aprint_error(": couldn't get %s regs\n", regname); 163 return; 164 } 165 error = bus_space_map(bst, addr, size, 0, &bsh); 166 if (error != 0) { 167 aprint_error(": couldn't map %s regs\n", regname); 168 return; 169 } 170 171 #define PREAD4(bst, bsh, reg) \ 172 bus_space_read_4((bst), (bsh), (reg)) 173 #define PWRITE4(bst, bsh, reg, val) \ 174 bus_space_write_4((bst), (bsh), (reg), (val)) 175 #define PSET4(bst, bsh, reg, bits) \ 176 PWRITE4((bst), (bsh), (reg), PREAD4((bst), (bsh), (reg)) | (bits)) 177 #define PCLR4(bst, bsh, reg, bits) \ 178 PWRITE4((bst), (bsh), (reg), PREAD4((bst), (bsh), (reg)) & ~(bits)) 179 180 /* Doorbell address must be below 4GB */ 181 KASSERT((sc->sc_msi_addr & ~0xffffffffUL) == 0); 182 183 int pwren_gpiolen, reset_gpiolen; 184 185 pwren_gpiolen = OF_getproplen(phandle, "pwren-gpios"); 186 reset_gpiolen = OF_getproplen(phandle, "reset-gpios"); 187 if (reset_gpiolen <= 0) 188 return; 189 190 /* 191 * Set things up such that we can share the 32 available MSIs 192 * across all ports. 193 */ 194 PWRITE4(bst, bsh, PCIE_PORT_MSI_CTRL, 195 PCIE_PORT_MSI_CTRL_32 | PCIE_PORT_MSI_CTRL_EN); 196 PWRITE4(bst, bsh, PCIE_PORT_MSI_REMAP, 0); 197 PWRITE4(bst, bsh, PCIE_PORT_MSI_DOORBELL, 198 __SHIFTOUT(sc->sc_msi_addr, __BITS(31, 0))); 199 200 /* Check if the link is already up. */ 201 uint32_t stat = PREAD4(bst, bsh, PCIE_PORT_LINK_STAT); 202 if (stat & PCIE_PORT_LINK_STAT_UP) { 203 aprint_debug_dev(dev, "link already up\n"); 204 return; 205 } 206 aprint_debug_dev(dev, "bringing link up\n"); 207 208 PSET4(bst, bsh, PCIE_PORT_APPCLK, PCIE_PORT_APPCLK_EN); 209 210 struct fdtbus_gpio_pin *gpio_reset = fdtbus_gpio_acquire(phandle, 211 "reset-gpios", GPIO_PIN_OUTPUT); 212 213 if (gpio_reset == NULL) { 214 aprint_debug_dev(dev, "failed to get reset-gpios\n"); 215 return; 216 } 217 218 fdtbus_gpio_write(gpio_reset, 1); 219 220 /* Power up the device if necessary. */ 221 if (pwren_gpiolen > 0) { 222 struct fdtbus_gpio_pin *gpio_pwren = fdtbus_gpio_acquire(phandle, 223 "pwren-gpios", GPIO_PIN_OUTPUT); 224 225 if (gpio_pwren == NULL) { 226 aprint_debug_dev(dev, "failed to get pwren-gpios\n"); 227 return; 228 } 229 230 fdtbus_gpio_write(gpio_pwren, 1); 231 } 232 233 /* Setup Refclk. */ 234 RSET4(sc, PCIE_CORE_LANE_CTRL(portno), PCIE_CORE_LANE_CTRL_CFGACC); 235 RSET4(sc, PCIE_CORE_LANE_CONF(portno), PCIE_CORE_LANE_CONF_REFCLK0REQ); 236 for (timo = 500; timo > 0; timo--) { 237 stat = RREAD4(sc, PCIE_CORE_LANE_CONF(portno)); 238 if (stat & PCIE_CORE_LANE_CONF_REFCLK0ACK) 239 break; 240 delay(100); 241 } 242 RSET4(sc, PCIE_CORE_LANE_CONF(portno), PCIE_CORE_LANE_CONF_REFCLK1REQ); 243 for (timo = 500; timo > 0; timo--) { 244 stat = RREAD4(sc, PCIE_CORE_LANE_CONF(portno)); 245 if (stat & PCIE_CORE_LANE_CONF_REFCLK1ACK) 246 break; 247 delay(100); 248 } 249 RCLR4(sc, PCIE_CORE_LANE_CTRL(portno), PCIE_CORE_LANE_CTRL_CFGACC); 250 RSET4(sc, PCIE_CORE_LANE_CONF(portno), 251 PCIE_CORE_LANE_CONF_REFCLK0EN | PCIE_CORE_LANE_CONF_REFCLK1EN); 252 PSET4(bst, bsh, PCIE_PORT_REFCLK, PCIE_PORT_REFCLK_EN); 253 254 /* 255 * PERST# must remain asserted for at least 100us after the 256 * reference clock becomes stable. But also has to remain 257 * active at least 100ms after power up. 258 */ 259 if (pwren_gpiolen > 0) 260 delay(100000); 261 else 262 delay(100); 263 264 /* Deassert PERST#. */ 265 PSET4(bst, bsh, PCIE_PORT_PERST, PCIE_PORT_PERST_DIS); 266 fdtbus_gpio_write(gpio_reset, 0); 267 268 for (timo = 2500; timo > 0; timo--) { 269 stat = PREAD4(bst, bsh, PCIE_PORT_STAT); 270 if (stat & PCIE_PORT_STAT_READY) 271 break; 272 delay(100); 273 } 274 if ((stat & PCIE_PORT_STAT_READY) == 0) { 275 aprint_debug_dev(dev, "link up\n"); 276 return; 277 } 278 279 PCLR4(bst, bsh, PCIE_PORT_REFCLK, PCIE_PORT_REFCLK_CGDIS); 280 PCLR4(bst, bsh, PCIE_PORT_APPCLK, PCIE_PORT_APPCLK_CGDIS); 281 282 /* Bring up the link. */ 283 PWRITE4(bst, bsh, PCIE_PORT_LTSSM_CTRL, PCIE_PORT_LTSSM_CTRL_START); 284 for (timo = 1000; timo > 0; timo--) { 285 stat = PREAD4(bst, bsh, PCIE_PORT_LINK_STAT); 286 if (stat & PCIE_PORT_LINK_STAT_UP) 287 break; 288 delay(100); 289 } 290 291 #undef PREAD4 292 #undef PWRITE4 293 #undef PCLR4 294 #undef PSET4 295 296 bus_space_unmap(bst, bsh, size); 297 } 298 299 static int 300 apple_pcie_match(device_t parent, cfdata_t cf, void *aux) 301 { 302 struct fdt_attach_args * const faa = aux; 303 304 return of_compatible_match(faa->faa_phandle, compat_data); 305 } 306 307 static void 308 apple_pcie_attach(device_t parent, device_t self, void *aux) 309 { 310 struct apple_pcie_softc * const asc = device_private(self); 311 struct pcihost_softc * const sc = &asc->sc_pcihost; 312 struct fdt_attach_args * const faa = aux; 313 const int phandle = faa->faa_phandle; 314 bus_addr_t cs_addr, rc_addr; 315 bus_size_t cs_size, rc_size; 316 int error; 317 318 if (fdtbus_get_reg_byname(phandle, "config", &cs_addr, &cs_size) != 0) { 319 aprint_error(": couldn't get registers (%s)\n", "config"); 320 return; 321 } 322 323 if (fdtbus_get_reg_byname(phandle, "rc", &rc_addr, &rc_size) != 0) { 324 aprint_error(": couldn't get registers (%s)\n", "rc"); 325 return; 326 } 327 328 sc->sc_dev = self; 329 sc->sc_dmat = faa->faa_dmat; 330 sc->sc_bst = asc->sc_rc_bst = faa->faa_bst; 331 /* 332 * Create a new bus tag for PCIe devices that does not inherit the 333 * nonposted MMIO flag from the host controller. 334 */ 335 sc->sc_pci_bst = &arm_generic_bs_tag; 336 sc->sc_phandle = phandle; 337 error = bus_space_map(faa->faa_bst, cs_addr, cs_size, 0, &sc->sc_bsh); 338 if (error) { 339 aprint_error(": couldn't map registers (%s): %d\n", "config", 340 error); 341 return; 342 } 343 error = bus_space_map(asc->sc_rc_bst, rc_addr, rc_size, 0, 344 &asc->sc_rc_bsh); 345 if (error) { 346 aprint_error(": couldn't map registers (%s): %d\n", "rc", 347 error); 348 return; 349 } 350 sc->sc_type = PCIHOST_ECAM; 351 352 if (apple_pcie_msi_init(asc) == 0) { 353 sc->sc_pci_flags |= PCI_FLAGS_MSI_OKAY; 354 #if notyet 355 sc->sc_pci_flags |= PCI_FLAGS_MSIX_OKAY; 356 #endif 357 } 358 359 aprint_naive("\n"); 360 aprint_normal(": Apple PCIe host controller\n"); 361 362 for (int node = OF_child(phandle); node; node = OF_peer(node)) 363 apple_pcie_setup_port(asc, node); 364 365 /* 366 * Must wait at least 100ms after link training completes 367 * before sending a configuration request to a device 368 * immediately below a port. 369 */ 370 delay(100000); 371 372 pcihost_init(&sc->sc_pc, sc); 373 374 sc->sc_pc.pc_attach_hook = apple_pcie_attach_hook; 375 pcihost_init2(sc); 376 } 377 378 379 380 static void 381 apple_pcie_attach_hook(device_t parent, device_t self, 382 struct pcibus_attach_args *pba) 383 { 384 struct apple_pcie_softc *sc = pba->pba_pc->pc_conf_v; 385 const int phandle = sc->sc_pcihost.sc_phandle; 386 bus_dma_tag_t dmat; 387 388 KASSERT(device_is_a(sc->sc_pcihost.sc_dev, "applepcie")); 389 390 /* XXX this should be per-device, not per-bus */ 391 const uint32_t rid = pba->pba_bus << 8; 392 393 dmat = fdtbus_iommu_map_pci(phandle, rid, sc->sc_pcihost.sc_dmat); 394 395 pba->pba_dmat = pba->pba_dmat64 = dmat; 396 } 397 398 static int 399 apple_pcie_msi_alloc_msi(struct apple_pcie_softc *sc, int count, 400 const struct pci_attach_args *pa) 401 { 402 struct pci_attach_args *new_pa; 403 int msi, n; 404 405 for (msi = 0; msi < sc->sc_nmsi; msi += n) { 406 /* Look for first empty slot */ 407 if (sc->sc_msi_pa[msi] != NULL) { 408 /* skip the used entry */ 409 n = 1; 410 continue; 411 } 412 413 /* Now check that 'count' entries are also empty */ 414 for (n = 1; n < count && msi + n < sc->sc_nmsi; n++) { 415 if (sc->sc_msi_pa[msi + n] != NULL) { 416 break; 417 } 418 } 419 /* 420 * If 'count' empty entries weren't found then the search 421 * continues. 422 */ 423 if (n != count) 424 continue; 425 for (n = 0; n < count; n++) { 426 new_pa = kmem_alloc(sizeof(*new_pa), KM_SLEEP); 427 memcpy(new_pa, pa, sizeof(*new_pa)); 428 sc->sc_msi_pa[msi + n] = new_pa; 429 } 430 431 return msi; 432 } 433 434 return -1; 435 } 436 437 static void 438 apple_pcie_msi_free_msi(struct apple_pcie_softc *sc, int msi) 439 { 440 struct pci_attach_args *pa; 441 442 pa = sc->sc_msi_pa[msi]; 443 sc->sc_msi_pa[msi] = NULL; 444 445 if (pa != NULL) { 446 kmem_free(pa, sizeof(*pa)); 447 } 448 } 449 450 static int 451 apple_pcie_msi_available_msi(struct apple_pcie_softc *sc) 452 { 453 int msi, n; 454 455 for (n = 0, msi = 0; msi < sc->sc_nmsi; msi++) { 456 if (sc->sc_msi_pa[msi] == NULL) { 457 n++; 458 } 459 } 460 461 return n; 462 } 463 464 static void 465 apple_pcie_msi_msi_enable(struct apple_pcie_softc *sc, int msi, int count) 466 { 467 const struct pci_attach_args *pa = sc->sc_msi_pa[msi]; 468 pci_chipset_tag_t pc = pa->pa_pc; 469 pcitag_t tag = pa->pa_tag; 470 pcireg_t ctl; 471 int off; 472 473 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) 474 panic("apple_pcie_msi_msi_enable: device is not MSI-capable"); 475 476 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 477 ctl &= ~PCI_MSI_CTL_MSI_ENABLE; 478 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 479 480 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 481 ctl &= ~PCI_MSI_CTL_MME_MASK; 482 ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK); 483 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 484 485 const uint64_t addr = sc->sc_msi_addr; 486 const uint32_t data = msi; 487 488 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 489 if (ctl & PCI_MSI_CTL_64BIT_ADDR) { 490 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, 491 __SHIFTOUT(addr, __BITS(31, 0))); 492 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, 493 __SHIFTOUT(addr, __BITS(63, 32))); 494 pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, data); 495 } else { 496 pci_conf_write(pc, tag, off + PCI_MSI_MADDR, 497 __SHIFTOUT(addr, __BITS(31, 0))); 498 pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data); 499 } 500 ctl |= PCI_MSI_CTL_MSI_ENABLE; 501 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 502 } 503 504 static void 505 apple_pcie_msi_msi_disable(struct apple_pcie_softc *sc, int msi) 506 { 507 const struct pci_attach_args *pa = sc->sc_msi_pa[msi]; 508 pci_chipset_tag_t pc = pa->pa_pc; 509 pcitag_t tag = pa->pa_tag; 510 pcireg_t ctl; 511 int off; 512 513 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) 514 panic("apple_pcie_msi_msi_disable: device is not MSI-capable"); 515 516 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 517 ctl &= ~PCI_MSI_CTL_MSI_ENABLE; 518 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 519 } 520 521 static void 522 apple_pcie_msi_msix_enable(struct apple_pcie_softc *sc, int msi, int msix_vec, 523 bus_space_tag_t bst, bus_space_handle_t bsh) 524 { 525 const struct pci_attach_args *pa = sc->sc_msi_pa[msi]; 526 pci_chipset_tag_t pc = pa->pa_pc; 527 pcitag_t tag = pa->pa_tag; 528 pcireg_t ctl; 529 uint32_t val; 530 int off; 531 532 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) 533 panic("apple_pcie_msi_msix_enable: device is not MSI-X-capable"); 534 535 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 536 ctl &= ~PCI_MSIX_CTL_ENABLE; 537 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 538 539 const uint64_t addr = sc->sc_msi_addr; 540 const uint32_t data = msi; 541 const uint64_t entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec; 542 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, 543 __SHIFTOUT(addr, __BITS(31, 0))); 544 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, 545 __SHIFTOUT(addr, __BITS(63, 32))); 546 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_DATA, 547 data); 548 val = bus_space_read_4(bst, bsh, 549 entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL); 550 val &= ~PCI_MSIX_VECTCTL_MASK; 551 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, 552 val); 553 554 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 555 ctl |= PCI_MSIX_CTL_ENABLE; 556 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 557 } 558 559 static void 560 apple_pcie_msi_msix_disable(struct apple_pcie_softc *sc, int msi) 561 { 562 const struct pci_attach_args *pa = sc->sc_msi_pa[msi]; 563 pci_chipset_tag_t pc = pa->pa_pc; 564 pcitag_t tag = pa->pa_tag; 565 pcireg_t ctl; 566 int off; 567 568 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) 569 panic("apple_pcie_msi_msix_disable: device is not MSI-X-capable"); 570 571 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 572 ctl &= ~PCI_MSIX_CTL_ENABLE; 573 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 574 } 575 576 static pci_intr_handle_t * 577 apple_pcie_msi_msi_alloc(struct arm_pci_msi *msi, int *count, 578 const struct pci_attach_args *pa, bool exact) 579 { 580 struct apple_pcie_softc * const sc = msi->msi_priv; 581 pci_intr_handle_t *vectors; 582 int n, off; 583 584 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSI, &off, NULL)) 585 return NULL; 586 587 const int avail = apple_pcie_msi_available_msi(sc); 588 if (avail == 0) 589 return NULL; 590 591 if (exact && *count > avail) 592 return NULL; 593 594 while (*count > avail) { 595 (*count) >>= 1; 596 } 597 if (*count == 0) 598 return NULL; 599 600 const int msi_base = apple_pcie_msi_alloc_msi(sc, *count, pa); 601 if (msi_base == -1) 602 return NULL; 603 604 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); 605 for (n = 0; n < *count; n++) { 606 const int msino = msi_base + n; 607 vectors[n] = ARM_PCI_INTR_MSI | 608 __SHIFTIN(msino, ARM_PCI_INTR_IRQ) | 609 __SHIFTIN(n, ARM_PCI_INTR_MSI_VEC) | 610 __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME); 611 } 612 613 apple_pcie_msi_msi_enable(sc, msi_base, *count); 614 615 return vectors; 616 } 617 618 static pci_intr_handle_t * 619 apple_pcie_msi_msix_alloc(struct arm_pci_msi *msi, u_int *table_indexes, 620 int *count, const struct pci_attach_args *pa, bool exact) 621 { 622 struct apple_pcie_softc * const sc = msi->msi_priv; 623 pci_intr_handle_t *vectors; 624 bus_space_tag_t bst; 625 bus_space_handle_t bsh; 626 bus_size_t bsz; 627 uint32_t table_offset, table_size; 628 int n, off, bar, error; 629 pcireg_t tbl; 630 631 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &off, NULL)) 632 return NULL; 633 634 const int avail = apple_pcie_msi_available_msi(sc); 635 if (exact && *count > avail) 636 return NULL; 637 638 while (*count > avail) { 639 (*count) >>= 1; 640 } 641 if (*count == 0) 642 return NULL; 643 644 tbl = pci_conf_read(pa->pa_pc, pa->pa_tag, off + PCI_MSIX_TBLOFFSET); 645 bar = PCI_BAR0 + (4 * (tbl & PCI_MSIX_TBLBIR_MASK)); 646 table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK; 647 table_size = pci_msix_count(pa->pa_pc, pa->pa_tag) * PCI_MSIX_TABLE_ENTRY_SIZE; 648 if (table_size == 0) 649 return NULL; 650 651 error = pci_mapreg_submap(pa, bar, pci_mapreg_type(pa->pa_pc, pa->pa_tag, bar), 652 BUS_SPACE_MAP_LINEAR, roundup(table_size, PAGE_SIZE), table_offset, 653 &bst, &bsh, NULL, &bsz); 654 if (error) 655 return NULL; 656 657 const int msi_base = apple_pcie_msi_alloc_msi(sc, *count, pa); 658 if (msi_base == -1) { 659 bus_space_unmap(bst, bsh, bsz); 660 return NULL; 661 } 662 663 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); 664 for (n = 0; n < *count; n++) { 665 const int msino = msi_base + n; 666 const int msix_vec = table_indexes ? table_indexes[n] : n; 667 vectors[msix_vec] = ARM_PCI_INTR_MSIX | 668 __SHIFTIN(msino, ARM_PCI_INTR_IRQ) | 669 __SHIFTIN(msix_vec, ARM_PCI_INTR_MSI_VEC) | 670 __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME); 671 672 apple_pcie_msi_msix_enable(sc, msino, msix_vec, bst, bsh); 673 } 674 675 bus_space_unmap(bst, bsh, bsz); 676 677 return vectors; 678 } 679 680 static void * 681 apple_pcie_msi_intr_establish(struct arm_pci_msi *msi, 682 pci_intr_handle_t ih, int ipl, int (*func)(void *), void *arg, const char *xname) 683 { 684 struct apple_pcie_softc * const sc = msi->msi_priv; 685 686 const int msino = __SHIFTOUT(ih, ARM_PCI_INTR_IRQ); 687 const int mpsafe = (ih & ARM_PCI_INTR_MPSAFE) ? FDT_INTR_MPSAFE : 0; 688 689 KASSERT(sc->sc_msi_ih[msino] == NULL); 690 sc->sc_msi_ih[msino] = intr_establish_xname(sc->sc_msi_start + msino, 691 ipl, IST_LEVEL | (mpsafe ? IST_MPSAFE : 0), func, arg, xname); 692 693 return sc->sc_msi_ih[msino]; 694 } 695 696 static void 697 apple_pcie_msi_intr_release(struct arm_pci_msi *msi, pci_intr_handle_t *pih, 698 int count) 699 { 700 struct apple_pcie_softc * const sc = msi->msi_priv; 701 int n; 702 703 for (n = 0; n < count; n++) { 704 const int msino = __SHIFTOUT(pih[n], ARM_PCI_INTR_IRQ); 705 if (pih[n] & ARM_PCI_INTR_MSIX) 706 apple_pcie_msi_msix_disable(sc, msino); 707 if (pih[n] & ARM_PCI_INTR_MSI) 708 apple_pcie_msi_msi_disable(sc, msino); 709 apple_pcie_msi_free_msi(sc, msino); 710 if (sc->sc_msi_ih[msino] != NULL) { 711 intr_disestablish(sc->sc_msi_ih[msino]); 712 sc->sc_msi_ih[msino] = NULL; 713 } 714 } 715 } 716 717 static int 718 apple_pcie_msi_init(struct apple_pcie_softc *sc) 719 { 720 struct arm_pci_msi *msi = &sc->sc_msi; 721 const int phandle = sc->sc_pcihost.sc_phandle; 722 int len; 723 724 const u_int *data = fdtbus_get_prop(phandle, "msi-ranges", &len); 725 switch (len) { 726 case 8: 727 /* two cells: start and count */ 728 sc->sc_msi_start = be32toh(data[0]); 729 sc->sc_nmsi = be32toh(data[1]); 730 break; 731 case 20: 732 /* 5 cells: xref, specifier (3 cells), and count */ 733 sc->sc_msi_start = be32toh(data[2]); 734 sc->sc_nmsi = be32toh(data[4]); 735 break; 736 default: 737 aprint_error_dev(sc->sc_pcihost.sc_dev, 738 "WARNING: bad msi-ranges property, MSI not enabled!\n"); 739 return ENXIO; 740 } 741 sc->sc_msi_pa = kmem_zalloc(sizeof(*sc->sc_msi_pa) * sc->sc_nmsi, 742 KM_SLEEP); 743 sc->sc_msi_ih = kmem_zalloc(sizeof(*sc->sc_msi_ih) * sc->sc_nmsi, 744 KM_SLEEP); 745 746 if (of_getprop_uint64(phandle, "msi-doorbell", &sc->sc_msi_addr)) { 747 sc->sc_msi_addr = 0xffff000ULL; 748 } 749 750 msi->msi_dev = sc->sc_pcihost.sc_dev; 751 msi->msi_priv = sc; 752 msi->msi_alloc = apple_pcie_msi_msi_alloc; 753 msi->msix_alloc = apple_pcie_msi_msix_alloc; 754 msi->msi_intr_establish = apple_pcie_msi_intr_establish; 755 msi->msi_intr_release = apple_pcie_msi_intr_release; 756 757 return arm_pci_msi_add(msi); 758 } 759
Indexes created Sat Sep 20 22:09:52 GMT 2025