Home | History | Annotate | Line # | Download | only in nvidia
      1 /* $NetBSD: tegra_pcie.c,v 1.40 2022/10/15 11:07:39 jmcneill Exp $ */
      2 
      3 /*-
      4  * Copyright (c) 2015 Jared D. McNeill <jmcneill (at) invisible.ca>
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
     21  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     22  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
     23  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     24  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     26  * SUCH DAMAGE.
     27  */
     28 
     29 #include <sys/cdefs.h>
     30 __KERNEL_RCSID(0, "$NetBSD: tegra_pcie.c,v 1.40 2022/10/15 11:07:39 jmcneill Exp $");
     31 
     32 #include <sys/param.h>
     33 
     34 #include <sys/bus.h>
     35 #include <sys/device.h>
     36 #include <sys/intr.h>
     37 #include <sys/kmem.h>
     38 #include <sys/kernel.h>
     39 #include <sys/lwp.h>
     40 #include <sys/mutex.h>
     41 #include <sys/queue.h>
     42 #include <sys/systm.h>
     43 
     44 #include <machine/cpu.h>
     45 
     46 #include <arm/cpufunc.h>
     47 
     48 #include <dev/pci/pcireg.h>
     49 #include <dev/pci/pcivar.h>
     50 #include <dev/pci/pciconf.h>
     51 
     52 #include <arm/nvidia/tegra_reg.h>
     53 #include <arm/nvidia/tegra_pciereg.h>
     54 #include <arm/nvidia/tegra_pmcreg.h>
     55 #include <arm/nvidia/tegra_var.h>
     56 
     57 #include <dev/fdt/fdtvar.h>
     58 
     59 /* Interrupt handle flags */
     60 #define	IH_MPSAFE	0x80000000
     61 
     62 static int	tegra_pcie_match(device_t, cfdata_t, void *);
     63 static void	tegra_pcie_attach(device_t, device_t, void *);
     64 
     65 #define TEGRA_PCIE_NBUS 256
     66 #define TEGRA_PCIE_ECFB (1<<(12 - 8))	/* extended conf frags per bus */
     67 
     68 enum tegra_pcie_type {
     69 	TEGRA_PCIE_124		= 0,
     70 	TEGRA_PCIE_210		= 1,
     71 };
     72 
     73 struct tegra_pcie_ih {
     74 	int			(*ih_callback)(void *);
     75 	void			*ih_arg;
     76 	int			ih_ipl;
     77 	int			ih_mpsafe;
     78 	TAILQ_ENTRY(tegra_pcie_ih) ih_entry;
     79 };
     80 
     81 struct tegra_pcie_softc {
     82 	device_t		sc_dev;
     83 	bus_dma_tag_t		sc_dmat;
     84 	bus_space_tag_t		sc_bst;
     85 	bus_space_handle_t	sc_bsh_afi;
     86 	bus_space_handle_t	sc_bsh_pads;
     87 	bus_space_handle_t	sc_bsh_rpconf;
     88 	int			sc_phandle;
     89 	enum tegra_pcie_type	sc_type;
     90 
     91 	struct arm32_pci_chipset sc_pc;
     92 
     93 	void			*sc_ih;
     94 
     95 	kmutex_t		sc_lock;
     96 
     97 	TAILQ_HEAD(, tegra_pcie_ih) sc_intrs;
     98 	u_int			sc_intrgen;
     99 
    100 	bus_space_handle_t	sc_bsh_extc[TEGRA_PCIE_NBUS-1][TEGRA_PCIE_ECFB];
    101 };
    102 
    103 static int	tegra_pcie_intr(void *);
    104 static void	tegra_pcie_init(pci_chipset_tag_t, void *);
    105 static void	tegra_pcie_enable(struct tegra_pcie_softc *);
    106 static void	tegra_pcie_enable_ports(struct tegra_pcie_softc *);
    107 static void	tegra_pcie_enable_clocks(struct tegra_pcie_softc *);
    108 static void	tegra_pcie_setup(struct tegra_pcie_softc * const);
    109 static void	tegra_pcie_conf_frag_map(struct tegra_pcie_softc * const,
    110 					 uint, uint);
    111 static void	tegra_pcie_conf_map_bus(struct tegra_pcie_softc * const, uint);
    112 static void	tegra_pcie_conf_map_buses(struct tegra_pcie_softc * const);
    113 
    114 static void	tegra_pcie_attach_hook(device_t, device_t,
    115 				       struct pcibus_attach_args *);
    116 static int	tegra_pcie_bus_maxdevs(void *, int);
    117 static pcitag_t	tegra_pcie_make_tag(void *, int, int, int);
    118 static void	tegra_pcie_decompose_tag(void *, pcitag_t, int *, int *, int *);
    119 static pcireg_t	tegra_pcie_conf_read(void *, pcitag_t, int);
    120 static void	tegra_pcie_conf_write(void *, pcitag_t, int, pcireg_t);
    121 static int	tegra_pcie_conf_hook(void *, int, int, int, pcireg_t);
    122 static void	tegra_pcie_conf_interrupt(void *, int, int, int, int, int *);
    123 
    124 static int	tegra_pcie_intr_map(const struct pci_attach_args *,
    125 				    pci_intr_handle_t *);
    126 static const char *tegra_pcie_intr_string(void *, pci_intr_handle_t,
    127 					  char *, size_t);
    128 const struct evcnt *tegra_pcie_intr_evcnt(void *, pci_intr_handle_t);
    129 static int	tegra_pcie_intr_setattr(void *, pci_intr_handle_t *, int,
    130 					uint64_t);
    131 static void *	tegra_pcie_intr_establish(void *, pci_intr_handle_t,
    132 					 int, int (*)(void *), void *,
    133 					 const char *);
    134 static void	tegra_pcie_intr_disestablish(void *, void *);
    135 
    136 CFATTACH_DECL_NEW(tegra_pcie, sizeof(struct tegra_pcie_softc),
    137 	tegra_pcie_match, tegra_pcie_attach, NULL, NULL);
    138 
    139 static const struct device_compatible_entry compat_data[] = {
    140 	{ .compat = "nvidia,tegra210-pcie",	.value = TEGRA_PCIE_210 },
    141 	{ .compat = "nvidia,tegra124-pcie",	.value = TEGRA_PCIE_124 },
    142 	DEVICE_COMPAT_EOL
    143 };
    144 
    145 static int
    146 tegra_pcie_match(device_t parent, cfdata_t cf, void *aux)
    147 {
    148 	struct fdt_attach_args * const faa = aux;
    149 
    150 	return of_compatible_match(faa->faa_phandle, compat_data);
    151 }
    152 
    153 static void
    154 tegra_pcie_attach(device_t parent, device_t self, void *aux)
    155 {
    156 	struct tegra_pcie_softc * const sc = device_private(self);
    157 	struct fdt_attach_args * const faa = aux;
    158 	const struct device_compatible_entry *dce;
    159 	struct pciconf_resources *pcires;
    160 	struct pcibus_attach_args pba;
    161 	bus_addr_t afi_addr, cs_addr, pads_addr;
    162 	bus_size_t afi_size, cs_size, pads_size;
    163 	char intrstr[128];
    164 	int error;
    165 
    166 	if (fdtbus_get_reg_byname(faa->faa_phandle, "afi", &afi_addr, &afi_size) != 0) {
    167 		aprint_error(": couldn't get afi registers\n");
    168 		return;
    169 	}
    170 	if (fdtbus_get_reg_byname(faa->faa_phandle, "pads", &pads_addr, &pads_size) != 0) {
    171 		aprint_error(": couldn't get pads registers\n");
    172 		return;
    173 	}
    174 #if notyet
    175 	if (fdtbus_get_reg(faa->faa_phandle, 2, &cs_addr, &cs_size) != 0) {
    176 		aprint_error(": couldn't get cs registers\n");
    177 		return;
    178 	}
    179 #else
    180 	cs_addr = TEGRA_PCIE_RPCONF_BASE;
    181 	cs_size = TEGRA_PCIE_RPCONF_SIZE;
    182 #endif
    183 
    184 	sc->sc_dev = self;
    185 	sc->sc_dmat = faa->faa_dmat;
    186 	sc->sc_bst = faa->faa_bst;
    187 	sc->sc_phandle = faa->faa_phandle;
    188 	error = bus_space_map(sc->sc_bst, afi_addr, afi_size, 0,
    189 	    &sc->sc_bsh_afi);
    190 	if (error) {
    191 		aprint_error(": couldn't map afi registers: %d\n", error);
    192 		return;
    193 	}
    194 	error = bus_space_map(sc->sc_bst, pads_addr, pads_size, 0,
    195 	    &sc->sc_bsh_pads);
    196 	if (error) {
    197 		aprint_error(": couldn't map pads registers: %d\n", error);
    198 		return;
    199 	}
    200 	error = bus_space_map(sc->sc_bst, cs_addr, cs_size,
    201 	    BUS_SPACE_MAP_NONPOSTED, &sc->sc_bsh_rpconf);
    202 	if (error) {
    203 		aprint_error(": couldn't map cs registers: %d\n", error);
    204 		return;
    205 	}
    206 
    207 	dce = of_compatible_lookup(faa->faa_phandle, compat_data);
    208 	KASSERT(dce != NULL);
    209 	sc->sc_type = dce->value;
    210 
    211 	tegra_pcie_conf_map_buses(sc);
    212 
    213 	TAILQ_INIT(&sc->sc_intrs);
    214 	mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_VM);
    215 
    216 	aprint_naive("\n");
    217 	aprint_normal(": PCIE\n");
    218 
    219 	tegra_pmc_power(PMC_PARTID_PCX, true);
    220 	tegra_pmc_remove_clamping(PMC_PARTID_PCX);
    221 
    222 	tegra_pcie_enable_clocks(sc);
    223 
    224 	if (!fdtbus_intr_str(faa->faa_phandle, 0, intrstr, sizeof(intrstr))) {
    225 		aprint_error_dev(self, "failed to decode interrupt\n");
    226 		return;
    227 	}
    228 
    229 	sc->sc_ih = fdtbus_intr_establish_xname(faa->faa_phandle, 0, IPL_VM,
    230 	    FDT_INTR_MPSAFE, tegra_pcie_intr, sc, device_xname(self));
    231 	if (sc->sc_ih == NULL) {
    232 		aprint_error_dev(self, "failed to establish interrupt on %s\n",
    233 		    intrstr);
    234 		return;
    235 	}
    236 	aprint_normal_dev(self, "interrupting on %s\n", intrstr);
    237 
    238 	tegra_pcie_setup(sc);
    239 
    240 	tegra_pcie_init(&sc->sc_pc, sc);
    241 
    242 	pcires = pciconf_resource_init();
    243 
    244 	pciconf_resource_add(pcires, PCICONF_RESOURCE_IO,
    245 	    TEGRA_PCIE_IO_BASE, TEGRA_PCIE_IO_SIZE);
    246 	pciconf_resource_add(pcires, PCICONF_RESOURCE_MEM,
    247 	    TEGRA_PCIE_MEM_BASE, TEGRA_PCIE_MEM_SIZE);
    248 	pciconf_resource_add(pcires, PCICONF_RESOURCE_PREFETCHABLE_MEM,
    249 	    TEGRA_PCIE_PMEM_BASE, TEGRA_PCIE_PMEM_SIZE);
    250 
    251 	error = pci_configure_bus(&sc->sc_pc, pcires, 0,
    252 	    arm_dcache_align);
    253 
    254 	pciconf_resource_fini(pcires);
    255 
    256 	if (error) {
    257 		aprint_error_dev(self, "configuration failed (%d)\n",
    258 		    error);
    259 		return;
    260 	}
    261 
    262 	tegra_pcie_enable(sc);
    263 
    264 	tegra_pcie_enable_ports(sc);
    265 
    266 	memset(&pba, 0, sizeof(pba));
    267 	pba.pba_flags = PCI_FLAGS_MRL_OKAY |
    268 			PCI_FLAGS_MRM_OKAY |
    269 			PCI_FLAGS_MWI_OKAY |
    270 			PCI_FLAGS_MEM_OKAY |
    271 			PCI_FLAGS_IO_OKAY;
    272 	pba.pba_iot = sc->sc_bst;
    273 	pba.pba_memt = sc->sc_bst;
    274 	pba.pba_dmat = sc->sc_dmat;
    275 	pba.pba_pc = &sc->sc_pc;
    276 	pba.pba_bus = 0;
    277 
    278 	config_found(self, &pba, pcibusprint,
    279 	    CFARGS(.devhandle = device_handle(self)));
    280 }
    281 
    282 static int
    283 tegra_pcie_legacy_intr(struct tegra_pcie_softc *sc)
    284 {
    285 	const uint32_t msg = bus_space_read_4(sc->sc_bst, sc->sc_bsh_afi,
    286 	    AFI_MSG_REG);
    287 	struct tegra_pcie_ih *pcie_ih;
    288 	int rv = 0;
    289 
    290 	if (msg & (AFI_MSG_INT0|AFI_MSG_INT1)) {
    291 		mutex_enter(&sc->sc_lock);
    292 		const u_int lastgen = sc->sc_intrgen;
    293 		TAILQ_FOREACH(pcie_ih, &sc->sc_intrs, ih_entry) {
    294 			int (*callback)(void *) = pcie_ih->ih_callback;
    295 			void *arg = pcie_ih->ih_arg;
    296 			const int mpsafe = pcie_ih->ih_mpsafe;
    297 			mutex_exit(&sc->sc_lock);
    298 
    299 			if (!mpsafe)
    300 				KERNEL_LOCK(1, curlwp);
    301 			rv += callback(arg);
    302 			if (!mpsafe)
    303 				KERNEL_UNLOCK_ONE(curlwp);
    304 
    305 			mutex_enter(&sc->sc_lock);
    306 			if (lastgen != sc->sc_intrgen)
    307 				break;
    308 		}
    309 		mutex_exit(&sc->sc_lock);
    310 	} else if (msg & (AFI_MSG_PM_PME0|AFI_MSG_PM_PME1)) {
    311 		device_printf(sc->sc_dev, "PM PME message; AFI_MSG=%08x\n",
    312 		    msg);
    313 	} else {
    314 		bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi, AFI_MSG_REG, msg);
    315 		rv = 1;
    316 	}
    317 
    318 	return rv;
    319 }
    320 
    321 static int
    322 tegra_pcie_intr(void *priv)
    323 {
    324 	struct tegra_pcie_softc *sc = priv;
    325 	int rv;
    326 
    327 	const uint32_t code = bus_space_read_4(sc->sc_bst, sc->sc_bsh_afi,
    328 	    AFI_INTR_CODE_REG);
    329 	const uint32_t sig = bus_space_read_4(sc->sc_bst, sc->sc_bsh_afi,
    330 	    AFI_INTR_SIGNATURE_REG);
    331 
    332 	switch (__SHIFTOUT(code, AFI_INTR_CODE_INT_CODE)) {
    333 	case AFI_INTR_CODE_SM_MSG:
    334 		rv = tegra_pcie_legacy_intr(sc);
    335 		break;
    336 	default:
    337 		device_printf(sc->sc_dev, "intr: code %#x sig %#x\n",
    338 		    code, sig);
    339 		rv = 1;
    340 		break;
    341 	}
    342 
    343 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi, AFI_INTR_CODE_REG, 0);
    344 
    345 	return rv;
    346 }
    347 
    348 static void
    349 tegra_pcie_enable_clocks(struct tegra_pcie_softc * const sc)
    350 {
    351 	const char *clock_names[] = { "pex", "afi", "pll_e", "cml" };
    352 	const char *reset_names[] = { "pex", "afi", "pcie_x" };
    353 	struct fdtbus_reset *rst;
    354 	struct clk *clk;
    355 	int n;
    356 
    357 	for (n = 0; n < __arraycount(clock_names); n++) {
    358 		clk = fdtbus_clock_get(sc->sc_phandle, clock_names[n]);
    359 		if (clk == NULL || clk_enable(clk) != 0)
    360 			aprint_error_dev(sc->sc_dev, "couldn't enable clock %s\n",
    361 			    clock_names[n]);
    362 	}
    363 
    364 	for (n = 0; n < __arraycount(reset_names); n++) {
    365 		rst = fdtbus_reset_get(sc->sc_phandle, reset_names[n]);
    366 		if (rst == NULL || fdtbus_reset_deassert(rst) != 0)
    367 			aprint_error_dev(sc->sc_dev, "couldn't de-assert reset %s\n",
    368 			    reset_names[n]);
    369 	}
    370 }
    371 
    372 #if 0
    373 static void
    374 tegra_pcie_reset_port(struct tegra_pcie_softc * const sc, int index)
    375 {
    376 	uint32_t val;
    377 
    378 	val = bus_space_read_4(sc->sc_bst, sc->sc_bsh_afi, AFI_PEXn_CTRL_REG(index));
    379 	val &= ~AFI_PEXn_CTRL_RST_L;
    380 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi, AFI_PEXn_CTRL_REG(index), val);
    381 
    382 	delay(2000);
    383 
    384 	val = bus_space_read_4(sc->sc_bst, sc->sc_bsh_afi, AFI_PEXn_CTRL_REG(index));
    385 	val |= AFI_PEXn_CTRL_RST_L;
    386 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi, AFI_PEXn_CTRL_REG(index), val);
    387 }
    388 #endif
    389 
    390 static void
    391 tegra_pcie_enable_ports(struct tegra_pcie_softc * const sc)
    392 {
    393 	struct fdtbus_phy *phy;
    394 	const u_int *data;
    395 	int child, len, n;
    396 	uint32_t val;
    397 
    398 	for (child = OF_child(sc->sc_phandle); child; child = OF_peer(child)) {
    399 		if (!fdtbus_status_okay(child))
    400 			continue;
    401 
    402 		/* Enable PHYs */
    403 		for (n = 0; (phy = fdtbus_phy_get_index(child, n)) != NULL; n++)
    404 			if (fdtbus_phy_enable(phy, true) != 0)
    405 				aprint_error_dev(sc->sc_dev, "couldn't enable %s phy #%d\n",
    406 				    fdtbus_get_string(child, "name"), n);
    407 
    408 		data = fdtbus_get_prop(child, "reg", &len);
    409 		if (data == NULL || len < 4)
    410 			continue;
    411 		const u_int index = ((be32toh(data[0]) >> 11) & 0x1f) - 1;
    412 
    413 		val = bus_space_read_4(sc->sc_bst, sc->sc_bsh_afi, AFI_PEXn_CTRL_REG(index));
    414 		val |= AFI_PEXn_CTRL_CLKREQ_EN;
    415 		val |= AFI_PEXn_CTRL_REFCLK_EN;
    416 		val |= AFI_PEXn_CTRL_REFCLK_OVERRIDE_EN;
    417 		bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi, AFI_PEXn_CTRL_REG(index), val);
    418 
    419 #if 0
    420 		tegra_pcie_reset_port(sc, index);
    421 #endif
    422 
    423 	}
    424 }
    425 
    426 static void
    427 tegra_pcie_setup(struct tegra_pcie_softc * const sc)
    428 {
    429 	uint32_t val, cfg, lanes;
    430 	int child, len;
    431 	const u_int *data;
    432 	size_t i;
    433 
    434 	/* Enable PLLE control */
    435 	val = bus_space_read_4(sc->sc_bst, sc->sc_bsh_afi, AFI_PLLE_CONTROL_REG);
    436 	val &= ~AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL;
    437 	val |= AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN;
    438 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi, AFI_PLLE_CONTROL_REG, val);
    439 
    440 	/* Disable PEX clock bias pad power down */
    441 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi, AFI_PEXBIAS_CTRL_REG, 0);
    442 
    443 	/* Configure PCIE mode and enable ports */
    444 	cfg = bus_space_read_4(sc->sc_bst, sc->sc_bsh_afi, AFI_PCIE_CONFIG_REG);
    445 	cfg |= AFI_PCIE_CONFIG_PCIECn_DISABLE_DEVICE(0);
    446 	cfg |= AFI_PCIE_CONFIG_PCIECn_DISABLE_DEVICE(1);
    447 	cfg &= ~AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG;
    448 
    449 	lanes = 0;
    450 	for (child = OF_child(sc->sc_phandle); child; child = OF_peer(child)) {
    451 		if (!fdtbus_status_okay(child))
    452 			continue;
    453 		data = fdtbus_get_prop(child, "reg", &len);
    454 		if (data == NULL || len < 4)
    455 			continue;
    456 		const u_int index = ((be32toh(data[0]) >> 11) & 0x1f) - 1;
    457 		if (of_getprop_uint32(child, "nvidia,num-lanes", &val) != 0)
    458 			continue;
    459 		lanes |= (val << (index << 3));
    460 		cfg &= ~AFI_PCIE_CONFIG_PCIECn_DISABLE_DEVICE(index);
    461 	}
    462 
    463 	switch (lanes) {
    464 	case 0x0104:
    465 		aprint_normal_dev(sc->sc_dev, "lane config: x4 x1\n");
    466 		cfg |= __SHIFTIN(AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_4_1,
    467 				 AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG);
    468 		break;
    469 	case 0x0102:
    470 		aprint_normal_dev(sc->sc_dev, "lane config: x2 x1\n");
    471 		cfg |= __SHIFTIN(AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_2_1,
    472 				 AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG);
    473 		break;
    474 	}
    475 
    476 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi, AFI_PCIE_CONFIG_REG, cfg);
    477 
    478 	/* Configure refclk pad */
    479 	if (sc->sc_type == TEGRA_PCIE_124) {
    480 		bus_space_write_4(sc->sc_bst, sc->sc_bsh_pads,
    481 		    PADS_REFCLK_CFG0_REG, 0x44ac44ac);
    482 	}
    483 	if (sc->sc_type == TEGRA_PCIE_210) {
    484 		bus_space_write_4(sc->sc_bst, sc->sc_bsh_pads,
    485 		    PADS_REFCLK_CFG0_REG, 0x90b890b8);
    486 	}
    487 
    488 	/*
    489 	 * Map PCI address spaces into ARM address space via
    490 	 * HyperTransport-like "FPCI".
    491 	 */
    492 	static const struct { uint32_t size, base, fpci; } pcie_init_table[] = {
    493 		/*
    494 		 * === BEWARE ===
    495 		 *
    496 		 * We depend on our TEGRA_PCIE_IO window overlaping the
    497 		 * TEGRA_PCIE_A1 window to allow us to use the same
    498 		 * bus_space_tag for both PCI IO and Memory spaces.
    499 		 *
    500 		 * 0xfdfc000000-0xfdfdffffff is the FPCI/HyperTransport
    501 		 * mapping for 0x0000000-0x1ffffff of PCI IO space.
    502 		 */
    503 		{ TEGRA_PCIE_IO_SIZE >> 12, TEGRA_PCIE_IO_BASE,
    504 		  (0xfdfc000000 + TEGRA_PCIE_IO_BASE) >> 8 | 0, },
    505 
    506 		/* HyperTransport Technology Type 1 Address Format */
    507 		{ TEGRA_PCIE_CONF_SIZE >> 12, TEGRA_PCIE_CONF_BASE,
    508 		  0xfdff000000 >> 8 | 0, },
    509 
    510 		/* 1:1 MMIO mapping */
    511 		{ TEGRA_PCIE_MEM_SIZE >> 12, TEGRA_PCIE_MEM_BASE,
    512 		  TEGRA_PCIE_MEM_BASE >> 8 | 1, },
    513 
    514 		/* Extended HyperTransport Technology Type 1 Address Format */
    515 		{ TEGRA_PCIE_EXTC_SIZE >> 12, TEGRA_PCIE_EXTC_BASE,
    516 		  0xfe10000000 >> 8 | 0, },
    517 
    518 		/* 1:1 prefetchable MMIO mapping */
    519 		{ TEGRA_PCIE_PMEM_SIZE >> 12, TEGRA_PCIE_PMEM_BASE,
    520 		  TEGRA_PCIE_PMEM_BASE >> 8 | 1, },
    521 	};
    522 
    523 	for (i = 0; i < AFI_AXI_NBAR; i++) {
    524 		bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi,
    525 		    AFI_AXI_BARi_SZ(i), 0);
    526 		bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi,
    527 		    AFI_AXI_BARi_START(i), 0);
    528 		bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi,
    529 		    AFI_FPCI_BARi(i), 0);
    530 	}
    531 
    532 	for (i = 0; i < __arraycount(pcie_init_table); i++) {
    533 		bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi,
    534 		    AFI_AXI_BARi_START(i), pcie_init_table[i].base);
    535 		bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi,
    536 		    AFI_FPCI_BARi(i), pcie_init_table[i].fpci);
    537 		bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi,
    538 		    AFI_AXI_BARi_SZ(i), pcie_init_table[i].size);
    539 	}
    540 }
    541 
    542 static void
    543 tegra_pcie_enable(struct tegra_pcie_softc *sc)
    544 {
    545 	/* disable MSI */
    546 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi,
    547 	    AFI_MSI_BAR_SZ_REG, 0);
    548 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi,
    549 	    AFI_MSI_FPCI_BAR_ST_REG, 0);
    550 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi,
    551 	    AFI_MSI_AXI_BAR_ST_REG, 0);
    552 
    553 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi,
    554 	    AFI_SM_INTR_ENABLE_REG, 0xffffffff);
    555 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi,
    556 	    AFI_AFI_INTR_ENABLE_REG, 0);
    557 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi, AFI_INTR_CODE_REG, 0);
    558 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_afi,
    559 	    AFI_INTR_MASK_REG, AFI_INTR_MASK_INT);
    560 }
    561 
    562 static void
    563 tegra_pcie_conf_frag_map(struct tegra_pcie_softc * const sc, uint bus,
    564     uint frg)
    565 {
    566 	bus_addr_t a;
    567 
    568 	KASSERT(bus >= 1);
    569 	KASSERT(bus < TEGRA_PCIE_NBUS);
    570 	KASSERT(frg < TEGRA_PCIE_ECFB);
    571 
    572 	if (sc->sc_bsh_extc[bus-1][frg] != 0) {
    573 		device_printf(sc->sc_dev, "bus %u fragment %#x already "
    574 		    "mapped\n", bus, frg);
    575 		return;
    576 	}
    577 
    578 	a = TEGRA_PCIE_EXTC_BASE + (bus << 16) + (frg << 24);
    579 	if (bus_space_map(sc->sc_bst, a, 1 << 16,
    580 	    BUS_SPACE_MAP_NONPOSTED,
    581 	    &sc->sc_bsh_extc[bus-1][frg]) != 0)
    582 		device_printf(sc->sc_dev, "couldn't map PCIE "
    583 		    "configuration for bus %u fragment %#x", bus, frg);
    584 }
    585 
    586 /* map non-non-extended configuration space for full bus range */
    587 static void
    588 tegra_pcie_conf_map_bus(struct tegra_pcie_softc * const sc, uint bus)
    589 {
    590 	uint i;
    591 
    592 	for (i = 1; i < TEGRA_PCIE_ECFB; i++) {
    593 		tegra_pcie_conf_frag_map(sc, bus, i);
    594 	}
    595 }
    596 
    597 /* map non-extended configuration space for full bus range */
    598 static void
    599 tegra_pcie_conf_map_buses(struct tegra_pcie_softc * const sc)
    600 {
    601 	uint b;
    602 
    603 	for (b = 1; b < TEGRA_PCIE_NBUS; b++) {
    604 		tegra_pcie_conf_frag_map(sc, b, 0);
    605 	}
    606 }
    607 
    608 void
    609 tegra_pcie_init(pci_chipset_tag_t pc, void *priv)
    610 {
    611 	pc->pc_conf_v = priv;
    612 	pc->pc_attach_hook = tegra_pcie_attach_hook;
    613 	pc->pc_bus_maxdevs = tegra_pcie_bus_maxdevs;
    614 	pc->pc_make_tag = tegra_pcie_make_tag;
    615 	pc->pc_decompose_tag = tegra_pcie_decompose_tag;
    616 	pc->pc_conf_read = tegra_pcie_conf_read;
    617 	pc->pc_conf_write = tegra_pcie_conf_write;
    618 	pc->pc_conf_hook = tegra_pcie_conf_hook;
    619 	pc->pc_conf_interrupt = tegra_pcie_conf_interrupt;
    620 
    621 	pc->pc_intr_v = priv;
    622 	pc->pc_intr_map = tegra_pcie_intr_map;
    623 	pc->pc_intr_string = tegra_pcie_intr_string;
    624 	pc->pc_intr_evcnt = tegra_pcie_intr_evcnt;
    625 	pc->pc_intr_setattr = tegra_pcie_intr_setattr;
    626 	pc->pc_intr_establish = tegra_pcie_intr_establish;
    627 	pc->pc_intr_disestablish = tegra_pcie_intr_disestablish;
    628 }
    629 
    630 static void
    631 tegra_pcie_attach_hook(device_t parent, device_t self,
    632     struct pcibus_attach_args *pba)
    633 {
    634 	const pci_chipset_tag_t pc = pba->pba_pc;
    635 	struct tegra_pcie_softc * const sc = pc->pc_conf_v;
    636 
    637 	if (pba->pba_bus >= 1) {
    638 		tegra_pcie_conf_map_bus(sc, pba->pba_bus);
    639 	}
    640 }
    641 
    642 static int
    643 tegra_pcie_bus_maxdevs(void *v, int busno)
    644 {
    645 	return busno == 0 ? 2 : 32;
    646 }
    647 
    648 static pcitag_t
    649 tegra_pcie_make_tag(void *v, int b, int d, int f)
    650 {
    651 	return (b << 16) | (d << 11) | (f << 8);
    652 }
    653 
    654 static void
    655 tegra_pcie_decompose_tag(void *v, pcitag_t tag, int *bp, int *dp, int *fp)
    656 {
    657 	if (bp)
    658 		*bp = (tag >> 16) & 0xff;
    659 	if (dp)
    660 		*dp = (tag >> 11) & 0x1f;
    661 	if (fp)
    662 		*fp = (tag >> 8) & 0x7;
    663 }
    664 
    665 static pcireg_t
    666 tegra_pcie_conf_read(void *v, pcitag_t tag, int offset)
    667 {
    668 	struct tegra_pcie_softc *sc = v;
    669 	bus_space_handle_t bsh;
    670 	int b, d, f;
    671 	u_int reg;
    672 
    673 	if ((unsigned int)offset >= PCI_EXTCONF_SIZE)
    674 		return (pcireg_t) -1;
    675 
    676 	tegra_pcie_decompose_tag(v, tag, &b, &d, &f);
    677 
    678 	if (b >= TEGRA_PCIE_NBUS)
    679 		return (pcireg_t) -1;
    680 
    681 	if (b == 0) {
    682 		if (d >= 2 || f != 0)
    683 			return (pcireg_t) -1;
    684 		reg = d * 0x1000 + offset;
    685 		bsh = sc->sc_bsh_rpconf;
    686 	} else {
    687 		reg = (d << 11) | (f << 8) | (offset & 0xff);
    688 		bsh = sc->sc_bsh_extc[b-1][(offset >> 8) & 0xf];
    689 		if (bsh == 0)
    690 			return (pcireg_t) -1;
    691 	}
    692 
    693 	return bus_space_read_4(sc->sc_bst, bsh, reg);
    694 }
    695 
    696 static void
    697 tegra_pcie_conf_write(void *v, pcitag_t tag, int offset, pcireg_t val)
    698 {
    699 	struct tegra_pcie_softc *sc = v;
    700 	bus_space_handle_t bsh;
    701 	int b, d, f;
    702 	u_int reg;
    703 
    704 	if ((unsigned int)offset >= PCI_EXTCONF_SIZE)
    705 		return;
    706 
    707 	tegra_pcie_decompose_tag(v, tag, &b, &d, &f);
    708 
    709 	if (b >= TEGRA_PCIE_NBUS)
    710 		return;
    711 
    712 	if (b == 0) {
    713 		if (d >= 2 || f != 0)
    714 			return;
    715 		reg = d * 0x1000 + offset;
    716 		bsh = sc->sc_bsh_rpconf;
    717 	} else {
    718 		reg = (d << 11) | (f << 8) | (offset & 0xff);
    719 		bsh = sc->sc_bsh_extc[b-1][(offset >> 8) & 0xf];
    720 		if (bsh == 0)
    721 			return;
    722 	}
    723 
    724 	bus_space_write_4(sc->sc_bst, bsh, reg, val);
    725 }
    726 
    727 static int
    728 tegra_pcie_conf_hook(void *v, int b, int d, int f, pcireg_t id)
    729 {
    730 	return PCI_CONF_DEFAULT & ~PCI_CONF_ENABLE_BM;
    731 }
    732 
    733 static void
    734 tegra_pcie_conf_interrupt(void *v, int bus, int dev, int ipin, int swiz,
    735     int *ilinep)
    736 {
    737 	*ilinep = 5;
    738 }
    739 
    740 static int
    741 tegra_pcie_intr_map(const struct pci_attach_args *pa, pci_intr_handle_t *ih)
    742 {
    743 	if (pa->pa_intrpin == 0)
    744 		return EINVAL;
    745 	*ih = pa->pa_intrpin;
    746 	return 0;
    747 }
    748 
    749 static const char *
    750 tegra_pcie_intr_string(void *v, pci_intr_handle_t ih, char *buf, size_t len)
    751 {
    752 	struct tegra_pcie_softc *sc = v;
    753 
    754 	if (ih == PCI_INTERRUPT_PIN_NONE)
    755 		return NULL;
    756 
    757 	if (!fdtbus_intr_str(sc->sc_phandle, 0, buf, len))
    758 		return NULL;
    759 
    760 	return buf;
    761 }
    762 
    763 const struct evcnt *
    764 tegra_pcie_intr_evcnt(void *v, pci_intr_handle_t ih)
    765 {
    766 	return NULL;
    767 }
    768 
    769 static int
    770 tegra_pcie_intr_setattr(void *v, pci_intr_handle_t *ih, int attr, uint64_t data)
    771 {
    772 	switch (attr) {
    773 	case PCI_INTR_MPSAFE:
    774 		if (data)
    775 			*ih |= IH_MPSAFE;
    776 		else
    777 			*ih &= ~IH_MPSAFE;
    778 		return 0;
    779 	default:
    780 		return ENODEV;
    781 	}
    782 }
    783 
    784 static void *
    785 tegra_pcie_intr_establish(void *v, pci_intr_handle_t ih, int ipl,
    786     int (*callback)(void *), void *arg, const char *xname)
    787 {
    788 	struct tegra_pcie_softc *sc = v;
    789 	struct tegra_pcie_ih *pcie_ih;
    790 
    791 	if (ih == 0)
    792 		return NULL;
    793 
    794 	pcie_ih = kmem_alloc(sizeof(*pcie_ih), KM_SLEEP);
    795 	pcie_ih->ih_callback = callback;
    796 	pcie_ih->ih_arg = arg;
    797 	pcie_ih->ih_ipl = ipl;
    798 	pcie_ih->ih_mpsafe = (ih & IH_MPSAFE) != 0;
    799 
    800 	mutex_enter(&sc->sc_lock);
    801 	TAILQ_INSERT_TAIL(&sc->sc_intrs, pcie_ih, ih_entry);
    802 	sc->sc_intrgen++;
    803 	mutex_exit(&sc->sc_lock);
    804 
    805 	return pcie_ih;
    806 }
    807 
    808 static void
    809 tegra_pcie_intr_disestablish(void *v, void *vih)
    810 {
    811 	struct tegra_pcie_softc *sc = v;
    812 	struct tegra_pcie_ih *pcie_ih = vih;
    813 
    814 	mutex_enter(&sc->sc_lock);
    815 	TAILQ_REMOVE(&sc->sc_intrs, pcie_ih, ih_entry);
    816 	mutex_exit(&sc->sc_lock);
    817 
    818 	kmem_free(pcie_ih, sizeof(*pcie_ih));
    819 }
    820