Home | History | Annotate | Line # | Download | only in rockchip
      1 /* $NetBSD: rk3399_pcie.c,v 1.23 2024/11/21 07:15:00 skrll Exp $ */
      2 /*
      3  * Copyright (c) 2018 Mark Kettenis <kettenis (at) openbsd.org>
      4  *
      5  * Permission to use, copy, modify, and distribute this software for any
      6  * purpose with or without fee is hereby granted, provided that the above
      7  * copyright notice and this permission notice appear in all copies.
      8  *
      9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
     10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     16  */
     17 
     18 #include <sys/cdefs.h>
     19 
     20 __KERNEL_RCSID(1, "$NetBSD: rk3399_pcie.c,v 1.23 2024/11/21 07:15:00 skrll Exp $");
     21 
     22 #include <sys/param.h>
     23 #include <sys/systm.h>
     24 #include <sys/bitops.h>
     25 #include <sys/device.h>
     26 #include <sys/kmem.h>
     27 
     28 #include <machine/intr.h>
     29 #include <sys/bus.h>
     30 #include <dev/fdt/fdtvar.h>
     31 #include <dev/fdt/syscon.h>
     32 #include <arm/cpufunc.h>
     33 
     34 #include <dev/pci/pcidevs.h>
     35 #include <dev/pci/pcireg.h>
     36 #include <dev/pci/pcivar.h>
     37 #include <dev/pci/pciconf.h>
     38 
     39 #include <arm/fdt/pcihost_fdtvar.h>
     40 #include <sys/gpio.h>
     41 
     42 #define SETREG(m, v)			((m)<<16|__SHIFTIN((v), (m)))
     43 #define GETREG(m, v)			(__SHIFTOUT((v), (m)))
     44 
     45 /* APB region */
     46 #define PCIE_CLIENT_BASE		0x000000
     47 #define PCIE_CLIENT_BASIC_STRAP_CONF	0x0000
     48 #define  PCBSC_PCIE_GEN_SEL		__BIT(7)
     49 #define   PCBSC_PGS_GEN1		SETREG(PCBSC_PCIE_GEN_SEL, 0)
     50 #define   PCBSC_PGS_GEN2		SETREG(PCBSC_PCIE_GEN_SEL, 1)
     51 #define  PCBSC_MODE_SELECT		__BIT(6)
     52 #define   PCBSC_MS_ENDPOINT		SETREG(PCBSC_MODE_SELECT, 0)
     53 #define   PCBSC_MS_ROOTPORT		SETREG(PCBSC_MODE_SELECT, 1)
     54 #define  PCBSC_LANE_COUNT		__BITS(5,4)
     55 #define   PCBSC_LC(x)			SETREG(PCBSC_LANE_COUNT, ilog2(x)) /* valid for x1,2,4 */
     56 #define  PCBSC_ARI_EN			SETREG(__BIT(3), 1) /* Alternate Routing ID Enable */
     57 #define  PCBSC_SR_IOV_EN		SETREG(__BIT(2), 1)
     58 #define  PCBSC_LINK_TRAIN_EN		SETREG(__BIT(1), 1)
     59 #define  PCBSC_CONF_EN			SETREG(__BIT(0), 1) /* Config enable */
     60 #define PCIE_CLIENT_DEBUG_OUT_0		0x003c
     61 #define PCIE_CLIENT_DEBUG_OUT_1		0x0040
     62 #define PCIE_CLIENT_BASIC_STATUS0	0x0044
     63 #define PCIE_CLIENT_BASIC_STATUS1	0x0048
     64 #define  PCBS1_LINK_ST(x)		(u_int)__SHIFTOUT((x), __BITS(21,20))
     65 #define   PCBS1_LS_NO_RECV		0	/* no receivers */
     66 #define   PCBS1_LS_TRAINING		1	/* link training */
     67 #define   PCBS1_LS_DL_INIT		2	/* link up, DL init progressing */
     68 #define   PCBS1_LS_DL_DONE		3	/* link up, DL init complete */
     69 #define PCIE_CLIENT_INT_MASK		0x004c
     70 #define   PCIM_INTx_MASK(x)		SETREG(__BIT((x)+5), 1)
     71 #define   PCIM_INTx_ENAB(x)		SETREG(__BIT((x)+5), 0)
     72 
     73 #define PCIE_CORE_BASE			0x800000
     74 #define PCIE_RC_NORMAL_BASE		(PCIE_CORE_BASE + 0x00000)
     75 
     76 #define PCIE_LM_BASE			0x900000
     77 #define PCIE_LM_CORE_CTRL		(PCIE_LM_BASE + 0x00)
     78 #define   PCIE_CORE_PL_CONF_SPEED_5G            0x00000008
     79 #define   PCIE_CORE_PL_CONF_SPEED_MASK          0x00000018
     80 #define   PCIE_CORE_PL_CONF_LANE_MASK           0x00000006
     81 #define   PCIE_CORE_PL_CONF_LANE_SHIFT          1
     82 #define PCIE_LM_PLC1			(PCIE_LM_BASE + 0x04)
     83 #define  PCIE_LM_PLC1_FTS_MASK			__BITS(23, 8)
     84 #define PCIE_LM_VENDOR_ID		(PCIE_LM_BASE + 0x44)
     85 #define PCIE_LM_LINKWIDTH		(PCIE_LM_BASE + 0x50)
     86 #define PCIE_LM_LANEMAP			(PCIE_LM_BASE + 0x200)
     87 #define PCIE_LM_DEBUG_MUX_CONTROL	(PCIE_LM_BASE + 0x208)
     88 #define PCIE_LM_RCBAR			(PCIE_LM_BASE + 0x300)
     89 #define  PCIE_LM_RCBARPME		__BIT(17)
     90 #define  PCIE_LM_RCBARPMS		__BIT(18)
     91 #define  PCIE_LM_RCBARPIE		__BIT(19)
     92 #define  PCIE_LM_RCBARPIS		__BIT(20)
     93 
     94 #define PCIE_RC_BASE			0xa00000
     95 #define PCIE_RC_CONFIG_DCSR		(PCIE_RC_BASE + 0x0c0 + PCIE_DCSR)
     96 #define PCIE_RC_PCIE_LCAP		(PCIE_RC_BASE + 0x0c0 + PCIE_LCAP)
     97 #define PCIE_RC_CONFIG_LCSR		(PCIE_RC_BASE + 0x0c0 + PCIE_LCSR)
     98 #define PCIE_RC_CONFIG_THP_CAP          (PCIE_RC_BASE + 0x274)
     99 #define   PCIE_RC_CONFIG_THP_CAP_NEXT_MASK      __BITS(31, 20)
    100 
    101 
    102 #define PCIE_ATR_BASE			0xc00000
    103 #define PCIE_ATR_OB_ADDR0(i)		(PCIE_ATR_BASE + 0x000 + (i) * 0x20)
    104 #define PCIE_ATR_OB_ADDR1(i)		(PCIE_ATR_BASE + 0x004 + (i) * 0x20)
    105 #define PCIE_ATR_OB_DESC0(i)		(PCIE_ATR_BASE + 0x008 + (i) * 0x20)
    106 #define PCIE_ATR_OB_DESC1(i)		(PCIE_ATR_BASE + 0x00c + (i) * 0x20)
    107 #define PCIE_ATR_IB_ADDR0(i)		(PCIE_ATR_BASE + 0x800 + (i) * 0x8)
    108 #define PCIE_ATR_IB_ADDR1(i)		(PCIE_ATR_BASE + 0x804 + (i) * 0x8)
    109 #define  PCIE_ATR_HDR_MEM		0x2
    110 #define  PCIE_ATR_HDR_IO		0x6
    111 #define  PCIE_ATR_HDR_CFG_TYPE0		0xa
    112 #define  PCIE_ATR_HDR_CFG_TYPE1		0xb
    113 #define  PCIE_ATR_HDR_RID		__BIT(23)
    114 
    115 /* AXI region */
    116 #define PCIE_ATR_OB_REGION0_SIZE	(32 * 1024 * 1024)
    117 #define PCIE_ATR_OB_REGION_SIZE		(1 * 1024 * 1024)
    118 
    119 #define HREAD4(sc, reg)							\
    120 	bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, (reg))
    121 #define HWRITE4(sc, reg, val)						\
    122 	bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, (reg), (val))
    123 #define AXIPEEK4(sc, reg, valp)						\
    124 	bus_space_peek_4((sc)->sc_iot, (sc)->sc_axi_ioh, (reg), (valp))
    125 #define AXIPOKE4(sc, reg, val)						\
    126 	bus_space_poke_4((sc)->sc_iot, (sc)->sc_axi_ioh, (reg), (val))
    127 
    128 struct rkpcie_softc {
    129 	struct pcihost_softc	sc_phsc;
    130 	bus_space_tag_t		sc_iot;
    131 	bus_space_handle_t	sc_ioh;
    132 	bus_space_handle_t	sc_axi_ioh;
    133 	bus_addr_t		sc_axi_addr;
    134 	bus_addr_t		sc_apb_addr;
    135 	bus_size_t		sc_axi_size;
    136 	bus_size_t		sc_apb_size;
    137 	kmutex_t		sc_conf_lock;
    138 };
    139 
    140 static int rkpcie_match(device_t, cfdata_t, void *);
    141 static void rkpcie_attach(device_t, device_t, void *);
    142 
    143 CFATTACH_DECL_NEW(rkpcie, sizeof(struct rkpcie_softc),
    144         rkpcie_match, rkpcie_attach, NULL, NULL);
    145 
    146 static const struct device_compatible_entry compat_data[] = {
    147 	{ .compat = "rockchip,rk3399-pcie" },
    148 	DEVICE_COMPAT_EOL
    149 };
    150 
    151 static int
    152 rkpcie_match(device_t parent, cfdata_t cf, void *aux)
    153 {
    154 	struct fdt_attach_args *faa = aux;
    155 
    156 	return of_compatible_match(faa->faa_phandle, compat_data);
    157 }
    158 
    159 static void	rkpcie_atr_init(struct rkpcie_softc *);
    160 
    161 static int	rkpcie_bus_maxdevs(void *, int);
    162 static pcitag_t rkpcie_make_tag(void *, int, int, int);
    163 static void	rkpcie_decompose_tag(void *, pcitag_t, int *, int *, int *);
    164 static pcireg_t rkpcie_conf_read(void *, pcitag_t, int);
    165 static void	rkpcie_conf_write(void *, pcitag_t, int, pcireg_t);
    166 static int	rkpcie_conf_hook(void *, int, int, int, pcireg_t);
    167 
    168 static struct fdtbus_interrupt_controller_func rkpcie_intrfuncs;
    169 
    170 static inline void
    171 clock_enable_all(int phandle)
    172 {
    173 	for (u_int i = 0; i < 4; i++) {
    174 		struct clk * clk = fdtbus_clock_get_index(phandle, i);
    175 		if (clk == NULL)
    176 			continue;
    177 		if (clk_enable(clk) != 0)
    178 			continue;
    179 	}
    180 }
    181 
    182 static void
    183 reset_assert(int phandle, const char *name)
    184 {
    185 	struct fdtbus_reset *rst;
    186 
    187 	rst = fdtbus_reset_get(phandle, name);
    188 	fdtbus_reset_assert(rst);
    189 	fdtbus_reset_put(rst);
    190 }
    191 
    192 static void
    193 reset_deassert(int phandle, const char *name)
    194 {
    195 	struct fdtbus_reset *rst;
    196 
    197 	rst = fdtbus_reset_get(phandle, name);
    198 	fdtbus_reset_deassert(rst);
    199 	fdtbus_reset_put(rst);
    200 }
    201 
    202 static void
    203 rkpcie_attach(device_t parent, device_t self, void *aux)
    204 {
    205 	struct rkpcie_softc *sc = device_private(self);
    206 	struct pcihost_softc * const phsc = &sc->sc_phsc;
    207 	struct fdt_attach_args *faa = aux;
    208 	struct fdtbus_gpio_pin *ep_gpio;
    209 	u_int max_link_speed, num_lanes, bus_scan_delay_ms;
    210 	struct fdtbus_phy *phy[4];
    211 	const u_int *bus_range;
    212 	uint32_t status;
    213 	uint32_t delayed_ms = 0;
    214 	int timo, len;
    215 
    216 	phsc->sc_dev = self;
    217 	phsc->sc_bst = faa->faa_bst;
    218 	phsc->sc_pci_bst = faa->faa_bst;
    219 	phsc->sc_dmat = faa->faa_dmat;
    220 	sc->sc_iot = phsc->sc_bst;
    221 	phsc->sc_phandle = faa->faa_phandle;
    222 	const int phandle = phsc->sc_phandle;
    223 
    224 	if (fdtbus_get_reg_byname(faa->faa_phandle, "axi-base", &sc->sc_axi_addr, &sc->sc_axi_size) != 0) {
    225 		aprint_error(": couldn't get axi registers\n");
    226 		return;
    227 	}
    228 	if (fdtbus_get_reg_byname(faa->faa_phandle, "apb-base", &sc->sc_apb_addr, &sc->sc_apb_size) != 0) {
    229 		aprint_error(": couldn't get apb registers\n");
    230 		sc->sc_axi_size = 0;
    231 		return;
    232 	}
    233 
    234 	const int mapflags = BUS_SPACE_MAP_NONPOSTED;
    235 	if (bus_space_map(sc->sc_iot, sc->sc_apb_addr, sc->sc_apb_size, mapflags, &sc->sc_ioh) != 0 ||
    236 	    bus_space_map(sc->sc_iot, sc->sc_axi_addr, sc->sc_axi_size, mapflags, &sc->sc_axi_ioh) != 0) {
    237 		printf(": can't map registers\n");
    238 		sc->sc_axi_size = 0;
    239 		sc->sc_apb_size = 0;
    240 		return;
    241 	}
    242 
    243 	aprint_naive("\n");
    244 	aprint_normal(": RK3399 PCIe\n");
    245 
    246 	struct fdtbus_regulator *regulator;
    247 	regulator = fdtbus_regulator_acquire(phandle, "vpcie3v3-supply");
    248 	if (regulator != NULL) {
    249 		fdtbus_regulator_enable(regulator);
    250 		fdtbus_regulator_release(regulator);
    251 	}
    252 
    253 	fdtbus_clock_assign(phandle);
    254 	clock_enable_all(phandle);
    255 
    256 	ep_gpio = fdtbus_gpio_acquire(phandle, "ep-gpios", GPIO_PIN_OUTPUT);
    257 
    258 	/*
    259 	 * Let board-specific properties override the default, which is set
    260 	 * to PCIe 1.x, due to errata in the RK3399 CPU.  We don't know exactly
    261 	 * what these errata involved (not public), but posts from the
    262 	 * @rock-chips.com domain to u-boot and linux-kernel lists indicate
    263 	 * that there is a errata related to this, and indeed, the Datasheet
    264 	 * since at least Rev 1.6 and including the latest Rev 1.8 say that the
    265 	 * PCIe can handle 2.5GT/s (ie, PCIe 1.x).
    266 	 */
    267 	if (of_getprop_uint32(phandle, "max-link-speed", &max_link_speed) != 0)
    268 		max_link_speed = 1;
    269 	if (of_getprop_uint32(phandle, "num-lanes", &num_lanes) != 0)
    270 		num_lanes = 1;
    271 
    272 	/*
    273 	 * If the DT has a "bus-scan-delay-ms" property, delay attaching the
    274 	 * PCI bus this many microseconds.
    275 	 */
    276 	if (of_getprop_uint32(phandle, "bus-scan-delay-ms",
    277 	    &bus_scan_delay_ms) != 0)
    278 		bus_scan_delay_ms = 0;
    279 
    280 again:
    281 	fdtbus_gpio_write(ep_gpio, 0);
    282 
    283 	reset_assert(phandle, "aclk");
    284 	reset_assert(phandle, "pclk");
    285 	reset_assert(phandle, "pm");
    286 
    287 	memset(phy, 0, sizeof(phy));
    288 	phy[0] = fdtbus_phy_get(phandle, "pcie-phy-0");
    289 	if (phy[0] == NULL) {
    290 		phy[0] = fdtbus_phy_get(phandle, "pcie-phy");
    291 	} else {
    292 		phy[1] = fdtbus_phy_get(phandle, "pcie-phy-1");
    293 		phy[2] = fdtbus_phy_get(phandle, "pcie-phy-2");
    294 		phy[3] = fdtbus_phy_get(phandle, "pcie-phy-3");
    295 	}
    296 
    297 	reset_assert(phandle, "core");
    298 	reset_assert(phandle, "mgmt");
    299 	reset_assert(phandle, "mgmt-sticky");
    300 	reset_assert(phandle, "pipe");
    301 
    302 	delay(1000);	/* TPERST. use 1ms */
    303 	delayed_ms += 1;
    304 
    305 	reset_deassert(phandle, "pm");
    306 	reset_deassert(phandle, "aclk");
    307 	reset_deassert(phandle, "pclk");
    308 
    309 	if (max_link_speed == 1)
    310 		HWRITE4(sc, PCIE_CLIENT_BASIC_STRAP_CONF, PCBSC_PGS_GEN1);
    311 	else
    312 		HWRITE4(sc, PCIE_CLIENT_BASIC_STRAP_CONF, PCBSC_PGS_GEN2);
    313 
    314 	/* Switch into Root Complex mode. */
    315 	HWRITE4(sc, PCIE_CLIENT_BASIC_STRAP_CONF,
    316 	    PCBSC_MS_ROOTPORT | PCBSC_CONF_EN | PCBSC_LC(num_lanes));
    317 
    318 	if (phy[3] && fdtbus_phy_enable(phy[3], true) != 0) {
    319 		aprint_error(": couldn't enable phy3\n");
    320 	}
    321 	if (phy[2] && fdtbus_phy_enable(phy[2], true) != 0) {
    322 		aprint_error(": couldn't enable phy2\n");
    323 	}
    324 	if (phy[1] && fdtbus_phy_enable(phy[1], true) != 0) {
    325 		aprint_error(": couldn't enable phy1\n");
    326 	}
    327 	if (phy[0] && fdtbus_phy_enable(phy[0], true) != 0) {
    328 		aprint_error(": couldn't enable phy0\n");
    329 	}
    330 
    331 	reset_deassert(phandle, "mgmt-sticky");
    332 	reset_deassert(phandle, "core");
    333 	reset_deassert(phandle, "mgmt");
    334 	reset_deassert(phandle, "pipe");
    335 
    336 	fdtbus_gpio_write(ep_gpio, 1);
    337 	delay(20000);	/* 20 ms according to PCI-e BS "Conventional Reset" */
    338 	delayed_ms += 20;
    339 
    340 	/* Start link training. */
    341 	HWRITE4(sc, PCIE_CLIENT_BASIC_STRAP_CONF, PCBSC_LINK_TRAIN_EN);
    342 
    343 	for (timo = 500; timo > 0; timo--) {
    344 		status = HREAD4(sc, PCIE_CLIENT_BASIC_STATUS1);
    345 		if (PCBS1_LINK_ST(status) == PCBS1_LS_DL_DONE)
    346 			break;
    347 		delay(1000);
    348 		delayed_ms += 1;
    349 	}
    350 	if (timo == 0) {
    351 		device_printf(self, "link training timeout (link_st %u)\n",
    352 		    PCBS1_LINK_ST(status));
    353 		if (max_link_speed > 1) {
    354 			--max_link_speed;
    355 			goto again;
    356 		}
    357 		return;
    358 	}
    359 
    360 	if (max_link_speed == 2) {
    361 		HWRITE4(sc, PCIE_RC_CONFIG_LCSR, HREAD4(sc, PCIE_RC_CONFIG_LCSR) | PCIE_LCSR_RETRAIN);
    362 		for (timo = 500; timo > 0; timo--) {
    363 			status = HREAD4(sc, PCIE_LM_CORE_CTRL);
    364 			if ((status & PCIE_CORE_PL_CONF_SPEED_MASK) == PCIE_CORE_PL_CONF_SPEED_5G)
    365 				break;
    366 			delay(1000);
    367 			delayed_ms += 1;
    368 		}
    369 		if (timo == 0) {
    370 			device_printf(self, "Gen2 link training timeout\n");
    371 			--max_link_speed;
    372 			goto again;
    373 		}
    374 	}
    375 	delay(80000);	/* wait 100 ms before CSR access. already waited 20. */
    376 	delayed_ms += 80;
    377 
    378 	fdtbus_gpio_release(ep_gpio);
    379 
    380 	HWRITE4(sc, PCIE_RC_BASE + PCI_CLASS_REG,
    381 	    PCI_CLASS_BRIDGE << PCI_CLASS_SHIFT |
    382 	    PCI_SUBCLASS_BRIDGE_PCI << PCI_SUBCLASS_SHIFT);
    383 
    384 	/* Initialize Root Complex registers. */
    385 	HWRITE4(sc, PCIE_LM_VENDOR_ID, PCI_VENDOR_ROCKCHIP);
    386 	HWRITE4(sc, PCIE_RC_BASE + PCI_CLASS_REG,
    387 	    PCI_CLASS_BRIDGE << PCI_CLASS_SHIFT |
    388 	    PCI_SUBCLASS_BRIDGE_PCI << PCI_SUBCLASS_SHIFT);
    389 	HWRITE4(sc, PCIE_LM_RCBAR, PCIE_LM_RCBARPIE | PCIE_LM_RCBARPIS);
    390 
    391 	/* remove L1 substate cap */
    392 	status = HREAD4(sc, PCIE_RC_CONFIG_THP_CAP);
    393 	status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
    394 	HWRITE4(sc, PCIE_RC_CONFIG_THP_CAP, status);
    395 
    396 	if (of_hasprop(phandle, "aspm-no-l0s")) {
    397 		status = HREAD4(sc, PCIE_RC_PCIE_LCAP);
    398 		status &= ~__SHIFTIN(1, PCIE_LCAP_ASPM);
    399 		HWRITE4(sc, PCIE_RC_PCIE_LCAP, status);
    400 	}
    401 
    402 	/* Default bus ranges */
    403 	sc->sc_phsc.sc_bus_min = 0;
    404 	sc->sc_phsc.sc_bus_max = 31;
    405 
    406 	/* Override bus range from DT */
    407 	bus_range = fdtbus_get_prop(phandle, "bus-range", &len);
    408 	if (len == 8) {
    409 		sc->sc_phsc.sc_bus_min = be32dec(&bus_range[0]);
    410 		sc->sc_phsc.sc_bus_max = be32dec(&bus_range[1]);
    411 	}
    412 
    413 	if (sc->sc_phsc.sc_bus_min != 0) {
    414 		aprint_error_dev(self, "bus-range doesn't start at 0\n");
    415 		return;
    416 	}
    417 
    418 	/* Configure Address Translation. */
    419 	rkpcie_atr_init(sc);
    420 
    421 	fdtbus_register_interrupt_controller(self,
    422 	    OF_child(sc->sc_phsc.sc_phandle), &rkpcie_intrfuncs);
    423 
    424 	sc->sc_phsc.sc_type = PCIHOST_ECAM;
    425 	sc->sc_phsc.sc_pci_flags |= PCI_FLAGS_MSI_OKAY;
    426 	sc->sc_phsc.sc_pci_flags |= PCI_FLAGS_MSIX_OKAY;
    427 	pcihost_init(&sc->sc_phsc.sc_pc, sc);
    428 	sc->sc_phsc.sc_pc.pc_bus_maxdevs = rkpcie_bus_maxdevs;
    429 	sc->sc_phsc.sc_pc.pc_make_tag = rkpcie_make_tag;
    430 	sc->sc_phsc.sc_pc.pc_decompose_tag = rkpcie_decompose_tag;
    431 	sc->sc_phsc.sc_pc.pc_conf_read = rkpcie_conf_read;
    432 	sc->sc_phsc.sc_pc.pc_conf_write = rkpcie_conf_write;
    433 	sc->sc_phsc.sc_pc.pc_conf_hook = rkpcie_conf_hook;
    434 
    435 	if (bus_scan_delay_ms > delayed_ms) {
    436 		uint32_t ms = bus_scan_delay_ms - delayed_ms;
    437 
    438 		aprint_verbose_dev(phsc->sc_dev,
    439 		    "waiting %u extra ms for reset (already waited %u)\n",
    440 		    ms, delayed_ms);
    441 		delay(ms * 1000);
    442 	}
    443 
    444 	mutex_init(&sc->sc_conf_lock, MUTEX_DEFAULT, IPL_HIGH);
    445 	pcihost_init2(&sc->sc_phsc);
    446 }
    447 
    448 static void
    449 rkpcie_atr_init(struct rkpcie_softc *sc)
    450 {
    451 	const u_int *ranges;
    452 	bus_addr_t aaddr;
    453 	bus_addr_t addr;
    454 	bus_size_t size, resid, offset;
    455 	uint32_t type;
    456 	int region, i, ranges_len;
    457 
    458 	/* Use region 0 to map PCI configuration space */
    459 	HWRITE4(sc, PCIE_ATR_OB_ADDR0(0), 20 - 1);
    460 	HWRITE4(sc, PCIE_ATR_OB_ADDR1(0), 0);
    461 	HWRITE4(sc, PCIE_ATR_OB_DESC0(0), PCIE_ATR_HDR_CFG_TYPE0 | PCIE_ATR_HDR_RID);
    462 	HWRITE4(sc, PCIE_ATR_OB_DESC1(0), 0);
    463 
    464 	ranges = fdtbus_get_prop(sc->sc_phsc.sc_phandle, "ranges", &ranges_len);
    465 	if (ranges == NULL)
    466 		goto fail;
    467 	const int ranges_cells = ranges_len / 4;
    468 
    469 	for (i = 0; i < ranges_cells; i += 7) {
    470 		/* Handle IO and MMIO. */
    471 		switch (be32toh(ranges[i]) & 0x03000000) {
    472 		case 0x01000000:
    473 			type = PCIE_ATR_HDR_IO;
    474 			break;
    475 		case 0x02000000:
    476 		case 0x03000000:
    477 			type = PCIE_ATR_HDR_MEM;
    478 			break;
    479 		default:
    480 			continue;
    481 		}
    482 
    483 		addr = ((uint64_t)be32toh(ranges[i + 1]) << 32) + be32toh(ranges[i + 2]);
    484 		aaddr = ((uint64_t)be32toh(ranges[i + 3]) << 32) + be32toh(ranges[i + 4]);
    485 		size = be32toh(ranges[i + 6]);
    486 
    487 		/* Only support mappings aligned on a region boundary. */
    488 		if (addr & (PCIE_ATR_OB_REGION_SIZE - 1))
    489 			goto fail;
    490 		if (aaddr & (PCIE_ATR_OB_REGION_SIZE - 1))
    491 			goto fail;
    492 		if (size & (PCIE_ATR_OB_REGION_SIZE - 1))
    493 			goto fail;
    494 
    495 		/* Mappings should lie in AXI region. */
    496 		if (aaddr < sc->sc_axi_addr)
    497 			goto fail;
    498 		if (aaddr + size > sc->sc_axi_addr + 64*1024*1024)
    499 			goto fail;
    500 
    501 		offset = addr - sc->sc_axi_addr - PCIE_ATR_OB_REGION0_SIZE;
    502 		region = 1 + (offset / PCIE_ATR_OB_REGION_SIZE);
    503 		resid = size;
    504 		while (resid > 0) {
    505 			HWRITE4(sc, PCIE_ATR_OB_ADDR0(region), 32 - 1);
    506 			HWRITE4(sc, PCIE_ATR_OB_ADDR1(region), 0);
    507 			HWRITE4(sc, PCIE_ATR_OB_DESC0(region), type | PCIE_ATR_HDR_RID);
    508 			HWRITE4(sc, PCIE_ATR_OB_DESC1(region), 0);
    509 
    510 			addr += PCIE_ATR_OB_REGION_SIZE;
    511 			resid -= PCIE_ATR_OB_REGION_SIZE;
    512 			region++;
    513 		}
    514 	}
    515 
    516 	/* Passthrough inbound translations unmodified. */
    517 	HWRITE4(sc, PCIE_ATR_IB_ADDR0(2), 32 - 1);
    518 	HWRITE4(sc, PCIE_ATR_IB_ADDR1(2), 0);
    519 
    520 	return;
    521 
    522 fail:
    523 	device_printf(sc->sc_phsc.sc_dev, "can't map ranges\n");
    524 }
    525 
    526 int
    527 rkpcie_bus_maxdevs(void *v, int bus)
    528 {
    529 	struct rkpcie_softc *rksc = v;
    530 	struct pcihost_softc *sc = &rksc->sc_phsc;
    531 
    532 	if (bus == sc->sc_bus_min || bus == sc->sc_bus_min + 1)
    533 		return 1;
    534 	return 32;
    535 }
    536 
    537 pcitag_t
    538 rkpcie_make_tag(void *v, int bus, int device, int function)
    539 {
    540 	/* Return ECAM address. */
    541 	return ((bus << 20) | (device << 15) | (function << 12));
    542 }
    543 
    544 void
    545 rkpcie_decompose_tag(void *v, pcitag_t tag, int *bp, int *dp, int *fp)
    546 {
    547 	if (bp != NULL)
    548 		*bp = (tag >> 20) & 0xff;
    549 	if (dp != NULL)
    550 		*dp = (tag >> 15) & 0x1f;
    551 	if (fp != NULL)
    552 		*fp = (tag >> 12) & 0x7;
    553 }
    554 
    555 /* Only one device on root port and the first subordinate port. */
    556 static bool
    557 rkpcie_conf_ok(int bus, int dev, int fn, int offset, struct rkpcie_softc *sc)
    558 {
    559 	int bus_min = sc->sc_phsc.sc_bus_min;
    560 
    561 	if ((unsigned int)offset >= (1<<12))
    562 		return false;
    563 	/* first two buses use type 0 cfg which doesn't use bus/device numbers */
    564 	if (dev != 0 && (bus == bus_min || bus == bus_min + 1))
    565 		return false;
    566 	return true;
    567 }
    568 
    569 pcireg_t
    570 rkpcie_conf_read(void *v, pcitag_t tag, int offset)
    571 {
    572 	struct rkpcie_softc *sc = v;
    573 	int bus_min = sc->sc_phsc.sc_bus_min;
    574 	int bus, dev, fn;
    575 	u_int reg;
    576 	int32_t val;
    577 
    578 	KASSERT(offset >= 0);
    579 	KASSERT(offset < PCI_EXTCONF_SIZE);
    580 
    581 	rkpcie_decompose_tag(sc, tag, &bus, &dev, &fn);
    582 	if (!rkpcie_conf_ok(bus, dev, fn, offset, sc))
    583 		return 0xffffffff;
    584 	reg = (dev << 15) | (fn << 12) | offset;
    585 
    586 	if (bus == bus_min)
    587 		val = HREAD4(sc, PCIE_RC_NORMAL_BASE + reg);
    588 	else {
    589 		mutex_spin_enter(&sc->sc_conf_lock);
    590 		HWRITE4(sc, PCIE_ATR_OB_ADDR0(0),
    591 		    (bus << 20) | (20 - 1));
    592 		HWRITE4(sc, PCIE_ATR_OB_DESC0(0),
    593 		    PCIE_ATR_HDR_RID | ((bus == bus_min + 1)
    594 		    ? PCIE_ATR_HDR_CFG_TYPE0 : PCIE_ATR_HDR_CFG_TYPE1));
    595 		bus_space_barrier(sc->sc_iot, sc->sc_ioh, 0, sc->sc_apb_size,
    596 		      BUS_SPACE_BARRIER_READ);
    597 		if (AXIPEEK4(sc, reg, &val) != 0)
    598 			val = 0xffffffff;
    599 		bus_space_barrier(sc->sc_iot, sc->sc_axi_ioh,
    600 		    0, sc->sc_axi_size,
    601 		    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
    602 		mutex_spin_exit(&sc->sc_conf_lock);
    603 	}
    604 	return val;
    605 }
    606 
    607 void
    608 rkpcie_conf_write(void *v, pcitag_t tag, int offset, pcireg_t data)
    609 {
    610 	struct rkpcie_softc *sc = v;
    611 	int bus_min = sc->sc_phsc.sc_bus_min;
    612 	int bus, dev, fn;
    613 	u_int reg;
    614 
    615 	KASSERT(offset >= 0);
    616 	KASSERT(offset < PCI_EXTCONF_SIZE);
    617 
    618 	rkpcie_decompose_tag(sc, tag, &bus, &dev, &fn);
    619 	if (!rkpcie_conf_ok(bus, dev, fn, offset, sc))
    620 		return;
    621 	reg = (dev << 15) | (fn << 12) | offset;
    622 
    623 	if (bus == bus_min)
    624 		HWRITE4(sc, PCIE_RC_NORMAL_BASE + reg, data);
    625 	else {
    626 		mutex_spin_enter(&sc->sc_conf_lock);
    627 		HWRITE4(sc, PCIE_ATR_OB_ADDR0(0),
    628 		    (bus << 20) | (20 - 1));
    629 		HWRITE4(sc, PCIE_ATR_OB_DESC0(0),
    630 		    PCIE_ATR_HDR_RID | ((bus == bus_min + 1)
    631 		    ? PCIE_ATR_HDR_CFG_TYPE0 : PCIE_ATR_HDR_CFG_TYPE1));
    632 		bus_space_barrier(sc->sc_iot, sc->sc_ioh, 0, sc->sc_apb_size,
    633 		    BUS_SPACE_BARRIER_WRITE);
    634 		AXIPOKE4(sc, reg, data);
    635 		bus_space_barrier(sc->sc_iot, sc->sc_axi_ioh,
    636 		    0, sc->sc_axi_size,
    637 		    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
    638 		mutex_spin_exit(&sc->sc_conf_lock);
    639 	}
    640 }
    641 
    642 static int
    643 rkpcie_conf_hook(void *v, int b, int d, int f, pcireg_t id)
    644 {
    645         return (PCI_CONF_DEFAULT & ~PCI_CONF_ENABLE_BM) | PCI_CONF_MAP_ROM;
    646 }
    647 
    648 /* INTx interrupt controller */
    649 static void *
    650 rkpcie_intx_establish(device_t dev, u_int *specifier, int ipl, int flags,
    651     int (*func)(void *), void *arg, const char *xname)
    652 {
    653 	struct rkpcie_softc *sc = device_private(dev);
    654 	void *cookie;
    655 
    656 #if notyet
    657 	const u_int pin = be32toh(specifier[0]);
    658 #endif
    659 
    660 	/* Unmask legacy interrupts. */
    661 	HWRITE4(sc, PCIE_CLIENT_INT_MASK,
    662 	    PCIM_INTx_ENAB(0) | PCIM_INTx_ENAB(1) |
    663 	    PCIM_INTx_ENAB(2) | PCIM_INTx_ENAB(3));
    664 
    665 	cookie = fdtbus_intr_establish_byname(sc->sc_phsc.sc_phandle,
    666 	    "legacy", ipl, flags, func, arg, xname);
    667 
    668 	return cookie;
    669 }
    670 
    671 static void
    672 rkpcie_intx_disestablish(device_t dev, void *ih)
    673 {
    674 	struct rkpcie_softc *sc = device_private(dev);
    675 	device_printf(dev, "%s\n", __func__);
    676 	fdtbus_intr_disestablish(sc->sc_phsc.sc_phandle, ih);
    677 }
    678 
    679 static bool
    680 rkpcie_intx_intrstr(device_t dev, u_int *specifier, char *buf, size_t buflen)
    681 {
    682 	struct rkpcie_softc *sc = device_private(dev);
    683 
    684 	fdtbus_intr_str(sc->sc_phsc.sc_phandle, 1, buf, buflen);
    685 
    686 	return true;
    687 }
    688 
    689 static struct fdtbus_interrupt_controller_func rkpcie_intrfuncs = {
    690 	.establish = rkpcie_intx_establish,
    691 	.disestablish = rkpcie_intx_disestablish,
    692 	.intrstr = rkpcie_intx_intrstr,
    693 };
    694