octeon_intr.c revision 1.3.2.6
1 1.3.2.6 skrll /* $NetBSD: octeon_intr.c,v 1.3.2.6 2017/08/28 17:51:45 skrll Exp $ */ 2 1.3.2.2 skrll /* 3 1.3.2.2 skrll * Copyright 2001, 2002 Wasabi Systems, Inc. 4 1.3.2.2 skrll * All rights reserved. 5 1.3.2.2 skrll * 6 1.3.2.2 skrll * Written by Jason R. Thorpe and Simon Burge for Wasabi Systems, Inc. 7 1.3.2.2 skrll * 8 1.3.2.2 skrll * Redistribution and use in source and binary forms, with or without 9 1.3.2.2 skrll * modification, are permitted provided that the following conditions 10 1.3.2.2 skrll * are met: 11 1.3.2.2 skrll * 1. Redistributions of source code must retain the above copyright 12 1.3.2.2 skrll * notice, this list of conditions and the following disclaimer. 13 1.3.2.2 skrll * 2. Redistributions in binary form must reproduce the above copyright 14 1.3.2.2 skrll * notice, this list of conditions and the following disclaimer in the 15 1.3.2.2 skrll * documentation and/or other materials provided with the distribution. 16 1.3.2.2 skrll * 3. All advertising materials mentioning features or use of this software 17 1.3.2.2 skrll * must display the following acknowledgement: 18 1.3.2.2 skrll * This product includes software developed for the NetBSD Project by 19 1.3.2.2 skrll * Wasabi Systems, Inc. 20 1.3.2.2 skrll * 4. The name of Wasabi Systems, Inc. may not be used to endorse 21 1.3.2.2 skrll * or promote products derived from this software without specific prior 22 1.3.2.2 skrll * written permission. 23 1.3.2.2 skrll * 24 1.3.2.2 skrll * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 25 1.3.2.2 skrll * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 1.3.2.2 skrll * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 1.3.2.2 skrll * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 28 1.3.2.2 skrll * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 1.3.2.2 skrll * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 1.3.2.2 skrll * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 1.3.2.2 skrll * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 1.3.2.2 skrll * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 1.3.2.2 skrll * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 1.3.2.2 skrll * POSSIBILITY OF SUCH DAMAGE. 35 1.3.2.2 skrll */ 36 1.3.2.2 skrll 37 1.3.2.2 skrll /* 38 1.3.2.2 skrll * Platform-specific interrupt support for the MIPS Malta. 39 1.3.2.2 skrll */ 40 1.3.2.2 skrll 41 1.3.2.2 skrll #include "opt_octeon.h" 42 1.3.2.4 skrll #include "opt_multiprocessor.h" 43 1.3.2.4 skrll 44 1.3.2.3 skrll #include "cpunode.h" 45 1.3.2.2 skrll #define __INTR_PRIVATE 46 1.3.2.2 skrll 47 1.3.2.2 skrll #include <sys/cdefs.h> 48 1.3.2.6 skrll __KERNEL_RCSID(0, "$NetBSD: octeon_intr.c,v 1.3.2.6 2017/08/28 17:51:45 skrll Exp $"); 49 1.3.2.2 skrll 50 1.3.2.2 skrll #include <sys/param.h> 51 1.3.2.2 skrll #include <sys/cpu.h> 52 1.3.2.2 skrll #include <sys/systm.h> 53 1.3.2.2 skrll #include <sys/device.h> 54 1.3.2.2 skrll #include <sys/intr.h> 55 1.3.2.2 skrll #include <sys/kernel.h> 56 1.3.2.2 skrll #include <sys/kmem.h> 57 1.3.2.2 skrll #include <sys/atomic.h> 58 1.3.2.2 skrll 59 1.3.2.2 skrll #include <lib/libkern/libkern.h> 60 1.3.2.2 skrll 61 1.3.2.2 skrll #include <mips/locore.h> 62 1.3.2.2 skrll 63 1.3.2.2 skrll #include <mips/cavium/dev/octeon_ciureg.h> 64 1.3.2.2 skrll #include <mips/cavium/octeonvar.h> 65 1.3.2.2 skrll 66 1.3.2.2 skrll /* 67 1.3.2.2 skrll * This is a mask of bits to clear in the SR when we go to a 68 1.3.2.2 skrll * given hardware interrupt priority level. 69 1.3.2.2 skrll */ 70 1.3.2.2 skrll static const struct ipl_sr_map octeon_ipl_sr_map = { 71 1.3.2.2 skrll .sr_bits = { 72 1.3.2.2 skrll [IPL_NONE] = 0, 73 1.3.2.2 skrll [IPL_SOFTCLOCK] = MIPS_SOFT_INT_MASK_0, 74 1.3.2.2 skrll [IPL_SOFTNET] = MIPS_SOFT_INT_MASK, 75 1.3.2.2 skrll [IPL_VM] = MIPS_SOFT_INT_MASK | MIPS_INT_MASK_0, 76 1.3.2.2 skrll [IPL_SCHED] = MIPS_SOFT_INT_MASK | MIPS_INT_MASK_0 77 1.3.2.5 skrll | MIPS_INT_MASK_1 | MIPS_INT_MASK_5, 78 1.3.2.2 skrll [IPL_DDB] = MIPS_SOFT_INT_MASK | MIPS_INT_MASK_0 79 1.3.2.2 skrll | MIPS_INT_MASK_1 | MIPS_INT_MASK_5, 80 1.3.2.2 skrll [IPL_HIGH] = MIPS_INT_MASK, 81 1.3.2.2 skrll }, 82 1.3.2.2 skrll }; 83 1.3.2.2 skrll 84 1.3.2.2 skrll const char * const octeon_intrnames[NIRQS] = { 85 1.3.2.2 skrll "workq 0", 86 1.3.2.2 skrll "workq 1", 87 1.3.2.2 skrll "workq 2", 88 1.3.2.2 skrll "workq 3", 89 1.3.2.2 skrll "workq 4", 90 1.3.2.2 skrll "workq 5", 91 1.3.2.2 skrll "workq 6", 92 1.3.2.2 skrll "workq 7", 93 1.3.2.2 skrll "workq 8", 94 1.3.2.2 skrll "workq 9", 95 1.3.2.2 skrll "workq 10", 96 1.3.2.2 skrll "workq 11", 97 1.3.2.2 skrll "workq 12", 98 1.3.2.2 skrll "workq 13", 99 1.3.2.2 skrll "workq 14", 100 1.3.2.2 skrll "workq 15", 101 1.3.2.2 skrll "gpio 0", 102 1.3.2.2 skrll "gpio 1", 103 1.3.2.2 skrll "gpio 2", 104 1.3.2.2 skrll "gpio 3", 105 1.3.2.2 skrll "gpio 4", 106 1.3.2.2 skrll "gpio 5", 107 1.3.2.2 skrll "gpio 6", 108 1.3.2.2 skrll "gpio 7", 109 1.3.2.2 skrll "gpio 8", 110 1.3.2.2 skrll "gpio 9", 111 1.3.2.2 skrll "gpio 10", 112 1.3.2.2 skrll "gpio 11", 113 1.3.2.2 skrll "gpio 12", 114 1.3.2.2 skrll "gpio 13", 115 1.3.2.2 skrll "gpio 14", 116 1.3.2.2 skrll "gpio 15", 117 1.3.2.2 skrll "mbox 0-15", 118 1.3.2.2 skrll "mbox 16-31", 119 1.3.2.2 skrll "uart 0", 120 1.3.2.2 skrll "uart 1", 121 1.3.2.2 skrll "pci inta", 122 1.3.2.2 skrll "pci intb", 123 1.3.2.2 skrll "pci intc", 124 1.3.2.2 skrll "pci intd", 125 1.3.2.2 skrll "pci msi 0-15", 126 1.3.2.2 skrll "pci msi 16-31", 127 1.3.2.2 skrll "pci msi 32-47", 128 1.3.2.2 skrll "pci msi 48-63", 129 1.3.2.2 skrll "wdog summary", 130 1.3.2.2 skrll "twsi", 131 1.3.2.2 skrll "rml", 132 1.3.2.2 skrll "trace", 133 1.3.2.2 skrll "gmx drop", 134 1.3.2.2 skrll "reserved", 135 1.3.2.2 skrll "ipd drop", 136 1.3.2.2 skrll "reserved", 137 1.3.2.2 skrll "timer 0", 138 1.3.2.2 skrll "timer 1", 139 1.3.2.2 skrll "timer 2", 140 1.3.2.2 skrll "timer 3", 141 1.3.2.2 skrll "usb", 142 1.3.2.2 skrll "pcm/tdm", 143 1.3.2.2 skrll "mpi/spi", 144 1.3.2.2 skrll "reserved", 145 1.3.2.2 skrll "reserved", 146 1.3.2.2 skrll "reserved", 147 1.3.2.2 skrll "reserved", 148 1.3.2.2 skrll "reserved", 149 1.3.2.2 skrll }; 150 1.3.2.2 skrll 151 1.3.2.2 skrll struct octeon_intrhand { 152 1.3.2.2 skrll int (*ih_func)(void *); 153 1.3.2.2 skrll void *ih_arg; 154 1.3.2.2 skrll int ih_irq; 155 1.3.2.2 skrll int ih_ipl; 156 1.3.2.2 skrll }; 157 1.3.2.2 skrll 158 1.3.2.2 skrll #ifdef MULTIPROCESSOR 159 1.3.2.2 skrll static int octeon_send_ipi(struct cpu_info *, int); 160 1.3.2.2 skrll static int octeon_ipi_intr(void *); 161 1.3.2.2 skrll 162 1.3.2.2 skrll struct octeon_intrhand ipi_intrhands[2] = { 163 1.3.2.2 skrll [0] = { 164 1.3.2.2 skrll .ih_func = octeon_ipi_intr, 165 1.3.2.2 skrll .ih_arg = (void *)(uintptr_t)__BITS(15,0), 166 1.3.2.2 skrll .ih_irq = _CIU_INT_MBOX_15_0_SHIFT, 167 1.3.2.2 skrll .ih_ipl = IPL_SCHED, 168 1.3.2.2 skrll }, 169 1.3.2.2 skrll [1] = { 170 1.3.2.2 skrll .ih_func = octeon_ipi_intr, 171 1.3.2.2 skrll .ih_arg = (void *)(uintptr_t)__BITS(31,16), 172 1.3.2.2 skrll .ih_irq = _CIU_INT_MBOX_31_16_SHIFT, 173 1.3.2.2 skrll .ih_ipl = IPL_HIGH, 174 1.3.2.2 skrll }, 175 1.3.2.2 skrll }; 176 1.3.2.2 skrll #endif 177 1.3.2.2 skrll 178 1.3.2.2 skrll struct octeon_intrhand *octeon_ciu_intrs[NIRQS] = { 179 1.3.2.2 skrll #ifdef MULTIPROCESSOR 180 1.3.2.2 skrll [_CIU_INT_MBOX_15_0_SHIFT] = &ipi_intrhands[0], 181 1.3.2.2 skrll [_CIU_INT_MBOX_31_16_SHIFT] = &ipi_intrhands[1], 182 1.3.2.2 skrll #endif 183 1.3.2.2 skrll }; 184 1.3.2.2 skrll 185 1.3.2.2 skrll kmutex_t octeon_intr_lock; 186 1.3.2.2 skrll 187 1.3.2.2 skrll #define X(a) MIPS_PHYS_TO_XKPHYS(OCTEON_CCA_NONE, (a)) 188 1.3.2.2 skrll 189 1.3.2.2 skrll struct cpu_softc octeon_cpu0_softc = { 190 1.3.2.2 skrll .cpu_ci = &cpu_info_store, 191 1.3.2.2 skrll .cpu_int0_sum0 = X(CIU_INT0_SUM0), 192 1.3.2.2 skrll .cpu_int1_sum0 = X(CIU_INT1_SUM0), 193 1.3.2.2 skrll .cpu_int2_sum0 = X(CIU_INT4_SUM0), 194 1.3.2.2 skrll 195 1.3.2.2 skrll .cpu_int0_en0 = X(CIU_INT0_EN0), 196 1.3.2.2 skrll .cpu_int1_en0 = X(CIU_INT1_EN0), 197 1.3.2.2 skrll .cpu_int2_en0 = X(CIU_INT4_EN00), 198 1.3.2.2 skrll 199 1.3.2.2 skrll .cpu_int0_en1 = X(CIU_INT0_EN1), 200 1.3.2.2 skrll .cpu_int1_en1 = X(CIU_INT1_EN1), 201 1.3.2.2 skrll .cpu_int2_en1 = X(CIU_INT4_EN01), 202 1.3.2.2 skrll 203 1.3.2.2 skrll .cpu_int32_en = X(CIU_INT32_EN0), 204 1.3.2.2 skrll 205 1.3.2.3 skrll .cpu_wdog = X(CIU_WDOG0), 206 1.3.2.3 skrll .cpu_pp_poke = X(CIU_PP_POKE0), 207 1.3.2.3 skrll 208 1.3.2.2 skrll #ifdef MULTIPROCESSOR 209 1.3.2.2 skrll .cpu_mbox_set = X(CIU_MBOX_SET0), 210 1.3.2.2 skrll .cpu_mbox_clr = X(CIU_MBOX_CLR0), 211 1.3.2.2 skrll #endif 212 1.3.2.2 skrll }; 213 1.3.2.2 skrll 214 1.3.2.2 skrll #ifdef MULTIPROCESSOR 215 1.3.2.2 skrll struct cpu_softc octeon_cpu1_softc = { 216 1.3.2.2 skrll .cpu_int0_sum0 = X(CIU_INT2_SUM0), 217 1.3.2.2 skrll .cpu_int1_sum0 = X(CIU_INT3_SUM0), 218 1.3.2.2 skrll .cpu_int2_sum0 = X(CIU_INT4_SUM1), 219 1.3.2.2 skrll 220 1.3.2.2 skrll .cpu_int0_en0 = X(CIU_INT2_EN0), 221 1.3.2.2 skrll .cpu_int1_en0 = X(CIU_INT3_EN0), 222 1.3.2.2 skrll .cpu_int2_en0 = X(CIU_INT4_EN10), 223 1.3.2.2 skrll 224 1.3.2.2 skrll .cpu_int0_en1 = X(CIU_INT2_EN1), 225 1.3.2.2 skrll .cpu_int1_en1 = X(CIU_INT3_EN1), 226 1.3.2.2 skrll .cpu_int2_en1 = X(CIU_INT4_EN11), 227 1.3.2.2 skrll 228 1.3.2.2 skrll .cpu_int32_en = X(CIU_INT32_EN1), 229 1.3.2.2 skrll 230 1.3.2.3 skrll .cpu_wdog = X(CIU_WDOG1), 231 1.3.2.3 skrll .cpu_pp_poke = X(CIU_PP_POKE1), 232 1.3.2.3 skrll 233 1.3.2.2 skrll .cpu_mbox_set = X(CIU_MBOX_SET1), 234 1.3.2.2 skrll .cpu_mbox_clr = X(CIU_MBOX_CLR1), 235 1.3.2.2 skrll }; 236 1.3.2.2 skrll #endif 237 1.3.2.2 skrll 238 1.3.2.3 skrll #ifdef DEBUG 239 1.3.2.3 skrll static void 240 1.3.2.3 skrll octeon_mbox_test(void) 241 1.3.2.3 skrll { 242 1.3.2.3 skrll const uint64_t mbox_clr0 = X(CIU_MBOX_CLR0); 243 1.3.2.3 skrll const uint64_t mbox_clr1 = X(CIU_MBOX_CLR1); 244 1.3.2.3 skrll const uint64_t mbox_set0 = X(CIU_MBOX_SET0); 245 1.3.2.3 skrll const uint64_t mbox_set1 = X(CIU_MBOX_SET1); 246 1.3.2.3 skrll const uint64_t int_sum0 = X(CIU_INT0_SUM0); 247 1.3.2.3 skrll const uint64_t int_sum1 = X(CIU_INT2_SUM0); 248 1.3.2.3 skrll const uint64_t sum_mbox_lo = __BIT(_CIU_INT_MBOX_15_0_SHIFT); 249 1.3.2.3 skrll const uint64_t sum_mbox_hi = __BIT(_CIU_INT_MBOX_31_16_SHIFT); 250 1.3.2.3 skrll 251 1.3.2.4 skrll mips3_sd(mbox_clr0, ~0ULL); 252 1.3.2.4 skrll mips3_sd(mbox_clr1, ~0ULL); 253 1.3.2.3 skrll 254 1.3.2.4 skrll uint32_t mbox0 = mips3_ld(mbox_set0); 255 1.3.2.4 skrll uint32_t mbox1 = mips3_ld(mbox_set1); 256 1.3.2.3 skrll 257 1.3.2.3 skrll KDASSERTMSG(mbox0 == 0, "mbox0 %#x mbox1 %#x", mbox0, mbox1); 258 1.3.2.3 skrll KDASSERTMSG(mbox1 == 0, "mbox0 %#x mbox1 %#x", mbox0, mbox1); 259 1.3.2.3 skrll 260 1.3.2.4 skrll mips3_sd(mbox_set0, __BIT(0)); 261 1.3.2.3 skrll 262 1.3.2.4 skrll mbox0 = mips3_ld(mbox_set0); 263 1.3.2.4 skrll mbox1 = mips3_ld(mbox_set1); 264 1.3.2.3 skrll 265 1.3.2.3 skrll KDASSERTMSG(mbox0 == 1, "mbox0 %#x mbox1 %#x", mbox0, mbox1); 266 1.3.2.3 skrll KDASSERTMSG(mbox1 == 0, "mbox0 %#x mbox1 %#x", mbox0, mbox1); 267 1.3.2.3 skrll 268 1.3.2.4 skrll uint64_t sum0 = mips3_ld(int_sum0); 269 1.3.2.4 skrll uint64_t sum1 = mips3_ld(int_sum1); 270 1.3.2.3 skrll 271 1.3.2.3 skrll KDASSERTMSG((sum0 & sum_mbox_lo) != 0, "sum0 %#"PRIx64, sum0); 272 1.3.2.3 skrll KDASSERTMSG((sum0 & sum_mbox_hi) == 0, "sum0 %#"PRIx64, sum0); 273 1.3.2.3 skrll 274 1.3.2.3 skrll KDASSERTMSG((sum1 & sum_mbox_lo) == 0, "sum1 %#"PRIx64, sum1); 275 1.3.2.3 skrll KDASSERTMSG((sum1 & sum_mbox_hi) == 0, "sum1 %#"PRIx64, sum1); 276 1.3.2.3 skrll 277 1.3.2.4 skrll mips3_sd(mbox_clr0, mbox0); 278 1.3.2.4 skrll mbox0 = mips3_ld(mbox_set0); 279 1.3.2.3 skrll KDASSERTMSG(mbox0 == 0, "mbox0 %#x", mbox0); 280 1.3.2.3 skrll 281 1.3.2.4 skrll mips3_sd(mbox_set0, __BIT(16)); 282 1.3.2.3 skrll 283 1.3.2.4 skrll mbox0 = mips3_ld(mbox_set0); 284 1.3.2.4 skrll mbox1 = mips3_ld(mbox_set1); 285 1.3.2.3 skrll 286 1.3.2.3 skrll KDASSERTMSG(mbox0 == __BIT(16), "mbox0 %#x", mbox0); 287 1.3.2.3 skrll KDASSERTMSG(mbox1 == 0, "mbox1 %#x", mbox1); 288 1.3.2.3 skrll 289 1.3.2.4 skrll sum0 = mips3_ld(int_sum0); 290 1.3.2.4 skrll sum1 = mips3_ld(int_sum1); 291 1.3.2.3 skrll 292 1.3.2.3 skrll KDASSERTMSG((sum0 & sum_mbox_lo) == 0, "sum0 %#"PRIx64, sum0); 293 1.3.2.3 skrll KDASSERTMSG((sum0 & sum_mbox_hi) != 0, "sum0 %#"PRIx64, sum0); 294 1.3.2.3 skrll 295 1.3.2.3 skrll KDASSERTMSG((sum1 & sum_mbox_lo) == 0, "sum1 %#"PRIx64, sum1); 296 1.3.2.3 skrll KDASSERTMSG((sum1 & sum_mbox_hi) == 0, "sum1 %#"PRIx64, sum1); 297 1.3.2.3 skrll } 298 1.3.2.3 skrll #endif 299 1.3.2.3 skrll 300 1.3.2.2 skrll #undef X 301 1.3.2.2 skrll 302 1.3.2.2 skrll void 303 1.3.2.2 skrll octeon_intr_init(struct cpu_info *ci) 304 1.3.2.2 skrll { 305 1.3.2.5 skrll #ifdef DIAGNOSTIC 306 1.3.2.2 skrll const int cpunum = cpu_index(ci); 307 1.3.2.5 skrll #endif 308 1.3.2.2 skrll const char * const xname = cpu_name(ci); 309 1.3.2.3 skrll struct cpu_softc *cpu = ci->ci_softc; 310 1.3.2.2 skrll 311 1.3.2.2 skrll 312 1.3.2.2 skrll if (ci->ci_cpuid == 0) { 313 1.3.2.3 skrll KASSERT(ci->ci_softc == &octeon_cpu0_softc); 314 1.3.2.3 skrll ipl_sr_map = octeon_ipl_sr_map; 315 1.3.2.2 skrll mutex_init(&octeon_intr_lock, MUTEX_DEFAULT, IPL_HIGH); 316 1.3.2.2 skrll #ifdef MULTIPROCESSOR 317 1.3.2.2 skrll mips_locoresw.lsw_send_ipi = octeon_send_ipi; 318 1.3.2.2 skrll #endif 319 1.3.2.3 skrll #ifdef DEBUG 320 1.3.2.3 skrll octeon_mbox_test(); 321 1.3.2.3 skrll #endif 322 1.3.2.2 skrll } else { 323 1.3.2.2 skrll KASSERT(cpunum == 1); 324 1.3.2.2 skrll #ifdef MULTIPROCESSOR 325 1.3.2.3 skrll KASSERT(ci->ci_softc == &octeon_cpu1_softc); 326 1.3.2.2 skrll #endif 327 1.3.2.2 skrll } 328 1.3.2.2 skrll 329 1.3.2.2 skrll #ifdef MULTIPROCESSOR 330 1.3.2.2 skrll // Enable the IPIs 331 1.3.2.5 skrll cpu->cpu_int1_enable0 |= __BIT(_CIU_INT_MBOX_15_0_SHIFT); 332 1.3.2.2 skrll cpu->cpu_int2_enable0 |= __BIT(_CIU_INT_MBOX_31_16_SHIFT); 333 1.3.2.2 skrll #endif 334 1.3.2.2 skrll 335 1.3.2.3 skrll if (ci->ci_dev) 336 1.3.2.6 skrll aprint_verbose_dev(ci->ci_dev, 337 1.3.2.6 skrll "enabling intr masks %#"PRIx64"/%#"PRIx64"/%#"PRIx64"\n", 338 1.3.2.6 skrll cpu->cpu_int0_enable0, cpu->cpu_int1_enable0, 339 1.3.2.6 skrll cpu->cpu_int2_enable0); 340 1.3.2.3 skrll 341 1.3.2.4 skrll mips3_sd(cpu->cpu_int0_en0, cpu->cpu_int0_enable0); 342 1.3.2.4 skrll mips3_sd(cpu->cpu_int1_en0, cpu->cpu_int1_enable0); 343 1.3.2.4 skrll mips3_sd(cpu->cpu_int2_en0, cpu->cpu_int2_enable0); 344 1.3.2.2 skrll 345 1.3.2.4 skrll mips3_sd(cpu->cpu_int32_en, 0); 346 1.3.2.2 skrll 347 1.3.2.4 skrll mips3_sd(cpu->cpu_int0_en1, 0); // WDOG IPL2 348 1.3.2.4 skrll mips3_sd(cpu->cpu_int1_en1, 0); // WDOG IPL3 349 1.3.2.4 skrll mips3_sd(cpu->cpu_int2_en1, 0); // WDOG IPL4 350 1.3.2.2 skrll 351 1.3.2.2 skrll #ifdef MULTIPROCESSOR 352 1.3.2.4 skrll mips3_sd(cpu->cpu_mbox_clr, __BITS(31,0)); 353 1.3.2.2 skrll #endif 354 1.3.2.2 skrll 355 1.3.2.2 skrll for (size_t i = 0; i < NIRQS; i++) { 356 1.3.2.2 skrll evcnt_attach_dynamic(&cpu->cpu_intr_evs[i], 357 1.3.2.2 skrll EVCNT_TYPE_INTR, NULL, xname, octeon_intrnames[i]); 358 1.3.2.2 skrll } 359 1.3.2.2 skrll } 360 1.3.2.2 skrll 361 1.3.2.2 skrll void 362 1.3.2.2 skrll octeon_cal_timer(int corefreq) 363 1.3.2.2 skrll { 364 1.3.2.2 skrll /* Compute the number of cycles per second. */ 365 1.3.2.2 skrll curcpu()->ci_cpu_freq = corefreq; 366 1.3.2.2 skrll 367 1.3.2.2 skrll /* Compute the number of ticks for hz. */ 368 1.3.2.2 skrll curcpu()->ci_cycles_per_hz = (curcpu()->ci_cpu_freq + hz / 2) / hz; 369 1.3.2.2 skrll 370 1.3.2.2 skrll /* Compute the delay divisor and reciprical. */ 371 1.3.2.2 skrll curcpu()->ci_divisor_delay = 372 1.3.2.2 skrll ((curcpu()->ci_cpu_freq + 500000) / 1000000); 373 1.3.2.2 skrll #if 0 374 1.3.2.2 skrll MIPS_SET_CI_RECIPRICAL(curcpu()); 375 1.3.2.2 skrll #endif 376 1.3.2.2 skrll 377 1.3.2.2 skrll mips3_cp0_count_write(0); 378 1.3.2.2 skrll mips3_cp0_compare_write(0); 379 1.3.2.2 skrll } 380 1.3.2.2 skrll 381 1.3.2.2 skrll void * 382 1.3.2.2 skrll octeon_intr_establish(int irq, int ipl, int (*func)(void *), void *arg) 383 1.3.2.2 skrll { 384 1.3.2.2 skrll struct octeon_intrhand *ih; 385 1.3.2.2 skrll 386 1.3.2.2 skrll if (irq >= NIRQS) 387 1.3.2.2 skrll panic("octeon_intr_establish: bogus IRQ %d", irq); 388 1.3.2.2 skrll if (ipl < IPL_VM) 389 1.3.2.2 skrll panic("octeon_intr_establish: bogus IPL %d", ipl); 390 1.3.2.2 skrll 391 1.3.2.2 skrll ih = kmem_zalloc(sizeof(*ih), KM_NOSLEEP); 392 1.3.2.2 skrll if (ih == NULL) 393 1.3.2.2 skrll return (NULL); 394 1.3.2.2 skrll 395 1.3.2.2 skrll ih->ih_func = func; 396 1.3.2.2 skrll ih->ih_arg = arg; 397 1.3.2.2 skrll ih->ih_irq = irq; 398 1.3.2.2 skrll ih->ih_ipl = ipl; 399 1.3.2.2 skrll 400 1.3.2.2 skrll mutex_enter(&octeon_intr_lock); 401 1.3.2.2 skrll 402 1.3.2.2 skrll /* 403 1.3.2.2 skrll * First, make it known. 404 1.3.2.2 skrll */ 405 1.3.2.2 skrll KASSERTMSG(octeon_ciu_intrs[irq] == NULL, "irq %d in use! (%p)", 406 1.3.2.2 skrll irq, octeon_ciu_intrs[irq]); 407 1.3.2.2 skrll 408 1.3.2.2 skrll octeon_ciu_intrs[irq] = ih; 409 1.3.2.2 skrll membar_producer(); 410 1.3.2.2 skrll 411 1.3.2.2 skrll /* 412 1.3.2.2 skrll * Now enable it. 413 1.3.2.2 skrll */ 414 1.3.2.2 skrll const uint64_t irq_mask = __BIT(irq); 415 1.3.2.2 skrll struct cpu_softc * const cpu0 = &octeon_cpu0_softc; 416 1.3.2.2 skrll #if MULTIPROCESSOR 417 1.3.2.2 skrll struct cpu_softc * const cpu1 = &octeon_cpu1_softc; 418 1.3.2.2 skrll #endif 419 1.3.2.2 skrll 420 1.3.2.2 skrll switch (ipl) { 421 1.3.2.2 skrll case IPL_VM: 422 1.3.2.2 skrll cpu0->cpu_int0_enable0 |= irq_mask; 423 1.3.2.4 skrll mips3_sd(cpu0->cpu_int0_en0, cpu0->cpu_int0_enable0); 424 1.3.2.2 skrll break; 425 1.3.2.2 skrll 426 1.3.2.2 skrll case IPL_SCHED: 427 1.3.2.2 skrll cpu0->cpu_int1_enable0 |= irq_mask; 428 1.3.2.4 skrll mips3_sd(cpu0->cpu_int1_en0, cpu0->cpu_int1_enable0); 429 1.3.2.2 skrll #ifdef MULTIPROCESSOR 430 1.3.2.2 skrll cpu1->cpu_int1_enable0 = cpu0->cpu_int1_enable0; 431 1.3.2.4 skrll mips3_sd(cpu1->cpu_int1_en0, cpu1->cpu_int1_enable0); 432 1.3.2.2 skrll #endif 433 1.3.2.2 skrll break; 434 1.3.2.2 skrll 435 1.3.2.2 skrll case IPL_DDB: 436 1.3.2.2 skrll case IPL_HIGH: 437 1.3.2.2 skrll cpu0->cpu_int2_enable0 |= irq_mask; 438 1.3.2.4 skrll mips3_sd(cpu0->cpu_int2_en0, cpu0->cpu_int2_enable0); 439 1.3.2.2 skrll #ifdef MULTIPROCESSOR 440 1.3.2.2 skrll cpu1->cpu_int2_enable0 = cpu0->cpu_int2_enable0; 441 1.3.2.4 skrll mips3_sd(cpu1->cpu_int2_en0, cpu1->cpu_int2_enable0); 442 1.3.2.2 skrll #endif 443 1.3.2.2 skrll break; 444 1.3.2.2 skrll } 445 1.3.2.2 skrll 446 1.3.2.2 skrll mutex_exit(&octeon_intr_lock); 447 1.3.2.2 skrll 448 1.3.2.2 skrll return ih; 449 1.3.2.2 skrll } 450 1.3.2.2 skrll 451 1.3.2.2 skrll void 452 1.3.2.2 skrll octeon_intr_disestablish(void *cookie) 453 1.3.2.2 skrll { 454 1.3.2.2 skrll struct octeon_intrhand * const ih = cookie; 455 1.3.2.2 skrll const int irq = ih->ih_irq & (NIRQS-1); 456 1.3.2.2 skrll const int ipl = ih->ih_ipl; 457 1.3.2.2 skrll 458 1.3.2.2 skrll mutex_enter(&octeon_intr_lock); 459 1.3.2.2 skrll 460 1.3.2.2 skrll /* 461 1.3.2.2 skrll * First disable it. 462 1.3.2.2 skrll */ 463 1.3.2.2 skrll const uint64_t irq_mask = ~__BIT(irq); 464 1.3.2.2 skrll struct cpu_softc * const cpu0 = &octeon_cpu0_softc; 465 1.3.2.2 skrll #if MULTIPROCESSOR 466 1.3.2.2 skrll struct cpu_softc * const cpu1 = &octeon_cpu1_softc; 467 1.3.2.2 skrll #endif 468 1.3.2.2 skrll 469 1.3.2.2 skrll switch (ipl) { 470 1.3.2.2 skrll case IPL_VM: 471 1.3.2.2 skrll cpu0->cpu_int0_enable0 &= ~irq_mask; 472 1.3.2.4 skrll mips3_sd(cpu0->cpu_int0_en0, cpu0->cpu_int0_enable0); 473 1.3.2.2 skrll break; 474 1.3.2.2 skrll 475 1.3.2.2 skrll case IPL_SCHED: 476 1.3.2.2 skrll cpu0->cpu_int1_enable0 &= ~irq_mask; 477 1.3.2.4 skrll mips3_sd(cpu0->cpu_int1_en0, cpu0->cpu_int1_enable0); 478 1.3.2.2 skrll #ifdef MULTIPROCESSOR 479 1.3.2.2 skrll cpu1->cpu_int1_enable0 = cpu0->cpu_int1_enable0; 480 1.3.2.4 skrll mips3_sd(cpu1->cpu_int1_en0, cpu1->cpu_int1_enable0); 481 1.3.2.2 skrll #endif 482 1.3.2.2 skrll break; 483 1.3.2.2 skrll 484 1.3.2.2 skrll case IPL_DDB: 485 1.3.2.2 skrll case IPL_HIGH: 486 1.3.2.2 skrll cpu0->cpu_int2_enable0 &= ~irq_mask; 487 1.3.2.4 skrll mips3_sd(cpu0->cpu_int2_en0, cpu0->cpu_int2_enable0); 488 1.3.2.2 skrll #ifdef MULTIPROCESSOR 489 1.3.2.2 skrll cpu1->cpu_int2_enable0 = cpu0->cpu_int2_enable0; 490 1.3.2.4 skrll mips3_sd(cpu1->cpu_int2_en0, cpu1->cpu_int2_enable0); 491 1.3.2.2 skrll #endif 492 1.3.2.2 skrll break; 493 1.3.2.2 skrll } 494 1.3.2.2 skrll 495 1.3.2.2 skrll /* 496 1.3.2.2 skrll * Now remove it since we shouldn't get interrupts for it. 497 1.3.2.2 skrll */ 498 1.3.2.2 skrll octeon_ciu_intrs[irq] = NULL; 499 1.3.2.2 skrll 500 1.3.2.2 skrll mutex_exit(&octeon_intr_lock); 501 1.3.2.2 skrll 502 1.3.2.2 skrll kmem_free(ih, sizeof(*ih)); 503 1.3.2.2 skrll } 504 1.3.2.2 skrll 505 1.3.2.2 skrll void 506 1.3.2.2 skrll octeon_iointr(int ipl, vaddr_t pc, uint32_t ipending) 507 1.3.2.2 skrll { 508 1.3.2.2 skrll struct cpu_info * const ci = curcpu(); 509 1.3.2.2 skrll struct cpu_softc * const cpu = ci->ci_softc; 510 1.3.2.2 skrll 511 1.3.2.3 skrll KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); 512 1.3.2.2 skrll KASSERT((ipending & ~MIPS_INT_MASK) == 0); 513 1.3.2.2 skrll KASSERT(ipending & MIPS_HARD_INT_MASK); 514 1.3.2.2 skrll uint64_t hwpend = 0; 515 1.3.2.2 skrll 516 1.3.2.2 skrll if (ipending & MIPS_INT_MASK_2) { 517 1.3.2.4 skrll hwpend = mips3_ld(cpu->cpu_int2_sum0) 518 1.3.2.2 skrll & cpu->cpu_int2_enable0; 519 1.3.2.2 skrll } else if (ipending & MIPS_INT_MASK_1) { 520 1.3.2.4 skrll hwpend = mips3_ld(cpu->cpu_int1_sum0) 521 1.3.2.2 skrll & cpu->cpu_int1_enable0; 522 1.3.2.2 skrll } else if (ipending & MIPS_INT_MASK_0) { 523 1.3.2.4 skrll hwpend = mips3_ld(cpu->cpu_int0_sum0) 524 1.3.2.2 skrll & cpu->cpu_int0_enable0; 525 1.3.2.2 skrll } else { 526 1.3.2.2 skrll panic("octeon_iointr: unexpected ipending %#x", ipending); 527 1.3.2.2 skrll } 528 1.3.2.2 skrll while (hwpend != 0) { 529 1.3.2.2 skrll const int irq = ffs64(hwpend) - 1; 530 1.3.2.2 skrll hwpend &= ~__BIT(irq); 531 1.3.2.3 skrll 532 1.3.2.2 skrll struct octeon_intrhand * const ih = octeon_ciu_intrs[irq]; 533 1.3.2.2 skrll cpu->cpu_intr_evs[irq].ev_count++; 534 1.3.2.2 skrll if (__predict_true(ih != NULL)) { 535 1.3.2.2 skrll #ifdef MULTIPROCESSOR 536 1.3.2.2 skrll if (ipl == IPL_VM) { 537 1.3.2.2 skrll KERNEL_LOCK(1, NULL); 538 1.3.2.2 skrll #endif 539 1.3.2.2 skrll (*ih->ih_func)(ih->ih_arg); 540 1.3.2.2 skrll #ifdef MULTIPROCESSOR 541 1.3.2.2 skrll KERNEL_UNLOCK_ONE(NULL); 542 1.3.2.2 skrll } else { 543 1.3.2.2 skrll (*ih->ih_func)(ih->ih_arg); 544 1.3.2.2 skrll } 545 1.3.2.2 skrll #endif 546 1.3.2.3 skrll KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); 547 1.3.2.2 skrll } 548 1.3.2.2 skrll } 549 1.3.2.3 skrll KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); 550 1.3.2.2 skrll } 551 1.3.2.2 skrll 552 1.3.2.2 skrll #ifdef MULTIPROCESSOR 553 1.3.2.2 skrll __CTASSERT(NIPIS < 16); 554 1.3.2.2 skrll 555 1.3.2.2 skrll int 556 1.3.2.2 skrll octeon_ipi_intr(void *arg) 557 1.3.2.2 skrll { 558 1.3.2.2 skrll struct cpu_info * const ci = curcpu(); 559 1.3.2.2 skrll struct cpu_softc * const cpu = ci->ci_softc; 560 1.3.2.3 skrll uint32_t ipi_mask = (uintptr_t) arg; 561 1.3.2.3 skrll 562 1.3.2.3 skrll KASSERTMSG((ipi_mask & __BITS(31,16)) == 0 || ci->ci_cpl >= IPL_SCHED, 563 1.3.2.3 skrll "ipi_mask %#"PRIx32" cpl %d", ipi_mask, ci->ci_cpl); 564 1.3.2.2 skrll 565 1.3.2.4 skrll ipi_mask &= mips3_ld(cpu->cpu_mbox_set); 566 1.3.2.3 skrll if (ipi_mask == 0) 567 1.3.2.3 skrll return 0; 568 1.3.2.3 skrll 569 1.3.2.4 skrll mips3_sd(cpu->cpu_mbox_clr, ipi_mask); 570 1.3.2.2 skrll 571 1.3.2.2 skrll ipi_mask |= (ipi_mask >> 16); 572 1.3.2.2 skrll ipi_mask &= __BITS(15,0); 573 1.3.2.2 skrll 574 1.3.2.2 skrll KASSERT(ipi_mask < __BIT(NIPIS)); 575 1.3.2.2 skrll 576 1.3.2.3 skrll #if NWDOG > 0 577 1.3.2.3 skrll // Handle WDOG requests ourselves. 578 1.3.2.3 skrll if (ipi_mask & __BIT(IPI_WDOG)) { 579 1.3.2.3 skrll softint_schedule(cpu->cpu_wdog_sih); 580 1.3.2.3 skrll atomic_and_64(&ci->ci_request_ipis, ~__BIT(IPI_WDOG)); 581 1.3.2.3 skrll ipi_mask &= ~__BIT(IPI_WDOG); 582 1.3.2.3 skrll ci->ci_evcnt_per_ipi[IPI_WDOG].ev_count++; 583 1.3.2.3 skrll if (__predict_true(ipi_mask == 0)) 584 1.3.2.3 skrll return 1; 585 1.3.2.3 skrll } 586 1.3.2.3 skrll #endif 587 1.3.2.3 skrll 588 1.3.2.2 skrll /* if the request is clear, it was previously processed */ 589 1.3.2.2 skrll if ((ci->ci_request_ipis & ipi_mask) == 0) 590 1.3.2.2 skrll return 0; 591 1.3.2.2 skrll 592 1.3.2.2 skrll atomic_or_64(&ci->ci_active_ipis, ipi_mask); 593 1.3.2.2 skrll atomic_and_64(&ci->ci_request_ipis, ~ipi_mask); 594 1.3.2.2 skrll 595 1.3.2.2 skrll ipi_process(ci, ipi_mask); 596 1.3.2.2 skrll 597 1.3.2.2 skrll atomic_and_64(&ci->ci_active_ipis, ~ipi_mask); 598 1.3.2.2 skrll 599 1.3.2.2 skrll return 1; 600 1.3.2.2 skrll } 601 1.3.2.2 skrll 602 1.3.2.2 skrll int 603 1.3.2.2 skrll octeon_send_ipi(struct cpu_info *ci, int req) 604 1.3.2.2 skrll { 605 1.3.2.2 skrll KASSERT(req < NIPIS); 606 1.3.2.2 skrll if (ci == NULL) { 607 1.3.2.3 skrll CPU_INFO_ITERATOR cii; 608 1.3.2.3 skrll for (CPU_INFO_FOREACH(cii, ci)) { 609 1.3.2.3 skrll if (ci != curcpu()) { 610 1.3.2.3 skrll octeon_send_ipi(ci, req); 611 1.3.2.3 skrll } 612 1.3.2.3 skrll } 613 1.3.2.3 skrll return 0; 614 1.3.2.2 skrll } 615 1.3.2.3 skrll KASSERT(cold || ci->ci_softc != NULL); 616 1.3.2.3 skrll if (ci->ci_softc == NULL) 617 1.3.2.3 skrll return -1; 618 1.3.2.2 skrll 619 1.3.2.2 skrll struct cpu_softc * const cpu = ci->ci_softc; 620 1.3.2.2 skrll uint64_t ipi_mask = __BIT(req); 621 1.3.2.2 skrll 622 1.3.2.4 skrll atomic_or_64(&ci->ci_request_ipis, ipi_mask); 623 1.3.2.4 skrll if (req == IPI_SUSPEND || req == IPI_WDOG) { 624 1.3.2.2 skrll ipi_mask <<= 16; 625 1.3.2.2 skrll } 626 1.3.2.2 skrll 627 1.3.2.4 skrll mips3_sd(cpu->cpu_mbox_set, ipi_mask); 628 1.3.2.2 skrll return 0; 629 1.3.2.2 skrll } 630 1.3.2.2 skrll #endif /* MULTIPROCESSOR */ 631
Indexes created Thu Oct 02 14:10:14 GMT 2025