octeon_intr.c revision 1.3.2.5
1 1.3.2.5 skrll /* $NetBSD: octeon_intr.c,v 1.3.2.5 2016/12/05 10:54:55 skrll Exp $ */ 2 1.3.2.2 skrll /* 3 1.3.2.2 skrll * Copyright 2001, 2002 Wasabi Systems, Inc. 4 1.3.2.2 skrll * All rights reserved. 5 1.3.2.2 skrll * 6 1.3.2.2 skrll * Written by Jason R. Thorpe and Simon Burge for Wasabi Systems, Inc. 7 1.3.2.2 skrll * 8 1.3.2.2 skrll * Redistribution and use in source and binary forms, with or without 9 1.3.2.2 skrll * modification, are permitted provided that the following conditions 10 1.3.2.2 skrll * are met: 11 1.3.2.2 skrll * 1. Redistributions of source code must retain the above copyright 12 1.3.2.2 skrll * notice, this list of conditions and the following disclaimer. 13 1.3.2.2 skrll * 2. Redistributions in binary form must reproduce the above copyright 14 1.3.2.2 skrll * notice, this list of conditions and the following disclaimer in the 15 1.3.2.2 skrll * documentation and/or other materials provided with the distribution. 16 1.3.2.2 skrll * 3. All advertising materials mentioning features or use of this software 17 1.3.2.2 skrll * must display the following acknowledgement: 18 1.3.2.2 skrll * This product includes software developed for the NetBSD Project by 19 1.3.2.2 skrll * Wasabi Systems, Inc. 20 1.3.2.2 skrll * 4. The name of Wasabi Systems, Inc. may not be used to endorse 21 1.3.2.2 skrll * or promote products derived from this software without specific prior 22 1.3.2.2 skrll * written permission. 23 1.3.2.2 skrll * 24 1.3.2.2 skrll * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 25 1.3.2.2 skrll * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 1.3.2.2 skrll * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 1.3.2.2 skrll * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 28 1.3.2.2 skrll * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 1.3.2.2 skrll * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 1.3.2.2 skrll * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 1.3.2.2 skrll * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 1.3.2.2 skrll * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 1.3.2.2 skrll * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 1.3.2.2 skrll * POSSIBILITY OF SUCH DAMAGE. 35 1.3.2.2 skrll */ 36 1.3.2.2 skrll 37 1.3.2.2 skrll /* 38 1.3.2.2 skrll * Platform-specific interrupt support for the MIPS Malta. 39 1.3.2.2 skrll */ 40 1.3.2.2 skrll 41 1.3.2.2 skrll #include "opt_octeon.h" 42 1.3.2.4 skrll #include "opt_multiprocessor.h" 43 1.3.2.4 skrll 44 1.3.2.3 skrll #include "cpunode.h" 45 1.3.2.2 skrll #define __INTR_PRIVATE 46 1.3.2.2 skrll 47 1.3.2.2 skrll #include <sys/cdefs.h> 48 1.3.2.5 skrll __KERNEL_RCSID(0, "$NetBSD: octeon_intr.c,v 1.3.2.5 2016/12/05 10:54:55 skrll Exp $"); 49 1.3.2.2 skrll 50 1.3.2.2 skrll #include <sys/param.h> 51 1.3.2.2 skrll #include <sys/cpu.h> 52 1.3.2.2 skrll #include <sys/systm.h> 53 1.3.2.2 skrll #include <sys/device.h> 54 1.3.2.2 skrll #include <sys/intr.h> 55 1.3.2.2 skrll #include <sys/kernel.h> 56 1.3.2.2 skrll #include <sys/kmem.h> 57 1.3.2.2 skrll #include <sys/atomic.h> 58 1.3.2.2 skrll 59 1.3.2.2 skrll #include <lib/libkern/libkern.h> 60 1.3.2.2 skrll 61 1.3.2.2 skrll #include <mips/locore.h> 62 1.3.2.2 skrll 63 1.3.2.2 skrll #include <mips/cavium/dev/octeon_ciureg.h> 64 1.3.2.2 skrll #include <mips/cavium/octeonvar.h> 65 1.3.2.2 skrll 66 1.3.2.2 skrll /* 67 1.3.2.2 skrll * This is a mask of bits to clear in the SR when we go to a 68 1.3.2.2 skrll * given hardware interrupt priority level. 69 1.3.2.2 skrll */ 70 1.3.2.2 skrll static const struct ipl_sr_map octeon_ipl_sr_map = { 71 1.3.2.2 skrll .sr_bits = { 72 1.3.2.2 skrll [IPL_NONE] = 0, 73 1.3.2.2 skrll [IPL_SOFTCLOCK] = MIPS_SOFT_INT_MASK_0, 74 1.3.2.2 skrll [IPL_SOFTNET] = MIPS_SOFT_INT_MASK, 75 1.3.2.2 skrll [IPL_VM] = MIPS_SOFT_INT_MASK | MIPS_INT_MASK_0, 76 1.3.2.2 skrll [IPL_SCHED] = MIPS_SOFT_INT_MASK | MIPS_INT_MASK_0 77 1.3.2.5 skrll | MIPS_INT_MASK_1 | MIPS_INT_MASK_5, 78 1.3.2.2 skrll [IPL_DDB] = MIPS_SOFT_INT_MASK | MIPS_INT_MASK_0 79 1.3.2.2 skrll | MIPS_INT_MASK_1 | MIPS_INT_MASK_5, 80 1.3.2.2 skrll [IPL_HIGH] = MIPS_INT_MASK, 81 1.3.2.2 skrll }, 82 1.3.2.2 skrll }; 83 1.3.2.2 skrll 84 1.3.2.2 skrll const char * const octeon_intrnames[NIRQS] = { 85 1.3.2.2 skrll "workq 0", 86 1.3.2.2 skrll "workq 1", 87 1.3.2.2 skrll "workq 2", 88 1.3.2.2 skrll "workq 3", 89 1.3.2.2 skrll "workq 4", 90 1.3.2.2 skrll "workq 5", 91 1.3.2.2 skrll "workq 6", 92 1.3.2.2 skrll "workq 7", 93 1.3.2.2 skrll "workq 8", 94 1.3.2.2 skrll "workq 9", 95 1.3.2.2 skrll "workq 10", 96 1.3.2.2 skrll "workq 11", 97 1.3.2.2 skrll "workq 12", 98 1.3.2.2 skrll "workq 13", 99 1.3.2.2 skrll "workq 14", 100 1.3.2.2 skrll "workq 15", 101 1.3.2.2 skrll "gpio 0", 102 1.3.2.2 skrll "gpio 1", 103 1.3.2.2 skrll "gpio 2", 104 1.3.2.2 skrll "gpio 3", 105 1.3.2.2 skrll "gpio 4", 106 1.3.2.2 skrll "gpio 5", 107 1.3.2.2 skrll "gpio 6", 108 1.3.2.2 skrll "gpio 7", 109 1.3.2.2 skrll "gpio 8", 110 1.3.2.2 skrll "gpio 9", 111 1.3.2.2 skrll "gpio 10", 112 1.3.2.2 skrll "gpio 11", 113 1.3.2.2 skrll "gpio 12", 114 1.3.2.2 skrll "gpio 13", 115 1.3.2.2 skrll "gpio 14", 116 1.3.2.2 skrll "gpio 15", 117 1.3.2.2 skrll "mbox 0-15", 118 1.3.2.2 skrll "mbox 16-31", 119 1.3.2.2 skrll "uart 0", 120 1.3.2.2 skrll "uart 1", 121 1.3.2.2 skrll "pci inta", 122 1.3.2.2 skrll "pci intb", 123 1.3.2.2 skrll "pci intc", 124 1.3.2.2 skrll "pci intd", 125 1.3.2.2 skrll "pci msi 0-15", 126 1.3.2.2 skrll "pci msi 16-31", 127 1.3.2.2 skrll "pci msi 32-47", 128 1.3.2.2 skrll "pci msi 48-63", 129 1.3.2.2 skrll "wdog summary", 130 1.3.2.2 skrll "twsi", 131 1.3.2.2 skrll "rml", 132 1.3.2.2 skrll "trace", 133 1.3.2.2 skrll "gmx drop", 134 1.3.2.2 skrll "reserved", 135 1.3.2.2 skrll "ipd drop", 136 1.3.2.2 skrll "reserved", 137 1.3.2.2 skrll "timer 0", 138 1.3.2.2 skrll "timer 1", 139 1.3.2.2 skrll "timer 2", 140 1.3.2.2 skrll "timer 3", 141 1.3.2.2 skrll "usb", 142 1.3.2.2 skrll "pcm/tdm", 143 1.3.2.2 skrll "mpi/spi", 144 1.3.2.2 skrll "reserved", 145 1.3.2.2 skrll "reserved", 146 1.3.2.2 skrll "reserved", 147 1.3.2.2 skrll "reserved", 148 1.3.2.2 skrll "reserved", 149 1.3.2.2 skrll }; 150 1.3.2.2 skrll 151 1.3.2.2 skrll struct octeon_intrhand { 152 1.3.2.2 skrll int (*ih_func)(void *); 153 1.3.2.2 skrll void *ih_arg; 154 1.3.2.2 skrll int ih_irq; 155 1.3.2.2 skrll int ih_ipl; 156 1.3.2.2 skrll }; 157 1.3.2.2 skrll 158 1.3.2.2 skrll #ifdef MULTIPROCESSOR 159 1.3.2.2 skrll static int octeon_send_ipi(struct cpu_info *, int); 160 1.3.2.2 skrll static int octeon_ipi_intr(void *); 161 1.3.2.2 skrll 162 1.3.2.2 skrll struct octeon_intrhand ipi_intrhands[2] = { 163 1.3.2.2 skrll [0] = { 164 1.3.2.2 skrll .ih_func = octeon_ipi_intr, 165 1.3.2.2 skrll .ih_arg = (void *)(uintptr_t)__BITS(15,0), 166 1.3.2.2 skrll .ih_irq = _CIU_INT_MBOX_15_0_SHIFT, 167 1.3.2.2 skrll .ih_ipl = IPL_SCHED, 168 1.3.2.2 skrll }, 169 1.3.2.2 skrll [1] = { 170 1.3.2.2 skrll .ih_func = octeon_ipi_intr, 171 1.3.2.2 skrll .ih_arg = (void *)(uintptr_t)__BITS(31,16), 172 1.3.2.2 skrll .ih_irq = _CIU_INT_MBOX_31_16_SHIFT, 173 1.3.2.2 skrll .ih_ipl = IPL_HIGH, 174 1.3.2.2 skrll }, 175 1.3.2.2 skrll }; 176 1.3.2.2 skrll #endif 177 1.3.2.2 skrll 178 1.3.2.2 skrll struct octeon_intrhand *octeon_ciu_intrs[NIRQS] = { 179 1.3.2.2 skrll #ifdef MULTIPROCESSOR 180 1.3.2.2 skrll [_CIU_INT_MBOX_15_0_SHIFT] = &ipi_intrhands[0], 181 1.3.2.2 skrll [_CIU_INT_MBOX_31_16_SHIFT] = &ipi_intrhands[1], 182 1.3.2.2 skrll #endif 183 1.3.2.2 skrll }; 184 1.3.2.2 skrll 185 1.3.2.2 skrll kmutex_t octeon_intr_lock; 186 1.3.2.2 skrll 187 1.3.2.2 skrll #define X(a) MIPS_PHYS_TO_XKPHYS(OCTEON_CCA_NONE, (a)) 188 1.3.2.2 skrll 189 1.3.2.2 skrll struct cpu_softc octeon_cpu0_softc = { 190 1.3.2.2 skrll .cpu_ci = &cpu_info_store, 191 1.3.2.2 skrll .cpu_int0_sum0 = X(CIU_INT0_SUM0), 192 1.3.2.2 skrll .cpu_int1_sum0 = X(CIU_INT1_SUM0), 193 1.3.2.2 skrll .cpu_int2_sum0 = X(CIU_INT4_SUM0), 194 1.3.2.2 skrll 195 1.3.2.2 skrll .cpu_int0_en0 = X(CIU_INT0_EN0), 196 1.3.2.2 skrll .cpu_int1_en0 = X(CIU_INT1_EN0), 197 1.3.2.2 skrll .cpu_int2_en0 = X(CIU_INT4_EN00), 198 1.3.2.2 skrll 199 1.3.2.2 skrll .cpu_int0_en1 = X(CIU_INT0_EN1), 200 1.3.2.2 skrll .cpu_int1_en1 = X(CIU_INT1_EN1), 201 1.3.2.2 skrll .cpu_int2_en1 = X(CIU_INT4_EN01), 202 1.3.2.2 skrll 203 1.3.2.2 skrll .cpu_int32_en = X(CIU_INT32_EN0), 204 1.3.2.2 skrll 205 1.3.2.3 skrll .cpu_wdog = X(CIU_WDOG0), 206 1.3.2.3 skrll .cpu_pp_poke = X(CIU_PP_POKE0), 207 1.3.2.3 skrll 208 1.3.2.2 skrll #ifdef MULTIPROCESSOR 209 1.3.2.2 skrll .cpu_mbox_set = X(CIU_MBOX_SET0), 210 1.3.2.2 skrll .cpu_mbox_clr = X(CIU_MBOX_CLR0), 211 1.3.2.2 skrll #endif 212 1.3.2.2 skrll }; 213 1.3.2.2 skrll 214 1.3.2.2 skrll #ifdef MULTIPROCESSOR 215 1.3.2.2 skrll struct cpu_softc octeon_cpu1_softc = { 216 1.3.2.2 skrll .cpu_int0_sum0 = X(CIU_INT2_SUM0), 217 1.3.2.2 skrll .cpu_int1_sum0 = X(CIU_INT3_SUM0), 218 1.3.2.2 skrll .cpu_int2_sum0 = X(CIU_INT4_SUM1), 219 1.3.2.2 skrll 220 1.3.2.2 skrll .cpu_int0_en0 = X(CIU_INT2_EN0), 221 1.3.2.2 skrll .cpu_int1_en0 = X(CIU_INT3_EN0), 222 1.3.2.2 skrll .cpu_int2_en0 = X(CIU_INT4_EN10), 223 1.3.2.2 skrll 224 1.3.2.2 skrll .cpu_int0_en1 = X(CIU_INT2_EN1), 225 1.3.2.2 skrll .cpu_int1_en1 = X(CIU_INT3_EN1), 226 1.3.2.2 skrll .cpu_int2_en1 = X(CIU_INT4_EN11), 227 1.3.2.2 skrll 228 1.3.2.2 skrll .cpu_int32_en = X(CIU_INT32_EN1), 229 1.3.2.2 skrll 230 1.3.2.3 skrll .cpu_wdog = X(CIU_WDOG1), 231 1.3.2.3 skrll .cpu_pp_poke = X(CIU_PP_POKE1), 232 1.3.2.3 skrll 233 1.3.2.2 skrll .cpu_mbox_set = X(CIU_MBOX_SET1), 234 1.3.2.2 skrll .cpu_mbox_clr = X(CIU_MBOX_CLR1), 235 1.3.2.2 skrll }; 236 1.3.2.2 skrll #endif 237 1.3.2.2 skrll 238 1.3.2.3 skrll #ifdef DEBUG 239 1.3.2.3 skrll static void 240 1.3.2.3 skrll octeon_mbox_test(void) 241 1.3.2.3 skrll { 242 1.3.2.3 skrll const uint64_t mbox_clr0 = X(CIU_MBOX_CLR0); 243 1.3.2.3 skrll const uint64_t mbox_clr1 = X(CIU_MBOX_CLR1); 244 1.3.2.3 skrll const uint64_t mbox_set0 = X(CIU_MBOX_SET0); 245 1.3.2.3 skrll const uint64_t mbox_set1 = X(CIU_MBOX_SET1); 246 1.3.2.3 skrll const uint64_t int_sum0 = X(CIU_INT0_SUM0); 247 1.3.2.3 skrll const uint64_t int_sum1 = X(CIU_INT2_SUM0); 248 1.3.2.3 skrll const uint64_t sum_mbox_lo = __BIT(_CIU_INT_MBOX_15_0_SHIFT); 249 1.3.2.3 skrll const uint64_t sum_mbox_hi = __BIT(_CIU_INT_MBOX_31_16_SHIFT); 250 1.3.2.3 skrll 251 1.3.2.4 skrll mips3_sd(mbox_clr0, ~0ULL); 252 1.3.2.4 skrll mips3_sd(mbox_clr1, ~0ULL); 253 1.3.2.3 skrll 254 1.3.2.4 skrll uint32_t mbox0 = mips3_ld(mbox_set0); 255 1.3.2.4 skrll uint32_t mbox1 = mips3_ld(mbox_set1); 256 1.3.2.3 skrll 257 1.3.2.3 skrll KDASSERTMSG(mbox0 == 0, "mbox0 %#x mbox1 %#x", mbox0, mbox1); 258 1.3.2.3 skrll KDASSERTMSG(mbox1 == 0, "mbox0 %#x mbox1 %#x", mbox0, mbox1); 259 1.3.2.3 skrll 260 1.3.2.4 skrll mips3_sd(mbox_set0, __BIT(0)); 261 1.3.2.3 skrll 262 1.3.2.4 skrll mbox0 = mips3_ld(mbox_set0); 263 1.3.2.4 skrll mbox1 = mips3_ld(mbox_set1); 264 1.3.2.3 skrll 265 1.3.2.3 skrll KDASSERTMSG(mbox0 == 1, "mbox0 %#x mbox1 %#x", mbox0, mbox1); 266 1.3.2.3 skrll KDASSERTMSG(mbox1 == 0, "mbox0 %#x mbox1 %#x", mbox0, mbox1); 267 1.3.2.3 skrll 268 1.3.2.4 skrll uint64_t sum0 = mips3_ld(int_sum0); 269 1.3.2.4 skrll uint64_t sum1 = mips3_ld(int_sum1); 270 1.3.2.3 skrll 271 1.3.2.3 skrll KDASSERTMSG((sum0 & sum_mbox_lo) != 0, "sum0 %#"PRIx64, sum0); 272 1.3.2.3 skrll KDASSERTMSG((sum0 & sum_mbox_hi) == 0, "sum0 %#"PRIx64, sum0); 273 1.3.2.3 skrll 274 1.3.2.3 skrll KDASSERTMSG((sum1 & sum_mbox_lo) == 0, "sum1 %#"PRIx64, sum1); 275 1.3.2.3 skrll KDASSERTMSG((sum1 & sum_mbox_hi) == 0, "sum1 %#"PRIx64, sum1); 276 1.3.2.3 skrll 277 1.3.2.4 skrll mips3_sd(mbox_clr0, mbox0); 278 1.3.2.4 skrll mbox0 = mips3_ld(mbox_set0); 279 1.3.2.3 skrll KDASSERTMSG(mbox0 == 0, "mbox0 %#x", mbox0); 280 1.3.2.3 skrll 281 1.3.2.4 skrll mips3_sd(mbox_set0, __BIT(16)); 282 1.3.2.3 skrll 283 1.3.2.4 skrll mbox0 = mips3_ld(mbox_set0); 284 1.3.2.4 skrll mbox1 = mips3_ld(mbox_set1); 285 1.3.2.3 skrll 286 1.3.2.3 skrll KDASSERTMSG(mbox0 == __BIT(16), "mbox0 %#x", mbox0); 287 1.3.2.3 skrll KDASSERTMSG(mbox1 == 0, "mbox1 %#x", mbox1); 288 1.3.2.3 skrll 289 1.3.2.4 skrll sum0 = mips3_ld(int_sum0); 290 1.3.2.4 skrll sum1 = mips3_ld(int_sum1); 291 1.3.2.3 skrll 292 1.3.2.3 skrll KDASSERTMSG((sum0 & sum_mbox_lo) == 0, "sum0 %#"PRIx64, sum0); 293 1.3.2.3 skrll KDASSERTMSG((sum0 & sum_mbox_hi) != 0, "sum0 %#"PRIx64, sum0); 294 1.3.2.3 skrll 295 1.3.2.3 skrll KDASSERTMSG((sum1 & sum_mbox_lo) == 0, "sum1 %#"PRIx64, sum1); 296 1.3.2.3 skrll KDASSERTMSG((sum1 & sum_mbox_hi) == 0, "sum1 %#"PRIx64, sum1); 297 1.3.2.3 skrll } 298 1.3.2.3 skrll #endif 299 1.3.2.3 skrll 300 1.3.2.2 skrll #undef X 301 1.3.2.2 skrll 302 1.3.2.2 skrll void 303 1.3.2.2 skrll octeon_intr_init(struct cpu_info *ci) 304 1.3.2.2 skrll { 305 1.3.2.5 skrll #ifdef DIAGNOSTIC 306 1.3.2.2 skrll const int cpunum = cpu_index(ci); 307 1.3.2.5 skrll #endif 308 1.3.2.2 skrll const char * const xname = cpu_name(ci); 309 1.3.2.3 skrll struct cpu_softc *cpu = ci->ci_softc; 310 1.3.2.2 skrll 311 1.3.2.2 skrll 312 1.3.2.2 skrll if (ci->ci_cpuid == 0) { 313 1.3.2.3 skrll KASSERT(ci->ci_softc == &octeon_cpu0_softc); 314 1.3.2.3 skrll ipl_sr_map = octeon_ipl_sr_map; 315 1.3.2.2 skrll mutex_init(&octeon_intr_lock, MUTEX_DEFAULT, IPL_HIGH); 316 1.3.2.2 skrll #ifdef MULTIPROCESSOR 317 1.3.2.2 skrll mips_locoresw.lsw_send_ipi = octeon_send_ipi; 318 1.3.2.2 skrll #endif 319 1.3.2.3 skrll #ifdef DEBUG 320 1.3.2.3 skrll octeon_mbox_test(); 321 1.3.2.3 skrll #endif 322 1.3.2.2 skrll } else { 323 1.3.2.2 skrll KASSERT(cpunum == 1); 324 1.3.2.2 skrll #ifdef MULTIPROCESSOR 325 1.3.2.3 skrll KASSERT(ci->ci_softc == &octeon_cpu1_softc); 326 1.3.2.2 skrll #endif 327 1.3.2.2 skrll } 328 1.3.2.2 skrll 329 1.3.2.2 skrll #ifdef MULTIPROCESSOR 330 1.3.2.2 skrll // Enable the IPIs 331 1.3.2.5 skrll cpu->cpu_int1_enable0 |= __BIT(_CIU_INT_MBOX_15_0_SHIFT); 332 1.3.2.2 skrll cpu->cpu_int2_enable0 |= __BIT(_CIU_INT_MBOX_31_16_SHIFT); 333 1.3.2.2 skrll #endif 334 1.3.2.2 skrll 335 1.3.2.3 skrll if (ci->ci_dev) 336 1.3.2.3 skrll aprint_verbose_dev(ci->ci_dev, 337 1.3.2.3 skrll "enabling intr masks %#"PRIx64"/%#"PRIx64"/%#"PRIx64"\n", 338 1.3.2.3 skrll cpu->cpu_int0_enable0, cpu->cpu_int1_enable0, cpu->cpu_int2_enable0); 339 1.3.2.3 skrll 340 1.3.2.4 skrll mips3_sd(cpu->cpu_int0_en0, cpu->cpu_int0_enable0); 341 1.3.2.4 skrll mips3_sd(cpu->cpu_int1_en0, cpu->cpu_int1_enable0); 342 1.3.2.4 skrll mips3_sd(cpu->cpu_int2_en0, cpu->cpu_int2_enable0); 343 1.3.2.2 skrll 344 1.3.2.4 skrll mips3_sd(cpu->cpu_int32_en, 0); 345 1.3.2.2 skrll 346 1.3.2.4 skrll mips3_sd(cpu->cpu_int0_en1, 0); // WDOG IPL2 347 1.3.2.4 skrll mips3_sd(cpu->cpu_int1_en1, 0); // WDOG IPL3 348 1.3.2.4 skrll mips3_sd(cpu->cpu_int2_en1, 0); // WDOG IPL4 349 1.3.2.2 skrll 350 1.3.2.2 skrll #ifdef MULTIPROCESSOR 351 1.3.2.4 skrll mips3_sd(cpu->cpu_mbox_clr, __BITS(31,0)); 352 1.3.2.2 skrll #endif 353 1.3.2.2 skrll 354 1.3.2.2 skrll for (size_t i = 0; i < NIRQS; i++) { 355 1.3.2.2 skrll evcnt_attach_dynamic(&cpu->cpu_intr_evs[i], 356 1.3.2.2 skrll EVCNT_TYPE_INTR, NULL, xname, octeon_intrnames[i]); 357 1.3.2.2 skrll } 358 1.3.2.2 skrll } 359 1.3.2.2 skrll 360 1.3.2.2 skrll void 361 1.3.2.2 skrll octeon_cal_timer(int corefreq) 362 1.3.2.2 skrll { 363 1.3.2.2 skrll /* Compute the number of cycles per second. */ 364 1.3.2.2 skrll curcpu()->ci_cpu_freq = corefreq; 365 1.3.2.2 skrll 366 1.3.2.2 skrll /* Compute the number of ticks for hz. */ 367 1.3.2.2 skrll curcpu()->ci_cycles_per_hz = (curcpu()->ci_cpu_freq + hz / 2) / hz; 368 1.3.2.2 skrll 369 1.3.2.2 skrll /* Compute the delay divisor and reciprical. */ 370 1.3.2.2 skrll curcpu()->ci_divisor_delay = 371 1.3.2.2 skrll ((curcpu()->ci_cpu_freq + 500000) / 1000000); 372 1.3.2.2 skrll #if 0 373 1.3.2.2 skrll MIPS_SET_CI_RECIPRICAL(curcpu()); 374 1.3.2.2 skrll #endif 375 1.3.2.2 skrll 376 1.3.2.2 skrll mips3_cp0_count_write(0); 377 1.3.2.2 skrll mips3_cp0_compare_write(0); 378 1.3.2.2 skrll } 379 1.3.2.2 skrll 380 1.3.2.2 skrll void * 381 1.3.2.2 skrll octeon_intr_establish(int irq, int ipl, int (*func)(void *), void *arg) 382 1.3.2.2 skrll { 383 1.3.2.2 skrll struct octeon_intrhand *ih; 384 1.3.2.2 skrll 385 1.3.2.2 skrll if (irq >= NIRQS) 386 1.3.2.2 skrll panic("octeon_intr_establish: bogus IRQ %d", irq); 387 1.3.2.2 skrll if (ipl < IPL_VM) 388 1.3.2.2 skrll panic("octeon_intr_establish: bogus IPL %d", ipl); 389 1.3.2.2 skrll 390 1.3.2.2 skrll ih = kmem_zalloc(sizeof(*ih), KM_NOSLEEP); 391 1.3.2.2 skrll if (ih == NULL) 392 1.3.2.2 skrll return (NULL); 393 1.3.2.2 skrll 394 1.3.2.2 skrll ih->ih_func = func; 395 1.3.2.2 skrll ih->ih_arg = arg; 396 1.3.2.2 skrll ih->ih_irq = irq; 397 1.3.2.2 skrll ih->ih_ipl = ipl; 398 1.3.2.2 skrll 399 1.3.2.2 skrll mutex_enter(&octeon_intr_lock); 400 1.3.2.2 skrll 401 1.3.2.2 skrll /* 402 1.3.2.2 skrll * First, make it known. 403 1.3.2.2 skrll */ 404 1.3.2.2 skrll KASSERTMSG(octeon_ciu_intrs[irq] == NULL, "irq %d in use! (%p)", 405 1.3.2.2 skrll irq, octeon_ciu_intrs[irq]); 406 1.3.2.2 skrll 407 1.3.2.2 skrll octeon_ciu_intrs[irq] = ih; 408 1.3.2.2 skrll membar_producer(); 409 1.3.2.2 skrll 410 1.3.2.2 skrll /* 411 1.3.2.2 skrll * Now enable it. 412 1.3.2.2 skrll */ 413 1.3.2.2 skrll const uint64_t irq_mask = __BIT(irq); 414 1.3.2.2 skrll struct cpu_softc * const cpu0 = &octeon_cpu0_softc; 415 1.3.2.2 skrll #if MULTIPROCESSOR 416 1.3.2.2 skrll struct cpu_softc * const cpu1 = &octeon_cpu1_softc; 417 1.3.2.2 skrll #endif 418 1.3.2.2 skrll 419 1.3.2.2 skrll switch (ipl) { 420 1.3.2.2 skrll case IPL_VM: 421 1.3.2.2 skrll cpu0->cpu_int0_enable0 |= irq_mask; 422 1.3.2.4 skrll mips3_sd(cpu0->cpu_int0_en0, cpu0->cpu_int0_enable0); 423 1.3.2.2 skrll break; 424 1.3.2.2 skrll 425 1.3.2.2 skrll case IPL_SCHED: 426 1.3.2.2 skrll cpu0->cpu_int1_enable0 |= irq_mask; 427 1.3.2.4 skrll mips3_sd(cpu0->cpu_int1_en0, cpu0->cpu_int1_enable0); 428 1.3.2.2 skrll #ifdef MULTIPROCESSOR 429 1.3.2.2 skrll cpu1->cpu_int1_enable0 = cpu0->cpu_int1_enable0; 430 1.3.2.4 skrll mips3_sd(cpu1->cpu_int1_en0, cpu1->cpu_int1_enable0); 431 1.3.2.2 skrll #endif 432 1.3.2.2 skrll break; 433 1.3.2.2 skrll 434 1.3.2.2 skrll case IPL_DDB: 435 1.3.2.2 skrll case IPL_HIGH: 436 1.3.2.2 skrll cpu0->cpu_int2_enable0 |= irq_mask; 437 1.3.2.4 skrll mips3_sd(cpu0->cpu_int2_en0, cpu0->cpu_int2_enable0); 438 1.3.2.2 skrll #ifdef MULTIPROCESSOR 439 1.3.2.2 skrll cpu1->cpu_int2_enable0 = cpu0->cpu_int2_enable0; 440 1.3.2.4 skrll mips3_sd(cpu1->cpu_int2_en0, cpu1->cpu_int2_enable0); 441 1.3.2.2 skrll #endif 442 1.3.2.2 skrll break; 443 1.3.2.2 skrll } 444 1.3.2.2 skrll 445 1.3.2.2 skrll mutex_exit(&octeon_intr_lock); 446 1.3.2.2 skrll 447 1.3.2.2 skrll return ih; 448 1.3.2.2 skrll } 449 1.3.2.2 skrll 450 1.3.2.2 skrll void 451 1.3.2.2 skrll octeon_intr_disestablish(void *cookie) 452 1.3.2.2 skrll { 453 1.3.2.2 skrll struct octeon_intrhand * const ih = cookie; 454 1.3.2.2 skrll const int irq = ih->ih_irq & (NIRQS-1); 455 1.3.2.2 skrll const int ipl = ih->ih_ipl; 456 1.3.2.2 skrll 457 1.3.2.2 skrll mutex_enter(&octeon_intr_lock); 458 1.3.2.2 skrll 459 1.3.2.2 skrll /* 460 1.3.2.2 skrll * First disable it. 461 1.3.2.2 skrll */ 462 1.3.2.2 skrll const uint64_t irq_mask = ~__BIT(irq); 463 1.3.2.2 skrll struct cpu_softc * const cpu0 = &octeon_cpu0_softc; 464 1.3.2.2 skrll #if MULTIPROCESSOR 465 1.3.2.2 skrll struct cpu_softc * const cpu1 = &octeon_cpu1_softc; 466 1.3.2.2 skrll #endif 467 1.3.2.2 skrll 468 1.3.2.2 skrll switch (ipl) { 469 1.3.2.2 skrll case IPL_VM: 470 1.3.2.2 skrll cpu0->cpu_int0_enable0 &= ~irq_mask; 471 1.3.2.4 skrll mips3_sd(cpu0->cpu_int0_en0, cpu0->cpu_int0_enable0); 472 1.3.2.2 skrll break; 473 1.3.2.2 skrll 474 1.3.2.2 skrll case IPL_SCHED: 475 1.3.2.2 skrll cpu0->cpu_int1_enable0 &= ~irq_mask; 476 1.3.2.4 skrll mips3_sd(cpu0->cpu_int1_en0, cpu0->cpu_int1_enable0); 477 1.3.2.2 skrll #ifdef MULTIPROCESSOR 478 1.3.2.2 skrll cpu1->cpu_int1_enable0 = cpu0->cpu_int1_enable0; 479 1.3.2.4 skrll mips3_sd(cpu1->cpu_int1_en0, cpu1->cpu_int1_enable0); 480 1.3.2.2 skrll #endif 481 1.3.2.2 skrll break; 482 1.3.2.2 skrll 483 1.3.2.2 skrll case IPL_DDB: 484 1.3.2.2 skrll case IPL_HIGH: 485 1.3.2.2 skrll cpu0->cpu_int2_enable0 &= ~irq_mask; 486 1.3.2.4 skrll mips3_sd(cpu0->cpu_int2_en0, cpu0->cpu_int2_enable0); 487 1.3.2.2 skrll #ifdef MULTIPROCESSOR 488 1.3.2.2 skrll cpu1->cpu_int2_enable0 = cpu0->cpu_int2_enable0; 489 1.3.2.4 skrll mips3_sd(cpu1->cpu_int2_en0, cpu1->cpu_int2_enable0); 490 1.3.2.2 skrll #endif 491 1.3.2.2 skrll break; 492 1.3.2.2 skrll } 493 1.3.2.2 skrll 494 1.3.2.2 skrll /* 495 1.3.2.2 skrll * Now remove it since we shouldn't get interrupts for it. 496 1.3.2.2 skrll */ 497 1.3.2.2 skrll octeon_ciu_intrs[irq] = NULL; 498 1.3.2.2 skrll 499 1.3.2.2 skrll mutex_exit(&octeon_intr_lock); 500 1.3.2.2 skrll 501 1.3.2.2 skrll kmem_free(ih, sizeof(*ih)); 502 1.3.2.2 skrll } 503 1.3.2.2 skrll 504 1.3.2.2 skrll void 505 1.3.2.2 skrll octeon_iointr(int ipl, vaddr_t pc, uint32_t ipending) 506 1.3.2.2 skrll { 507 1.3.2.2 skrll struct cpu_info * const ci = curcpu(); 508 1.3.2.2 skrll struct cpu_softc * const cpu = ci->ci_softc; 509 1.3.2.2 skrll 510 1.3.2.3 skrll KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); 511 1.3.2.2 skrll KASSERT((ipending & ~MIPS_INT_MASK) == 0); 512 1.3.2.2 skrll KASSERT(ipending & MIPS_HARD_INT_MASK); 513 1.3.2.2 skrll uint64_t hwpend = 0; 514 1.3.2.2 skrll 515 1.3.2.2 skrll if (ipending & MIPS_INT_MASK_2) { 516 1.3.2.4 skrll hwpend = mips3_ld(cpu->cpu_int2_sum0) 517 1.3.2.2 skrll & cpu->cpu_int2_enable0; 518 1.3.2.2 skrll } else if (ipending & MIPS_INT_MASK_1) { 519 1.3.2.4 skrll hwpend = mips3_ld(cpu->cpu_int1_sum0) 520 1.3.2.2 skrll & cpu->cpu_int1_enable0; 521 1.3.2.2 skrll } else if (ipending & MIPS_INT_MASK_0) { 522 1.3.2.4 skrll hwpend = mips3_ld(cpu->cpu_int0_sum0) 523 1.3.2.2 skrll & cpu->cpu_int0_enable0; 524 1.3.2.2 skrll } else { 525 1.3.2.2 skrll panic("octeon_iointr: unexpected ipending %#x", ipending); 526 1.3.2.2 skrll } 527 1.3.2.2 skrll while (hwpend != 0) { 528 1.3.2.2 skrll const int irq = ffs64(hwpend) - 1; 529 1.3.2.2 skrll hwpend &= ~__BIT(irq); 530 1.3.2.3 skrll 531 1.3.2.2 skrll struct octeon_intrhand * const ih = octeon_ciu_intrs[irq]; 532 1.3.2.2 skrll cpu->cpu_intr_evs[irq].ev_count++; 533 1.3.2.2 skrll if (__predict_true(ih != NULL)) { 534 1.3.2.2 skrll #ifdef MULTIPROCESSOR 535 1.3.2.2 skrll if (ipl == IPL_VM) { 536 1.3.2.2 skrll KERNEL_LOCK(1, NULL); 537 1.3.2.2 skrll #endif 538 1.3.2.2 skrll (*ih->ih_func)(ih->ih_arg); 539 1.3.2.2 skrll #ifdef MULTIPROCESSOR 540 1.3.2.2 skrll KERNEL_UNLOCK_ONE(NULL); 541 1.3.2.2 skrll } else { 542 1.3.2.2 skrll (*ih->ih_func)(ih->ih_arg); 543 1.3.2.2 skrll } 544 1.3.2.2 skrll #endif 545 1.3.2.3 skrll KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); 546 1.3.2.2 skrll } 547 1.3.2.2 skrll } 548 1.3.2.3 skrll KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); 549 1.3.2.2 skrll } 550 1.3.2.2 skrll 551 1.3.2.2 skrll #ifdef MULTIPROCESSOR 552 1.3.2.2 skrll __CTASSERT(NIPIS < 16); 553 1.3.2.2 skrll 554 1.3.2.2 skrll int 555 1.3.2.2 skrll octeon_ipi_intr(void *arg) 556 1.3.2.2 skrll { 557 1.3.2.2 skrll struct cpu_info * const ci = curcpu(); 558 1.3.2.2 skrll struct cpu_softc * const cpu = ci->ci_softc; 559 1.3.2.3 skrll uint32_t ipi_mask = (uintptr_t) arg; 560 1.3.2.3 skrll 561 1.3.2.3 skrll KASSERTMSG((ipi_mask & __BITS(31,16)) == 0 || ci->ci_cpl >= IPL_SCHED, 562 1.3.2.3 skrll "ipi_mask %#"PRIx32" cpl %d", ipi_mask, ci->ci_cpl); 563 1.3.2.2 skrll 564 1.3.2.4 skrll ipi_mask &= mips3_ld(cpu->cpu_mbox_set); 565 1.3.2.3 skrll if (ipi_mask == 0) 566 1.3.2.3 skrll return 0; 567 1.3.2.3 skrll 568 1.3.2.4 skrll mips3_sd(cpu->cpu_mbox_clr, ipi_mask); 569 1.3.2.2 skrll 570 1.3.2.2 skrll ipi_mask |= (ipi_mask >> 16); 571 1.3.2.2 skrll ipi_mask &= __BITS(15,0); 572 1.3.2.2 skrll 573 1.3.2.2 skrll KASSERT(ipi_mask < __BIT(NIPIS)); 574 1.3.2.2 skrll 575 1.3.2.3 skrll #if NWDOG > 0 576 1.3.2.3 skrll // Handle WDOG requests ourselves. 577 1.3.2.3 skrll if (ipi_mask & __BIT(IPI_WDOG)) { 578 1.3.2.3 skrll softint_schedule(cpu->cpu_wdog_sih); 579 1.3.2.3 skrll atomic_and_64(&ci->ci_request_ipis, ~__BIT(IPI_WDOG)); 580 1.3.2.3 skrll ipi_mask &= ~__BIT(IPI_WDOG); 581 1.3.2.3 skrll ci->ci_evcnt_per_ipi[IPI_WDOG].ev_count++; 582 1.3.2.3 skrll if (__predict_true(ipi_mask == 0)) 583 1.3.2.3 skrll return 1; 584 1.3.2.3 skrll } 585 1.3.2.3 skrll #endif 586 1.3.2.3 skrll 587 1.3.2.2 skrll /* if the request is clear, it was previously processed */ 588 1.3.2.2 skrll if ((ci->ci_request_ipis & ipi_mask) == 0) 589 1.3.2.2 skrll return 0; 590 1.3.2.2 skrll 591 1.3.2.2 skrll atomic_or_64(&ci->ci_active_ipis, ipi_mask); 592 1.3.2.2 skrll atomic_and_64(&ci->ci_request_ipis, ~ipi_mask); 593 1.3.2.2 skrll 594 1.3.2.2 skrll ipi_process(ci, ipi_mask); 595 1.3.2.2 skrll 596 1.3.2.2 skrll atomic_and_64(&ci->ci_active_ipis, ~ipi_mask); 597 1.3.2.2 skrll 598 1.3.2.2 skrll return 1; 599 1.3.2.2 skrll } 600 1.3.2.2 skrll 601 1.3.2.2 skrll int 602 1.3.2.2 skrll octeon_send_ipi(struct cpu_info *ci, int req) 603 1.3.2.2 skrll { 604 1.3.2.2 skrll KASSERT(req < NIPIS); 605 1.3.2.2 skrll if (ci == NULL) { 606 1.3.2.3 skrll CPU_INFO_ITERATOR cii; 607 1.3.2.3 skrll for (CPU_INFO_FOREACH(cii, ci)) { 608 1.3.2.3 skrll if (ci != curcpu()) { 609 1.3.2.3 skrll octeon_send_ipi(ci, req); 610 1.3.2.3 skrll } 611 1.3.2.3 skrll } 612 1.3.2.3 skrll return 0; 613 1.3.2.2 skrll } 614 1.3.2.3 skrll KASSERT(cold || ci->ci_softc != NULL); 615 1.3.2.3 skrll if (ci->ci_softc == NULL) 616 1.3.2.3 skrll return -1; 617 1.3.2.2 skrll 618 1.3.2.2 skrll struct cpu_softc * const cpu = ci->ci_softc; 619 1.3.2.2 skrll uint64_t ipi_mask = __BIT(req); 620 1.3.2.2 skrll 621 1.3.2.4 skrll atomic_or_64(&ci->ci_request_ipis, ipi_mask); 622 1.3.2.4 skrll if (req == IPI_SUSPEND || req == IPI_WDOG) { 623 1.3.2.2 skrll ipi_mask <<= 16; 624 1.3.2.2 skrll } 625 1.3.2.2 skrll 626 1.3.2.4 skrll mips3_sd(cpu->cpu_mbox_set, ipi_mask); 627 1.3.2.2 skrll return 0; 628 1.3.2.2 skrll } 629 1.3.2.2 skrll #endif /* MULTIPROCESSOR */ 630
Indexes created Thu Oct 02 14:10:14 GMT 2025