pic.c revision 1.10
1 /* $NetBSD: pic.c,v 1.10 2012/07/07 08:05:48 skrll Exp $ */ 2 /*- 3 * Copyright (c) 2008 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Matt Thomas. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * POSSIBILITY OF SUCH DAMAGE. 29 */ 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.10 2012/07/07 08:05:48 skrll Exp $"); 32 33 #define _INTR_PRIVATE 34 #include <sys/param.h> 35 #include <sys/evcnt.h> 36 #include <sys/atomic.h> 37 #include <sys/malloc.h> 38 #include <sys/mallocvar.h> 39 #include <sys/atomic.h> 40 41 #include <arm/armreg.h> 42 #include <arm/cpu.h> 43 #include <arm/cpufunc.h> 44 45 #include <arm/pic/picvar.h> 46 47 MALLOC_DEFINE(M_INTRSOURCE, "intrsource", "interrupt source"); 48 49 static uint32_t 50 pic_find_pending_irqs_by_ipl(struct pic_softc *, size_t, uint32_t, int); 51 static struct pic_softc * 52 pic_list_find_pic_by_pending_ipl(uint32_t); 53 static void 54 pic_deliver_irqs(struct pic_softc *, int, void *); 55 static void 56 pic_list_deliver_irqs(register_t, int, void *); 57 58 struct pic_softc *pic_list[PIC_MAXPICS]; 59 #if PIC_MAXPICS > 32 60 #error PIC_MAXPICS > 32 not supported 61 #endif 62 volatile uint32_t pic_blocked_pics; 63 volatile uint32_t pic_pending_pics; 64 volatile uint32_t pic_pending_ipls; 65 struct intrsource *pic_sources[PIC_MAXMAXSOURCES]; 66 struct intrsource *pic__iplsources[PIC_MAXMAXSOURCES]; 67 struct intrsource **pic_iplsource[NIPL] = { 68 [0 ... NIPL-1] = pic__iplsources, 69 }; 70 size_t pic_ipl_offset[NIPL+1]; 71 size_t pic_sourcebase; 72 static struct evcnt pic_deferral_ev = 73 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "deferred", "intr"); 74 EVCNT_ATTACH_STATIC(pic_deferral_ev); 75 76 77 78 int 79 pic_handle_intr(void *arg) 80 { 81 struct pic_softc * const pic = arg; 82 int rv; 83 84 rv = (*pic->pic_ops->pic_find_pending_irqs)(pic); 85 86 return rv > 0; 87 } 88 89 void 90 pic_mark_pending_source(struct pic_softc *pic, struct intrsource *is) 91 { 92 const uint32_t ipl_mask = __BIT(is->is_ipl); 93 94 atomic_or_32(&pic->pic_pending_irqs[is->is_irq >> 5], 95 __BIT(is->is_irq & 0x1f)); 96 97 atomic_or_32(&pic->pic_pending_ipls, ipl_mask); 98 atomic_or_32(&pic_pending_ipls, ipl_mask); 99 atomic_or_32(&pic_pending_pics, __BIT(pic->pic_id)); 100 } 101 102 void 103 pic_mark_pending(struct pic_softc *pic, int irq) 104 { 105 struct intrsource * const is = pic->pic_sources[irq]; 106 107 KASSERT(irq < pic->pic_maxsources); 108 KASSERT(is != NULL); 109 110 pic_mark_pending_source(pic, is); 111 } 112 113 uint32_t 114 pic_mark_pending_sources(struct pic_softc *pic, size_t irq_base, 115 uint32_t pending) 116 { 117 struct intrsource ** const isbase = &pic->pic_sources[irq_base]; 118 struct intrsource *is; 119 volatile uint32_t *ipending = &pic->pic_pending_irqs[irq_base >> 5]; 120 uint32_t ipl_mask = 0; 121 122 if (pending == 0) 123 return ipl_mask; 124 125 KASSERT((irq_base & 31) == 0); 126 127 (*pic->pic_ops->pic_block_irqs)(pic, irq_base, pending); 128 129 atomic_or_32(ipending, pending); 130 while (pending != 0) { 131 int n = ffs(pending); 132 if (n-- == 0) 133 break; 134 is = isbase[n]; 135 KASSERT(is != NULL); 136 KASSERT(irq_base <= is->is_irq && is->is_irq < irq_base + 32); 137 pending &= ~__BIT(n); 138 ipl_mask |= __BIT(is->is_ipl); 139 } 140 141 atomic_or_32(&pic->pic_pending_ipls, ipl_mask); 142 atomic_or_32(&pic_pending_ipls, ipl_mask); 143 atomic_or_32(&pic_pending_pics, __BIT(pic->pic_id)); 144 145 return ipl_mask; 146 } 147 148 uint32_t 149 pic_find_pending_irqs_by_ipl(struct pic_softc *pic, size_t irq_base, 150 uint32_t pending, int ipl) 151 { 152 uint32_t ipl_irq_mask = 0; 153 uint32_t irq_mask; 154 155 for (;;) { 156 int irq = ffs(pending); 157 if (irq-- == 0) 158 return ipl_irq_mask; 159 160 irq_mask = __BIT(irq); 161 #if 1 162 KASSERTMSG(pic->pic_sources[irq_base + irq] != NULL, 163 "%s: irq_base %zu irq %d\n", __func__, irq_base, irq); 164 #else 165 if (pic->pic_sources[irq_base + irq] == NULL) { 166 aprint_error("stray interrupt? irq_base=%zu irq=%d\n", 167 irq_base, irq); 168 } else 169 #endif 170 if (pic->pic_sources[irq_base + irq]->is_ipl == ipl) 171 ipl_irq_mask |= irq_mask; 172 173 pending &= ~irq_mask; 174 } 175 } 176 177 void 178 pic_dispatch(struct intrsource *is, void *frame) 179 { 180 int rv; 181 182 if (__predict_false(is->is_arg == NULL) 183 && __predict_true(frame != NULL)) { 184 rv = (*is->is_func)(frame); 185 } else if (__predict_true(is->is_arg != NULL)) { 186 rv = (*is->is_func)(is->is_arg); 187 } else { 188 pic_deferral_ev.ev_count++; 189 return; 190 } 191 is->is_ev.ev_count++; 192 } 193 194 void 195 pic_deliver_irqs(struct pic_softc *pic, int ipl, void *frame) 196 { 197 const uint32_t ipl_mask = __BIT(ipl); 198 struct intrsource *is; 199 volatile uint32_t *ipending = pic->pic_pending_irqs; 200 volatile uint32_t *iblocked = pic->pic_blocked_irqs; 201 size_t irq_base; 202 #if PIC_MAXSOURCES > 32 203 size_t irq_count; 204 int poi = 0; /* Possibility of interrupting */ 205 #endif 206 uint32_t pending_irqs; 207 uint32_t blocked_irqs; 208 int irq; 209 bool progress = false; 210 211 KASSERT(pic->pic_pending_ipls & ipl_mask); 212 213 irq_base = 0; 214 #if PIC_MAXSOURCES > 32 215 irq_count = 0; 216 #endif 217 218 for (;;) { 219 pending_irqs = pic_find_pending_irqs_by_ipl(pic, irq_base, 220 *ipending, ipl); 221 KASSERT((pending_irqs & *ipending) == pending_irqs); 222 KASSERT((pending_irqs & ~(*ipending)) == 0); 223 if (pending_irqs == 0) { 224 #if PIC_MAXSOURCES > 32 225 irq_count += 32; 226 if (__predict_true(irq_count >= pic->pic_maxsources)) { 227 if (!poi) 228 /*Interrupt at this level was handled.*/ 229 break; 230 irq_base = 0; 231 irq_count = 0; 232 poi = 0; 233 ipending = pic->pic_pending_irqs; 234 iblocked = pic->pic_blocked_irqs; 235 } else { 236 irq_base += 32; 237 ipending++; 238 iblocked++; 239 KASSERT(irq_base <= pic->pic_maxsources); 240 } 241 continue; 242 #else 243 break; 244 #endif 245 } 246 progress = true; 247 blocked_irqs = 0; 248 do { 249 irq = ffs(pending_irqs) - 1; 250 KASSERT(irq >= 0); 251 252 atomic_and_32(ipending, ~__BIT(irq)); 253 is = pic->pic_sources[irq_base + irq]; 254 if (is != NULL) { 255 cpsie(I32_bit); 256 pic_dispatch(is, frame); 257 cpsid(I32_bit); 258 #if PIC_MAXSOURCES > 32 259 /* 260 * There is a possibility of interrupting 261 * from cpsie() to cpsid(). 262 */ 263 poi = 1; 264 #endif 265 blocked_irqs |= __BIT(irq); 266 } else { 267 KASSERT(0); 268 } 269 pending_irqs = pic_find_pending_irqs_by_ipl(pic, 270 irq_base, *ipending, ipl); 271 } while (pending_irqs); 272 if (blocked_irqs) { 273 atomic_or_32(iblocked, blocked_irqs); 274 atomic_or_32(&pic_blocked_pics, __BIT(pic->pic_id)); 275 } 276 } 277 278 KASSERT(progress); 279 /* 280 * Since interrupts are disabled, we don't have to be too careful 281 * about these. 282 */ 283 if (atomic_and_32_nv(&pic->pic_pending_ipls, ~ipl_mask) == 0) 284 atomic_and_32(&pic_pending_pics, ~__BIT(pic->pic_id)); 285 } 286 287 static void 288 pic_list_unblock_irqs(void) 289 { 290 uint32_t blocked_pics = pic_blocked_pics; 291 292 pic_blocked_pics = 0; 293 for (;;) { 294 struct pic_softc *pic; 295 #if PIC_MAXSOURCES > 32 296 volatile uint32_t *iblocked; 297 uint32_t blocked; 298 size_t irq_base; 299 #endif 300 301 int pic_id = ffs(blocked_pics); 302 if (pic_id-- == 0) 303 return; 304 305 pic = pic_list[pic_id]; 306 KASSERT(pic != NULL); 307 #if PIC_MAXSOURCES > 32 308 for (irq_base = 0, iblocked = pic->pic_blocked_irqs; 309 irq_base < pic->pic_maxsources; 310 irq_base += 32, iblocked++) { 311 if ((blocked = *iblocked) != 0) { 312 (*pic->pic_ops->pic_unblock_irqs)(pic, 313 irq_base, blocked); 314 atomic_and_32(iblocked, ~blocked); 315 } 316 } 317 #else 318 KASSERT(pic->pic_blocked_irqs[0] != 0); 319 (*pic->pic_ops->pic_unblock_irqs)(pic, 320 0, pic->pic_blocked_irqs[0]); 321 pic->pic_blocked_irqs[0] = 0; 322 #endif 323 blocked_pics &= ~__BIT(pic_id); 324 } 325 } 326 327 328 struct pic_softc * 329 pic_list_find_pic_by_pending_ipl(uint32_t ipl_mask) 330 { 331 uint32_t pending_pics = pic_pending_pics; 332 struct pic_softc *pic; 333 334 for (;;) { 335 int pic_id = ffs(pending_pics); 336 if (pic_id-- == 0) 337 return NULL; 338 339 pic = pic_list[pic_id]; 340 KASSERT(pic != NULL); 341 if (pic->pic_pending_ipls & ipl_mask) 342 return pic; 343 pending_pics &= ~__BIT(pic_id); 344 } 345 } 346 347 void 348 pic_list_deliver_irqs(register_t psw, int ipl, void *frame) 349 { 350 const uint32_t ipl_mask = __BIT(ipl); 351 struct pic_softc *pic; 352 353 while ((pic = pic_list_find_pic_by_pending_ipl(ipl_mask)) != NULL) { 354 pic_deliver_irqs(pic, ipl, frame); 355 KASSERT((pic->pic_pending_ipls & ipl_mask) == 0); 356 } 357 atomic_and_32(&pic_pending_ipls, ~ipl_mask); 358 } 359 360 void 361 pic_do_pending_ints(register_t psw, int newipl, void *frame) 362 { 363 struct cpu_info * const ci = curcpu(); 364 if (__predict_false(newipl == IPL_HIGH)) 365 return; 366 while ((pic_pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) { 367 KASSERT(pic_pending_ipls < __BIT(NIPL)); 368 for (;;) { 369 int ipl = 31 - __builtin_clz(pic_pending_ipls); 370 KASSERT(ipl < NIPL); 371 if (ipl <= newipl) 372 break; 373 374 ci->ci_cpl = ipl; 375 pic_list_deliver_irqs(psw, ipl, frame); 376 pic_list_unblock_irqs(); 377 } 378 } 379 if (ci->ci_cpl != newipl) 380 ci->ci_cpl = newipl; 381 #ifdef __HAVE_FAST_SOFTINTS 382 cpu_dosoftints(); 383 #endif 384 } 385 386 void 387 pic_add(struct pic_softc *pic, int irqbase) 388 { 389 int slot, maybe_slot = -1; 390 391 for (slot = 0; slot < PIC_MAXPICS; slot++) { 392 struct pic_softc * const xpic = pic_list[slot]; 393 if (xpic == NULL) { 394 if (maybe_slot < 0) 395 maybe_slot = slot; 396 if (irqbase < 0) 397 break; 398 continue; 399 } 400 if (irqbase < 0 || xpic->pic_irqbase < 0) 401 continue; 402 if (irqbase >= xpic->pic_irqbase + xpic->pic_maxsources) 403 continue; 404 if (irqbase + pic->pic_maxsources <= xpic->pic_irqbase) 405 continue; 406 panic("pic_add: pic %s (%zu sources @ irq %u) conflicts" 407 " with pic %s (%zu sources @ irq %u)", 408 pic->pic_name, pic->pic_maxsources, irqbase, 409 xpic->pic_name, xpic->pic_maxsources, xpic->pic_irqbase); 410 } 411 slot = maybe_slot; 412 #if 0 413 printf("%s: pic_sourcebase=%zu pic_maxsources=%zu\n", 414 pic->pic_name, pic_sourcebase, pic->pic_maxsources); 415 #endif 416 KASSERT(pic->pic_maxsources <= PIC_MAXSOURCES); 417 KASSERT(pic_sourcebase + pic->pic_maxsources <= PIC_MAXMAXSOURCES); 418 419 pic->pic_sources = &pic_sources[pic_sourcebase]; 420 pic->pic_irqbase = irqbase; 421 pic_sourcebase += pic->pic_maxsources; 422 pic->pic_id = slot; 423 pic_list[slot] = pic; 424 } 425 426 int 427 pic_alloc_irq(struct pic_softc *pic) 428 { 429 int irq; 430 431 for (irq = 0; irq < pic->pic_maxsources; irq++) { 432 if (pic->pic_sources[irq] == NULL) 433 return irq; 434 } 435 436 return -1; 437 } 438 439 void * 440 pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type, 441 int (*func)(void *), void *arg) 442 { 443 struct intrsource *is; 444 int off, nipl; 445 446 if (pic->pic_sources[irq]) { 447 printf("pic_establish_intr: pic %s irq %d already present\n", 448 pic->pic_name, irq); 449 return NULL; 450 } 451 452 is = malloc(sizeof(*is), M_INTRSOURCE, M_NOWAIT|M_ZERO); 453 if (is == NULL) 454 return NULL; 455 456 is->is_pic = pic; 457 is->is_irq = irq; 458 is->is_ipl = ipl; 459 is->is_type = type; 460 is->is_func = func; 461 is->is_arg = arg; 462 463 if (pic->pic_ops->pic_source_name) 464 (*pic->pic_ops->pic_source_name)(pic, irq, is->is_source, 465 sizeof(is->is_source)); 466 else 467 snprintf(is->is_source, sizeof(is->is_source), "irq %d", irq); 468 469 evcnt_attach_dynamic(&is->is_ev, EVCNT_TYPE_INTR, NULL, 470 pic->pic_name, is->is_source); 471 472 pic->pic_sources[irq] = is; 473 474 /* 475 * First try to use an existing slot which is empty. 476 */ 477 for (off = pic_ipl_offset[ipl]; off < pic_ipl_offset[ipl+1]; off++) { 478 if (pic__iplsources[off] == NULL) { 479 is->is_iplidx = off - pic_ipl_offset[ipl]; 480 pic__iplsources[off] = is; 481 return is; 482 } 483 } 484 485 /* 486 * Move up all the sources by one. 487 */ 488 if (ipl < NIPL) { 489 off = pic_ipl_offset[ipl+1]; 490 memmove(&pic__iplsources[off+1], &pic__iplsources[off], 491 sizeof(pic__iplsources[0]) * (pic_ipl_offset[NIPL] - off)); 492 } 493 494 /* 495 * Advance the offset of all IPLs higher than this. Include an 496 * extra one as well. Thus the number of sources per ipl is 497 * pic_ipl_offset[ipl+1] - pic_ipl_offset[ipl]. 498 */ 499 for (nipl = ipl + 1; nipl <= NIPL; nipl++) 500 pic_ipl_offset[nipl]++; 501 502 /* 503 * Insert into the previously made position at the end of this IPL's 504 * sources. 505 */ 506 off = pic_ipl_offset[ipl + 1] - 1; 507 is->is_iplidx = off - pic_ipl_offset[ipl]; 508 pic__iplsources[off] = is; 509 510 (*pic->pic_ops->pic_establish_irq)(pic, is); 511 512 (*pic->pic_ops->pic_unblock_irqs)(pic, is->is_irq & ~0x1f, 513 __BIT(is->is_irq & 0x1f)); 514 515 /* We're done. */ 516 return is; 517 } 518 519 void 520 pic_disestablish_source(struct intrsource *is) 521 { 522 struct pic_softc * const pic = is->is_pic; 523 const int irq = is->is_irq; 524 525 (*pic->pic_ops->pic_block_irqs)(pic, irq & ~31, __BIT(irq)); 526 pic->pic_sources[irq] = NULL; 527 pic__iplsources[pic_ipl_offset[is->is_ipl] + is->is_iplidx] = NULL; 528 evcnt_detach(&is->is_ev); 529 530 free(is, M_INTRSOURCE); 531 } 532 533 void * 534 intr_establish(int irq, int ipl, int type, int (*func)(void *), void *arg) 535 { 536 int slot; 537 538 for (slot = 0; slot < PIC_MAXPICS; slot++) { 539 struct pic_softc * const pic = pic_list[slot]; 540 if (pic == NULL || pic->pic_irqbase < 0) 541 continue; 542 if (pic->pic_irqbase <= irq 543 && irq < pic->pic_irqbase + pic->pic_maxsources) { 544 return pic_establish_intr(pic, irq - pic->pic_irqbase, 545 ipl, type, func, arg); 546 } 547 } 548 549 return NULL; 550 } 551 552 void 553 intr_disestablish(void *ih) 554 { 555 struct intrsource * const is = ih; 556 pic_disestablish_source(is); 557 } 558
Indexes created Mon Sep 29 18:09:42 GMT 2025