pic.c revision 1.12
1 /* $NetBSD: pic.c,v 1.12 2012/07/20 21:53:57 matt Exp $ */ 2 /*- 3 * Copyright (c) 2008 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Matt Thomas. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * POSSIBILITY OF SUCH DAMAGE. 29 */ 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.12 2012/07/20 21:53:57 matt Exp $"); 32 33 #define _INTR_PRIVATE 34 #include <sys/param.h> 35 #include <sys/evcnt.h> 36 #include <sys/atomic.h> 37 #include <sys/kmem.h> 38 #include <sys/atomic.h> 39 #include <sys/cpu.h> 40 41 #include <arm/armreg.h> 42 #include <arm/cpufunc.h> 43 44 #include <arm/pic/picvar.h> 45 46 static uint32_t 47 pic_find_pending_irqs_by_ipl(struct pic_softc *, size_t, uint32_t, int); 48 static struct pic_softc * 49 pic_list_find_pic_by_pending_ipl(uint32_t); 50 static void 51 pic_deliver_irqs(struct pic_softc *, int, void *); 52 static void 53 pic_list_deliver_irqs(register_t, int, void *); 54 55 struct pic_softc *pic_list[PIC_MAXPICS]; 56 #if PIC_MAXPICS > 32 57 #error PIC_MAXPICS > 32 not supported 58 #endif 59 volatile uint32_t pic_blocked_pics; 60 volatile uint32_t pic_pending_pics; 61 volatile uint32_t pic_pending_ipls; 62 struct intrsource *pic_sources[PIC_MAXMAXSOURCES]; 63 struct intrsource *pic__iplsources[PIC_MAXMAXSOURCES]; 64 struct intrsource **pic_iplsource[NIPL] = { 65 [0 ... NIPL-1] = pic__iplsources, 66 }; 67 size_t pic_ipl_offset[NIPL+1]; 68 size_t pic_sourcebase; 69 static struct evcnt pic_deferral_ev = 70 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "deferred", "intr"); 71 EVCNT_ATTACH_STATIC(pic_deferral_ev); 72 73 #ifdef __HAVE_PIC_SET_PRIORITY 74 void 75 pic_set_priority(struct cpu_info *ci, int newipl) 76 { 77 register_t psw = disable_interrupts(I32_bit); 78 ci->ci_cpl = newipl; 79 (pic_list[0]->pic_set_priority)(newipl); 80 restore_interrupts(psw); 81 } 82 #endif 83 84 int 85 pic_handle_intr(void *arg) 86 { 87 struct pic_softc * const pic = arg; 88 int rv; 89 90 rv = (*pic->pic_ops->pic_find_pending_irqs)(pic); 91 92 return rv > 0; 93 } 94 95 void 96 pic_mark_pending_source(struct pic_softc *pic, struct intrsource *is) 97 { 98 const uint32_t ipl_mask = __BIT(is->is_ipl); 99 100 atomic_or_32(&pic->pic_pending_irqs[is->is_irq >> 5], 101 __BIT(is->is_irq & 0x1f)); 102 103 atomic_or_32(&pic->pic_pending_ipls, ipl_mask); 104 atomic_or_32(&pic_pending_ipls, ipl_mask); 105 atomic_or_32(&pic_pending_pics, __BIT(pic->pic_id)); 106 } 107 108 void 109 pic_mark_pending(struct pic_softc *pic, int irq) 110 { 111 struct intrsource * const is = pic->pic_sources[irq]; 112 113 KASSERT(irq < pic->pic_maxsources); 114 KASSERT(is != NULL); 115 116 pic_mark_pending_source(pic, is); 117 } 118 119 uint32_t 120 pic_mark_pending_sources(struct pic_softc *pic, size_t irq_base, 121 uint32_t pending) 122 { 123 struct intrsource ** const isbase = &pic->pic_sources[irq_base]; 124 struct intrsource *is; 125 volatile uint32_t *ipending = &pic->pic_pending_irqs[irq_base >> 5]; 126 uint32_t ipl_mask = 0; 127 128 if (pending == 0) 129 return ipl_mask; 130 131 KASSERT((irq_base & 31) == 0); 132 133 (*pic->pic_ops->pic_block_irqs)(pic, irq_base, pending); 134 135 atomic_or_32(ipending, pending); 136 while (pending != 0) { 137 int n = ffs(pending); 138 if (n-- == 0) 139 break; 140 is = isbase[n]; 141 KASSERT(is != NULL); 142 KASSERT(irq_base <= is->is_irq && is->is_irq < irq_base + 32); 143 pending &= ~__BIT(n); 144 ipl_mask |= __BIT(is->is_ipl); 145 } 146 147 atomic_or_32(&pic->pic_pending_ipls, ipl_mask); 148 atomic_or_32(&pic_pending_ipls, ipl_mask); 149 atomic_or_32(&pic_pending_pics, __BIT(pic->pic_id)); 150 151 return ipl_mask; 152 } 153 154 uint32_t 155 pic_find_pending_irqs_by_ipl(struct pic_softc *pic, size_t irq_base, 156 uint32_t pending, int ipl) 157 { 158 uint32_t ipl_irq_mask = 0; 159 uint32_t irq_mask; 160 161 for (;;) { 162 int irq = ffs(pending); 163 if (irq-- == 0) 164 return ipl_irq_mask; 165 166 irq_mask = __BIT(irq); 167 #if 1 168 KASSERTMSG(pic->pic_sources[irq_base + irq] != NULL, 169 "%s: irq_base %zu irq %d\n", __func__, irq_base, irq); 170 #else 171 if (pic->pic_sources[irq_base + irq] == NULL) { 172 aprint_error("stray interrupt? irq_base=%zu irq=%d\n", 173 irq_base, irq); 174 } else 175 #endif 176 if (pic->pic_sources[irq_base + irq]->is_ipl == ipl) 177 ipl_irq_mask |= irq_mask; 178 179 pending &= ~irq_mask; 180 } 181 } 182 183 void 184 pic_dispatch(struct intrsource *is, void *frame) 185 { 186 int rv; 187 188 if (__predict_false(is->is_arg == NULL) 189 && __predict_true(frame != NULL)) { 190 rv = (*is->is_func)(frame); 191 } else if (__predict_true(is->is_arg != NULL)) { 192 rv = (*is->is_func)(is->is_arg); 193 } else { 194 pic_deferral_ev.ev_count++; 195 return; 196 } 197 is->is_ev.ev_count++; 198 } 199 200 void 201 pic_deliver_irqs(struct pic_softc *pic, int ipl, void *frame) 202 { 203 const uint32_t ipl_mask = __BIT(ipl); 204 struct intrsource *is; 205 volatile uint32_t *ipending = pic->pic_pending_irqs; 206 volatile uint32_t *iblocked = pic->pic_blocked_irqs; 207 size_t irq_base; 208 #if PIC_MAXSOURCES > 32 209 size_t irq_count; 210 int poi = 0; /* Possibility of interrupting */ 211 #endif 212 uint32_t pending_irqs; 213 uint32_t blocked_irqs; 214 int irq; 215 bool progress = false; 216 217 KASSERT(pic->pic_pending_ipls & ipl_mask); 218 219 irq_base = 0; 220 #if PIC_MAXSOURCES > 32 221 irq_count = 0; 222 #endif 223 224 for (;;) { 225 pending_irqs = pic_find_pending_irqs_by_ipl(pic, irq_base, 226 *ipending, ipl); 227 KASSERT((pending_irqs & *ipending) == pending_irqs); 228 KASSERT((pending_irqs & ~(*ipending)) == 0); 229 if (pending_irqs == 0) { 230 #if PIC_MAXSOURCES > 32 231 irq_count += 32; 232 if (__predict_true(irq_count >= pic->pic_maxsources)) { 233 if (!poi) 234 /*Interrupt at this level was handled.*/ 235 break; 236 irq_base = 0; 237 irq_count = 0; 238 poi = 0; 239 ipending = pic->pic_pending_irqs; 240 iblocked = pic->pic_blocked_irqs; 241 } else { 242 irq_base += 32; 243 ipending++; 244 iblocked++; 245 KASSERT(irq_base <= pic->pic_maxsources); 246 } 247 continue; 248 #else 249 break; 250 #endif 251 } 252 progress = true; 253 blocked_irqs = 0; 254 do { 255 irq = ffs(pending_irqs) - 1; 256 KASSERT(irq >= 0); 257 258 atomic_and_32(ipending, ~__BIT(irq)); 259 is = pic->pic_sources[irq_base + irq]; 260 if (is != NULL) { 261 cpsie(I32_bit); 262 pic_dispatch(is, frame); 263 cpsid(I32_bit); 264 #if PIC_MAXSOURCES > 32 265 /* 266 * There is a possibility of interrupting 267 * from cpsie() to cpsid(). 268 */ 269 poi = 1; 270 #endif 271 blocked_irqs |= __BIT(irq); 272 } else { 273 KASSERT(0); 274 } 275 pending_irqs = pic_find_pending_irqs_by_ipl(pic, 276 irq_base, *ipending, ipl); 277 } while (pending_irqs); 278 if (blocked_irqs) { 279 atomic_or_32(iblocked, blocked_irqs); 280 atomic_or_32(&pic_blocked_pics, __BIT(pic->pic_id)); 281 } 282 } 283 284 KASSERT(progress); 285 /* 286 * Since interrupts are disabled, we don't have to be too careful 287 * about these. 288 */ 289 if (atomic_and_32_nv(&pic->pic_pending_ipls, ~ipl_mask) == 0) 290 atomic_and_32(&pic_pending_pics, ~__BIT(pic->pic_id)); 291 } 292 293 static void 294 pic_list_unblock_irqs(void) 295 { 296 uint32_t blocked_pics = pic_blocked_pics; 297 298 pic_blocked_pics = 0; 299 for (;;) { 300 struct pic_softc *pic; 301 #if PIC_MAXSOURCES > 32 302 volatile uint32_t *iblocked; 303 uint32_t blocked; 304 size_t irq_base; 305 #endif 306 307 int pic_id = ffs(blocked_pics); 308 if (pic_id-- == 0) 309 return; 310 311 pic = pic_list[pic_id]; 312 KASSERT(pic != NULL); 313 #if PIC_MAXSOURCES > 32 314 for (irq_base = 0, iblocked = pic->pic_blocked_irqs; 315 irq_base < pic->pic_maxsources; 316 irq_base += 32, iblocked++) { 317 if ((blocked = *iblocked) != 0) { 318 (*pic->pic_ops->pic_unblock_irqs)(pic, 319 irq_base, blocked); 320 atomic_and_32(iblocked, ~blocked); 321 } 322 } 323 #else 324 KASSERT(pic->pic_blocked_irqs[0] != 0); 325 (*pic->pic_ops->pic_unblock_irqs)(pic, 326 0, pic->pic_blocked_irqs[0]); 327 pic->pic_blocked_irqs[0] = 0; 328 #endif 329 blocked_pics &= ~__BIT(pic_id); 330 } 331 } 332 333 334 struct pic_softc * 335 pic_list_find_pic_by_pending_ipl(uint32_t ipl_mask) 336 { 337 uint32_t pending_pics = pic_pending_pics; 338 struct pic_softc *pic; 339 340 for (;;) { 341 int pic_id = ffs(pending_pics); 342 if (pic_id-- == 0) 343 return NULL; 344 345 pic = pic_list[pic_id]; 346 KASSERT(pic != NULL); 347 if (pic->pic_pending_ipls & ipl_mask) 348 return pic; 349 pending_pics &= ~__BIT(pic_id); 350 } 351 } 352 353 void 354 pic_list_deliver_irqs(register_t psw, int ipl, void *frame) 355 { 356 const uint32_t ipl_mask = __BIT(ipl); 357 struct pic_softc *pic; 358 359 while ((pic = pic_list_find_pic_by_pending_ipl(ipl_mask)) != NULL) { 360 pic_deliver_irqs(pic, ipl, frame); 361 KASSERT((pic->pic_pending_ipls & ipl_mask) == 0); 362 } 363 atomic_and_32(&pic_pending_ipls, ~ipl_mask); 364 } 365 366 void 367 pic_do_pending_ints(register_t psw, int newipl, void *frame) 368 { 369 struct cpu_info * const ci = curcpu(); 370 if (__predict_false(newipl == IPL_HIGH)) 371 return; 372 while ((pic_pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) { 373 KASSERT(pic_pending_ipls < __BIT(NIPL)); 374 for (;;) { 375 int ipl = 31 - __builtin_clz(pic_pending_ipls); 376 KASSERT(ipl < NIPL); 377 if (ipl <= newipl) 378 break; 379 380 pic_set_priority(ci, ipl); 381 pic_list_deliver_irqs(psw, ipl, frame); 382 pic_list_unblock_irqs(); 383 } 384 } 385 if (ci->ci_cpl != newipl) 386 pic_set_priority(ci, newipl); 387 #ifdef __HAVE_FAST_SOFTINTS 388 cpu_dosoftints(); 389 #endif 390 } 391 392 void 393 pic_add(struct pic_softc *pic, int irqbase) 394 { 395 int slot, maybe_slot = -1; 396 397 for (slot = 0; slot < PIC_MAXPICS; slot++) { 398 struct pic_softc * const xpic = pic_list[slot]; 399 if (xpic == NULL) { 400 if (maybe_slot < 0) 401 maybe_slot = slot; 402 if (irqbase < 0) 403 break; 404 continue; 405 } 406 if (irqbase < 0 || xpic->pic_irqbase < 0) 407 continue; 408 if (irqbase >= xpic->pic_irqbase + xpic->pic_maxsources) 409 continue; 410 if (irqbase + pic->pic_maxsources <= xpic->pic_irqbase) 411 continue; 412 panic("pic_add: pic %s (%zu sources @ irq %u) conflicts" 413 " with pic %s (%zu sources @ irq %u)", 414 pic->pic_name, pic->pic_maxsources, irqbase, 415 xpic->pic_name, xpic->pic_maxsources, xpic->pic_irqbase); 416 } 417 slot = maybe_slot; 418 #if 0 419 printf("%s: pic_sourcebase=%zu pic_maxsources=%zu\n", 420 pic->pic_name, pic_sourcebase, pic->pic_maxsources); 421 #endif 422 KASSERT(pic->pic_maxsources <= PIC_MAXSOURCES); 423 KASSERT(pic_sourcebase + pic->pic_maxsources <= PIC_MAXMAXSOURCES); 424 425 pic->pic_sources = &pic_sources[pic_sourcebase]; 426 pic->pic_irqbase = irqbase; 427 pic_sourcebase += pic->pic_maxsources; 428 pic->pic_id = slot; 429 pic_list[slot] = pic; 430 } 431 432 int 433 pic_alloc_irq(struct pic_softc *pic) 434 { 435 int irq; 436 437 for (irq = 0; irq < pic->pic_maxsources; irq++) { 438 if (pic->pic_sources[irq] == NULL) 439 return irq; 440 } 441 442 return -1; 443 } 444 445 void * 446 pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type, 447 int (*func)(void *), void *arg) 448 { 449 struct intrsource *is; 450 int off, nipl; 451 452 if (pic->pic_sources[irq]) { 453 printf("pic_establish_intr: pic %s irq %d already present\n", 454 pic->pic_name, irq); 455 return NULL; 456 } 457 458 is = kmem_zalloc(sizeof(*is), KM_SLEEP); 459 if (is == NULL) 460 return NULL; 461 462 is->is_pic = pic; 463 is->is_irq = irq; 464 is->is_ipl = ipl; 465 is->is_type = type; 466 is->is_func = func; 467 is->is_arg = arg; 468 469 if (pic->pic_ops->pic_source_name) 470 (*pic->pic_ops->pic_source_name)(pic, irq, is->is_source, 471 sizeof(is->is_source)); 472 else 473 snprintf(is->is_source, sizeof(is->is_source), "irq %d", irq); 474 475 evcnt_attach_dynamic(&is->is_ev, EVCNT_TYPE_INTR, NULL, 476 pic->pic_name, is->is_source); 477 478 pic->pic_sources[irq] = is; 479 480 /* 481 * First try to use an existing slot which is empty. 482 */ 483 for (off = pic_ipl_offset[ipl]; off < pic_ipl_offset[ipl+1]; off++) { 484 if (pic__iplsources[off] == NULL) { 485 is->is_iplidx = off - pic_ipl_offset[ipl]; 486 pic__iplsources[off] = is; 487 return is; 488 } 489 } 490 491 /* 492 * Move up all the sources by one. 493 */ 494 if (ipl < NIPL) { 495 off = pic_ipl_offset[ipl+1]; 496 memmove(&pic__iplsources[off+1], &pic__iplsources[off], 497 sizeof(pic__iplsources[0]) * (pic_ipl_offset[NIPL] - off)); 498 } 499 500 /* 501 * Advance the offset of all IPLs higher than this. Include an 502 * extra one as well. Thus the number of sources per ipl is 503 * pic_ipl_offset[ipl+1] - pic_ipl_offset[ipl]. 504 */ 505 for (nipl = ipl + 1; nipl <= NIPL; nipl++) 506 pic_ipl_offset[nipl]++; 507 508 /* 509 * Insert into the previously made position at the end of this IPL's 510 * sources. 511 */ 512 off = pic_ipl_offset[ipl + 1] - 1; 513 is->is_iplidx = off - pic_ipl_offset[ipl]; 514 pic__iplsources[off] = is; 515 516 (*pic->pic_ops->pic_establish_irq)(pic, is); 517 518 (*pic->pic_ops->pic_unblock_irqs)(pic, is->is_irq & ~0x1f, 519 __BIT(is->is_irq & 0x1f)); 520 521 /* We're done. */ 522 return is; 523 } 524 525 void 526 pic_disestablish_source(struct intrsource *is) 527 { 528 struct pic_softc * const pic = is->is_pic; 529 const int irq = is->is_irq; 530 531 (*pic->pic_ops->pic_block_irqs)(pic, irq & ~31, __BIT(irq)); 532 pic->pic_sources[irq] = NULL; 533 pic__iplsources[pic_ipl_offset[is->is_ipl] + is->is_iplidx] = NULL; 534 evcnt_detach(&is->is_ev); 535 536 kmem_free(is, sizeof(*is)); 537 } 538 539 void * 540 intr_establish(int irq, int ipl, int type, int (*func)(void *), void *arg) 541 { 542 int slot; 543 544 KASSERT(!cpu_intr_p()); 545 KASSERT(!cpu_softintr_p()); 546 547 for (slot = 0; slot < PIC_MAXPICS; slot++) { 548 struct pic_softc * const pic = pic_list[slot]; 549 if (pic == NULL || pic->pic_irqbase < 0) 550 continue; 551 if (pic->pic_irqbase <= irq 552 && irq < pic->pic_irqbase + pic->pic_maxsources) { 553 return pic_establish_intr(pic, irq - pic->pic_irqbase, 554 ipl, type, func, arg); 555 } 556 } 557 558 return NULL; 559 } 560 561 void 562 intr_disestablish(void *ih) 563 { 564 struct intrsource * const is = ih; 565 pic_disestablish_source(is); 566 } 567
Indexes created Sun Sep 28 16:09:52 GMT 2025