pic.c revision 1.28
1 /* $NetBSD: pic.c,v 1.28 2015/04/08 21:43:30 matt Exp $ */ 2 /*- 3 * Copyright (c) 2008 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Matt Thomas. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #define _INTR_PRIVATE 32 #include "opt_ddb.h" 33 #include "opt_multiprocessor.h" 34 35 #include <sys/cdefs.h> 36 __KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.28 2015/04/08 21:43:30 matt Exp $"); 37 38 #include <sys/param.h> 39 #include <sys/atomic.h> 40 #include <sys/cpu.h> 41 #include <sys/evcnt.h> 42 #include <sys/intr.h> 43 #include <sys/kernel.h> 44 #include <sys/kmem.h> 45 #include <sys/xcall.h> 46 #include <sys/ipi.h> 47 48 #if defined(__arm__) 49 #include <arm/armreg.h> 50 #include <arm/cpufunc.h> 51 #elif defined(__aarch64__) 52 #include <aarch64/locore.h> 53 #define I32_bit DAIF_I 54 #define F32_bit DAIF_F 55 #endif 56 57 #ifdef DDB 58 #include <arm/db_machdep.h> 59 #endif 60 61 #include <arm/pic/picvar.h> 62 63 #if defined(__HAVE_PIC_PENDING_INTRS) 64 static uint32_t 65 pic_find_pending_irqs_by_ipl(struct pic_softc *, size_t, uint32_t, int); 66 static struct pic_softc * 67 pic_list_find_pic_by_pending_ipl(uint32_t); 68 static void 69 pic_deliver_irqs(struct pic_softc *, int, void *); 70 static void 71 pic_list_deliver_irqs(register_t, int, void *); 72 volatile uint32_t pic_blocked_pics; 73 volatile uint32_t pic_pending_pics; 74 volatile uint32_t pic_pending_ipls; 75 #endif /* __HAVE_PIC_PENDING_INTRS */ 76 77 struct pic_softc *pic_list[PIC_MAXPICS]; 78 #if PIC_MAXPICS > 32 79 #error PIC_MAXPICS > 32 not supported 80 #endif 81 struct intrsource *pic_sources[PIC_MAXMAXSOURCES]; 82 struct intrsource *pic__iplsources[PIC_MAXMAXSOURCES]; 83 struct intrsource **pic_iplsource[NIPL] = { 84 [0 ... NIPL-1] = pic__iplsources, 85 }; 86 size_t pic_ipl_offset[NIPL+1]; 87 size_t pic_sourcebase; 88 static struct evcnt pic_deferral_ev = 89 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "deferred", "intr"); 90 EVCNT_ATTACH_STATIC(pic_deferral_ev); 91 92 #ifdef __HAVE_PIC_SET_PRIORITY 93 void 94 pic_set_priority(struct cpu_info *ci, int newipl) 95 { 96 register_t psw = cpsid(I32_bit); 97 if (pic_list[0] != NULL) 98 (pic_list[0]->pic_ops->pic_set_priority)(pic_list[0], newipl); 99 ci->ci_cpl = newipl; 100 if ((psw & I32_bit) == 0) 101 cpsie(I32_bit); 102 } 103 #endif 104 105 #ifdef MULTIPROCESSOR 106 int 107 pic_ipi_nop(void *arg) 108 { 109 /* do nothing */ 110 return 1; 111 } 112 113 int 114 pic_ipi_xcall(void *arg) 115 { 116 xc_ipi_handler(); 117 return 1; 118 } 119 120 int 121 pic_ipi_generic(void *arg) 122 { 123 ipi_cpu_handler(); 124 return 1; 125 } 126 127 #ifdef DDB 128 int 129 pic_ipi_ddb(void *arg) 130 { 131 // printf("%s: %s: tf=%p\n", __func__, curcpu()->ci_cpuname, arg); 132 kdb_trap(-1, arg); 133 return 1; 134 } 135 #endif 136 137 void 138 intr_cpu_init(struct cpu_info *ci) 139 { 140 for (size_t slot = 0; slot < PIC_MAXPICS; slot++) { 141 struct pic_softc * const pic = pic_list[slot]; 142 if (pic != NULL && pic->pic_ops->pic_cpu_init != NULL) { 143 (*pic->pic_ops->pic_cpu_init)(pic, ci); 144 } 145 } 146 } 147 148 typedef void (*pic_ipi_send_func_t)(struct pic_softc *, u_long); 149 150 static struct pic_softc * 151 pic_ipi_sender(void) 152 { 153 for (size_t slot = 0; slot < PIC_MAXPICS; slot++) { 154 struct pic_softc * const pic = pic_list[slot]; 155 if (pic != NULL && pic->pic_ops->pic_ipi_send != NULL) { 156 return pic; 157 } 158 } 159 return NULL; 160 } 161 162 void 163 intr_ipi_send(const kcpuset_t *kcp, u_long ipi) 164 { 165 struct pic_softc * const pic = pic_ipi_sender(); 166 KASSERT(ipi < NIPI); 167 if (cold && pic == NULL) 168 return; 169 KASSERT(pic != NULL); 170 (*pic->pic_ops->pic_ipi_send)(pic, kcp, ipi); 171 } 172 #endif /* MULTIPROCESSOR */ 173 174 #ifdef __HAVE_PIC_FAST_SOFTINTS 175 int 176 pic_handle_softint(void *arg) 177 { 178 void softint_switch(lwp_t *, int); 179 struct cpu_info * const ci = curcpu(); 180 const size_t softint = (size_t) arg; 181 int s = splhigh(); 182 ci->ci_intr_depth--; // don't count these as interrupts 183 softint_switch(ci->ci_softlwps[softint], s); 184 ci->ci_intr_depth++; 185 splx(s); 186 return 1; 187 } 188 #endif 189 190 int 191 pic_handle_intr(void *arg) 192 { 193 struct pic_softc * const pic = arg; 194 int rv; 195 196 rv = (*pic->pic_ops->pic_find_pending_irqs)(pic); 197 198 return rv > 0; 199 } 200 201 #if defined(__HAVE_PIC_PENDING_INTRS) 202 void 203 pic_mark_pending_source(struct pic_softc *pic, struct intrsource *is) 204 { 205 const uint32_t ipl_mask = __BIT(is->is_ipl); 206 207 atomic_or_32(&pic->pic_pending_irqs[is->is_irq >> 5], 208 __BIT(is->is_irq & 0x1f)); 209 210 atomic_or_32(&pic->pic_pending_ipls, ipl_mask); 211 atomic_or_32(&pic_pending_ipls, ipl_mask); 212 atomic_or_32(&pic_pending_pics, __BIT(pic->pic_id)); 213 } 214 215 void 216 pic_mark_pending(struct pic_softc *pic, int irq) 217 { 218 struct intrsource * const is = pic->pic_sources[irq]; 219 220 KASSERT(irq < pic->pic_maxsources); 221 KASSERT(is != NULL); 222 223 pic_mark_pending_source(pic, is); 224 } 225 226 uint32_t 227 pic_mark_pending_sources(struct pic_softc *pic, size_t irq_base, 228 uint32_t pending) 229 { 230 struct intrsource ** const isbase = &pic->pic_sources[irq_base]; 231 struct intrsource *is; 232 volatile uint32_t *ipending = &pic->pic_pending_irqs[irq_base >> 5]; 233 uint32_t ipl_mask = 0; 234 235 if (pending == 0) 236 return ipl_mask; 237 238 KASSERT((irq_base & 31) == 0); 239 240 (*pic->pic_ops->pic_block_irqs)(pic, irq_base, pending); 241 242 atomic_or_32(ipending, pending); 243 while (pending != 0) { 244 int n = ffs(pending); 245 if (n-- == 0) 246 break; 247 is = isbase[n]; 248 KASSERT(is != NULL); 249 KASSERT(irq_base <= is->is_irq && is->is_irq < irq_base + 32); 250 pending &= ~__BIT(n); 251 ipl_mask |= __BIT(is->is_ipl); 252 } 253 254 atomic_or_32(&pic->pic_pending_ipls, ipl_mask); 255 atomic_or_32(&pic_pending_ipls, ipl_mask); 256 atomic_or_32(&pic_pending_pics, __BIT(pic->pic_id)); 257 258 return ipl_mask; 259 } 260 261 uint32_t 262 pic_find_pending_irqs_by_ipl(struct pic_softc *pic, size_t irq_base, 263 uint32_t pending, int ipl) 264 { 265 uint32_t ipl_irq_mask = 0; 266 uint32_t irq_mask; 267 268 for (;;) { 269 int irq = ffs(pending); 270 if (irq-- == 0) 271 return ipl_irq_mask; 272 273 irq_mask = __BIT(irq); 274 #if 1 275 KASSERTMSG(pic->pic_sources[irq_base + irq] != NULL, 276 "%s: irq_base %zu irq %d\n", __func__, irq_base, irq); 277 #else 278 if (pic->pic_sources[irq_base + irq] == NULL) { 279 aprint_error("stray interrupt? irq_base=%zu irq=%d\n", 280 irq_base, irq); 281 } else 282 #endif 283 if (pic->pic_sources[irq_base + irq]->is_ipl == ipl) 284 ipl_irq_mask |= irq_mask; 285 286 pending &= ~irq_mask; 287 } 288 } 289 #endif /* __HAVE_PIC_PENDING_INTRS */ 290 291 void 292 pic_dispatch(struct intrsource *is, void *frame) 293 { 294 int (*func)(void *) = is->is_func; 295 void *arg = is->is_arg; 296 297 if (__predict_false(arg == NULL)) { 298 if (__predict_false(frame == NULL)) { 299 pic_deferral_ev.ev_count++; 300 return; 301 } 302 arg = frame; 303 } 304 305 #ifdef MULTIPROCESSOR 306 if (!is->is_mpsafe) { 307 KERNEL_LOCK(1, NULL); 308 const u_int ci_blcnt __diagused = curcpu()->ci_biglock_count; 309 const u_int l_blcnt __diagused = curlwp->l_blcnt; 310 (void)(*func)(arg); 311 KASSERT(ci_blcnt == curcpu()->ci_biglock_count); 312 KASSERT(l_blcnt == curlwp->l_blcnt); 313 KERNEL_UNLOCK_ONE(NULL); 314 } else 315 #endif 316 (void)(*func)(arg); 317 318 319 struct pic_percpu * const pcpu = percpu_getref(is->is_pic->pic_percpu); 320 KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC); 321 pcpu->pcpu_evs[is->is_irq].ev_count++; 322 percpu_putref(is->is_pic->pic_percpu); 323 } 324 325 #if defined(__HAVE_PIC_PENDING_INTRS) 326 void 327 pic_deliver_irqs(struct pic_softc *pic, int ipl, void *frame) 328 { 329 const uint32_t ipl_mask = __BIT(ipl); 330 struct intrsource *is; 331 volatile uint32_t *ipending = pic->pic_pending_irqs; 332 volatile uint32_t *iblocked = pic->pic_blocked_irqs; 333 size_t irq_base; 334 #if PIC_MAXSOURCES > 32 335 size_t irq_count; 336 int poi = 0; /* Possibility of interrupting */ 337 #endif 338 uint32_t pending_irqs; 339 uint32_t blocked_irqs; 340 int irq; 341 bool progress __diagused = false; 342 343 KASSERT(pic->pic_pending_ipls & ipl_mask); 344 345 irq_base = 0; 346 #if PIC_MAXSOURCES > 32 347 irq_count = 0; 348 #endif 349 350 for (;;) { 351 pending_irqs = pic_find_pending_irqs_by_ipl(pic, irq_base, 352 *ipending, ipl); 353 KASSERT((pending_irqs & *ipending) == pending_irqs); 354 KASSERT((pending_irqs & ~(*ipending)) == 0); 355 if (pending_irqs == 0) { 356 #if PIC_MAXSOURCES > 32 357 irq_count += 32; 358 if (__predict_true(irq_count >= pic->pic_maxsources)) { 359 if (!poi) 360 /*Interrupt at this level was handled.*/ 361 break; 362 irq_base = 0; 363 irq_count = 0; 364 poi = 0; 365 ipending = pic->pic_pending_irqs; 366 iblocked = pic->pic_blocked_irqs; 367 } else { 368 irq_base += 32; 369 ipending++; 370 iblocked++; 371 KASSERT(irq_base <= pic->pic_maxsources); 372 } 373 continue; 374 #else 375 break; 376 #endif 377 } 378 progress = true; 379 blocked_irqs = 0; 380 do { 381 irq = ffs(pending_irqs) - 1; 382 KASSERT(irq >= 0); 383 384 atomic_and_32(ipending, ~__BIT(irq)); 385 is = pic->pic_sources[irq_base + irq]; 386 if (is != NULL) { 387 cpsie(I32_bit); 388 pic_dispatch(is, frame); 389 cpsid(I32_bit); 390 #if PIC_MAXSOURCES > 32 391 /* 392 * There is a possibility of interrupting 393 * from cpsie() to cpsid(). 394 */ 395 poi = 1; 396 #endif 397 blocked_irqs |= __BIT(irq); 398 } else { 399 KASSERT(0); 400 } 401 pending_irqs = pic_find_pending_irqs_by_ipl(pic, 402 irq_base, *ipending, ipl); 403 } while (pending_irqs); 404 if (blocked_irqs) { 405 atomic_or_32(iblocked, blocked_irqs); 406 atomic_or_32(&pic_blocked_pics, __BIT(pic->pic_id)); 407 } 408 } 409 410 KASSERT(progress); 411 /* 412 * Since interrupts are disabled, we don't have to be too careful 413 * about these. 414 */ 415 if (atomic_and_32_nv(&pic->pic_pending_ipls, ~ipl_mask) == 0) 416 atomic_and_32(&pic_pending_pics, ~__BIT(pic->pic_id)); 417 } 418 419 static void 420 pic_list_unblock_irqs(void) 421 { 422 uint32_t blocked_pics = pic_blocked_pics; 423 424 pic_blocked_pics = 0; 425 for (;;) { 426 struct pic_softc *pic; 427 #if PIC_MAXSOURCES > 32 428 volatile uint32_t *iblocked; 429 uint32_t blocked; 430 size_t irq_base; 431 #endif 432 433 int pic_id = ffs(blocked_pics); 434 if (pic_id-- == 0) 435 return; 436 437 pic = pic_list[pic_id]; 438 KASSERT(pic != NULL); 439 #if PIC_MAXSOURCES > 32 440 for (irq_base = 0, iblocked = pic->pic_blocked_irqs; 441 irq_base < pic->pic_maxsources; 442 irq_base += 32, iblocked++) { 443 if ((blocked = *iblocked) != 0) { 444 (*pic->pic_ops->pic_unblock_irqs)(pic, 445 irq_base, blocked); 446 atomic_and_32(iblocked, ~blocked); 447 } 448 } 449 #else 450 KASSERT(pic->pic_blocked_irqs[0] != 0); 451 (*pic->pic_ops->pic_unblock_irqs)(pic, 452 0, pic->pic_blocked_irqs[0]); 453 pic->pic_blocked_irqs[0] = 0; 454 #endif 455 blocked_pics &= ~__BIT(pic_id); 456 } 457 } 458 459 460 struct pic_softc * 461 pic_list_find_pic_by_pending_ipl(uint32_t ipl_mask) 462 { 463 uint32_t pending_pics = pic_pending_pics; 464 struct pic_softc *pic; 465 466 for (;;) { 467 int pic_id = ffs(pending_pics); 468 if (pic_id-- == 0) 469 return NULL; 470 471 pic = pic_list[pic_id]; 472 KASSERT(pic != NULL); 473 if (pic->pic_pending_ipls & ipl_mask) 474 return pic; 475 pending_pics &= ~__BIT(pic_id); 476 } 477 } 478 479 void 480 pic_list_deliver_irqs(register_t psw, int ipl, void *frame) 481 { 482 const uint32_t ipl_mask = __BIT(ipl); 483 struct pic_softc *pic; 484 485 while ((pic = pic_list_find_pic_by_pending_ipl(ipl_mask)) != NULL) { 486 pic_deliver_irqs(pic, ipl, frame); 487 KASSERT((pic->pic_pending_ipls & ipl_mask) == 0); 488 } 489 atomic_and_32(&pic_pending_ipls, ~ipl_mask); 490 } 491 #endif /* __HAVE_PIC_PENDING_INTRS */ 492 493 void 494 pic_do_pending_ints(register_t psw, int newipl, void *frame) 495 { 496 struct cpu_info * const ci = curcpu(); 497 if (__predict_false(newipl == IPL_HIGH)) { 498 KASSERTMSG(ci->ci_cpl == IPL_HIGH, "cpl %d", ci->ci_cpl); 499 return; 500 } 501 #if defined(__HAVE_PIC_PENDING_INTRS) 502 while ((pic_pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) { 503 KASSERT(pic_pending_ipls < __BIT(NIPL)); 504 for (;;) { 505 int ipl = 31 - __builtin_clz(pic_pending_ipls); 506 KASSERT(ipl < NIPL); 507 if (ipl <= newipl) 508 break; 509 510 pic_set_priority(ci, ipl); 511 pic_list_deliver_irqs(psw, ipl, frame); 512 pic_list_unblock_irqs(); 513 } 514 } 515 #endif /* __HAVE_PIC_PENDING_INTRS */ 516 #ifdef __HAVE_PREEEMPTION 517 if (newipl == IPL_NONE && (ci->ci_astpending & __BIT(1))) { 518 pic_set_priority(ci, IPL_SCHED); 519 kpreempt(0); 520 } 521 #endif 522 if (ci->ci_cpl != newipl) 523 pic_set_priority(ci, newipl); 524 } 525 526 static void 527 pic_percpu_allocate(void *v0, void *v1, struct cpu_info *ci) 528 { 529 struct pic_percpu * const pcpu = v0; 530 struct pic_softc * const pic = v1; 531 532 pcpu->pcpu_evs = kmem_zalloc(pic->pic_maxsources * sizeof(pcpu->pcpu_evs[0]), 533 KM_SLEEP); 534 KASSERT(pcpu->pcpu_evs != NULL); 535 536 #define PCPU_NAMELEN 32 537 #ifdef DIAGNOSTIC 538 const size_t namelen = strlen(pic->pic_name) + 4 + strlen(ci->ci_data.cpu_name); 539 #endif 540 541 KASSERT(namelen < PCPU_NAMELEN); 542 pcpu->pcpu_name = kmem_alloc(PCPU_NAMELEN, KM_SLEEP); 543 #ifdef MULTIPROCESSOR 544 snprintf(pcpu->pcpu_name, PCPU_NAMELEN, 545 "%s (%s)", pic->pic_name, ci->ci_data.cpu_name); 546 #else 547 strlcpy(pcpu->pcpu_name, pic->pic_name, PCPU_NAMELEN); 548 #endif 549 pcpu->pcpu_magic = PICPERCPU_MAGIC; 550 #if 0 551 printf("%s: %s %s: <%s>\n", 552 __func__, ci->ci_data.cpu_name, pic->pic_name, 553 pcpu->pcpu_name); 554 #endif 555 } 556 557 void 558 pic_add(struct pic_softc *pic, int irqbase) 559 { 560 int slot, maybe_slot = -1; 561 562 KASSERT(strlen(pic->pic_name) > 0); 563 564 for (slot = 0; slot < PIC_MAXPICS; slot++) { 565 struct pic_softc * const xpic = pic_list[slot]; 566 if (xpic == NULL) { 567 if (maybe_slot < 0) 568 maybe_slot = slot; 569 if (irqbase < 0) 570 break; 571 continue; 572 } 573 if (irqbase < 0 || xpic->pic_irqbase < 0) 574 continue; 575 if (irqbase >= xpic->pic_irqbase + xpic->pic_maxsources) 576 continue; 577 if (irqbase + pic->pic_maxsources <= xpic->pic_irqbase) 578 continue; 579 panic("pic_add: pic %s (%zu sources @ irq %u) conflicts" 580 " with pic %s (%zu sources @ irq %u)", 581 pic->pic_name, pic->pic_maxsources, irqbase, 582 xpic->pic_name, xpic->pic_maxsources, xpic->pic_irqbase); 583 } 584 slot = maybe_slot; 585 #if 0 586 printf("%s: pic_sourcebase=%zu pic_maxsources=%zu\n", 587 pic->pic_name, pic_sourcebase, pic->pic_maxsources); 588 #endif 589 KASSERTMSG(pic->pic_maxsources <= PIC_MAXSOURCES, "%zu", 590 pic->pic_maxsources); 591 KASSERT(pic_sourcebase + pic->pic_maxsources <= PIC_MAXMAXSOURCES); 592 593 /* 594 * Allocate a pointer to each cpu's evcnts and then, for each cpu, 595 * allocate its evcnts and then attach an evcnt for each pin. 596 * We can't allocate the evcnt structures directly since 597 * percpu will move the contents of percpu memory around and 598 * corrupt the pointers in the evcnts themselves. Remember, any 599 * problem can be solved with sufficient indirection. 600 */ 601 pic->pic_percpu = percpu_alloc(sizeof(struct pic_percpu)); 602 KASSERT(pic->pic_percpu != NULL); 603 604 /* 605 * Now allocate the per-cpu evcnts. 606 */ 607 percpu_foreach(pic->pic_percpu, pic_percpu_allocate, pic); 608 609 pic->pic_sources = &pic_sources[pic_sourcebase]; 610 pic->pic_irqbase = irqbase; 611 pic_sourcebase += pic->pic_maxsources; 612 pic->pic_id = slot; 613 #ifdef __HAVE_PIC_SET_PRIORITY 614 KASSERT((slot == 0) == (pic->pic_ops->pic_set_priority != NULL)); 615 #endif 616 #ifdef MULTIPROCESSOR 617 KASSERT((slot == 0) == (pic->pic_ops->pic_ipi_send != NULL)); 618 #endif 619 pic_list[slot] = pic; 620 } 621 622 int 623 pic_alloc_irq(struct pic_softc *pic) 624 { 625 int irq; 626 627 for (irq = 0; irq < pic->pic_maxsources; irq++) { 628 if (pic->pic_sources[irq] == NULL) 629 return irq; 630 } 631 632 return -1; 633 } 634 635 static void 636 pic_percpu_evcnt_attach(void *v0, void *v1, struct cpu_info *ci) 637 { 638 struct pic_percpu * const pcpu = v0; 639 struct intrsource * const is = v1; 640 641 KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC); 642 evcnt_attach_dynamic(&pcpu->pcpu_evs[is->is_irq], EVCNT_TYPE_INTR, NULL, 643 pcpu->pcpu_name, is->is_source); 644 } 645 646 void * 647 pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type, 648 int (*func)(void *), void *arg) 649 { 650 struct intrsource *is; 651 int off, nipl; 652 653 if (pic->pic_sources[irq]) { 654 printf("pic_establish_intr: pic %s irq %d already present\n", 655 pic->pic_name, irq); 656 return NULL; 657 } 658 659 is = kmem_zalloc(sizeof(*is), KM_SLEEP); 660 if (is == NULL) 661 return NULL; 662 663 is->is_pic = pic; 664 is->is_irq = irq; 665 is->is_ipl = ipl; 666 is->is_type = type & 0xff; 667 is->is_func = func; 668 is->is_arg = arg; 669 #ifdef MULTIPROCESSOR 670 is->is_mpsafe = (type & IST_MPSAFE) || ipl != IPL_VM; 671 #endif 672 673 if (pic->pic_ops->pic_source_name) 674 (*pic->pic_ops->pic_source_name)(pic, irq, is->is_source, 675 sizeof(is->is_source)); 676 else 677 snprintf(is->is_source, sizeof(is->is_source), "irq %d", irq); 678 679 /* 680 * Now attach the per-cpu evcnts. 681 */ 682 percpu_foreach(pic->pic_percpu, pic_percpu_evcnt_attach, is); 683 684 pic->pic_sources[irq] = is; 685 686 /* 687 * First try to use an existing slot which is empty. 688 */ 689 for (off = pic_ipl_offset[ipl]; off < pic_ipl_offset[ipl+1]; off++) { 690 if (pic__iplsources[off] == NULL) { 691 is->is_iplidx = off - pic_ipl_offset[ipl]; 692 pic__iplsources[off] = is; 693 return is; 694 } 695 } 696 697 /* 698 * Move up all the sources by one. 699 */ 700 if (ipl < NIPL) { 701 off = pic_ipl_offset[ipl+1]; 702 memmove(&pic__iplsources[off+1], &pic__iplsources[off], 703 sizeof(pic__iplsources[0]) * (pic_ipl_offset[NIPL] - off)); 704 } 705 706 /* 707 * Advance the offset of all IPLs higher than this. Include an 708 * extra one as well. Thus the number of sources per ipl is 709 * pic_ipl_offset[ipl+1] - pic_ipl_offset[ipl]. 710 */ 711 for (nipl = ipl + 1; nipl <= NIPL; nipl++) 712 pic_ipl_offset[nipl]++; 713 714 /* 715 * Insert into the previously made position at the end of this IPL's 716 * sources. 717 */ 718 off = pic_ipl_offset[ipl + 1] - 1; 719 is->is_iplidx = off - pic_ipl_offset[ipl]; 720 pic__iplsources[off] = is; 721 722 (*pic->pic_ops->pic_establish_irq)(pic, is); 723 724 (*pic->pic_ops->pic_unblock_irqs)(pic, is->is_irq & ~0x1f, 725 __BIT(is->is_irq & 0x1f)); 726 727 /* We're done. */ 728 return is; 729 } 730 731 static void 732 pic_percpu_evcnt_deattach(void *v0, void *v1, struct cpu_info *ci) 733 { 734 struct pic_percpu * const pcpu = v0; 735 struct intrsource * const is = v1; 736 737 KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC); 738 evcnt_detach(&pcpu->pcpu_evs[is->is_irq]); 739 } 740 741 void 742 pic_disestablish_source(struct intrsource *is) 743 { 744 struct pic_softc * const pic = is->is_pic; 745 const int irq = is->is_irq; 746 747 KASSERT(is == pic->pic_sources[irq]); 748 749 (*pic->pic_ops->pic_block_irqs)(pic, irq & ~0x1f, __BIT(irq & 0x1f)); 750 pic->pic_sources[irq] = NULL; 751 pic__iplsources[pic_ipl_offset[is->is_ipl] + is->is_iplidx] = NULL; 752 /* 753 * Now detach the per-cpu evcnts. 754 */ 755 percpu_foreach(pic->pic_percpu, pic_percpu_evcnt_deattach, is); 756 757 kmem_free(is, sizeof(*is)); 758 } 759 760 void * 761 intr_establish(int irq, int ipl, int type, int (*func)(void *), void *arg) 762 { 763 KASSERT(!cpu_intr_p()); 764 KASSERT(!cpu_softintr_p()); 765 766 for (size_t slot = 0; slot < PIC_MAXPICS; slot++) { 767 struct pic_softc * const pic = pic_list[slot]; 768 if (pic == NULL || pic->pic_irqbase < 0) 769 continue; 770 if (pic->pic_irqbase <= irq 771 && irq < pic->pic_irqbase + pic->pic_maxsources) { 772 return pic_establish_intr(pic, irq - pic->pic_irqbase, 773 ipl, type, func, arg); 774 } 775 } 776 777 return NULL; 778 } 779 780 void 781 intr_disestablish(void *ih) 782 { 783 struct intrsource * const is = ih; 784 785 KASSERT(!cpu_intr_p()); 786 KASSERT(!cpu_softintr_p()); 787 788 pic_disestablish_source(is); 789 } 790
Indexes created Tue Sep 30 20:09:53 GMT 2025