pic.c revision 1.25.2.1
1 /* $NetBSD: pic.c,v 1.25.2.1 2015/04/06 15:17:53 skrll Exp $ */ 2 /*- 3 * Copyright (c) 2008 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Matt Thomas. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #define _INTR_PRIVATE 32 #include "opt_ddb.h" 33 #include "opt_multiprocessor.h" 34 35 #include <sys/cdefs.h> 36 __KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.25.2.1 2015/04/06 15:17:53 skrll Exp $"); 37 38 #include <sys/param.h> 39 #include <sys/atomic.h> 40 #include <sys/cpu.h> 41 #include <sys/evcnt.h> 42 #include <sys/intr.h> 43 #include <sys/kernel.h> 44 #include <sys/kmem.h> 45 #include <sys/xcall.h> 46 #include <sys/ipi.h> 47 48 #if defined(__arm__) 49 #include <arm/armreg.h> 50 #include <arm/cpufunc.h> 51 #elif defined(__aarch64__) 52 #include <aarch64/locore.h> 53 #define I32_bit DAIF_I 54 #define F32_bit DAIF_F 55 #endif 56 57 #ifdef DDB 58 #include <arm/db_machdep.h> 59 #endif 60 61 #include <arm/pic/picvar.h> 62 63 static uint32_t 64 pic_find_pending_irqs_by_ipl(struct pic_softc *, size_t, uint32_t, int); 65 static struct pic_softc * 66 pic_list_find_pic_by_pending_ipl(uint32_t); 67 static void 68 pic_deliver_irqs(struct pic_softc *, int, void *); 69 static void 70 pic_list_deliver_irqs(register_t, int, void *); 71 72 struct pic_softc *pic_list[PIC_MAXPICS]; 73 #if PIC_MAXPICS > 32 74 #error PIC_MAXPICS > 32 not supported 75 #endif 76 volatile uint32_t pic_blocked_pics; 77 volatile uint32_t pic_pending_pics; 78 volatile uint32_t pic_pending_ipls; 79 struct intrsource *pic_sources[PIC_MAXMAXSOURCES]; 80 struct intrsource *pic__iplsources[PIC_MAXMAXSOURCES]; 81 struct intrsource **pic_iplsource[NIPL] = { 82 [0 ... NIPL-1] = pic__iplsources, 83 }; 84 size_t pic_ipl_offset[NIPL+1]; 85 size_t pic_sourcebase; 86 static struct evcnt pic_deferral_ev = 87 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "deferred", "intr"); 88 EVCNT_ATTACH_STATIC(pic_deferral_ev); 89 90 #ifdef __HAVE_PIC_SET_PRIORITY 91 void 92 pic_set_priority(struct cpu_info *ci, int newipl) 93 { 94 register_t psw = cpsid(I32_bit); 95 if (pic_list[0] != NULL) 96 (pic_list[0]->pic_ops->pic_set_priority)(pic_list[0], newipl); 97 ci->ci_cpl = newipl; 98 if ((psw & I32_bit) == 0) 99 cpsie(I32_bit); 100 } 101 #endif 102 103 #ifdef MULTIPROCESSOR 104 int 105 pic_ipi_nop(void *arg) 106 { 107 /* do nothing */ 108 return 1; 109 } 110 111 int 112 pic_ipi_xcall(void *arg) 113 { 114 xc_ipi_handler(); 115 return 1; 116 } 117 118 int 119 pic_ipi_generic(void *arg) 120 { 121 ipi_cpu_handler(); 122 return 1; 123 } 124 125 #ifdef DDB 126 int 127 pic_ipi_ddb(void *arg) 128 { 129 // printf("%s: %s: tf=%p\n", __func__, curcpu()->ci_cpuname, arg); 130 kdb_trap(-1, arg); 131 return 1; 132 } 133 #endif 134 135 void 136 intr_cpu_init(struct cpu_info *ci) 137 { 138 for (size_t slot = 0; slot < PIC_MAXPICS; slot++) { 139 struct pic_softc * const pic = pic_list[slot]; 140 if (pic != NULL && pic->pic_ops->pic_cpu_init != NULL) { 141 (*pic->pic_ops->pic_cpu_init)(pic, ci); 142 } 143 } 144 } 145 146 typedef void (*pic_ipi_send_func_t)(struct pic_softc *, u_long); 147 148 static struct pic_softc * 149 pic_ipi_sender(void) 150 { 151 for (size_t slot = 0; slot < PIC_MAXPICS; slot++) { 152 struct pic_softc * const pic = pic_list[slot]; 153 if (pic != NULL && pic->pic_ops->pic_ipi_send != NULL) { 154 return pic; 155 } 156 } 157 return NULL; 158 } 159 160 void 161 intr_ipi_send(const kcpuset_t *kcp, u_long ipi) 162 { 163 struct pic_softc * const pic = pic_ipi_sender(); 164 KASSERT(ipi < NIPI); 165 if (cold && pic == NULL) 166 return; 167 KASSERT(pic != NULL); 168 (*pic->pic_ops->pic_ipi_send)(pic, kcp, ipi); 169 } 170 #endif /* MULTIPROCESSOR */ 171 172 #ifdef __HAVE_PIC_FAST_SOFTINTS 173 int 174 pic_handle_softint(void *arg) 175 { 176 void softint_switch(lwp_t *, int); 177 struct cpu_info * const ci = curcpu(); 178 const size_t softint = (size_t) arg; 179 int s = splhigh(); 180 ci->ci_intr_depth--; // don't count these as interrupts 181 softint_switch(ci->ci_softlwps[softint], s); 182 ci->ci_intr_depth++; 183 splx(s); 184 return 1; 185 } 186 #endif 187 188 int 189 pic_handle_intr(void *arg) 190 { 191 struct pic_softc * const pic = arg; 192 int rv; 193 194 rv = (*pic->pic_ops->pic_find_pending_irqs)(pic); 195 196 return rv > 0; 197 } 198 199 void 200 pic_mark_pending_source(struct pic_softc *pic, struct intrsource *is) 201 { 202 const uint32_t ipl_mask = __BIT(is->is_ipl); 203 204 atomic_or_32(&pic->pic_pending_irqs[is->is_irq >> 5], 205 __BIT(is->is_irq & 0x1f)); 206 207 atomic_or_32(&pic->pic_pending_ipls, ipl_mask); 208 atomic_or_32(&pic_pending_ipls, ipl_mask); 209 atomic_or_32(&pic_pending_pics, __BIT(pic->pic_id)); 210 } 211 212 void 213 pic_mark_pending(struct pic_softc *pic, int irq) 214 { 215 struct intrsource * const is = pic->pic_sources[irq]; 216 217 KASSERT(irq < pic->pic_maxsources); 218 KASSERT(is != NULL); 219 220 pic_mark_pending_source(pic, is); 221 } 222 223 uint32_t 224 pic_mark_pending_sources(struct pic_softc *pic, size_t irq_base, 225 uint32_t pending) 226 { 227 struct intrsource ** const isbase = &pic->pic_sources[irq_base]; 228 struct intrsource *is; 229 volatile uint32_t *ipending = &pic->pic_pending_irqs[irq_base >> 5]; 230 uint32_t ipl_mask = 0; 231 232 if (pending == 0) 233 return ipl_mask; 234 235 KASSERT((irq_base & 31) == 0); 236 237 (*pic->pic_ops->pic_block_irqs)(pic, irq_base, pending); 238 239 atomic_or_32(ipending, pending); 240 while (pending != 0) { 241 int n = ffs(pending); 242 if (n-- == 0) 243 break; 244 is = isbase[n]; 245 KASSERT(is != NULL); 246 KASSERT(irq_base <= is->is_irq && is->is_irq < irq_base + 32); 247 pending &= ~__BIT(n); 248 ipl_mask |= __BIT(is->is_ipl); 249 } 250 251 atomic_or_32(&pic->pic_pending_ipls, ipl_mask); 252 atomic_or_32(&pic_pending_ipls, ipl_mask); 253 atomic_or_32(&pic_pending_pics, __BIT(pic->pic_id)); 254 255 return ipl_mask; 256 } 257 258 uint32_t 259 pic_find_pending_irqs_by_ipl(struct pic_softc *pic, size_t irq_base, 260 uint32_t pending, int ipl) 261 { 262 uint32_t ipl_irq_mask = 0; 263 uint32_t irq_mask; 264 265 for (;;) { 266 int irq = ffs(pending); 267 if (irq-- == 0) 268 return ipl_irq_mask; 269 270 irq_mask = __BIT(irq); 271 #if 1 272 KASSERTMSG(pic->pic_sources[irq_base + irq] != NULL, 273 "%s: irq_base %zu irq %d\n", __func__, irq_base, irq); 274 #else 275 if (pic->pic_sources[irq_base + irq] == NULL) { 276 aprint_error("stray interrupt? irq_base=%zu irq=%d\n", 277 irq_base, irq); 278 } else 279 #endif 280 if (pic->pic_sources[irq_base + irq]->is_ipl == ipl) 281 ipl_irq_mask |= irq_mask; 282 283 pending &= ~irq_mask; 284 } 285 } 286 287 void 288 pic_dispatch(struct intrsource *is, void *frame) 289 { 290 int (*func)(void *) = is->is_func; 291 void *arg = is->is_arg; 292 293 if (__predict_false(arg == NULL)) { 294 if (__predict_false(frame == NULL)) { 295 pic_deferral_ev.ev_count++; 296 return; 297 } 298 arg = frame; 299 } 300 301 #ifdef MULTIPROCESSOR 302 if (!is->is_mpsafe) { 303 KERNEL_LOCK(1, NULL); 304 const u_int ci_blcnt __diagused = curcpu()->ci_biglock_count; 305 const u_int l_blcnt __diagused = curlwp->l_blcnt; 306 (void)(*func)(arg); 307 KASSERT(ci_blcnt == curcpu()->ci_biglock_count); 308 KASSERT(l_blcnt == curlwp->l_blcnt); 309 KERNEL_UNLOCK_ONE(NULL); 310 } else 311 #endif 312 (void)(*func)(arg); 313 314 315 struct pic_percpu * const pcpu = percpu_getref(is->is_pic->pic_percpu); 316 KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC); 317 pcpu->pcpu_evs[is->is_irq].ev_count++; 318 percpu_putref(is->is_pic->pic_percpu); 319 } 320 321 void 322 pic_deliver_irqs(struct pic_softc *pic, int ipl, void *frame) 323 { 324 const uint32_t ipl_mask = __BIT(ipl); 325 struct intrsource *is; 326 volatile uint32_t *ipending = pic->pic_pending_irqs; 327 volatile uint32_t *iblocked = pic->pic_blocked_irqs; 328 size_t irq_base; 329 #if PIC_MAXSOURCES > 32 330 size_t irq_count; 331 int poi = 0; /* Possibility of interrupting */ 332 #endif 333 uint32_t pending_irqs; 334 uint32_t blocked_irqs; 335 int irq; 336 bool progress __diagused = false; 337 338 KASSERT(pic->pic_pending_ipls & ipl_mask); 339 340 irq_base = 0; 341 #if PIC_MAXSOURCES > 32 342 irq_count = 0; 343 #endif 344 345 for (;;) { 346 pending_irqs = pic_find_pending_irqs_by_ipl(pic, irq_base, 347 *ipending, ipl); 348 KASSERT((pending_irqs & *ipending) == pending_irqs); 349 KASSERT((pending_irqs & ~(*ipending)) == 0); 350 if (pending_irqs == 0) { 351 #if PIC_MAXSOURCES > 32 352 irq_count += 32; 353 if (__predict_true(irq_count >= pic->pic_maxsources)) { 354 if (!poi) 355 /*Interrupt at this level was handled.*/ 356 break; 357 irq_base = 0; 358 irq_count = 0; 359 poi = 0; 360 ipending = pic->pic_pending_irqs; 361 iblocked = pic->pic_blocked_irqs; 362 } else { 363 irq_base += 32; 364 ipending++; 365 iblocked++; 366 KASSERT(irq_base <= pic->pic_maxsources); 367 } 368 continue; 369 #else 370 break; 371 #endif 372 } 373 progress = true; 374 blocked_irqs = 0; 375 do { 376 irq = ffs(pending_irqs) - 1; 377 KASSERT(irq >= 0); 378 379 atomic_and_32(ipending, ~__BIT(irq)); 380 is = pic->pic_sources[irq_base + irq]; 381 if (is != NULL) { 382 cpsie(I32_bit); 383 pic_dispatch(is, frame); 384 cpsid(I32_bit); 385 #if PIC_MAXSOURCES > 32 386 /* 387 * There is a possibility of interrupting 388 * from cpsie() to cpsid(). 389 */ 390 poi = 1; 391 #endif 392 blocked_irqs |= __BIT(irq); 393 } else { 394 KASSERT(0); 395 } 396 pending_irqs = pic_find_pending_irqs_by_ipl(pic, 397 irq_base, *ipending, ipl); 398 } while (pending_irqs); 399 if (blocked_irqs) { 400 atomic_or_32(iblocked, blocked_irqs); 401 atomic_or_32(&pic_blocked_pics, __BIT(pic->pic_id)); 402 } 403 } 404 405 KASSERT(progress); 406 /* 407 * Since interrupts are disabled, we don't have to be too careful 408 * about these. 409 */ 410 if (atomic_and_32_nv(&pic->pic_pending_ipls, ~ipl_mask) == 0) 411 atomic_and_32(&pic_pending_pics, ~__BIT(pic->pic_id)); 412 } 413 414 static void 415 pic_list_unblock_irqs(void) 416 { 417 uint32_t blocked_pics = pic_blocked_pics; 418 419 pic_blocked_pics = 0; 420 for (;;) { 421 struct pic_softc *pic; 422 #if PIC_MAXSOURCES > 32 423 volatile uint32_t *iblocked; 424 uint32_t blocked; 425 size_t irq_base; 426 #endif 427 428 int pic_id = ffs(blocked_pics); 429 if (pic_id-- == 0) 430 return; 431 432 pic = pic_list[pic_id]; 433 KASSERT(pic != NULL); 434 #if PIC_MAXSOURCES > 32 435 for (irq_base = 0, iblocked = pic->pic_blocked_irqs; 436 irq_base < pic->pic_maxsources; 437 irq_base += 32, iblocked++) { 438 if ((blocked = *iblocked) != 0) { 439 (*pic->pic_ops->pic_unblock_irqs)(pic, 440 irq_base, blocked); 441 atomic_and_32(iblocked, ~blocked); 442 } 443 } 444 #else 445 KASSERT(pic->pic_blocked_irqs[0] != 0); 446 (*pic->pic_ops->pic_unblock_irqs)(pic, 447 0, pic->pic_blocked_irqs[0]); 448 pic->pic_blocked_irqs[0] = 0; 449 #endif 450 blocked_pics &= ~__BIT(pic_id); 451 } 452 } 453 454 455 struct pic_softc * 456 pic_list_find_pic_by_pending_ipl(uint32_t ipl_mask) 457 { 458 uint32_t pending_pics = pic_pending_pics; 459 struct pic_softc *pic; 460 461 for (;;) { 462 int pic_id = ffs(pending_pics); 463 if (pic_id-- == 0) 464 return NULL; 465 466 pic = pic_list[pic_id]; 467 KASSERT(pic != NULL); 468 if (pic->pic_pending_ipls & ipl_mask) 469 return pic; 470 pending_pics &= ~__BIT(pic_id); 471 } 472 } 473 474 void 475 pic_list_deliver_irqs(register_t psw, int ipl, void *frame) 476 { 477 const uint32_t ipl_mask = __BIT(ipl); 478 struct pic_softc *pic; 479 480 while ((pic = pic_list_find_pic_by_pending_ipl(ipl_mask)) != NULL) { 481 pic_deliver_irqs(pic, ipl, frame); 482 KASSERT((pic->pic_pending_ipls & ipl_mask) == 0); 483 } 484 atomic_and_32(&pic_pending_ipls, ~ipl_mask); 485 } 486 487 void 488 pic_do_pending_ints(register_t psw, int newipl, void *frame) 489 { 490 struct cpu_info * const ci = curcpu(); 491 if (__predict_false(newipl == IPL_HIGH)) { 492 KASSERTMSG(ci->ci_cpl == IPL_HIGH, "cpl %d", ci->ci_cpl); 493 return; 494 } 495 while ((pic_pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) { 496 KASSERT(pic_pending_ipls < __BIT(NIPL)); 497 for (;;) { 498 int ipl = 31 - __builtin_clz(pic_pending_ipls); 499 KASSERT(ipl < NIPL); 500 if (ipl <= newipl) 501 break; 502 503 pic_set_priority(ci, ipl); 504 pic_list_deliver_irqs(psw, ipl, frame); 505 pic_list_unblock_irqs(); 506 } 507 } 508 if (ci->ci_cpl != newipl) 509 pic_set_priority(ci, newipl); 510 } 511 512 static void 513 pic_percpu_allocate(void *v0, void *v1, struct cpu_info *ci) 514 { 515 struct pic_percpu * const pcpu = v0; 516 struct pic_softc * const pic = v1; 517 518 pcpu->pcpu_evs = kmem_zalloc(pic->pic_maxsources * sizeof(pcpu->pcpu_evs[0]), 519 KM_SLEEP); 520 KASSERT(pcpu->pcpu_evs != NULL); 521 522 #define PCPU_NAMELEN 32 523 #ifdef DIAGNOSTIC 524 const size_t namelen = strlen(pic->pic_name) + 4 + strlen(ci->ci_data.cpu_name); 525 #endif 526 527 KASSERT(namelen < PCPU_NAMELEN); 528 pcpu->pcpu_name = kmem_alloc(PCPU_NAMELEN, KM_SLEEP); 529 #ifdef MULTIPROCESSOR 530 snprintf(pcpu->pcpu_name, PCPU_NAMELEN, 531 "%s (%s)", pic->pic_name, ci->ci_data.cpu_name); 532 #else 533 strlcpy(pcpu->pcpu_name, pic->pic_name, PCPU_NAMELEN); 534 #endif 535 pcpu->pcpu_magic = PICPERCPU_MAGIC; 536 #if 0 537 printf("%s: %s %s: <%s>\n", 538 __func__, ci->ci_data.cpu_name, pic->pic_name, 539 pcpu->pcpu_name); 540 #endif 541 } 542 543 void 544 pic_add(struct pic_softc *pic, int irqbase) 545 { 546 int slot, maybe_slot = -1; 547 548 KASSERT(strlen(pic->pic_name) > 0); 549 550 for (slot = 0; slot < PIC_MAXPICS; slot++) { 551 struct pic_softc * const xpic = pic_list[slot]; 552 if (xpic == NULL) { 553 if (maybe_slot < 0) 554 maybe_slot = slot; 555 if (irqbase < 0) 556 break; 557 continue; 558 } 559 if (irqbase < 0 || xpic->pic_irqbase < 0) 560 continue; 561 if (irqbase >= xpic->pic_irqbase + xpic->pic_maxsources) 562 continue; 563 if (irqbase + pic->pic_maxsources <= xpic->pic_irqbase) 564 continue; 565 panic("pic_add: pic %s (%zu sources @ irq %u) conflicts" 566 " with pic %s (%zu sources @ irq %u)", 567 pic->pic_name, pic->pic_maxsources, irqbase, 568 xpic->pic_name, xpic->pic_maxsources, xpic->pic_irqbase); 569 } 570 slot = maybe_slot; 571 #if 0 572 printf("%s: pic_sourcebase=%zu pic_maxsources=%zu\n", 573 pic->pic_name, pic_sourcebase, pic->pic_maxsources); 574 #endif 575 KASSERTMSG(pic->pic_maxsources <= PIC_MAXSOURCES, "%zu", 576 pic->pic_maxsources); 577 KASSERT(pic_sourcebase + pic->pic_maxsources <= PIC_MAXMAXSOURCES); 578 579 /* 580 * Allocate a pointer to each cpu's evcnts and then, for each cpu, 581 * allocate its evcnts and then attach an evcnt for each pin. 582 * We can't allocate the evcnt structures directly since 583 * percpu will move the contents of percpu memory around and 584 * corrupt the pointers in the evcnts themselves. Remember, any 585 * problem can be solved with sufficient indirection. 586 */ 587 pic->pic_percpu = percpu_alloc(sizeof(struct pic_percpu)); 588 KASSERT(pic->pic_percpu != NULL); 589 590 /* 591 * Now allocate the per-cpu evcnts. 592 */ 593 percpu_foreach(pic->pic_percpu, pic_percpu_allocate, pic); 594 595 pic->pic_sources = &pic_sources[pic_sourcebase]; 596 pic->pic_irqbase = irqbase; 597 pic_sourcebase += pic->pic_maxsources; 598 pic->pic_id = slot; 599 #ifdef __HAVE_PIC_SET_PRIORITY 600 KASSERT((slot == 0) == (pic->pic_ops->pic_set_priority != NULL)); 601 #endif 602 #ifdef MULTIPROCESSOR 603 KASSERT((slot == 0) == (pic->pic_ops->pic_ipi_send != NULL)); 604 #endif 605 pic_list[slot] = pic; 606 } 607 608 int 609 pic_alloc_irq(struct pic_softc *pic) 610 { 611 int irq; 612 613 for (irq = 0; irq < pic->pic_maxsources; irq++) { 614 if (pic->pic_sources[irq] == NULL) 615 return irq; 616 } 617 618 return -1; 619 } 620 621 static void 622 pic_percpu_evcnt_attach(void *v0, void *v1, struct cpu_info *ci) 623 { 624 struct pic_percpu * const pcpu = v0; 625 struct intrsource * const is = v1; 626 627 KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC); 628 evcnt_attach_dynamic(&pcpu->pcpu_evs[is->is_irq], EVCNT_TYPE_INTR, NULL, 629 pcpu->pcpu_name, is->is_source); 630 } 631 632 void * 633 pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type, 634 int (*func)(void *), void *arg) 635 { 636 struct intrsource *is; 637 int off, nipl; 638 639 if (pic->pic_sources[irq]) { 640 printf("pic_establish_intr: pic %s irq %d already present\n", 641 pic->pic_name, irq); 642 return NULL; 643 } 644 645 is = kmem_zalloc(sizeof(*is), KM_SLEEP); 646 if (is == NULL) 647 return NULL; 648 649 is->is_pic = pic; 650 is->is_irq = irq; 651 is->is_ipl = ipl; 652 is->is_type = type & 0xff; 653 is->is_func = func; 654 is->is_arg = arg; 655 #ifdef MULTIPROCESSOR 656 is->is_mpsafe = (type & IST_MPSAFE) || ipl != IPL_VM; 657 #endif 658 659 if (pic->pic_ops->pic_source_name) 660 (*pic->pic_ops->pic_source_name)(pic, irq, is->is_source, 661 sizeof(is->is_source)); 662 else 663 snprintf(is->is_source, sizeof(is->is_source), "irq %d", irq); 664 665 /* 666 * Now attach the per-cpu evcnts. 667 */ 668 percpu_foreach(pic->pic_percpu, pic_percpu_evcnt_attach, is); 669 670 pic->pic_sources[irq] = is; 671 672 /* 673 * First try to use an existing slot which is empty. 674 */ 675 for (off = pic_ipl_offset[ipl]; off < pic_ipl_offset[ipl+1]; off++) { 676 if (pic__iplsources[off] == NULL) { 677 is->is_iplidx = off - pic_ipl_offset[ipl]; 678 pic__iplsources[off] = is; 679 return is; 680 } 681 } 682 683 /* 684 * Move up all the sources by one. 685 */ 686 if (ipl < NIPL) { 687 off = pic_ipl_offset[ipl+1]; 688 memmove(&pic__iplsources[off+1], &pic__iplsources[off], 689 sizeof(pic__iplsources[0]) * (pic_ipl_offset[NIPL] - off)); 690 } 691 692 /* 693 * Advance the offset of all IPLs higher than this. Include an 694 * extra one as well. Thus the number of sources per ipl is 695 * pic_ipl_offset[ipl+1] - pic_ipl_offset[ipl]. 696 */ 697 for (nipl = ipl + 1; nipl <= NIPL; nipl++) 698 pic_ipl_offset[nipl]++; 699 700 /* 701 * Insert into the previously made position at the end of this IPL's 702 * sources. 703 */ 704 off = pic_ipl_offset[ipl + 1] - 1; 705 is->is_iplidx = off - pic_ipl_offset[ipl]; 706 pic__iplsources[off] = is; 707 708 (*pic->pic_ops->pic_establish_irq)(pic, is); 709 710 (*pic->pic_ops->pic_unblock_irqs)(pic, is->is_irq & ~0x1f, 711 __BIT(is->is_irq & 0x1f)); 712 713 /* We're done. */ 714 return is; 715 } 716 717 static void 718 pic_percpu_evcnt_deattach(void *v0, void *v1, struct cpu_info *ci) 719 { 720 struct pic_percpu * const pcpu = v0; 721 struct intrsource * const is = v1; 722 723 KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC); 724 evcnt_detach(&pcpu->pcpu_evs[is->is_irq]); 725 } 726 727 void 728 pic_disestablish_source(struct intrsource *is) 729 { 730 struct pic_softc * const pic = is->is_pic; 731 const int irq = is->is_irq; 732 733 KASSERT(is == pic->pic_sources[irq]); 734 735 (*pic->pic_ops->pic_block_irqs)(pic, irq & ~0x1f, __BIT(irq & 0x1f)); 736 pic->pic_sources[irq] = NULL; 737 pic__iplsources[pic_ipl_offset[is->is_ipl] + is->is_iplidx] = NULL; 738 /* 739 * Now detach the per-cpu evcnts. 740 */ 741 percpu_foreach(pic->pic_percpu, pic_percpu_evcnt_deattach, is); 742 743 kmem_free(is, sizeof(*is)); 744 } 745 746 void * 747 intr_establish(int irq, int ipl, int type, int (*func)(void *), void *arg) 748 { 749 KASSERT(!cpu_intr_p()); 750 KASSERT(!cpu_softintr_p()); 751 752 for (size_t slot = 0; slot < PIC_MAXPICS; slot++) { 753 struct pic_softc * const pic = pic_list[slot]; 754 if (pic == NULL || pic->pic_irqbase < 0) 755 continue; 756 if (pic->pic_irqbase <= irq 757 && irq < pic->pic_irqbase + pic->pic_maxsources) { 758 return pic_establish_intr(pic, irq - pic->pic_irqbase, 759 ipl, type, func, arg); 760 } 761 } 762 763 return NULL; 764 } 765 766 void 767 intr_disestablish(void *ih) 768 { 769 struct intrsource * const is = ih; 770 771 KASSERT(!cpu_intr_p()); 772 KASSERT(!cpu_softintr_p()); 773 774 pic_disestablish_source(is); 775 } 776
Indexes created Wed Oct 29 09:09:48 GMT 2025