pci_msi_machdep.c revision 1.11
1 /* $NetBSD: pci_msi_machdep.c,v 1.11 2017/04/14 09:34:46 knakahara Exp $ */ 2 3 /* 4 * Copyright (c) 2015 Internet Initiative Japan Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * TODO 31 * 32 * - PBA (Pending Bit Array) support 33 * - HyperTransport mapping support 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: pci_msi_machdep.c,v 1.11 2017/04/14 09:34:46 knakahara Exp $"); 38 39 #include "opt_intrdebug.h" 40 #include "ioapic.h" 41 42 #include <sys/types.h> 43 #include <sys/param.h> 44 #include <sys/time.h> 45 #include <sys/systm.h> 46 #include <sys/cpu.h> 47 #include <sys/errno.h> 48 #include <sys/device.h> 49 #include <sys/intr.h> 50 #include <sys/kmem.h> 51 #include <sys/malloc.h> 52 53 #include <dev/pci/pcivar.h> 54 55 #include <machine/i82093var.h> 56 #include <machine/pic.h> 57 58 #include <x86/pci/msipic.h> 59 #include <x86/pci/pci_msi_machdep.h> 60 61 #ifdef INTRDEBUG 62 #define MSIDEBUG 63 #endif 64 65 #ifdef MSIDEBUG 66 #define DPRINTF(msg) printf msg 67 #else 68 #define DPRINTF(msg) 69 #endif 70 71 static pci_intr_handle_t 72 pci_msi_calculate_handle(struct pic *msi_pic, int vector) 73 { 74 pci_intr_handle_t pih; 75 76 KASSERT(msipic_is_msi_pic(msi_pic)); 77 78 pih = __SHIFTIN((uint64_t)msipic_get_devid(msi_pic), MSI_INT_DEV_MASK) 79 | __SHIFTIN((uint64_t)vector, MSI_INT_VEC_MASK) 80 | APIC_INT_VIA_MSI; 81 if (msi_pic->pic_type == PIC_MSI) 82 MSI_INT_MAKE_MSI(pih); 83 else if (msi_pic->pic_type == PIC_MSIX) 84 MSI_INT_MAKE_MSIX(pih); 85 else 86 panic("%s: Unexpected pic_type: %d\n", __func__, 87 msi_pic->pic_type); 88 89 return pih; 90 } 91 92 #ifdef __HAVE_PCI_MSI_MSIX 93 static pci_intr_handle_t * 94 pci_msi_alloc_vectors(struct pic *msi_pic, uint *table_indexes, int *count) 95 { 96 struct intrsource *isp; 97 pci_intr_handle_t *vectors, pih; 98 int i; 99 const char *intrstr; 100 char intrstr_buf[INTRIDBUF]; 101 102 vectors = kmem_zalloc(sizeof(vectors[0]) * (*count), KM_SLEEP); 103 if (vectors == NULL) { 104 DPRINTF(("cannot allocate vectors\n")); 105 return NULL; 106 } 107 108 mutex_enter(&cpu_lock); 109 for (i = 0; i < *count; i++) { 110 u_int table_index; 111 112 if (table_indexes == NULL) 113 table_index = i; 114 else 115 table_index = table_indexes[i]; 116 117 pih = pci_msi_calculate_handle(msi_pic, table_index); 118 119 intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf, 120 sizeof(intrstr_buf)); 121 isp = intr_allocate_io_intrsource(intrstr); 122 if (isp == NULL) { 123 mutex_exit(&cpu_lock); 124 DPRINTF(("can't allocate io_intersource\n")); 125 kmem_free(vectors, sizeof(vectors[0]) * (*count)); 126 return NULL; 127 } 128 129 vectors[i] = pih; 130 } 131 mutex_exit(&cpu_lock); 132 133 return vectors; 134 } 135 #endif /* __HAVE_PCI_MSI_MSIX */ 136 137 static void 138 pci_msi_free_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs, int count) 139 { 140 pci_intr_handle_t pih; 141 int i; 142 const char *intrstr; 143 char intrstr_buf[INTRIDBUF]; 144 145 mutex_enter(&cpu_lock); 146 for (i = 0; i < count; i++) { 147 pih = pci_msi_calculate_handle(msi_pic, i); 148 intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf, 149 sizeof(intrstr_buf)); 150 intr_free_io_intrsource(intrstr); 151 } 152 mutex_exit(&cpu_lock); 153 154 kmem_free(pihs, sizeof(pihs[0]) * count); 155 } 156 157 #ifdef __HAVE_PCI_MSI_MSIX 158 static int 159 pci_msi_alloc_common(pci_intr_handle_t **ihps, int *count, 160 const struct pci_attach_args *pa, bool exact) 161 { 162 struct pic *msi_pic; 163 pci_intr_handle_t *vectors; 164 int error, i; 165 166 #if NIOAPIC > 0 167 if (nioapics == 0) { 168 DPRINTF(("no IOAPIC.\n")); 169 return ENODEV; 170 } 171 #endif 172 173 if ((pa->pa_flags & PCI_FLAGS_MSI_OKAY) == 0) { 174 DPRINTF(("PCI host bridge does not support MSI.\n")); 175 return ENODEV; 176 } 177 178 msi_pic = msipic_construct_msi_pic(pa); 179 if (msi_pic == NULL) { 180 DPRINTF(("cannot allocate MSI pic.\n")); 181 return EINVAL; 182 } 183 184 vectors = NULL; 185 while (*count > 0) { 186 vectors = pci_msi_alloc_vectors(msi_pic, NULL, count); 187 if (vectors != NULL) 188 break; 189 190 if (exact) { 191 DPRINTF(("cannot allocate MSI vectors.\n")); 192 msipic_destruct_msi_pic(msi_pic); 193 return ENOMEM; 194 } else { 195 (*count) >>= 1; /* must be power of 2. */ 196 continue; 197 } 198 } 199 if (vectors == NULL) { 200 DPRINTF(("cannot allocate MSI vectors.\n")); 201 msipic_destruct_msi_pic(msi_pic); 202 return ENOMEM; 203 } 204 205 for (i = 0; i < *count; i++) { 206 MSI_INT_MAKE_MSI(vectors[i]); 207 } 208 209 error = msipic_set_msi_vectors(msi_pic, NULL, *count); 210 if (error) { 211 pci_msi_free_vectors(msi_pic, vectors, *count); 212 msipic_destruct_msi_pic(msi_pic); 213 return error; 214 } 215 216 *ihps = vectors; 217 return 0; 218 } 219 #endif /* __HAVE_PCI_MSI_MSIX */ 220 221 static void * 222 pci_msi_common_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 223 int level, int (*func)(void *), void *arg, struct pic *pic, 224 const char *xname) 225 { 226 int irq, pin; 227 bool mpsafe; 228 229 KASSERT(INT_VIA_MSI(ih)); 230 231 irq = -1; 232 pin = MSI_INT_VEC(ih); 233 mpsafe = ((ih & MPSAFE_MASK) != 0); 234 235 return intr_establish_xname(irq, pic, pin, IST_EDGE, level, func, arg, 236 mpsafe, xname); 237 } 238 239 static void 240 pci_msi_common_disestablish(pci_chipset_tag_t pc, void *cookie) 241 { 242 243 intr_disestablish(cookie); 244 } 245 246 #ifdef __HAVE_PCI_MSI_MSIX 247 static int 248 pci_msix_alloc_common(pci_intr_handle_t **ihps, u_int *table_indexes, 249 int *count, const struct pci_attach_args *pa, bool exact) 250 { 251 struct pic *msix_pic; 252 pci_intr_handle_t *vectors; 253 int error, i; 254 255 #if NIOAPIC > 0 256 if (nioapics == 0) { 257 DPRINTF(("no IOAPIC.\n")); 258 return ENODEV; 259 } 260 #endif 261 262 if ((pa->pa_flags & PCI_FLAGS_MSIX_OKAY) == 0) { 263 DPRINTF(("PCI host bridge does not support MSI-X.\n")); 264 return ENODEV; 265 } 266 267 msix_pic = msipic_construct_msix_pic(pa); 268 if (msix_pic == NULL) 269 return EINVAL; 270 271 vectors = NULL; 272 while (*count > 0) { 273 vectors = pci_msi_alloc_vectors(msix_pic, table_indexes, count); 274 if (vectors != NULL) 275 break; 276 277 if (exact) { 278 DPRINTF(("cannot allocate MSI-X vectors.\n")); 279 msipic_destruct_msix_pic(msix_pic); 280 return ENOMEM; 281 } else { 282 (*count)--; 283 continue; 284 } 285 } 286 if (vectors == NULL) { 287 DPRINTF(("cannot allocate MSI-X vectors.\n")); 288 msipic_destruct_msix_pic(msix_pic); 289 return ENOMEM; 290 } 291 292 for (i = 0; i < *count; i++) { 293 MSI_INT_MAKE_MSIX(vectors[i]); 294 } 295 296 error = msipic_set_msi_vectors(msix_pic, vectors, *count); 297 if (error) { 298 pci_msi_free_vectors(msix_pic, vectors, *count); 299 msipic_destruct_msix_pic(msix_pic); 300 return error; 301 } 302 303 *ihps = vectors; 304 return 0; 305 } 306 307 static int 308 x86_pci_msi_alloc(pci_intr_handle_t **ihps, int *count, 309 const struct pci_attach_args *pa) 310 { 311 312 return pci_msi_alloc_common(ihps, count, pa, false); 313 } 314 315 static int 316 x86_pci_msi_alloc_exact(pci_intr_handle_t **ihps, int count, 317 const struct pci_attach_args *pa) 318 { 319 320 return pci_msi_alloc_common(ihps, &count, pa, true); 321 } 322 #endif /* __HAVE_PCI_MSI_MSIX */ 323 324 static void 325 x86_pci_msi_release_internal(pci_intr_handle_t *pihs, int count) 326 { 327 struct pic *pic; 328 329 pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0])); 330 if (pic == NULL) 331 return; 332 333 pci_msi_free_vectors(pic, pihs, count); 334 msipic_destruct_msi_pic(pic); 335 } 336 337 #ifdef __HAVE_PCI_MSI_MSIX 338 static int 339 x86_pci_msix_alloc(pci_intr_handle_t **ihps, int *count, 340 const struct pci_attach_args *pa) 341 { 342 343 return pci_msix_alloc_common(ihps, NULL, count, pa, false); 344 } 345 346 static int 347 x86_pci_msix_alloc_exact(pci_intr_handle_t **ihps, int count, 348 const struct pci_attach_args *pa) 349 { 350 351 return pci_msix_alloc_common(ihps, NULL, &count, pa, true); 352 } 353 354 static int 355 x86_pci_msix_alloc_map(pci_intr_handle_t **ihps, u_int *table_indexes, 356 int count, const struct pci_attach_args *pa) 357 { 358 359 return pci_msix_alloc_common(ihps, table_indexes, &count, pa, true); 360 } 361 #endif /* __HAVE_PCI_MSI_MSIX */ 362 363 static void 364 x86_pci_msix_release_internal(pci_intr_handle_t *pihs, int count) 365 { 366 struct pic *pic; 367 368 pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0])); 369 if (pic == NULL) 370 return; 371 372 pci_msi_free_vectors(pic, pihs, count); 373 msipic_destruct_msix_pic(pic); 374 } 375 376 /*****************************************************************************/ 377 /* 378 * extern for MD code. 379 */ 380 381 /* 382 * Return intrid for a MSI/MSI-X device. 383 * "buf" must be allocated by caller. 384 */ 385 const char * 386 x86_pci_msi_string(pci_chipset_tag_t pc, pci_intr_handle_t ih, char *buf, 387 size_t len) 388 { 389 int dev, vec; 390 391 KASSERT(INT_VIA_MSI(ih)); 392 393 dev = MSI_INT_DEV(ih); 394 vec = MSI_INT_VEC(ih); 395 if (MSI_INT_IS_MSIX(ih)) 396 snprintf(buf, len, "msix%d vec %d", dev, vec); 397 else 398 snprintf(buf, len, "msi%d vec %d", dev, vec); 399 400 return buf; 401 } 402 403 /* 404 * Release MSI handles. 405 */ 406 void 407 x86_pci_msi_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count) 408 { 409 410 if (count < 1) 411 return; 412 413 return x86_pci_msi_release_internal(pihs, count); 414 } 415 416 /* 417 * Establish a MSI handle. 418 * If multiple MSI handle is requied to establish, device driver must call 419 * this function for each handle. 420 */ 421 void * 422 x86_pci_msi_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 423 int level, int (*func)(void *), void *arg, const char *xname) 424 { 425 struct pic *pic; 426 427 pic = msipic_find_msi_pic(MSI_INT_DEV(ih)); 428 if (pic == NULL) { 429 DPRINTF(("pci_intr_handler has no msi_pic\n")); 430 return NULL; 431 } 432 433 return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname); 434 } 435 436 /* 437 * Disestablish a MSI handle. 438 * If multiple MSI handle is requied to disestablish, device driver must call 439 * this function for each handle. 440 */ 441 void 442 x86_pci_msi_disestablish(pci_chipset_tag_t pc, void *cookie) 443 { 444 445 pci_msi_common_disestablish(pc, cookie); 446 } 447 448 /* 449 * Release MSI-X handles. 450 */ 451 void 452 x86_pci_msix_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count) 453 { 454 455 if (count < 1) 456 return; 457 458 return x86_pci_msix_release_internal(pihs, count); 459 } 460 461 /* 462 * Establish a MSI-X handle. 463 * If multiple MSI-X handle is requied to establish, device driver must call 464 * this function for each handle. 465 */ 466 void * 467 x86_pci_msix_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 468 int level, int (*func)(void *), void *arg, const char *xname) 469 { 470 struct pic *pic; 471 472 pic = msipic_find_msi_pic(MSI_INT_DEV(ih)); 473 if (pic == NULL) { 474 DPRINTF(("pci_intr_handler has no msi_pic\n")); 475 return NULL; 476 } 477 478 return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname); 479 } 480 481 /* 482 * Disestablish a MSI-X handle. 483 * If multiple MSI-X handle is requied to disestablish, device driver must call 484 * this function for each handle. 485 */ 486 void 487 x86_pci_msix_disestablish(pci_chipset_tag_t pc, void *cookie) 488 { 489 490 pci_msi_common_disestablish(pc, cookie); 491 } 492 493 #ifdef __HAVE_PCI_MSI_MSIX 494 /*****************************************************************************/ 495 /* 496 * extern for MI code. 497 */ 498 499 /* 500 * This function is used by device drivers like pci_intr_map(). 501 * 502 * "ihps" is the array of vector numbers which MSI used instead of IRQ number. 503 * "count" must be power of 2. 504 * "count" can decrease if struct intrsource cannot be allocated. 505 * if count == 0, return non-zero value. 506 */ 507 int 508 pci_msi_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 509 int *count) 510 { 511 int hw_max; 512 513 /* MSI vector count must be power of 2. */ 514 KASSERT(*count > 0); 515 KASSERT(((*count - 1) & *count) == 0); 516 517 hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag); 518 if (hw_max == 0) 519 return ENODEV; 520 521 if (*count > hw_max) { 522 DPRINTF(("cut off MSI count to %d\n", hw_max)); 523 *count = hw_max; /* cut off hw_max */ 524 } 525 526 return x86_pci_msi_alloc(ihps, count, pa); 527 } 528 529 /* 530 * This function is used by device drivers like pci_intr_map(). 531 * 532 * "ihps" is the array of vector numbers which MSI used instead of IRQ number. 533 * "count" must be power of 2. 534 * "count" can not decrease. 535 * If "count" struct intrsources cannot be allocated, return non-zero value. 536 */ 537 int 538 pci_msi_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 539 int count) 540 { 541 int hw_max; 542 543 /* MSI vector count must be power of 2. */ 544 KASSERT(count > 0); 545 KASSERT(((count - 1) & count) == 0); 546 547 hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag); 548 if (hw_max == 0) 549 return ENODEV; 550 551 if (count > hw_max) { 552 DPRINTF(("over hardware max MSI count %d\n", hw_max)); 553 return EINVAL; 554 } 555 556 return x86_pci_msi_alloc_exact(ihps, count, pa); 557 } 558 559 /* 560 * This function is used by device drivers like pci_intr_map(). 561 * 562 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 563 * "count" can decrease if enough struct intrsources cannot be allocated. 564 * if count == 0, return non-zero value. 565 */ 566 int 567 pci_msix_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 568 int *count) 569 { 570 int hw_max; 571 572 KASSERT(*count > 0); 573 574 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag); 575 if (hw_max == 0) 576 return ENODEV; 577 578 if (*count > hw_max) { 579 DPRINTF(("cut off MSI-X count to %d\n", hw_max)); 580 *count = hw_max; /* cut off hw_max */ 581 } 582 583 return x86_pci_msix_alloc(ihps, count, pa); 584 } 585 586 /* 587 * This function is used by device drivers like pci_intr_map(). 588 * 589 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 590 * "count" can not decrease. 591 * If "count" struct intrsource cannot be allocated, return non-zero value. 592 */ 593 int 594 pci_msix_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 595 int count) 596 { 597 int hw_max; 598 599 KASSERT(count > 0); 600 601 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag); 602 if (hw_max == 0) 603 return ENODEV; 604 605 if (count > hw_max) { 606 DPRINTF(("over hardware max MSI-X count %d\n", hw_max)); 607 return EINVAL; 608 } 609 610 return x86_pci_msix_alloc_exact(ihps, count, pa); 611 } 612 613 /* 614 * This function is used by device drivers like pci_intr_map(). 615 * Futhermore, this function can map each handle to a MSI-X table index. 616 * 617 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 618 * "count" can not decrease. 619 * "map" size must be equal to "count". 620 * If "count" struct intrsource cannot be allocated, return non-zero value. 621 * e.g. 622 * If "map" = { 1, 4, 0 }, 623 * 1st handle is bound to MSI-X index 1 624 * 2nd handle is bound to MSI-X index 4 625 * 3rd handle is bound to MSI-X index 0 626 */ 627 int 628 pci_msix_alloc_map(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 629 u_int *table_indexes, int count) 630 { 631 int hw_max, i, j; 632 633 KASSERT(count > 0); 634 635 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag); 636 if (hw_max == 0) 637 return ENODEV; 638 639 if (count > hw_max) { 640 DPRINTF(("over hardware max MSI-X count %d\n", hw_max)); 641 return EINVAL; 642 } 643 644 /* check not to duplicate table_index */ 645 for (i = 0; i < count; i++) { 646 u_int basing = table_indexes[i]; 647 648 KASSERT(table_indexes[i] < PCI_MSIX_MAX_VECTORS); 649 if (basing >= hw_max) { 650 DPRINTF(("table index is over hardware max MSI-X index %d\n", 651 hw_max - 1)); 652 return EINVAL; 653 } 654 655 for (j = i + 1; j < count; j++) { 656 if (basing == table_indexes[j]) { 657 DPRINTF(("MSI-X table index duplicated\n")); 658 return EINVAL; 659 } 660 } 661 } 662 663 return x86_pci_msix_alloc_map(ihps, table_indexes, count, pa); 664 } 665 #endif /* __HAVE_PCI_MSI_MSIX */ 666
Indexes created Sat Sep 27 22:09:54 GMT 2025