pci_msi_machdep.c revision 1.5.2.3
1 /* $NetBSD: pci_msi_machdep.c,v 1.5.2.3 2015/09/22 12:05:54 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2015 Internet Initiative Japan Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * TODO 31 * 32 * - PBA (Pending Bit Array) support 33 * - HyperTransport mapping support 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: pci_msi_machdep.c,v 1.5.2.3 2015/09/22 12:05:54 skrll Exp $"); 38 39 #include "opt_intrdebug.h" 40 41 #include <sys/types.h> 42 #include <sys/param.h> 43 #include <sys/time.h> 44 #include <sys/systm.h> 45 #include <sys/cpu.h> 46 #include <sys/errno.h> 47 #include <sys/device.h> 48 #include <sys/intr.h> 49 #include <sys/kmem.h> 50 #include <sys/malloc.h> 51 52 #include <dev/pci/pcivar.h> 53 54 #include <machine/i82093var.h> 55 #include <machine/pic.h> 56 57 #include <x86/pci/msipic.h> 58 #include <x86/pci/pci_msi_machdep.h> 59 60 #ifdef INTRDEBUG 61 #define MSIDEBUG 62 #endif 63 64 #ifdef MSIDEBUG 65 #define DPRINTF(msg) printf msg 66 #else 67 #define DPRINTF(msg) 68 #endif 69 70 static pci_intr_handle_t 71 pci_msi_calculate_handle(struct pic *msi_pic, int vector) 72 { 73 pci_intr_handle_t pih; 74 75 KASSERT(msipic_is_msi_pic(msi_pic)); 76 77 pih = __SHIFTIN((uint64_t)msipic_get_devid(msi_pic), MSI_INT_DEV_MASK) 78 | __SHIFTIN((uint64_t)vector, MSI_INT_VEC_MASK) 79 | APIC_INT_VIA_MSI; 80 if (msi_pic->pic_type == PIC_MSI) 81 MSI_INT_MAKE_MSI(pih); 82 else if (msi_pic->pic_type == PIC_MSIX) 83 MSI_INT_MAKE_MSIX(pih); 84 else 85 panic("%s: Unexpected pic_type: %d\n", __func__, 86 msi_pic->pic_type); 87 88 return pih; 89 } 90 91 static pci_intr_handle_t * 92 pci_msi_alloc_vectors(struct pic *msi_pic, uint *table_indexes, int *count) 93 { 94 struct intrsource *isp; 95 pci_intr_handle_t *vectors, pih; 96 int i; 97 const char *intrstr; 98 char intrstr_buf[INTRIDBUF]; 99 100 vectors = kmem_zalloc(sizeof(vectors[0]) * (*count), KM_SLEEP); 101 if (vectors == NULL) { 102 DPRINTF(("cannot allocate vectors\n")); 103 return NULL; 104 } 105 106 mutex_enter(&cpu_lock); 107 for (i = 0; i < *count; i++) { 108 u_int table_index; 109 110 if (table_indexes == NULL) 111 table_index = i; 112 else 113 table_index = table_indexes[i]; 114 115 pih = pci_msi_calculate_handle(msi_pic, table_index); 116 117 intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf, 118 sizeof(intrstr_buf)); 119 isp = intr_allocate_io_intrsource(intrstr); 120 if (isp == NULL) { 121 mutex_exit(&cpu_lock); 122 DPRINTF(("can't allocate io_intersource\n")); 123 kmem_free(vectors, sizeof(vectors[0]) * (*count)); 124 return NULL; 125 } 126 127 vectors[i] = pih; 128 } 129 mutex_exit(&cpu_lock); 130 131 return vectors; 132 } 133 134 static void 135 pci_msi_free_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs, int count) 136 { 137 pci_intr_handle_t pih; 138 int i; 139 const char *intrstr; 140 char intrstr_buf[INTRIDBUF]; 141 142 mutex_enter(&cpu_lock); 143 for (i = 0; i < count; i++) { 144 pih = pci_msi_calculate_handle(msi_pic, i); 145 intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf, 146 sizeof(intrstr_buf)); 147 intr_free_io_intrsource(intrstr); 148 } 149 mutex_exit(&cpu_lock); 150 151 kmem_free(pihs, sizeof(pihs[0]) * count); 152 } 153 154 static int 155 pci_msi_alloc_common(pci_intr_handle_t **ihps, int *count, 156 const struct pci_attach_args *pa, bool exact) 157 { 158 struct pic *msi_pic; 159 pci_intr_handle_t *vectors; 160 int error, i; 161 162 if ((pa->pa_flags & PCI_FLAGS_MSI_OKAY) == 0) { 163 DPRINTF(("PCI host bridge does not support MSI.\n")); 164 return ENODEV; 165 } 166 167 msi_pic = msipic_construct_msi_pic(pa); 168 if (msi_pic == NULL) { 169 DPRINTF(("cannot allocate MSI pic.\n")); 170 return EINVAL; 171 } 172 173 vectors = NULL; 174 while (*count > 0) { 175 vectors = pci_msi_alloc_vectors(msi_pic, NULL, count); 176 if (vectors != NULL) 177 break; 178 179 if (exact) { 180 DPRINTF(("cannot allocate MSI vectors.\n")); 181 msipic_destruct_msi_pic(msi_pic); 182 return ENOMEM; 183 } else { 184 (*count) >>= 1; /* must be power of 2. */ 185 continue; 186 } 187 } 188 if (vectors == NULL) { 189 DPRINTF(("cannot allocate MSI vectors.\n")); 190 msipic_destruct_msi_pic(msi_pic); 191 return ENOMEM; 192 } 193 194 for (i = 0; i < *count; i++) { 195 MSI_INT_MAKE_MSI(vectors[i]); 196 } 197 198 error = msipic_set_msi_vectors(msi_pic, NULL, *count); 199 if (error) { 200 pci_msi_free_vectors(msi_pic, vectors, *count); 201 msipic_destruct_msi_pic(msi_pic); 202 return error; 203 } 204 205 *ihps = vectors; 206 return 0; 207 } 208 209 static void * 210 pci_msi_common_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 211 int level, int (*func)(void *), void *arg, struct pic *pic, 212 const char *xname) 213 { 214 int irq, pin; 215 bool mpsafe; 216 217 KASSERT(INT_VIA_MSI(ih)); 218 219 irq = -1; 220 pin = MSI_INT_VEC(ih); 221 mpsafe = ((ih & MPSAFE_MASK) != 0); 222 223 return intr_establish_xname(irq, pic, pin, IST_EDGE, level, func, arg, 224 mpsafe, xname); 225 } 226 227 static void 228 pci_msi_common_disestablish(pci_chipset_tag_t pc, void *cookie) 229 { 230 231 intr_disestablish(cookie); 232 } 233 234 static int 235 pci_msix_alloc_common(pci_intr_handle_t **ihps, u_int *table_indexes, 236 int *count, const struct pci_attach_args *pa, bool exact) 237 { 238 struct pic *msix_pic; 239 pci_intr_handle_t *vectors; 240 int error, i; 241 242 if ((pa->pa_flags & PCI_FLAGS_MSIX_OKAY) == 0) { 243 DPRINTF(("PCI host bridge does not support MSI-X.\n")); 244 return ENODEV; 245 } 246 247 msix_pic = msipic_construct_msix_pic(pa); 248 if (msix_pic == NULL) 249 return EINVAL; 250 251 vectors = NULL; 252 while (*count > 0) { 253 vectors = pci_msi_alloc_vectors(msix_pic, table_indexes, count); 254 if (vectors != NULL) 255 break; 256 257 if (exact) { 258 DPRINTF(("cannot allocate MSI-X vectors.\n")); 259 msipic_destruct_msix_pic(msix_pic); 260 return ENOMEM; 261 } else { 262 (*count)--; 263 continue; 264 } 265 } 266 if (vectors == NULL) { 267 DPRINTF(("cannot allocate MSI-X vectors.\n")); 268 msipic_destruct_msix_pic(msix_pic); 269 return ENOMEM; 270 } 271 272 for (i = 0; i < *count; i++) { 273 MSI_INT_MAKE_MSIX(vectors[i]); 274 } 275 276 error = msipic_set_msi_vectors(msix_pic, vectors, *count); 277 if (error) { 278 pci_msi_free_vectors(msix_pic, vectors, *count); 279 msipic_destruct_msix_pic(msix_pic); 280 return error; 281 } 282 283 *ihps = vectors; 284 return 0; 285 } 286 287 static int 288 x86_pci_msi_alloc(pci_intr_handle_t **ihps, int *count, 289 const struct pci_attach_args *pa) 290 { 291 292 return pci_msi_alloc_common(ihps, count, pa, false); 293 } 294 295 static int 296 x86_pci_msi_alloc_exact(pci_intr_handle_t **ihps, int count, 297 const struct pci_attach_args *pa) 298 { 299 300 return pci_msi_alloc_common(ihps, &count, pa, true); 301 } 302 303 static void 304 x86_pci_msi_release_internal(pci_intr_handle_t *pihs, int count) 305 { 306 struct pic *pic; 307 308 pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0])); 309 if (pic == NULL) 310 return; 311 312 pci_msi_free_vectors(pic, pihs, count); 313 msipic_destruct_msi_pic(pic); 314 } 315 316 static int 317 x86_pci_msix_alloc(pci_intr_handle_t **ihps, int *count, 318 const struct pci_attach_args *pa) 319 { 320 321 return pci_msix_alloc_common(ihps, NULL, count, pa, false); 322 } 323 324 static int 325 x86_pci_msix_alloc_exact(pci_intr_handle_t **ihps, int count, 326 const struct pci_attach_args *pa) 327 { 328 329 return pci_msix_alloc_common(ihps, NULL, &count, pa, true); 330 } 331 332 static int 333 x86_pci_msix_alloc_map(pci_intr_handle_t **ihps, u_int *table_indexes, 334 int count, const struct pci_attach_args *pa) 335 { 336 337 return pci_msix_alloc_common(ihps, table_indexes, &count, pa, true); 338 } 339 340 static void 341 x86_pci_msix_release_internal(pci_intr_handle_t *pihs, int count) 342 { 343 struct pic *pic; 344 345 pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0])); 346 if (pic == NULL) 347 return; 348 349 pci_msi_free_vectors(pic, pihs, count); 350 msipic_destruct_msix_pic(pic); 351 } 352 353 /*****************************************************************************/ 354 /* 355 * extern for MD code. 356 */ 357 358 /* 359 * Return intrid for a MSI/MSI-X device. 360 * "buf" must be allocated by caller. 361 */ 362 const char * 363 x86_pci_msi_string(pci_chipset_tag_t pc, pci_intr_handle_t ih, char *buf, 364 size_t len) 365 { 366 int dev, vec; 367 368 KASSERT(INT_VIA_MSI(ih)); 369 370 dev = MSI_INT_DEV(ih); 371 vec = MSI_INT_VEC(ih); 372 if (MSI_INT_IS_MSIX(ih)) 373 snprintf(buf, len, "msix%d vec %d", dev, vec); 374 else 375 snprintf(buf, len, "msi%d vec %d", dev, vec); 376 377 return buf; 378 } 379 380 /* 381 * Release MSI handles. 382 */ 383 void 384 x86_pci_msi_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count) 385 { 386 387 if (count < 1) 388 return; 389 390 return x86_pci_msi_release_internal(pihs, count); 391 } 392 393 /* 394 * Establish a MSI handle. 395 * If multiple MSI handle is requied to establish, device driver must call 396 * this function for each handle. 397 */ 398 void * 399 x86_pci_msi_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 400 int level, int (*func)(void *), void *arg, const char *xname) 401 { 402 struct pic *pic; 403 404 pic = msipic_find_msi_pic(MSI_INT_DEV(ih)); 405 if (pic == NULL) { 406 DPRINTF(("pci_intr_handler has no msi_pic\n")); 407 return NULL; 408 } 409 410 return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname); 411 } 412 413 /* 414 * Disestablish a MSI handle. 415 * If multiple MSI handle is requied to disestablish, device driver must call 416 * this function for each handle. 417 */ 418 void 419 x86_pci_msi_disestablish(pci_chipset_tag_t pc, void *cookie) 420 { 421 422 pci_msi_common_disestablish(pc, cookie); 423 } 424 425 /* 426 * Release MSI-X handles. 427 */ 428 void 429 x86_pci_msix_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count) 430 { 431 432 if (count < 1) 433 return; 434 435 return x86_pci_msix_release_internal(pihs, count); 436 } 437 438 /* 439 * Establish a MSI-X handle. 440 * If multiple MSI-X handle is requied to establish, device driver must call 441 * this function for each handle. 442 */ 443 void * 444 x86_pci_msix_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 445 int level, int (*func)(void *), void *arg, const char *xname) 446 { 447 struct pic *pic; 448 449 pic = msipic_find_msi_pic(MSI_INT_DEV(ih)); 450 if (pic == NULL) { 451 DPRINTF(("pci_intr_handler has no msi_pic\n")); 452 return NULL; 453 } 454 455 return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname); 456 } 457 458 /* 459 * Disestablish a MSI-X handle. 460 * If multiple MSI-X handle is requied to disestablish, device driver must call 461 * this function for each handle. 462 */ 463 void 464 x86_pci_msix_disestablish(pci_chipset_tag_t pc, void *cookie) 465 { 466 467 pci_msi_common_disestablish(pc, cookie); 468 } 469 470 /*****************************************************************************/ 471 /* 472 * extern for MI code. 473 */ 474 475 /* 476 * This function is used by device drivers like pci_intr_map(). 477 * 478 * "ihps" is the array of vector numbers which MSI used instead of IRQ number. 479 * "count" must be power of 2. 480 * "count" can decrease if struct intrsource cannot be allocated. 481 * if count == 0, return non-zero value. 482 */ 483 int 484 pci_msi_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 485 int *count) 486 { 487 int hw_max; 488 489 /* MSI vector count must be power of 2. */ 490 KASSERT(*count > 0); 491 KASSERT(((*count - 1) & *count) == 0); 492 493 hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag); 494 if (hw_max == 0) 495 return ENODEV; 496 497 if (*count > hw_max) { 498 DPRINTF(("cut off MSI count to %d\n", hw_max)); 499 *count = hw_max; /* cut off hw_max */ 500 } 501 502 return x86_pci_msi_alloc(ihps, count, pa); 503 } 504 505 /* 506 * This function is used by device drivers like pci_intr_map(). 507 * 508 * "ihps" is the array of vector numbers which MSI used instead of IRQ number. 509 * "count" must be power of 2. 510 * "count" can not decrease. 511 * If "count" struct intrsources cannot be allocated, return non-zero value. 512 */ 513 int 514 pci_msi_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 515 int count) 516 { 517 int hw_max; 518 519 /* MSI vector count must be power of 2. */ 520 KASSERT(count > 0); 521 KASSERT(((count - 1) & count) == 0); 522 523 hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag); 524 if (hw_max == 0) 525 return ENODEV; 526 527 if (count > hw_max) { 528 DPRINTF(("over hardware max MSI count %d\n", hw_max)); 529 return EINVAL; 530 } 531 532 return x86_pci_msi_alloc_exact(ihps, count, pa); 533 } 534 535 /* 536 * This function is used by device drivers like pci_intr_map(). 537 * 538 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 539 * "count" can decrease if enough struct intrsources cannot be allocated. 540 * if count == 0, return non-zero value. 541 */ 542 int 543 pci_msix_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 544 int *count) 545 { 546 int hw_max; 547 548 KASSERT(*count > 0); 549 550 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag); 551 if (hw_max == 0) 552 return ENODEV; 553 554 if (*count > hw_max) { 555 DPRINTF(("cut off MSI-X count to %d\n", hw_max)); 556 *count = hw_max; /* cut off hw_max */ 557 } 558 559 return x86_pci_msix_alloc(ihps, count, pa); 560 } 561 562 /* 563 * This function is used by device drivers like pci_intr_map(). 564 * 565 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 566 * "count" can not decrease. 567 * If "count" struct intrsource cannot be allocated, return non-zero value. 568 */ 569 int 570 pci_msix_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 571 int count) 572 { 573 int hw_max; 574 575 KASSERT(count > 0); 576 577 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag); 578 if (hw_max == 0) 579 return ENODEV; 580 581 if (count > hw_max) { 582 DPRINTF(("over hardware max MSI-X count %d\n", hw_max)); 583 return EINVAL; 584 } 585 586 return x86_pci_msix_alloc_exact(ihps, count, pa); 587 } 588 589 /* 590 * This function is used by device drivers like pci_intr_map(). 591 * Futhermore, this function can map each handle to a MSI-X table index. 592 * 593 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 594 * "count" can not decrease. 595 * "map" size must be equal to "count". 596 * If "count" struct intrsource cannot be allocated, return non-zero value. 597 * e.g. 598 * If "map" = { 1, 4, 0 }, 599 * 1st handle is bound to MSI-X index 1 600 * 2nd handle is bound to MSI-X index 4 601 * 3rd handle is bound to MSI-X index 0 602 */ 603 int 604 pci_msix_alloc_map(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 605 u_int *table_indexes, int count) 606 { 607 int hw_max, i, j; 608 609 KASSERT(count > 0); 610 611 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag); 612 if (hw_max == 0) 613 return ENODEV; 614 615 if (count > hw_max) { 616 DPRINTF(("over hardware max MSI-X count %d\n", hw_max)); 617 return EINVAL; 618 } 619 620 /* check not to duplicate table_index */ 621 for (i = 0; i < count; i++) { 622 u_int basing = table_indexes[i]; 623 624 KASSERT(table_indexes[i] < PCI_MSIX_MAX_VECTORS); 625 if (basing >= hw_max) { 626 DPRINTF(("table index is over hardware max MSI-X index %d\n", 627 hw_max - 1)); 628 return EINVAL; 629 } 630 631 for (j = i + 1; j < count; j++) { 632 if (basing == table_indexes[j]) { 633 DPRINTF(("MSI-X table index duplicated\n")); 634 return EINVAL; 635 } 636 } 637 } 638 639 return x86_pci_msix_alloc_map(ihps, table_indexes, count, pa); 640 } 641
Indexes created Tue Oct 14 21:09:58 GMT 2025