1 /* $NetBSD: gicv3_its.c,v 1.43 2026/09/20 11:00:31 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jared McNeill <jmcneill (at) invisible.ca>. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #define _INTR_PRIVATE 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: gicv3_its.c,v 1.43 2026/09/20 11:00:31 skrll Exp $"); 36 37 #include <sys/param.h> 38 #include <sys/kmem.h> 39 #include <sys/bus.h> 40 #include <sys/cpu.h> 41 #include <sys/bitops.h> 42 43 #include <uvm/uvm.h> 44 45 #include <dev/pci/pcireg.h> 46 #include <dev/pci/pcivar.h> 47 48 #include <machine/cpufunc.h> 49 50 #include <arm/pic/picvar.h> 51 #include <arm/cortex/gicv3_its.h> 52 53 #ifdef ITS_DEBUG 54 #define DPRINTF(x) printf x 55 #else 56 #define DPRINTF(x) __nothing 57 #endif 58 59 /* 60 * ITS translation table sizes 61 */ 62 #define GITS_COMMANDS_SIZE 0x1000 63 #define GITS_COMMANDS_ALIGN 0x10000 64 65 #define GITS_ITT_ALIGN 0x100 66 67 #define GITS_INDIRECT_ENTRY_SIZE 8 68 69 /* 70 * IIDR values used for errata 71 */ 72 #define GITS_IIDR_PID_CAVIUM_THUNDERX 0xa1 73 #define GITS_IIDR_IMP_CAVIUM 0x34c 74 #define GITS_IIDR_CAVIUM_ERRATA_MASK (GITS_IIDR_Implementor|GITS_IIDR_ProductID|GITS_IIDR_Variant) 75 #define GITS_IIDR_CAVIUM_ERRATA_VALUE \ 76 (__SHIFTIN(GITS_IIDR_IMP_CAVIUM, GITS_IIDR_Implementor) | \ 77 __SHIFTIN(GITS_IIDR_PID_CAVIUM_THUNDERX, GITS_IIDR_ProductID) | \ 78 __SHIFTIN(0, GITS_IIDR_Variant)) 79 80 static const char * gits_cache_type[] = { 81 [GITS_Cache_DEVICE_nGnRnE] = "Device-nGnRnE", 82 [GITS_Cache_NORMAL_NC] = "Non-cacheable", 83 [GITS_Cache_NORMAL_RA_WT] = "Cacheable RA WT", 84 [GITS_Cache_NORMAL_RA_WB] = "Cacheable RA WB", 85 [GITS_Cache_NORMAL_WA_WT] = "Cacheable WA WT", 86 [GITS_Cache_NORMAL_WA_WB] = "Cacheable WA WB", 87 [GITS_Cache_NORMAL_RA_WA_WT] = "Cacheable RA WA WT", 88 [GITS_Cache_NORMAL_RA_WA_WB] = "Cacheable RA WA WB", 89 }; 90 91 static const char * gits_share_type[] = { 92 [GITS_Shareability_NS] = "Non-shareable", 93 [GITS_Shareability_IS] = "Inner shareable", 94 [GITS_Shareability_OS] = "Outer shareable", 95 [3] = "(Reserved)", 96 }; 97 98 static inline uint32_t 99 gits_read_4(struct gicv3_its *its, bus_size_t reg) 100 { 101 return bus_space_read_4(its->its_bst, its->its_bsh, reg); 102 } 103 104 static inline void 105 gits_write_4(struct gicv3_its *its, bus_size_t reg, uint32_t val) 106 { 107 bus_space_write_4(its->its_bst, its->its_bsh, reg, val); 108 } 109 110 static inline uint64_t 111 gits_read_8(struct gicv3_its *its, bus_size_t reg) 112 { 113 return bus_space_read_8(its->its_bst, its->its_bsh, reg); 114 } 115 116 static inline void 117 gits_write_8(struct gicv3_its *its, bus_size_t reg, uint64_t val) 118 { 119 bus_space_write_8(its->its_bst, its->its_bsh, reg, val); 120 } 121 122 static int 123 gits_command(struct gicv3_its *its, const struct gicv3_its_command *cmd) 124 { 125 uint64_t cwriter, creadr; 126 u_int woff; 127 128 creadr = gits_read_8(its, GITS_CREADR); 129 if (ISSET(creadr, GITS_CREADR_Stalled)) { 130 DPRINTF(("ITS: stalled! GITS_CREADR = 0x%lx\n", creadr)); 131 return EIO; 132 } 133 134 cwriter = gits_read_8(its, GITS_CWRITER); 135 woff = cwriter & GITS_CWRITER_Offset; 136 137 uint64_t *dw = (uint64_t *)(its->its_cmd.base + woff); 138 for (int i = 0; i < __arraycount(cmd->dw); i++) { 139 dw[i] = htole64(cmd->dw[i]); 140 DPRINTF(("ITS: dw[%u] = 0x%016lx\n", i, cmd->dw[i])); 141 } 142 143 if (its->its_cmd_flush) { 144 cpu_dcache_wb_range((vaddr_t)dw, sizeof(cmd->dw)); 145 } 146 dsb(sy); 147 148 woff += sizeof(cmd->dw); 149 if (woff == its->its_cmd.len) 150 woff = 0; 151 152 gits_write_8(its, GITS_CWRITER, woff); 153 154 return 0; 155 } 156 157 static int 158 gits_command_mapc(struct gicv3_its *its, uint16_t icid, uint64_t rdbase, bool v) 159 { 160 struct gicv3_its_command cmd; 161 162 KASSERT((rdbase & 0xffff) == 0); 163 164 /* 165 * Map a collection table entry (ICID) to the target redistributor (RDbase). 166 */ 167 memset(&cmd, 0, sizeof(cmd)); 168 cmd.dw[0] = GITS_CMD_MAPC; 169 cmd.dw[2] = icid; 170 if (v) { 171 cmd.dw[2] |= rdbase; 172 cmd.dw[2] |= __BIT(63); 173 } 174 175 DPRINTF(("ITS #%u: MAPC icid 0x%x rdbase 0x%lx valid %u\n", 176 its->its_id, icid, rdbase, v)); 177 178 return gits_command(its, &cmd); 179 } 180 181 static int 182 gits_command_mapd(struct gicv3_its *its, uint32_t deviceid, uint64_t itt_addr, u_int size, bool v) 183 { 184 struct gicv3_its_command cmd; 185 186 KASSERT((itt_addr & 0xff) == 0); 187 188 /* 189 * Map a device table entry (DeviceID) to its associated ITT (ITT_addr). 190 */ 191 memset(&cmd, 0, sizeof(cmd)); 192 cmd.dw[0] = GITS_CMD_MAPD | ((uint64_t)deviceid << 32); 193 if (v) { 194 cmd.dw[1] = size; 195 cmd.dw[2] = itt_addr | __BIT(63); 196 } 197 198 DPRINTF(("ITS #%u: MAPD deviceid 0x%x itt_addr 0x%lx size %u valid %u\n", 199 its->its_id, deviceid, itt_addr, size, v)); 200 201 return gits_command(its, &cmd); 202 } 203 204 static int 205 gits_command_mapti(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid, uint32_t pintid, uint16_t icid) 206 { 207 struct gicv3_its_command cmd; 208 209 /* 210 * Map the event defined by EventID and DeviceID to its associated ITE, defined by ICID and pINTID 211 * in the ITT associated with DeviceID. 212 */ 213 memset(&cmd, 0, sizeof(cmd)); 214 cmd.dw[0] = GITS_CMD_MAPTI | ((uint64_t)deviceid << 32); 215 cmd.dw[1] = eventid | ((uint64_t)pintid << 32); 216 cmd.dw[2] = icid; 217 218 DPRINTF(("ITS #%u: MAPTI deviceid 0x%x eventid 0x%x pintid 0x%x icid 0x%x\n", 219 its->its_id, deviceid, eventid, pintid, icid)); 220 221 return gits_command(its, &cmd); 222 } 223 224 static int 225 gits_command_movi(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid, uint16_t icid) 226 { 227 struct gicv3_its_command cmd; 228 229 /* 230 * Update the ICID field in the ITT entry for the event defined by DeviceID and 231 * EventID. 232 */ 233 memset(&cmd, 0, sizeof(cmd)); 234 cmd.dw[0] = GITS_CMD_MOVI | ((uint64_t)deviceid << 32); 235 cmd.dw[1] = eventid; 236 cmd.dw[2] = icid; 237 238 DPRINTF(("ITS #%u: MOVI deviceid 0x%x eventid 0x%x icid 0x%x\n", 239 its->its_id, deviceid, eventid, icid)); 240 241 return gits_command(its, &cmd); 242 } 243 244 static int 245 gits_command_inv(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid) 246 { 247 struct gicv3_its_command cmd; 248 249 /* 250 * Ensure any caching in the redistributors associated with the specified 251 * EventID is consistent with the LPI configuration tables. 252 */ 253 memset(&cmd, 0, sizeof(cmd)); 254 cmd.dw[0] = GITS_CMD_INV | ((uint64_t)deviceid << 32); 255 cmd.dw[1] = eventid; 256 257 DPRINTF(("ITS #%u: INV deviceid 0x%x eventid 0x%x\n", 258 its->its_id, deviceid, eventid)); 259 260 return gits_command(its, &cmd); 261 } 262 263 static int 264 gits_command_invall(struct gicv3_its *its, uint16_t icid) 265 { 266 struct gicv3_its_command cmd; 267 268 /* 269 * Ensure any caching associated with this ICID is consistent with LPI 270 * configuration tables for all redistributors. 271 */ 272 memset(&cmd, 0, sizeof(cmd)); 273 cmd.dw[0] = GITS_CMD_INVALL; 274 cmd.dw[2] = icid; 275 276 DPRINTF(("ITS #%u: INVALL icid 0x%x\n", its->its_id, icid)); 277 278 return gits_command(its, &cmd); 279 } 280 281 static int 282 gits_command_sync(struct gicv3_its *its, uint64_t rdbase) 283 { 284 struct gicv3_its_command cmd; 285 286 KASSERT((rdbase & 0xffff) == 0); 287 288 /* 289 * Ensure all outstanding ITS operations associated with physical interrupts 290 * for the specified redistributor (RDbase) are globally observed before 291 * further ITS commands are executed. 292 */ 293 memset(&cmd, 0, sizeof(cmd)); 294 cmd.dw[0] = GITS_CMD_SYNC; 295 cmd.dw[2] = rdbase; 296 297 DPRINTF(("ITS #%u: SYNC rdbase 0x%lx\n", its->its_id, rdbase)); 298 299 return gits_command(its, &cmd); 300 } 301 302 #if 0 303 static int 304 gits_command_int(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid) 305 { 306 struct gicv3_its_command cmd; 307 308 /* 309 * Translate the deviceid and eventid into an icid and pintid through 310 * the device table and ITT. Mark the pintid as pending 311 * on the redistributor. 312 * If the interrupt is not configured the command queue stalls. 313 */ 314 memset(&cmd, 0, sizeof(cmd)); 315 cmd.dw[0] = GITS_CMD_INT | ((uint64_t)deviceid << 32); 316 cmd.dw[1] = eventid; 317 318 DPRINTF(("ITS #%u: INT deviceid 0x%x eventid 0x%x\n", 319 its->its_id, deviceid, eventid)); 320 321 return gits_command(its, &cmd); 322 } 323 #endif 324 325 static int 326 gits_wait(struct gicv3_its *its) 327 { 328 u_int woff, roff; 329 int retry = 100000; 330 331 /* 332 * The ITS command queue is empty when CWRITER and CREADR specify the 333 * same base address offset value. 334 */ 335 for (retry = 1000; retry > 0; retry--) { 336 woff = gits_read_8(its, GITS_CWRITER) & GITS_CWRITER_Offset; 337 roff = gits_read_8(its, GITS_CREADR) & GITS_CREADR_Offset; 338 if (woff == roff) 339 break; 340 delay(100); 341 } 342 if (retry == 0) { 343 device_printf(its->its_gic->sc_dev, 344 "ITS command queue timeout! CREADR=0x%lx CWRITER=0x%lx\n", 345 gits_read_8(its, GITS_CREADR), gits_read_8(its, GITS_CWRITER)); 346 return ETIMEDOUT; 347 } 348 349 return 0; 350 } 351 352 static int 353 gicv3_its_msi_alloc_lpi(struct gicv3_its *its, 354 const struct pci_attach_args *pa) 355 { 356 struct pci_attach_args *new_pa; 357 vmem_addr_t n; 358 359 KASSERT(its->its_gic->sc_lpi_pool != NULL); 360 361 if (vmem_alloc(its->its_gic->sc_lpi_pool, 1, VM_INSTANTFIT|VM_SLEEP, &n) != 0) 362 return -1; 363 364 KASSERT(its->its_pa[n] == NULL); 365 366 new_pa = kmem_alloc(sizeof(*new_pa), KM_SLEEP); 367 memcpy(new_pa, pa, sizeof(*new_pa)); 368 its->its_pa[n] = new_pa; 369 return n + its->its_pic->pic_irqbase; 370 } 371 372 static void 373 gicv3_its_msi_free_lpi(struct gicv3_its *its, int lpi) 374 { 375 struct pci_attach_args *pa; 376 377 KASSERT(its->its_gic->sc_lpi_pool != NULL); 378 KASSERT(lpi >= its->its_pic->pic_irqbase); 379 380 pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 381 its->its_pa[lpi - its->its_pic->pic_irqbase] = NULL; 382 kmem_free(pa, sizeof(*pa)); 383 384 vmem_free(its->its_gic->sc_lpi_pool, lpi - its->its_pic->pic_irqbase, 1); 385 } 386 387 static uint32_t 388 gicv3_its_alloc_eventid(struct gicv3_its_device *dev, unsigned count) 389 { 390 uint32_t eventid = ITS_INVALID_EVENTID; 391 unsigned index, bit, n; 392 unsigned nalloc = 0; 393 394 KASSERT(count > 0); 395 396 for (index = 0; index < __arraycount(dev->dev_eventid); index++) { 397 for (bit = 0; bit < 32; bit++) { 398 if ((dev->dev_eventid[index] & __BIT(bit)) == 0) { 399 if (nalloc++ == 0) { 400 eventid = index * 32 + bit; 401 } 402 if (nalloc == count) { 403 break; 404 } 405 } else if (nalloc != 0) { 406 nalloc = 0; 407 eventid = ITS_INVALID_EVENTID; 408 } 409 } 410 if (nalloc == count) { 411 break; 412 } 413 } 414 if (nalloc != count) { 415 eventid = ITS_INVALID_EVENTID; 416 } else { 417 for (n = 0; n < count; n++) { 418 index = (eventid + n) / 32; 419 bit = (eventid + n) % 32; 420 421 KASSERT((dev->dev_eventid[index] & __BIT(bit)) == 0); 422 dev->dev_eventid[index] |= __BIT(bit); 423 } 424 } 425 426 if (__predict_false(eventid == ITS_INVALID_EVENTID)) { 427 panic("failed to allocate %u event ID(s) for device ID 0x%x; " 428 "increase ITS_MAX_EVENTS", count, dev->dev_id); 429 } 430 431 return eventid; 432 } 433 434 static void 435 gicv3_its_free_eventid(struct gicv3_its_device *dev, uint32_t eventid) 436 { 437 unsigned index, bit; 438 439 KASSERT(eventid < ITS_MAX_EVENTS); 440 441 index = eventid / 32; 442 bit = eventid % 32; 443 444 KASSERT((dev->dev_eventid[index] & __BIT(bit)) != 0); 445 dev->dev_eventid[index] &= ~__BIT(bit); 446 } 447 448 static uint32_t 449 gicv3_its_devid(pci_chipset_tag_t pc, pcitag_t tag) 450 { 451 uint32_t devid; 452 int b, d, f; 453 454 pci_decompose_tag(pc, tag, &b, &d, &f); 455 456 devid = (b << 8) | (d << 3) | f; 457 458 return pci_get_devid(pc, devid); 459 } 460 461 static struct gicv3_its_device * 462 gicv3_its_device_map(struct gicv3_its *its, uint32_t devid, u_int count) 463 { 464 struct gicv3_its_device *dev; 465 struct gicv3_its_table *itstab = &its->its_tab_device; 466 u_int vectors; 467 int error; 468 469 vectors = MAX(2, count); 470 while (!powerof2(vectors)) 471 vectors++; 472 473 const uint64_t typer = gits_read_8(its, GITS_TYPER); 474 const u_int itt_entry_size = __SHIFTOUT(typer, GITS_TYPER_ITT_entry_size) + 1; 475 476 /* 477 * Compute the size of the ITT. To simplify things, allocate enough 478 * space for ITS_MAX_EVENTS even if it's more than supported by 479 * hardware. Then use the smaller of the two to determin the 480 * effective supported ID bits for this device. 481 */ 482 const u_int itt_size = roundup(ITS_MAX_EVENTS * itt_entry_size, GITS_ITT_ALIGN); 483 const u_int id_bits = uimin(31 - __builtin_clz(ITS_MAX_EVENTS) - 1, 484 __SHIFTOUT(typer, GITS_TYPER_ID_bits)); 485 486 LIST_FOREACH(dev, &its->its_devices, dev_list) 487 if (dev->dev_id == devid) { 488 return itt_size <= dev->dev_size ? dev : NULL; 489 } 490 491 if (itstab->tab_indirect) { 492 uint64_t *l1_tab = itstab->tab_l1; 493 const u_int index = devid / itstab->tab_l2_num_ids; 494 495 if ((l1_tab[index] & GITS_BASER_Valid) == 0) { 496 /* Need to allocate the L2 table. */ 497 struct gicv3_its_page_table *pt; 498 499 pt = kmem_alloc(sizeof(*pt), KM_SLEEP); 500 pt->pt_index = index; 501 gicv3_dma_alloc(its->its_gic, &pt->pt_dma, itstab->tab_l2_entry_size, 502 itstab->tab_page_size); 503 LIST_INSERT_HEAD(&itstab->tab_pt, pt, pt_list); 504 505 if (!itstab->tab_shareable) { 506 cpu_dcache_wb_range((vaddr_t)pt->pt_dma.base, 507 itstab->tab_l2_entry_size); 508 } 509 l1_tab[index] = pt->pt_dma.segs[0].ds_addr | GITS_BASER_Valid; 510 if (!itstab->tab_shareable) { 511 cpu_dcache_wb_range((vaddr_t)&l1_tab[index], 512 sizeof(l1_tab[index])); 513 } 514 dsb(sy); 515 516 DPRINTF(("ITS: Allocated L2 entry at index %u\n", index)); 517 } 518 } 519 520 dev = kmem_alloc(sizeof(*dev), KM_SLEEP); 521 dev->dev_id = devid; 522 dev->dev_size = itt_size; 523 memset(dev->dev_eventid, 0, sizeof(dev->dev_eventid)); 524 gicv3_dma_alloc(its->its_gic, &dev->dev_itt, itt_size, GITS_ITT_ALIGN); 525 LIST_INSERT_HEAD(&its->its_devices, dev, dev_list); 526 527 if (its->its_cmd_flush) { 528 cpu_dcache_wb_range((vaddr_t)dev->dev_itt.base, itt_size); 529 } 530 dsb(sy); 531 532 /* 533 * Map the device to the ITT 534 */ 535 mutex_enter(its->its_lock); 536 error = gits_command_mapd(its, devid, dev->dev_itt.segs[0].ds_addr, id_bits, true); 537 if (error == 0) { 538 error = gits_wait(its); 539 } 540 mutex_exit(its->its_lock); 541 542 return error == 0 ? dev : NULL; 543 } 544 545 static void 546 gicv3_its_msi_enable(struct gicv3_its *its, int lpi, uint32_t data, int count) 547 { 548 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 549 pci_chipset_tag_t pc = pa->pa_pc; 550 pcitag_t tag = pa->pa_tag; 551 pcireg_t ctl; 552 int off; 553 554 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) 555 panic("gicv3_its_msi_enable: device is not MSI-capable"); 556 557 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 558 ctl &= ~PCI_MSI_CTL_MME_MASK; 559 ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK); 560 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 561 562 const uint64_t addr = its->its_base + GITS_TRANSLATER; 563 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 564 if (ctl & PCI_MSI_CTL_64BIT_ADDR) { 565 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, 566 addr & 0xffffffff); 567 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, 568 (addr >> 32) & 0xffffffff); 569 pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, data); 570 } else { 571 KASSERT((addr >> 32) == 0); 572 pci_conf_write(pc, tag, off + PCI_MSI_MADDR, 573 addr & 0xffffffff); 574 pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data); 575 } 576 ctl |= PCI_MSI_CTL_MSI_ENABLE; 577 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 578 } 579 580 static void 581 gicv3_its_msi_disable(struct gicv3_its *its, int lpi) 582 { 583 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 584 pci_chipset_tag_t pc = pa->pa_pc; 585 pcitag_t tag = pa->pa_tag; 586 pcireg_t ctl; 587 int off; 588 589 if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) 590 panic("gicv3_its_msi_enable: device is not MSI-capable"); 591 592 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 593 ctl &= ~PCI_MSI_CTL_MSI_ENABLE; 594 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 595 } 596 597 static void 598 gicv3_its_msix_enable(struct gicv3_its *its, int lpi, int msix_vec, 599 uint32_t data, bus_space_tag_t bst, bus_space_handle_t bsh) 600 { 601 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 602 pci_chipset_tag_t pc = pa->pa_pc; 603 pcitag_t tag = pa->pa_tag; 604 pcireg_t ctl; 605 uint32_t val; 606 int off; 607 608 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) 609 panic("gicv3_its_msix_enable: device is not MSI-X-capable"); 610 611 const uint64_t addr = its->its_base + GITS_TRANSLATER; 612 const uint64_t entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec; 613 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, 614 (uint32_t)addr); 615 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, 616 (uint32_t)(addr >> 32)); 617 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_DATA, 618 data); 619 val = bus_space_read_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL); 620 val &= ~PCI_MSIX_VECTCTL_MASK; 621 bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, val); 622 623 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 624 ctl |= PCI_MSIX_CTL_ENABLE; 625 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 626 } 627 628 static void 629 gicv3_its_msix_disable(struct gicv3_its *its, int lpi) 630 { 631 const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; 632 pci_chipset_tag_t pc = pa->pa_pc; 633 pcitag_t tag = pa->pa_tag; 634 pcireg_t ctl; 635 int off; 636 637 if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) 638 panic("gicv3_its_msix_disable: device is not MSI-X-capable"); 639 640 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 641 ctl &= ~PCI_MSIX_CTL_ENABLE; 642 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 643 } 644 645 static pci_intr_handle_t * 646 gicv3_its_msi_alloc(struct md_pci_msi *msi, int *count, 647 const struct pci_attach_args *pa, bool exact) 648 { 649 struct gicv3_its * const its = msi->msi_priv; 650 struct gicv3_its_device *dev; 651 struct cpu_info * const ci = cpu_lookup(0); 652 pci_intr_handle_t *vectors; 653 int n, off, error; 654 uint32_t eventid; 655 656 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSI, &off, NULL)) 657 return NULL; 658 659 const uint64_t typer = gits_read_8(its, GITS_TYPER); 660 const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1; 661 if (*count == 0 || *count > (1 << id_bits)) 662 return NULL; 663 664 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); 665 666 dev = gicv3_its_device_map(its, devid, *count); 667 if (dev == NULL) 668 return NULL; 669 670 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); 671 mutex_enter(its->its_lock); 672 eventid = gicv3_its_alloc_eventid(dev, *count); 673 KASSERT(eventid != ITS_INVALID_EVENTID); 674 for (n = 0; n < *count; n++) { 675 const int lpi = gicv3_its_msi_alloc_lpi(its, pa); 676 KASSERT(lpi >= 0); 677 vectors[n] = MD_PCI_INTR_MSI | 678 __SHIFTIN(lpi, MD_PCI_INTR_IRQ) | 679 __SHIFTIN(eventid + n, MD_PCI_INTR_MSI_VEC) | 680 __SHIFTIN(msi->msi_id, MD_PCI_INTR_FRAME); 681 682 if (n == 0) 683 gicv3_its_msi_enable(its, lpi, eventid + n, *count); 684 685 /* 686 * Record devid, eventid, dev, and target PE 687 */ 688 its->its_devid[lpi - its->its_pic->pic_irqbase] = devid; 689 its->its_eventid[lpi - its->its_pic->pic_irqbase] = eventid + n; 690 its->its_lpitodev[lpi - its->its_pic->pic_irqbase] = dev; 691 its->its_targets[lpi - its->its_pic->pic_irqbase] = ci; 692 693 /* 694 * Map event 695 */ 696 gits_command_mapti(its, devid, eventid + n, lpi, cpu_index(ci)); 697 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); 698 } 699 error = gits_wait(its); 700 mutex_exit(its->its_lock); 701 702 if (error != 0) { 703 kmem_free(vectors, sizeof(*vectors) * *count); 704 vectors = NULL; 705 } 706 707 return vectors; 708 } 709 710 static pci_intr_handle_t * 711 gicv3_its_msix_alloc(struct md_pci_msi *msi, u_int *table_indexes, int *count, 712 const struct pci_attach_args *pa, bool exact) 713 { 714 struct gicv3_its * const its = msi->msi_priv; 715 struct gicv3_its_device *dev; 716 struct cpu_info *ci = cpu_lookup(0); 717 pci_intr_handle_t *vectors; 718 bus_space_tag_t bst; 719 bus_space_handle_t bsh; 720 bus_size_t bsz; 721 uint32_t table_offset, table_size; 722 uint32_t eventid; 723 int n, off, bar, error; 724 pcireg_t tbl; 725 726 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &off, NULL)) 727 return NULL; 728 729 const uint64_t typer = gits_read_8(its, GITS_TYPER); 730 const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1; 731 if (*count == 0 || *count > (1 << id_bits)) 732 return NULL; 733 734 tbl = pci_conf_read(pa->pa_pc, pa->pa_tag, off + PCI_MSIX_TBLOFFSET); 735 bar = PCI_BAR0 + (4 * (tbl & PCI_MSIX_TBLBIR_MASK)); 736 table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK; 737 table_size = pci_msix_count(pa->pa_pc, pa->pa_tag) * PCI_MSIX_TABLE_ENTRY_SIZE; 738 if (table_size == 0) 739 return NULL; 740 741 error = pci_mapreg_submap(pa, bar, pci_mapreg_type(pa->pa_pc, pa->pa_tag, bar), 742 BUS_SPACE_MAP_LINEAR, roundup(table_size, PAGE_SIZE), table_offset, 743 &bst, &bsh, NULL, &bsz); 744 if (error) 745 return NULL; 746 747 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); 748 749 dev = gicv3_its_device_map(its, devid, *count); 750 if (dev == NULL) { 751 bus_space_unmap(bst, bsh, bsz); 752 return NULL; 753 } 754 755 vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); 756 mutex_enter(its->its_lock); 757 eventid = gicv3_its_alloc_eventid(dev, *count); 758 KASSERT(eventid != ITS_INVALID_EVENTID); 759 for (n = 0; n < *count; n++) { 760 const int lpi = gicv3_its_msi_alloc_lpi(its, pa); 761 KASSERT(lpi >= 0); 762 const int msix_vec = table_indexes ? table_indexes[n] : n; 763 vectors[msix_vec] = MD_PCI_INTR_MSIX | 764 __SHIFTIN(lpi, MD_PCI_INTR_IRQ) | 765 __SHIFTIN(eventid + n, MD_PCI_INTR_MSI_VEC) | 766 __SHIFTIN(msi->msi_id, MD_PCI_INTR_FRAME); 767 768 gicv3_its_msix_enable(its, lpi, msix_vec, eventid + n, bst, bsh); 769 770 /* 771 * Record devid, eventid, dev, and target PE 772 */ 773 its->its_devid[lpi - its->its_pic->pic_irqbase] = devid; 774 its->its_eventid[lpi - its->its_pic->pic_irqbase] = eventid + n; 775 its->its_lpitodev[lpi - its->its_pic->pic_irqbase] = dev; 776 its->its_targets[lpi - its->its_pic->pic_irqbase] = ci; 777 778 /* 779 * Map event 780 */ 781 gits_command_mapti(its, devid, eventid + n, lpi, cpu_index(ci)); 782 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); 783 } 784 gits_wait(its); 785 mutex_exit(its->its_lock); 786 787 bus_space_unmap(bst, bsh, bsz); 788 789 return vectors; 790 } 791 792 static void * 793 gicv3_its_msi_intr_establish(struct md_pci_msi *msi, 794 pci_intr_handle_t ih, int ipl, int (*func)(void *), void *arg, const char *xname) 795 { 796 struct gicv3_its * const its = msi->msi_priv; 797 void *intrh; 798 799 const int lpi = __SHIFTOUT(ih, MD_PCI_INTR_IRQ); 800 const uint32_t eventid = __SHIFTOUT(ih, MD_PCI_INTR_MSI_VEC); 801 const int mpsafe = (ih & MD_PCI_INTR_MPSAFE) ? IST_MPSAFE : 0; 802 803 intrh = pic_establish_intr(its->its_pic, lpi - its->its_pic->pic_irqbase, ipl, 804 IST_EDGE | mpsafe, func, arg, xname); 805 if (intrh == NULL) 806 return NULL; 807 808 /* Invalidate LPI configuration tables */ 809 KASSERT(its->its_pa[lpi - its->its_pic->pic_irqbase] != NULL); 810 const uint32_t devid = its->its_devid[lpi - its->its_pic->pic_irqbase]; 811 gits_command_inv(its, devid, eventid); 812 813 return intrh; 814 } 815 816 static void 817 gicv3_its_msi_intr_release(struct md_pci_msi *msi, pci_intr_handle_t *pih, 818 int count) 819 { 820 struct gicv3_its * const its = msi->msi_priv; 821 int n; 822 823 for (n = 0; n < count; n++) { 824 const int lpi = __SHIFTOUT(pih[n], MD_PCI_INTR_IRQ); 825 const uint32_t eventid = __SHIFTOUT(pih[n], MD_PCI_INTR_MSI_VEC); 826 KASSERT(lpi >= its->its_pic->pic_irqbase); 827 struct gicv3_its_device *dev = 828 its->its_lpitodev[lpi - its->its_pic->pic_irqbase]; 829 KASSERT(dev != NULL); 830 if (pih[n] & MD_PCI_INTR_MSIX) 831 gicv3_its_msix_disable(its, lpi); 832 if (pih[n] & MD_PCI_INTR_MSI) 833 gicv3_its_msi_disable(its, lpi); 834 gicv3_its_free_eventid(dev, eventid); 835 gicv3_its_msi_free_lpi(its, lpi); 836 its->its_targets[lpi - its->its_pic->pic_irqbase] = NULL; 837 its->its_devid[lpi - its->its_pic->pic_irqbase] = 0; 838 its->its_eventid[lpi - its->its_pic->pic_irqbase] = ITS_INVALID_EVENTID; 839 its->its_lpitodev[lpi - its->its_pic->pic_irqbase] = NULL; 840 struct intrsource * const is = 841 its->its_pic->pic_sources[lpi - its->its_pic->pic_irqbase]; 842 if (is != NULL) 843 pic_disestablish_source(is); 844 } 845 } 846 847 static void 848 gicv3_its_command_init(struct gicv3_softc *sc, struct gicv3_its *its) 849 { 850 uint64_t cbaser, tmp; 851 852 gicv3_dma_alloc(sc, &its->its_cmd, GITS_COMMANDS_SIZE, GITS_COMMANDS_ALIGN); 853 if (its->its_cmd_flush) { 854 cpu_dcache_wb_range((vaddr_t)its->its_cmd.base, GITS_COMMANDS_SIZE); 855 } 856 dsb(sy); 857 858 KASSERT((gits_read_4(its, GITS_CTLR) & GITS_CTLR_Enabled) == 0); 859 KASSERT((gits_read_4(its, GITS_CTLR) & GITS_CTLR_Quiescent) != 0); 860 861 cbaser = its->its_cmd.segs[0].ds_addr; 862 cbaser |= __SHIFTIN((its->its_cmd.len / 4096) - 1, GITS_CBASER_Size); 863 cbaser |= GITS_CBASER_Valid; 864 865 cbaser |= __SHIFTIN(GITS_Cache_NORMAL_WA_WB, GITS_CBASER_InnerCache); 866 cbaser |= __SHIFTIN(GITS_Shareability_IS, GITS_CBASER_Shareability); 867 gits_write_8(its, GITS_CBASER, cbaser); 868 869 tmp = gits_read_8(its, GITS_CBASER); 870 if (__SHIFTOUT(tmp, GITS_CBASER_Shareability) != GITS_Shareability_IS) { 871 if (__SHIFTOUT(tmp, GITS_CBASER_InnerCache) == GITS_Shareability_NS) { 872 cbaser &= ~GITS_CBASER_InnerCache; 873 cbaser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_CBASER_InnerCache); 874 cbaser &= ~GITS_CBASER_Shareability; 875 cbaser |= __SHIFTIN(GITS_Shareability_NS, GITS_CBASER_Shareability); 876 gits_write_8(its, GITS_CBASER, cbaser); 877 } 878 879 its->its_cmd_flush = true; 880 } 881 aprint_normal_dev(sc->sc_dev, "ITS command table @ %#lx/%#lx, %s, %s\n", 882 its->its_cmd.segs[0].ds_addr, its->its_cmd.len, 883 gits_cache_type[__SHIFTOUT(cbaser, GITS_BASER_InnerCache)], 884 gits_share_type[__SHIFTOUT(cbaser, GITS_BASER_Shareability)]); 885 886 gits_write_8(its, GITS_CWRITER, 0); 887 } 888 889 static void 890 gicv3_its_table_params(struct gicv3_softc *sc, struct gicv3_its *its, 891 u_int *devbits, u_int *innercache, u_int *share) 892 { 893 894 const uint64_t typer = gits_read_8(its, GITS_TYPER); 895 const uint32_t iidr = gits_read_4(its, GITS_IIDR); 896 897 /* Default values */ 898 *devbits = __SHIFTOUT(typer, GITS_TYPER_Devbits) + 1; 899 *innercache = GITS_Cache_NORMAL_WA_WB; 900 *share = GITS_Shareability_IS; 901 902 /* Cavium ThunderX errata */ 903 if ((iidr & GITS_IIDR_CAVIUM_ERRATA_MASK) == GITS_IIDR_CAVIUM_ERRATA_VALUE) { 904 *devbits = 20; /* 8Mb */ 905 *innercache = GITS_Cache_DEVICE_nGnRnE; 906 aprint_normal_dev(sc->sc_dev, "Cavium ThunderX errata detected\n"); 907 } 908 } 909 910 static bool 911 gicv3_its_table_probe_indirect(struct gicv3_its *its, int tab) 912 { 913 uint64_t baser; 914 915 baser = gits_read_8(its, GITS_BASERn(tab)); 916 baser |= GITS_BASER_Indirect; 917 gits_write_8(its, GITS_BASERn(tab), baser); 918 919 baser = gits_read_8(its, GITS_BASERn(tab)); 920 921 return (baser & GITS_BASER_Indirect) != 0; 922 } 923 924 static void 925 gicv3_its_table_init(struct gicv3_softc *sc, struct gicv3_its *its) 926 { 927 u_int page_size, table_align; 928 u_int devbits, innercache, share; 929 const char *table_type; 930 uint64_t baser; 931 int tab; 932 933 gicv3_its_table_params(sc, its, &devbits, &innercache, &share); 934 935 DPRINTF(("ITS: devbits = %u\n", devbits)); 936 937 for (tab = 0; tab < 8; tab++) { 938 struct gicv3_its_table *itstab; 939 bool indirect = false; 940 uint64_t l1_entry_size, l2_entry_size; 941 uint64_t l1_num_ids, l2_num_ids; 942 uint64_t table_size; 943 944 baser = gits_read_8(its, GITS_BASERn(tab)); 945 946 l1_entry_size = __SHIFTOUT(baser, GITS_BASER_Entry_Size) + 1; 947 l2_entry_size = 0; 948 l2_num_ids = 0; 949 950 switch (__SHIFTOUT(baser, GITS_BASER_Page_Size)) { 951 case GITS_Page_Size_64KB: 952 page_size = 65536; 953 break; 954 case GITS_Page_Size_16KB: 955 page_size = 16384; 956 break; 957 case GITS_Page_Size_4KB: 958 default: 959 page_size = 4096; 960 } 961 table_align = page_size; 962 963 switch (__SHIFTOUT(baser, GITS_BASER_Type)) { 964 case GITS_Type_Devices: 965 /* 966 * Table size scales with the width of the DeviceID. 967 */ 968 l1_num_ids = 1ULL << devbits; 969 DPRINTF(("ITS: l1_num_ids = %lu\n", l1_num_ids)); 970 indirect = 971 gicv3_its_table_probe_indirect(its, tab); 972 if (indirect) { 973 DPRINTF(("ITS: indirect\n")); 974 l2_entry_size = l1_entry_size; 975 l2_num_ids = page_size / l2_entry_size; 976 l1_num_ids = l1_num_ids / l2_num_ids; 977 l1_entry_size = GITS_INDIRECT_ENTRY_SIZE; 978 } 979 table_size = roundup2(l1_entry_size * l1_num_ids, page_size); 980 if (howmany(table_size, page_size) > GITS_BASER_Size + 1) { 981 DPRINTF(("ITS: clamp table size 0x%lx -> ", table_size)); 982 table_size = (GITS_BASER_Size + 1) * page_size; 983 DPRINTF(("0x%lx\n", table_size)); 984 } 985 table_type = "Devices"; 986 987 DPRINTF(("ITS: table_size is 0x%lx\n", table_size)); 988 989 itstab = &its->its_tab_device; 990 itstab->tab_page_size = page_size; 991 itstab->tab_l1_entry_size = l1_entry_size; 992 itstab->tab_l1_num_ids = l1_num_ids; 993 itstab->tab_l2_entry_size = l2_entry_size; 994 itstab->tab_l2_num_ids = l2_num_ids; 995 itstab->tab_indirect = indirect; 996 LIST_INIT(&itstab->tab_pt); 997 break; 998 case GITS_Type_InterruptCollections: 999 /* 1000 * Allocate space for one interrupt collection per CPU. 1001 */ 1002 table_size = roundup(l1_entry_size * ncpu, page_size); 1003 table_type = "Collections"; 1004 break; 1005 default: 1006 table_size = 0; 1007 break; 1008 } 1009 1010 if (table_size == 0) 1011 continue; 1012 1013 gicv3_dma_alloc(sc, &its->its_tab[tab], table_size, table_align); 1014 if (its->its_cmd_flush) { 1015 cpu_dcache_wb_range((vaddr_t)its->its_tab[tab].base, table_size); 1016 } 1017 dsb(sy); 1018 1019 baser &= ~GITS_BASER_Size; 1020 baser |= __SHIFTIN(howmany(table_size, page_size) - 1, GITS_BASER_Size); 1021 baser &= ~GITS_BASER_Physical_Address; 1022 baser |= its->its_tab[tab].segs[0].ds_addr; 1023 baser &= ~GITS_BASER_InnerCache; 1024 baser |= __SHIFTIN(innercache, GITS_BASER_InnerCache); 1025 baser &= ~GITS_BASER_Shareability; 1026 baser |= __SHIFTIN(share, GITS_BASER_Shareability); 1027 baser |= GITS_BASER_Valid; 1028 if (indirect) { 1029 baser |= GITS_BASER_Indirect; 1030 } else { 1031 baser &= ~GITS_BASER_Indirect; 1032 } 1033 1034 gits_write_8(its, GITS_BASERn(tab), baser); 1035 1036 baser = gits_read_8(its, GITS_BASERn(tab)); 1037 if (__SHIFTOUT(baser, GITS_BASER_Shareability) == GITS_Shareability_NS) { 1038 baser &= ~GITS_BASER_InnerCache; 1039 baser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_BASER_InnerCache); 1040 1041 gits_write_8(its, GITS_BASERn(tab), baser); 1042 } 1043 1044 baser = gits_read_8(its, GITS_BASERn(tab)); 1045 aprint_normal_dev(sc->sc_dev, "ITS [#%d] %s table @ %#lx/%#lx, %s, %s%s\n", 1046 tab, table_type, its->its_tab[tab].segs[0].ds_addr, table_size, 1047 gits_cache_type[__SHIFTOUT(baser, GITS_BASER_InnerCache)], 1048 gits_share_type[__SHIFTOUT(baser, GITS_BASER_Shareability)], 1049 indirect ? ", indirect" : ""); 1050 1051 if (__SHIFTOUT(baser, GITS_BASER_Type) == GITS_Type_Devices) { 1052 its->its_tab_device.tab_l1 = its->its_tab[tab].base; 1053 its->its_tab_device.tab_shareable = 1054 __SHIFTOUT(baser, GITS_BASER_Shareability) != GITS_Shareability_NS; 1055 } 1056 1057 } 1058 } 1059 1060 static void 1061 gicv3_its_enable(struct gicv3_softc *sc, struct gicv3_its *its) 1062 { 1063 uint32_t ctlr; 1064 1065 ctlr = gits_read_4(its, GITS_CTLR); 1066 ctlr |= GITS_CTLR_Enabled; 1067 gits_write_4(its, GITS_CTLR, ctlr); 1068 } 1069 1070 static void 1071 gicv3_its_cpu_init(void *priv, struct cpu_info *ci) 1072 { 1073 struct gicv3_its * const its = priv; 1074 struct gicv3_softc * const sc = its->its_gic; 1075 uint64_t rdbase; 1076 size_t irq; 1077 1078 const uint64_t typer = bus_space_read_8(sc->sc_bst, its->its_bsh, GITS_TYPER); 1079 if (typer & GITS_TYPER_PTA) { 1080 void *va = bus_space_vaddr(sc->sc_bst, sc->sc_bsh_r[ci->ci_gic_redist]); 1081 rdbase = vtophys((vaddr_t)va); 1082 } else { 1083 rdbase = (uint64_t)sc->sc_processor_id[cpu_index(ci)] << 16; 1084 } 1085 its->its_rdbase[cpu_index(ci)] = rdbase; 1086 1087 /* 1088 * Map collection ID of this CPU's index to this CPU's redistributor. 1089 */ 1090 mutex_enter(its->its_lock); 1091 gits_command_mapc(its, cpu_index(ci), rdbase, true); 1092 gits_command_invall(its, cpu_index(ci)); 1093 gits_wait(its); 1094 1095 /* 1096 * Update routing for LPIs targetting this CPU 1097 */ 1098 for (irq = 0; irq < its->its_pic->pic_maxsources; irq++) { 1099 if (its->its_targets[irq] != ci) 1100 continue; 1101 KASSERT(its->its_pa[irq] != NULL); 1102 1103 const uint32_t devid = its->its_devid[irq]; 1104 const uint32_t eventid = its->its_eventid[irq]; 1105 KASSERT(eventid != ITS_INVALID_EVENTID); 1106 gits_command_movi(its, devid, eventid, cpu_index(ci)); 1107 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); 1108 } 1109 gits_wait(its); 1110 mutex_exit(its->its_lock); 1111 1112 its->its_cpuonline[cpu_index(ci)] = true; 1113 } 1114 1115 static void 1116 gicv3_its_get_affinity(void *priv, size_t irq, kcpuset_t *affinity) 1117 { 1118 struct gicv3_its * const its = priv; 1119 struct cpu_info *ci; 1120 1121 ci = its->its_targets[irq]; 1122 if (ci) 1123 kcpuset_set(affinity, cpu_index(ci)); 1124 } 1125 1126 static int 1127 gicv3_its_set_affinity(void *priv, size_t irq, const kcpuset_t *affinity) 1128 { 1129 struct gicv3_its * const its = priv; 1130 const struct pci_attach_args *pa; 1131 struct cpu_info *ci; 1132 1133 const int set = kcpuset_countset(affinity); 1134 if (set != 1) 1135 return EINVAL; 1136 1137 pa = its->its_pa[irq]; 1138 if (pa == NULL) 1139 return EPASSTHROUGH; 1140 1141 ci = cpu_lookup(kcpuset_ffs(affinity) - 1); 1142 its->its_targets[irq] = ci; 1143 1144 if (its->its_cpuonline[cpu_index(ci)] == true) { 1145 const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); 1146 const uint32_t eventid = its->its_eventid[irq]; 1147 KASSERT(eventid != ITS_INVALID_EVENTID); 1148 mutex_enter(its->its_lock); 1149 gits_command_movi(its, devid, eventid, cpu_index(ci)); 1150 gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); 1151 mutex_exit(its->its_lock); 1152 } 1153 1154 return 0; 1155 } 1156 1157 int 1158 gicv3_its_init(struct gicv3_softc *sc, bus_space_handle_t bsh, 1159 uint64_t its_base, uint32_t its_id) 1160 { 1161 struct gicv3_its *its; 1162 struct md_pci_msi *msi; 1163 1164 const uint64_t typer = bus_space_read_8(sc->sc_bst, bsh, GITS_TYPER); 1165 if ((typer & GITS_TYPER_Physical) == 0) 1166 return ENXIO; 1167 1168 its = kmem_zalloc(sizeof(*its), KM_SLEEP); 1169 its->its_id = its_id; 1170 its->its_bst = sc->sc_bst; 1171 its->its_bsh = bsh; 1172 its->its_dmat = sc->sc_dmat; 1173 its->its_base = its_base; 1174 its->its_pic = &sc->sc_lpi; 1175 snprintf(its->its_pic->pic_name, sizeof(its->its_pic->pic_name), "gicv3-its"); 1176 KASSERT(its->its_pic->pic_maxsources > 0); 1177 its->its_pa = kmem_zalloc(sizeof(struct pci_attach_args *) * its->its_pic->pic_maxsources, KM_SLEEP); 1178 its->its_targets = kmem_zalloc(sizeof(struct cpu_info *) * its->its_pic->pic_maxsources, KM_SLEEP); 1179 its->its_devid = kmem_zalloc(sizeof(uint32_t) * its->its_pic->pic_maxsources, KM_SLEEP); 1180 its->its_eventid = kmem_alloc(sizeof(uint32_t) * its->its_pic->pic_maxsources, KM_SLEEP); 1181 memset(its->its_eventid, 0xff, sizeof(uint32_t) * its->its_pic->pic_maxsources); 1182 its->its_lpitodev = kmem_alloc(sizeof(struct gicv3_its_device *) * its->its_pic->pic_maxsources, KM_SLEEP); 1183 its->its_gic = sc; 1184 its->its_rdbase = kmem_zalloc(sizeof(*its->its_rdbase) * ncpu, KM_SLEEP); 1185 its->its_cpuonline = kmem_zalloc(sizeof(*its->its_cpuonline) * ncpu, KM_SLEEP); 1186 its->its_cb.cpu_init = gicv3_its_cpu_init; 1187 its->its_cb.get_affinity = gicv3_its_get_affinity; 1188 its->its_cb.set_affinity = gicv3_its_set_affinity; 1189 its->its_cb.priv = its; 1190 LIST_INIT(&its->its_devices); 1191 LIST_INSERT_HEAD(&sc->sc_lpi_callbacks, &its->its_cb, list); 1192 its->its_lock = mutex_obj_alloc(MUTEX_SPIN, IPL_NONE); 1193 1194 gicv3_its_command_init(sc, its); 1195 gicv3_its_table_init(sc, its); 1196 1197 gicv3_its_enable(sc, its); 1198 1199 gicv3_its_cpu_init(its, curcpu()); 1200 1201 msi = &its->its_msi; 1202 msi->msi_id = its_id; 1203 msi->msi_dev = sc->sc_dev; 1204 msi->msi_priv = its; 1205 msi->msi_alloc = gicv3_its_msi_alloc; 1206 msi->msix_alloc = gicv3_its_msix_alloc; 1207 msi->msi_intr_establish = gicv3_its_msi_intr_establish; 1208 msi->msi_intr_release = gicv3_its_msi_intr_release; 1209 1210 return md_pci_msi_add(msi); 1211 } 1212
Indexes created Sun Oct 25 00:26:45 UTC 2026