kfd_events.c revision 1.2
1 /* $NetBSD: kfd_events.c,v 1.2 2018/08/27 04:58:20 riastradh Exp $ */ 2 3 /* 4 * Copyright 2014 Advanced Micro Devices, Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 */ 24 25 #include <sys/cdefs.h> 26 __KERNEL_RCSID(0, "$NetBSD: kfd_events.c,v 1.2 2018/08/27 04:58:20 riastradh Exp $"); 27 28 #include <linux/mm_types.h> 29 #include <linux/slab.h> 30 #include <linux/types.h> 31 #include <linux/sched.h> 32 #include <linux/uaccess.h> 33 #include <linux/mm.h> 34 #include <linux/mman.h> 35 #include <linux/memory.h> 36 #include "kfd_priv.h" 37 #include "kfd_events.h" 38 #include <linux/device.h> 39 40 /* 41 * A task can only be on a single wait_queue at a time, but we need to support 42 * waiting on multiple events (any/all). 43 * Instead of each event simply having a wait_queue with sleeping tasks, it 44 * has a singly-linked list of tasks. 45 * A thread that wants to sleep creates an array of these, one for each event 46 * and adds one to each event's waiter chain. 47 */ 48 struct kfd_event_waiter { 49 struct list_head waiters; 50 struct task_struct *sleeping_task; 51 52 /* Transitions to true when the event this belongs to is signaled. */ 53 bool activated; 54 55 /* Event */ 56 struct kfd_event *event; 57 uint32_t input_index; 58 }; 59 60 /* 61 * Over-complicated pooled allocator for event notification slots. 62 * 63 * Each signal event needs a 64-bit signal slot where the signaler will write 64 * a 1 before sending an interrupt.l (This is needed because some interrupts 65 * do not contain enough spare data bits to identify an event.) 66 * We get whole pages from vmalloc and map them to the process VA. 67 * Individual signal events are then allocated a slot in a page. 68 */ 69 70 struct signal_page { 71 struct list_head event_pages; /* kfd_process.signal_event_pages */ 72 uint64_t *kernel_address; 73 uint64_t __user *user_address; 74 uint32_t page_index; /* Index into the mmap aperture. */ 75 unsigned int free_slots; 76 unsigned long used_slot_bitmap[0]; 77 }; 78 79 #define SLOTS_PER_PAGE KFD_SIGNAL_EVENT_LIMIT 80 #define SLOT_BITMAP_SIZE BITS_TO_LONGS(SLOTS_PER_PAGE) 81 #define BITS_PER_PAGE (ilog2(SLOTS_PER_PAGE)+1) 82 #define SIGNAL_PAGE_SIZE (sizeof(struct signal_page) + \ 83 SLOT_BITMAP_SIZE * sizeof(long)) 84 85 /* 86 * For signal events, the event ID is used as the interrupt user data. 87 * For SQ s_sendmsg interrupts, this is limited to 8 bits. 88 */ 89 90 #define INTERRUPT_DATA_BITS 8 91 #define SIGNAL_EVENT_ID_SLOT_SHIFT 0 92 93 static uint64_t *page_slots(struct signal_page *page) 94 { 95 return page->kernel_address; 96 } 97 98 static bool allocate_free_slot(struct kfd_process *process, 99 struct signal_page **out_page, 100 unsigned int *out_slot_index) 101 { 102 struct signal_page *page; 103 104 list_for_each_entry(page, &process->signal_event_pages, event_pages) { 105 if (page->free_slots > 0) { 106 unsigned int slot = 107 find_first_zero_bit(page->used_slot_bitmap, 108 SLOTS_PER_PAGE); 109 110 __set_bit(slot, page->used_slot_bitmap); 111 page->free_slots--; 112 113 page_slots(page)[slot] = UNSIGNALED_EVENT_SLOT; 114 115 *out_page = page; 116 *out_slot_index = slot; 117 118 pr_debug("allocated event signal slot in page %p, slot %d\n", 119 page, slot); 120 121 return true; 122 } 123 } 124 125 pr_debug("No free event signal slots were found for process %p\n", 126 process); 127 128 return false; 129 } 130 131 #define list_tail_entry(head, type, member) \ 132 list_entry((head)->prev, type, member) 133 134 static bool allocate_signal_page(struct file *devkfd, struct kfd_process *p) 135 { 136 void *backing_store; 137 struct signal_page *page; 138 139 page = kzalloc(SIGNAL_PAGE_SIZE, GFP_KERNEL); 140 if (!page) 141 goto fail_alloc_signal_page; 142 143 page->free_slots = SLOTS_PER_PAGE; 144 145 backing_store = (void *) __get_free_pages(GFP_KERNEL | __GFP_ZERO, 146 get_order(KFD_SIGNAL_EVENT_LIMIT * 8)); 147 if (!backing_store) 148 goto fail_alloc_signal_store; 149 150 /* prevent user-mode info leaks */ 151 memset(backing_store, (uint8_t) UNSIGNALED_EVENT_SLOT, 152 KFD_SIGNAL_EVENT_LIMIT * 8); 153 154 page->kernel_address = backing_store; 155 156 if (list_empty(&p->signal_event_pages)) 157 page->page_index = 0; 158 else 159 page->page_index = list_tail_entry(&p->signal_event_pages, 160 struct signal_page, 161 event_pages)->page_index + 1; 162 163 pr_debug("allocated new event signal page at %p, for process %p\n", 164 page, p); 165 pr_debug("page index is %d\n", page->page_index); 166 167 list_add(&page->event_pages, &p->signal_event_pages); 168 169 return true; 170 171 fail_alloc_signal_store: 172 kfree(page); 173 fail_alloc_signal_page: 174 return false; 175 } 176 177 static bool allocate_event_notification_slot(struct file *devkfd, 178 struct kfd_process *p, 179 struct signal_page **page, 180 unsigned int *signal_slot_index) 181 { 182 bool ret; 183 184 ret = allocate_free_slot(p, page, signal_slot_index); 185 if (ret == false) { 186 ret = allocate_signal_page(devkfd, p); 187 if (ret == true) 188 ret = allocate_free_slot(p, page, signal_slot_index); 189 } 190 191 return ret; 192 } 193 194 /* Assumes that the process's event_mutex is locked. */ 195 static void release_event_notification_slot(struct signal_page *page, 196 size_t slot_index) 197 { 198 __clear_bit(slot_index, page->used_slot_bitmap); 199 page->free_slots++; 200 201 /* We don't free signal pages, they are retained by the process 202 * and reused until it exits. */ 203 } 204 205 static struct signal_page *lookup_signal_page_by_index(struct kfd_process *p, 206 unsigned int page_index) 207 { 208 struct signal_page *page; 209 210 /* 211 * This is safe because we don't delete signal pages until the 212 * process exits. 213 */ 214 list_for_each_entry(page, &p->signal_event_pages, event_pages) 215 if (page->page_index == page_index) 216 return page; 217 218 return NULL; 219 } 220 221 /* 222 * Assumes that p->event_mutex is held and of course that p is not going 223 * away (current or locked). 224 */ 225 static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id) 226 { 227 struct kfd_event *ev; 228 229 hash_for_each_possible(p->events, ev, events, id) 230 if (ev->event_id == id) 231 return ev; 232 233 return NULL; 234 } 235 236 static u32 make_signal_event_id(struct signal_page *page, 237 unsigned int signal_slot_index) 238 { 239 return page->page_index | 240 (signal_slot_index << SIGNAL_EVENT_ID_SLOT_SHIFT); 241 } 242 243 /* 244 * Produce a kfd event id for a nonsignal event. 245 * These are arbitrary numbers, so we do a sequential search through 246 * the hash table for an unused number. 247 */ 248 static u32 make_nonsignal_event_id(struct kfd_process *p) 249 { 250 u32 id; 251 252 for (id = p->next_nonsignal_event_id; 253 id < KFD_LAST_NONSIGNAL_EVENT_ID && 254 lookup_event_by_id(p, id) != NULL; 255 id++) 256 ; 257 258 if (id < KFD_LAST_NONSIGNAL_EVENT_ID) { 259 260 /* 261 * What if id == LAST_NONSIGNAL_EVENT_ID - 1? 262 * Then next_nonsignal_event_id = LAST_NONSIGNAL_EVENT_ID so 263 * the first loop fails immediately and we proceed with the 264 * wraparound loop below. 265 */ 266 p->next_nonsignal_event_id = id + 1; 267 268 return id; 269 } 270 271 for (id = KFD_FIRST_NONSIGNAL_EVENT_ID; 272 id < KFD_LAST_NONSIGNAL_EVENT_ID && 273 lookup_event_by_id(p, id) != NULL; 274 id++) 275 ; 276 277 278 if (id < KFD_LAST_NONSIGNAL_EVENT_ID) { 279 p->next_nonsignal_event_id = id + 1; 280 return id; 281 } 282 283 p->next_nonsignal_event_id = KFD_FIRST_NONSIGNAL_EVENT_ID; 284 return 0; 285 } 286 287 static struct kfd_event *lookup_event_by_page_slot(struct kfd_process *p, 288 struct signal_page *page, 289 unsigned int signal_slot) 290 { 291 return lookup_event_by_id(p, make_signal_event_id(page, signal_slot)); 292 } 293 294 static int create_signal_event(struct file *devkfd, 295 struct kfd_process *p, 296 struct kfd_event *ev) 297 { 298 if (p->signal_event_count == KFD_SIGNAL_EVENT_LIMIT) { 299 pr_warn("amdkfd: Signal event wasn't created because limit was reached\n"); 300 return -ENOMEM; 301 } 302 303 if (!allocate_event_notification_slot(devkfd, p, &ev->signal_page, 304 &ev->signal_slot_index)) { 305 pr_warn("amdkfd: Signal event wasn't created because out of kernel memory\n"); 306 return -ENOMEM; 307 } 308 309 p->signal_event_count++; 310 311 ev->user_signal_address = 312 &ev->signal_page->user_address[ev->signal_slot_index]; 313 314 ev->event_id = make_signal_event_id(ev->signal_page, 315 ev->signal_slot_index); 316 317 pr_debug("signal event number %zu created with id %d, address %p\n", 318 p->signal_event_count, ev->event_id, 319 ev->user_signal_address); 320 321 pr_debug("signal event number %zu created with id %d, address %p\n", 322 p->signal_event_count, ev->event_id, 323 ev->user_signal_address); 324 325 return 0; 326 } 327 328 /* 329 * No non-signal events are supported yet. 330 * We create them as events that never signal. 331 * Set event calls from user-mode are failed. 332 */ 333 static int create_other_event(struct kfd_process *p, struct kfd_event *ev) 334 { 335 ev->event_id = make_nonsignal_event_id(p); 336 if (ev->event_id == 0) 337 return -ENOMEM; 338 339 return 0; 340 } 341 342 void kfd_event_init_process(struct kfd_process *p) 343 { 344 mutex_init(&p->event_mutex); 345 hash_init(p->events); 346 INIT_LIST_HEAD(&p->signal_event_pages); 347 p->next_nonsignal_event_id = KFD_FIRST_NONSIGNAL_EVENT_ID; 348 p->signal_event_count = 0; 349 } 350 351 static void destroy_event(struct kfd_process *p, struct kfd_event *ev) 352 { 353 if (ev->signal_page != NULL) { 354 release_event_notification_slot(ev->signal_page, 355 ev->signal_slot_index); 356 p->signal_event_count--; 357 } 358 359 /* 360 * Abandon the list of waiters. Individual waiting threads will 361 * clean up their own data. 362 */ 363 list_del(&ev->waiters); 364 365 hash_del(&ev->events); 366 kfree(ev); 367 } 368 369 static void destroy_events(struct kfd_process *p) 370 { 371 struct kfd_event *ev; 372 struct hlist_node *tmp; 373 unsigned int hash_bkt; 374 375 hash_for_each_safe(p->events, hash_bkt, tmp, ev, events) 376 destroy_event(p, ev); 377 } 378 379 /* 380 * We assume that the process is being destroyed and there is no need to 381 * unmap the pages or keep bookkeeping data in order. 382 */ 383 static void shutdown_signal_pages(struct kfd_process *p) 384 { 385 struct signal_page *page, *tmp; 386 387 list_for_each_entry_safe(page, tmp, &p->signal_event_pages, 388 event_pages) { 389 free_pages((unsigned long)page->kernel_address, 390 get_order(KFD_SIGNAL_EVENT_LIMIT * 8)); 391 kfree(page); 392 } 393 } 394 395 void kfd_event_free_process(struct kfd_process *p) 396 { 397 destroy_events(p); 398 shutdown_signal_pages(p); 399 } 400 401 static bool event_can_be_gpu_signaled(const struct kfd_event *ev) 402 { 403 return ev->type == KFD_EVENT_TYPE_SIGNAL || 404 ev->type == KFD_EVENT_TYPE_DEBUG; 405 } 406 407 static bool event_can_be_cpu_signaled(const struct kfd_event *ev) 408 { 409 return ev->type == KFD_EVENT_TYPE_SIGNAL; 410 } 411 412 int kfd_event_create(struct file *devkfd, struct kfd_process *p, 413 uint32_t event_type, bool auto_reset, uint32_t node_id, 414 uint32_t *event_id, uint32_t *event_trigger_data, 415 uint64_t *event_page_offset, uint32_t *event_slot_index) 416 { 417 int ret = 0; 418 struct kfd_event *ev = kzalloc(sizeof(*ev), GFP_KERNEL); 419 420 if (!ev) 421 return -ENOMEM; 422 423 ev->type = event_type; 424 ev->auto_reset = auto_reset; 425 ev->signaled = false; 426 427 INIT_LIST_HEAD(&ev->waiters); 428 429 *event_page_offset = 0; 430 431 mutex_lock(&p->event_mutex); 432 433 switch (event_type) { 434 case KFD_EVENT_TYPE_SIGNAL: 435 case KFD_EVENT_TYPE_DEBUG: 436 ret = create_signal_event(devkfd, p, ev); 437 if (!ret) { 438 *event_page_offset = (ev->signal_page->page_index | 439 KFD_MMAP_EVENTS_MASK); 440 *event_page_offset <<= PAGE_SHIFT; 441 *event_slot_index = ev->signal_slot_index; 442 } 443 break; 444 default: 445 ret = create_other_event(p, ev); 446 break; 447 } 448 449 if (!ret) { 450 hash_add(p->events, &ev->events, ev->event_id); 451 452 *event_id = ev->event_id; 453 *event_trigger_data = ev->event_id; 454 } else { 455 kfree(ev); 456 } 457 458 mutex_unlock(&p->event_mutex); 459 460 return ret; 461 } 462 463 /* Assumes that p is current. */ 464 int kfd_event_destroy(struct kfd_process *p, uint32_t event_id) 465 { 466 struct kfd_event *ev; 467 int ret = 0; 468 469 mutex_lock(&p->event_mutex); 470 471 ev = lookup_event_by_id(p, event_id); 472 473 if (ev) 474 destroy_event(p, ev); 475 else 476 ret = -EINVAL; 477 478 mutex_unlock(&p->event_mutex); 479 return ret; 480 } 481 482 static void set_event(struct kfd_event *ev) 483 { 484 struct kfd_event_waiter *waiter; 485 struct kfd_event_waiter *next; 486 487 /* Auto reset if the list is non-empty and we're waking someone. */ 488 ev->signaled = !ev->auto_reset || list_empty(&ev->waiters); 489 490 list_for_each_entry_safe(waiter, next, &ev->waiters, waiters) { 491 waiter->activated = true; 492 493 /* _init because free_waiters will call list_del */ 494 list_del_init(&waiter->waiters); 495 496 wake_up_process(waiter->sleeping_task); 497 } 498 } 499 500 /* Assumes that p is current. */ 501 int kfd_set_event(struct kfd_process *p, uint32_t event_id) 502 { 503 int ret = 0; 504 struct kfd_event *ev; 505 506 mutex_lock(&p->event_mutex); 507 508 ev = lookup_event_by_id(p, event_id); 509 510 if (ev && event_can_be_cpu_signaled(ev)) 511 set_event(ev); 512 else 513 ret = -EINVAL; 514 515 mutex_unlock(&p->event_mutex); 516 return ret; 517 } 518 519 static void reset_event(struct kfd_event *ev) 520 { 521 ev->signaled = false; 522 } 523 524 /* Assumes that p is current. */ 525 int kfd_reset_event(struct kfd_process *p, uint32_t event_id) 526 { 527 int ret = 0; 528 struct kfd_event *ev; 529 530 mutex_lock(&p->event_mutex); 531 532 ev = lookup_event_by_id(p, event_id); 533 534 if (ev && event_can_be_cpu_signaled(ev)) 535 reset_event(ev); 536 else 537 ret = -EINVAL; 538 539 mutex_unlock(&p->event_mutex); 540 return ret; 541 542 } 543 544 static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev) 545 { 546 page_slots(ev->signal_page)[ev->signal_slot_index] = 547 UNSIGNALED_EVENT_SLOT; 548 } 549 550 static bool is_slot_signaled(struct signal_page *page, unsigned int index) 551 { 552 return page_slots(page)[index] != UNSIGNALED_EVENT_SLOT; 553 } 554 555 static void set_event_from_interrupt(struct kfd_process *p, 556 struct kfd_event *ev) 557 { 558 if (ev && event_can_be_gpu_signaled(ev)) { 559 acknowledge_signal(p, ev); 560 set_event(ev); 561 } 562 } 563 564 void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id, 565 uint32_t valid_id_bits) 566 { 567 struct kfd_event *ev; 568 569 /* 570 * Because we are called from arbitrary context (workqueue) as opposed 571 * to process context, kfd_process could attempt to exit while we are 572 * running so the lookup function returns a locked process. 573 */ 574 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); 575 576 if (!p) 577 return; /* Presumably process exited. */ 578 579 mutex_lock(&p->event_mutex); 580 581 if (valid_id_bits >= INTERRUPT_DATA_BITS) { 582 /* Partial ID is a full ID. */ 583 ev = lookup_event_by_id(p, partial_id); 584 set_event_from_interrupt(p, ev); 585 } else { 586 /* 587 * Partial ID is in fact partial. For now we completely 588 * ignore it, but we could use any bits we did receive to 589 * search faster. 590 */ 591 struct signal_page *page; 592 unsigned i; 593 594 list_for_each_entry(page, &p->signal_event_pages, event_pages) 595 for (i = 0; i < SLOTS_PER_PAGE; i++) 596 if (is_slot_signaled(page, i)) { 597 ev = lookup_event_by_page_slot(p, 598 page, i); 599 set_event_from_interrupt(p, ev); 600 } 601 } 602 603 mutex_unlock(&p->event_mutex); 604 mutex_unlock(&p->mutex); 605 } 606 607 static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events) 608 { 609 struct kfd_event_waiter *event_waiters; 610 uint32_t i; 611 612 event_waiters = kmalloc_array(num_events, 613 sizeof(struct kfd_event_waiter), 614 GFP_KERNEL); 615 616 for (i = 0; (event_waiters) && (i < num_events) ; i++) { 617 INIT_LIST_HEAD(&event_waiters[i].waiters); 618 event_waiters[i].sleeping_task = current; 619 event_waiters[i].activated = false; 620 } 621 622 return event_waiters; 623 } 624 625 static int init_event_waiter(struct kfd_process *p, 626 struct kfd_event_waiter *waiter, 627 uint32_t event_id, 628 uint32_t input_index) 629 { 630 struct kfd_event *ev = lookup_event_by_id(p, event_id); 631 632 if (!ev) 633 return -EINVAL; 634 635 waiter->event = ev; 636 waiter->input_index = input_index; 637 waiter->activated = ev->signaled; 638 ev->signaled = ev->signaled && !ev->auto_reset; 639 640 list_add(&waiter->waiters, &ev->waiters); 641 642 return 0; 643 } 644 645 static bool test_event_condition(bool all, uint32_t num_events, 646 struct kfd_event_waiter *event_waiters) 647 { 648 uint32_t i; 649 uint32_t activated_count = 0; 650 651 for (i = 0; i < num_events; i++) { 652 if (event_waiters[i].activated) { 653 if (!all) 654 return true; 655 656 activated_count++; 657 } 658 } 659 660 return activated_count == num_events; 661 } 662 663 /* 664 * Copy event specific data, if defined. 665 * Currently only memory exception events have additional data to copy to user 666 */ 667 static bool copy_signaled_event_data(uint32_t num_events, 668 struct kfd_event_waiter *event_waiters, 669 struct kfd_event_data __user *data) 670 { 671 struct kfd_hsa_memory_exception_data *src; 672 struct kfd_hsa_memory_exception_data __user *dst; 673 struct kfd_event_waiter *waiter; 674 struct kfd_event *event; 675 uint32_t i; 676 677 for (i = 0; i < num_events; i++) { 678 waiter = &event_waiters[i]; 679 event = waiter->event; 680 if (waiter->activated && event->type == KFD_EVENT_TYPE_MEMORY) { 681 dst = &data[waiter->input_index].memory_exception_data; 682 src = &event->memory_exception_data; 683 if (copy_to_user(dst, src, 684 sizeof(struct kfd_hsa_memory_exception_data))) 685 return false; 686 } 687 } 688 689 return true; 690 691 } 692 693 694 695 static long user_timeout_to_jiffies(uint32_t user_timeout_ms) 696 { 697 if (user_timeout_ms == KFD_EVENT_TIMEOUT_IMMEDIATE) 698 return 0; 699 700 if (user_timeout_ms == KFD_EVENT_TIMEOUT_INFINITE) 701 return MAX_SCHEDULE_TIMEOUT; 702 703 /* 704 * msecs_to_jiffies interprets all values above 2^31-1 as infinite, 705 * but we consider them finite. 706 * This hack is wrong, but nobody is likely to notice. 707 */ 708 user_timeout_ms = min_t(uint32_t, user_timeout_ms, 0x7FFFFFFF); 709 710 return msecs_to_jiffies(user_timeout_ms) + 1; 711 } 712 713 static void free_waiters(uint32_t num_events, struct kfd_event_waiter *waiters) 714 { 715 uint32_t i; 716 717 for (i = 0; i < num_events; i++) 718 list_del(&waiters[i].waiters); 719 720 kfree(waiters); 721 } 722 723 int kfd_wait_on_events(struct kfd_process *p, 724 uint32_t num_events, void __user *data, 725 bool all, uint32_t user_timeout_ms, 726 enum kfd_event_wait_result *wait_result) 727 { 728 struct kfd_event_data __user *events = 729 (struct kfd_event_data __user *) data; 730 uint32_t i; 731 int ret = 0; 732 struct kfd_event_waiter *event_waiters = NULL; 733 long timeout = user_timeout_to_jiffies(user_timeout_ms); 734 735 mutex_lock(&p->event_mutex); 736 737 event_waiters = alloc_event_waiters(num_events); 738 if (!event_waiters) { 739 ret = -ENOMEM; 740 goto fail; 741 } 742 743 for (i = 0; i < num_events; i++) { 744 struct kfd_event_data event_data; 745 746 if (copy_from_user(&event_data, &events[i], 747 sizeof(struct kfd_event_data))) { 748 ret = -EFAULT; 749 goto fail; 750 } 751 752 ret = init_event_waiter(p, &event_waiters[i], 753 event_data.event_id, i); 754 if (ret) 755 goto fail; 756 } 757 758 mutex_unlock(&p->event_mutex); 759 760 while (true) { 761 if (fatal_signal_pending(current)) { 762 ret = -EINTR; 763 break; 764 } 765 766 if (signal_pending(current)) { 767 /* 768 * This is wrong when a nonzero, non-infinite timeout 769 * is specified. We need to use 770 * ERESTARTSYS_RESTARTBLOCK, but struct restart_block 771 * contains a union with data for each user and it's 772 * in generic kernel code that I don't want to 773 * touch yet. 774 */ 775 ret = -ERESTARTSYS; 776 break; 777 } 778 779 if (test_event_condition(all, num_events, event_waiters)) { 780 if (copy_signaled_event_data(num_events, 781 event_waiters, events)) 782 *wait_result = KFD_WAIT_COMPLETE; 783 else 784 *wait_result = KFD_WAIT_ERROR; 785 break; 786 } 787 788 if (timeout <= 0) { 789 *wait_result = KFD_WAIT_TIMEOUT; 790 break; 791 } 792 793 timeout = schedule_timeout_interruptible(timeout); 794 } 795 __set_current_state(TASK_RUNNING); 796 797 mutex_lock(&p->event_mutex); 798 free_waiters(num_events, event_waiters); 799 mutex_unlock(&p->event_mutex); 800 801 return ret; 802 803 fail: 804 if (event_waiters) 805 free_waiters(num_events, event_waiters); 806 807 mutex_unlock(&p->event_mutex); 808 809 *wait_result = KFD_WAIT_ERROR; 810 811 return ret; 812 } 813 814 int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma) 815 { 816 817 unsigned int page_index; 818 unsigned long pfn; 819 struct signal_page *page; 820 821 /* check required size is logical */ 822 if (get_order(KFD_SIGNAL_EVENT_LIMIT * 8) != 823 get_order(vma->vm_end - vma->vm_start)) { 824 pr_err("amdkfd: event page mmap requested illegal size\n"); 825 return -EINVAL; 826 } 827 828 page_index = vma->vm_pgoff; 829 830 page = lookup_signal_page_by_index(p, page_index); 831 if (!page) { 832 /* Probably KFD bug, but mmap is user-accessible. */ 833 pr_debug("signal page could not be found for page_index %u\n", 834 page_index); 835 return -EINVAL; 836 } 837 838 pfn = __pa(page->kernel_address); 839 pfn >>= PAGE_SHIFT; 840 841 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE 842 | VM_DONTDUMP | VM_PFNMAP; 843 844 pr_debug("mapping signal page\n"); 845 pr_debug(" start user address == 0x%08lx\n", vma->vm_start); 846 pr_debug(" end user address == 0x%08lx\n", vma->vm_end); 847 pr_debug(" pfn == 0x%016lX\n", pfn); 848 pr_debug(" vm_flags == 0x%08lX\n", vma->vm_flags); 849 pr_debug(" size == 0x%08lX\n", 850 vma->vm_end - vma->vm_start); 851 852 page->user_address = (uint64_t __user *)vma->vm_start; 853 854 /* mapping the page to user process */ 855 return remap_pfn_range(vma, vma->vm_start, pfn, 856 vma->vm_end - vma->vm_start, vma->vm_page_prot); 857 } 858 859 /* 860 * Assumes that p->event_mutex is held and of course 861 * that p is not going away (current or locked). 862 */ 863 static void lookup_events_by_type_and_signal(struct kfd_process *p, 864 int type, void *event_data) 865 { 866 struct kfd_hsa_memory_exception_data *ev_data; 867 struct kfd_event *ev; 868 int bkt; 869 bool send_signal = true; 870 871 ev_data = (struct kfd_hsa_memory_exception_data *) event_data; 872 873 hash_for_each(p->events, bkt, ev, events) 874 if (ev->type == type) { 875 send_signal = false; 876 dev_dbg(kfd_device, 877 "Event found: id %X type %d", 878 ev->event_id, ev->type); 879 set_event(ev); 880 if (ev->type == KFD_EVENT_TYPE_MEMORY && ev_data) 881 ev->memory_exception_data = *ev_data; 882 } 883 884 /* Send SIGTERM no event of type "type" has been found*/ 885 if (send_signal) { 886 if (send_sigterm) { 887 dev_warn(kfd_device, 888 "Sending SIGTERM to HSA Process with PID %d ", 889 p->lead_thread->pid); 890 send_sig(SIGTERM, p->lead_thread, 0); 891 } else { 892 dev_err(kfd_device, 893 "HSA Process (PID %d) got unhandled exception", 894 p->lead_thread->pid); 895 } 896 } 897 } 898 899 void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid, 900 unsigned long address, bool is_write_requested, 901 bool is_execute_requested) 902 { 903 struct kfd_hsa_memory_exception_data memory_exception_data; 904 struct vm_area_struct *vma; 905 906 /* 907 * Because we are called from arbitrary context (workqueue) as opposed 908 * to process context, kfd_process could attempt to exit while we are 909 * running so the lookup function returns a locked process. 910 */ 911 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); 912 913 if (!p) 914 return; /* Presumably process exited. */ 915 916 memset(&memory_exception_data, 0, sizeof(memory_exception_data)); 917 918 down_read(&p->mm->mmap_sem); 919 vma = find_vma(p->mm, address); 920 921 memory_exception_data.gpu_id = dev->id; 922 memory_exception_data.va = address; 923 /* Set failure reason */ 924 memory_exception_data.failure.NotPresent = 1; 925 memory_exception_data.failure.NoExecute = 0; 926 memory_exception_data.failure.ReadOnly = 0; 927 if (vma) { 928 if (vma->vm_start > address) { 929 memory_exception_data.failure.NotPresent = 1; 930 memory_exception_data.failure.NoExecute = 0; 931 memory_exception_data.failure.ReadOnly = 0; 932 } else { 933 memory_exception_data.failure.NotPresent = 0; 934 if (is_write_requested && !(vma->vm_flags & VM_WRITE)) 935 memory_exception_data.failure.ReadOnly = 1; 936 else 937 memory_exception_data.failure.ReadOnly = 0; 938 if (is_execute_requested && !(vma->vm_flags & VM_EXEC)) 939 memory_exception_data.failure.NoExecute = 1; 940 else 941 memory_exception_data.failure.NoExecute = 0; 942 } 943 } 944 945 up_read(&p->mm->mmap_sem); 946 947 mutex_lock(&p->event_mutex); 948 949 /* Lookup events by type and signal them */ 950 lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_MEMORY, 951 &memory_exception_data); 952 953 mutex_unlock(&p->event_mutex); 954 mutex_unlock(&p->mutex); 955 } 956 957 void kfd_signal_hw_exception_event(unsigned int pasid) 958 { 959 /* 960 * Because we are called from arbitrary context (workqueue) as opposed 961 * to process context, kfd_process could attempt to exit while we are 962 * running so the lookup function returns a locked process. 963 */ 964 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); 965 966 if (!p) 967 return; /* Presumably process exited. */ 968 969 mutex_lock(&p->event_mutex); 970 971 /* Lookup events by type and signal them */ 972 lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_HW_EXCEPTION, NULL); 973 974 mutex_unlock(&p->event_mutex); 975 mutex_unlock(&p->mutex); 976 } 977
Indexes created Thu Oct 23 22:10:10 GMT 2025