Home | History | Annotate | Line # | Download | only in amdgpu 
      1 /*	$NetBSD: amdgpu_amdkfd_fence.c,v 1.2 2021/12/18 23:44:58 riastradh Exp $	*/
      2 
      3 /*
      4  * Copyright 2016-2018 Advanced Micro Devices, Inc.
      5  *
      6  * Permission is hereby granted, free of charge, to any person obtaining a
      7  * copy of this software and associated documentation files (the "Software"),
      8  * to deal in the Software without restriction, including without limitation
      9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
     10  * and/or sell copies of the Software, and to permit persons to whom the
     11  * Software is furnished to do so, subject to the following conditions:
     12  *
     13  * The above copyright notice and this permission notice shall be included in
     14  * all copies or substantial portions of the Software.
     15  *
     16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
     20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
     21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
     22  * OTHER DEALINGS IN THE SOFTWARE.
     23  */
     24 
     25 #include <sys/cdefs.h>
     26 __KERNEL_RCSID(0, "$NetBSD: amdgpu_amdkfd_fence.c,v 1.2 2021/12/18 23:44:58 riastradh Exp $");
     27 
     28 #include <linux/dma-fence.h>
     29 #include <linux/spinlock.h>
     30 #include <linux/atomic.h>
     31 #include <linux/stacktrace.h>
     32 #include <linux/sched.h>
     33 #include <linux/slab.h>
     34 #include <linux/sched/mm.h>
     35 #include "amdgpu_amdkfd.h"
     36 
     37 static const struct dma_fence_ops amdkfd_fence_ops;
     38 static atomic_t fence_seq = ATOMIC_INIT(0);
     39 
     40 /* Eviction Fence
     41  * Fence helper functions to deal with KFD memory eviction.
     42  * Big Idea - Since KFD submissions are done by user queues, a BO cannot be
     43  *  evicted unless all the user queues for that process are evicted.
     44  *
     45  * All the BOs in a process share an eviction fence. When process X wants
     46  * to map VRAM memory but TTM can't find enough space, TTM will attempt to
     47  * evict BOs from its LRU list. TTM checks if the BO is valuable to evict
     48  * by calling ttm_bo_driver->eviction_valuable().
     49  *
     50  * ttm_bo_driver->eviction_valuable() - will return false if the BO belongs
     51  *  to process X. Otherwise, it will return true to indicate BO can be
     52  *  evicted by TTM.
     53  *
     54  * If ttm_bo_driver->eviction_valuable returns true, then TTM will continue
     55  * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move
     56  * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler.
     57  *
     58  * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to
     59  *  nofity when the BO is free to move. fence_add_callback --> enable_signaling
     60  *  --> amdgpu_amdkfd_fence.enable_signaling
     61  *
     62  * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce
     63  * user queues and signal fence. The work item will also start another delayed
     64  * work item to restore BOs
     65  */
     66 
     67 struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
     68 						       struct mm_struct *mm)
     69 {
     70 	struct amdgpu_amdkfd_fence *fence;
     71 
     72 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
     73 	if (fence == NULL)
     74 		return NULL;
     75 
     76 	/* This reference gets released in amdkfd_fence_release */
     77 	mmgrab(mm);
     78 	fence->mm = mm;
     79 	get_task_comm(fence->timeline_name, current);
     80 	spin_lock_init(&fence->lock);
     81 
     82 	dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,
     83 		   context, atomic_inc_return(&fence_seq));
     84 
     85 	return fence;
     86 }
     87 
     88 struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)
     89 {
     90 	struct amdgpu_amdkfd_fence *fence;
     91 
     92 	if (!f)
     93 		return NULL;
     94 
     95 	fence = container_of(f, struct amdgpu_amdkfd_fence, base);
     96 	if (fence && f->ops == &amdkfd_fence_ops)
     97 		return fence;
     98 
     99 	return NULL;
    100 }
    101 
    102 static const char *amdkfd_fence_get_driver_name(struct dma_fence *f)
    103 {
    104 	return "amdgpu_amdkfd_fence";
    105 }
    106 
    107 static const char *amdkfd_fence_get_timeline_name(struct dma_fence *f)
    108 {
    109 	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
    110 
    111 	return fence->timeline_name;
    112 }
    113 
    114 /**
    115  * amdkfd_fence_enable_signaling - This gets called when TTM wants to evict
    116  *  a KFD BO and schedules a job to move the BO.
    117  *  If fence is already signaled return true.
    118  *  If fence is not signaled schedule a evict KFD process work item.
    119  */
    120 static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
    121 {
    122 	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
    123 
    124 	if (!fence)
    125 		return false;
    126 
    127 	if (dma_fence_is_signaled(f))
    128 		return true;
    129 
    130 	if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f))
    131 		return true;
    132 
    133 	return false;
    134 }
    135 
    136 /**
    137  * amdkfd_fence_release - callback that fence can be freed
    138  *
    139  * @fence: fence
    140  *
    141  * This function is called when the reference count becomes zero.
    142  * Drops the mm_struct reference and RCU schedules freeing up the fence.
    143  */
    144 static void amdkfd_fence_release(struct dma_fence *f)
    145 {
    146 	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
    147 
    148 	/* Unconditionally signal the fence. The process is getting
    149 	 * terminated.
    150 	 */
    151 	if (WARN_ON(!fence))
    152 		return; /* Not an amdgpu_amdkfd_fence */
    153 
    154 	mmdrop(fence->mm);
    155 	kfree_rcu(f, rcu);
    156 }
    157 
    158 /**
    159  * amdkfd_fence_check_mm - Check if @mm is same as that of the fence @f
    160  *  if same return TRUE else return FALSE.
    161  *
    162  * @f: [IN] fence
    163  * @mm: [IN] mm that needs to be verified
    164  */
    165 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
    166 {
    167 	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
    168 
    169 	if (!fence)
    170 		return false;
    171 	else if (fence->mm == mm)
    172 		return true;
    173 
    174 	return false;
    175 }
    176 
    177 static const struct dma_fence_ops amdkfd_fence_ops = {
    178 	.get_driver_name = amdkfd_fence_get_driver_name,
    179 	.get_timeline_name = amdkfd_fence_get_timeline_name,
    180 	.enable_signaling = amdkfd_fence_enable_signaling,
    181 	.release = amdkfd_fence_release,
    182 };
    183