Home | History | Annotate | Line # | Download | only in amdgpu 
      1 /*	$NetBSD: amdgpu_mn.c,v 1.3 2021/12/18 23:44:58 riastradh Exp $	*/
      2 
      3 /*
      4  * Copyright 2014 Advanced Micro Devices, Inc.
      5  * All Rights Reserved.
      6  *
      7  * Permission is hereby granted, free of charge, to any person obtaining a
      8  * copy of this software and associated documentation files (the
      9  * "Software"), to deal in the Software without restriction, including
     10  * without limitation the rights to use, copy, modify, merge, publish,
     11  * distribute, sub license, and/or sell copies of the Software, and to
     12  * permit persons to whom the Software is furnished to do so, subject to
     13  * the following conditions:
     14  *
     15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
     18  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
     19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
     20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
     21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
     22  *
     23  * The above copyright notice and this permission notice (including the
     24  * next paragraph) shall be included in all copies or substantial portions
     25  * of the Software.
     26  *
     27  */
     28 /*
     29  * Authors:
     30  *    Christian Knig <christian.koenig (at) amd.com>
     31  */
     32 
     33 /**
     34  * DOC: MMU Notifier
     35  *
     36  * For coherent userptr handling registers an MMU notifier to inform the driver
     37  * about updates on the page tables of a process.
     38  *
     39  * When somebody tries to invalidate the page tables we block the update until
     40  * all operations on the pages in question are completed, then those pages are
     41  * marked as accessed and also dirty if it wasn't a read only access.
     42  *
     43  * New command submissions using the userptrs in question are delayed until all
     44  * page table invalidation are completed and we once more see a coherent process
     45  * address space.
     46  */
     47 
     48 #include <sys/cdefs.h>
     49 __KERNEL_RCSID(0, "$NetBSD: amdgpu_mn.c,v 1.3 2021/12/18 23:44:58 riastradh Exp $");
     50 
     51 #include <linux/firmware.h>
     52 #include <linux/module.h>
     53 #include <drm/drm.h>
     54 
     55 #include "amdgpu.h"
     56 #include "amdgpu_amdkfd.h"
     57 
     58 /**
     59  * amdgpu_mn_invalidate_gfx - callback to notify about mm change
     60  *
     61  * @mni: the range (mm) is about to update
     62  * @range: details on the invalidation
     63  * @cur_seq: Value to pass to mmu_interval_set_seq()
     64  *
     65  * Block for operations on BOs to finish and mark pages as accessed and
     66  * potentially dirty.
     67  */
     68 static bool amdgpu_mn_invalidate_gfx(struct mmu_interval_notifier *mni,
     69 				     const struct mmu_notifier_range *range,
     70 				     unsigned long cur_seq)
     71 {
     72 	struct amdgpu_bo *bo = container_of(mni, struct amdgpu_bo, notifier);
     73 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
     74 	long r;
     75 
     76 	if (!mmu_notifier_range_blockable(range))
     77 		return false;
     78 
     79 	mutex_lock(&adev->notifier_lock);
     80 
     81 	mmu_interval_set_seq(mni, cur_seq);
     82 
     83 	r = dma_resv_wait_timeout_rcu(bo->tbo.base.resv, true, false,
     84 				      MAX_SCHEDULE_TIMEOUT);
     85 	mutex_unlock(&adev->notifier_lock);
     86 	if (r <= 0)
     87 		DRM_ERROR("(%ld) failed to wait for user bo\n", r);
     88 	return true;
     89 }
     90 
     91 static const struct mmu_interval_notifier_ops amdgpu_mn_gfx_ops = {
     92 	.invalidate = amdgpu_mn_invalidate_gfx,
     93 };
     94 
     95 /**
     96  * amdgpu_mn_invalidate_hsa - callback to notify about mm change
     97  *
     98  * @mni: the range (mm) is about to update
     99  * @range: details on the invalidation
    100  * @cur_seq: Value to pass to mmu_interval_set_seq()
    101  *
    102  * We temporarily evict the BO attached to this range. This necessitates
    103  * evicting all user-mode queues of the process.
    104  */
    105 static bool amdgpu_mn_invalidate_hsa(struct mmu_interval_notifier *mni,
    106 				     const struct mmu_notifier_range *range,
    107 				     unsigned long cur_seq)
    108 {
    109 	struct amdgpu_bo *bo = container_of(mni, struct amdgpu_bo, notifier);
    110 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
    111 
    112 	if (!mmu_notifier_range_blockable(range))
    113 		return false;
    114 
    115 	mutex_lock(&adev->notifier_lock);
    116 
    117 	mmu_interval_set_seq(mni, cur_seq);
    118 
    119 	amdgpu_amdkfd_evict_userptr(bo->kfd_bo, bo->notifier.mm);
    120 	mutex_unlock(&adev->notifier_lock);
    121 
    122 	return true;
    123 }
    124 
    125 static const struct mmu_interval_notifier_ops amdgpu_mn_hsa_ops = {
    126 	.invalidate = amdgpu_mn_invalidate_hsa,
    127 };
    128 
    129 /**
    130  * amdgpu_mn_register - register a BO for notifier updates
    131  *
    132  * @bo: amdgpu buffer object
    133  * @addr: userptr addr we should monitor
    134  *
    135  * Registers a mmu_notifier for the given BO at the specified address.
    136  * Returns 0 on success, -ERRNO if anything goes wrong.
    137  */
    138 int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
    139 {
    140 	if (bo->kfd_bo)
    141 		return mmu_interval_notifier_insert(&bo->notifier, current->mm,
    142 						    addr, amdgpu_bo_size(bo),
    143 						    &amdgpu_mn_hsa_ops);
    144 	return mmu_interval_notifier_insert(&bo->notifier, current->mm, addr,
    145 					    amdgpu_bo_size(bo),
    146 					    &amdgpu_mn_gfx_ops);
    147 }
    148 
    149 /**
    150  * amdgpu_mn_unregister - unregister a BO for notifier updates
    151  *
    152  * @bo: amdgpu buffer object
    153  *
    154  * Remove any registration of mmu notifier updates from the buffer object.
    155  */
    156 void amdgpu_mn_unregister(struct amdgpu_bo *bo)
    157 {
    158 	if (!bo->notifier.mm)
    159 		return;
    160 	mmu_interval_notifier_remove(&bo->notifier);
    161 	bo->notifier.mm = NULL;
    162 }
    163