Home | History | Annotate | Line # | Download | only in amdkfd 
      1 /*	$NetBSD: kfd_doorbell.c,v 1.3 2021/12/18 23:44:59 riastradh Exp $	*/
      2 
      3 /*
      4  * Copyright 2014 Advanced Micro Devices, Inc.
      5  *
      6  * Permission is hereby granted, free of charge, to any person obtaining a
      7  * copy of this software and associated documentation files (the "Software"),
      8  * to deal in the Software without restriction, including without limitation
      9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
     10  * and/or sell copies of the Software, and to permit persons to whom the
     11  * Software is furnished to do so, subject to the following conditions:
     12  *
     13  * The above copyright notice and this permission notice shall be included in
     14  * all copies or substantial portions of the Software.
     15  *
     16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
     20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
     21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
     22  * OTHER DEALINGS IN THE SOFTWARE.
     23  */
     24 #include <sys/cdefs.h>
     25 __KERNEL_RCSID(0, "$NetBSD: kfd_doorbell.c,v 1.3 2021/12/18 23:44:59 riastradh Exp $");
     26 
     27 #include "kfd_priv.h"
     28 #include <linux/mm.h>
     29 #include <linux/mman.h>
     30 #include <linux/slab.h>
     31 #include <linux/io.h>
     32 #include <linux/idr.h>
     33 
     34 /*
     35  * This extension supports a kernel level doorbells management for the
     36  * kernel queues using the first doorbell page reserved for the kernel.
     37  */
     38 
     39 static DEFINE_IDA(doorbell_ida);
     40 static unsigned int max_doorbell_slices;
     41 
     42 /*
     43  * Each device exposes a doorbell aperture, a PCI MMIO aperture that
     44  * receives 32-bit writes that are passed to queues as wptr values.
     45  * The doorbells are intended to be written by applications as part
     46  * of queueing work on user-mode queues.
     47  * We assign doorbells to applications in PAGE_SIZE-sized and aligned chunks.
     48  * We map the doorbell address space into user-mode when a process creates
     49  * its first queue on each device.
     50  * Although the mapping is done by KFD, it is equivalent to an mmap of
     51  * the /dev/kfd with the particular device encoded in the mmap offset.
     52  * There will be other uses for mmap of /dev/kfd, so only a range of
     53  * offsets (KFD_MMAP_DOORBELL_START-END) is used for doorbells.
     54  */
     55 
     56 /* # of doorbell bytes allocated for each process. */
     57 size_t kfd_doorbell_process_slice(struct kfd_dev *kfd)
     58 {
     59 	return roundup(kfd->device_info->doorbell_size *
     60 			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
     61 			PAGE_SIZE);
     62 }
     63 
     64 /* Doorbell calculations for device init. */
     65 int kfd_doorbell_init(struct kfd_dev *kfd)
     66 {
     67 	size_t doorbell_start_offset;
     68 	size_t doorbell_aperture_size;
     69 	size_t doorbell_process_limit;
     70 
     71 	/*
     72 	 * We start with calculations in bytes because the input data might
     73 	 * only be byte-aligned.
     74 	 * Only after we have done the rounding can we assume any alignment.
     75 	 */
     76 
     77 	doorbell_start_offset =
     78 			roundup(kfd->shared_resources.doorbell_start_offset,
     79 					kfd_doorbell_process_slice(kfd));
     80 
     81 	doorbell_aperture_size =
     82 			rounddown(kfd->shared_resources.doorbell_aperture_size,
     83 					kfd_doorbell_process_slice(kfd));
     84 
     85 	if (doorbell_aperture_size > doorbell_start_offset)
     86 		doorbell_process_limit =
     87 			(doorbell_aperture_size - doorbell_start_offset) /
     88 						kfd_doorbell_process_slice(kfd);
     89 	else
     90 		return -ENOSPC;
     91 
     92 	if (!max_doorbell_slices ||
     93 	    doorbell_process_limit < max_doorbell_slices)
     94 		max_doorbell_slices = doorbell_process_limit;
     95 
     96 	kfd->doorbell_base = kfd->shared_resources.doorbell_physical_address +
     97 				doorbell_start_offset;
     98 
     99 	kfd->doorbell_base_dw_offset = doorbell_start_offset / sizeof(u32);
    100 
    101 	kfd->doorbell_kernel_ptr = ioremap(kfd->doorbell_base,
    102 					   kfd_doorbell_process_slice(kfd));
    103 
    104 	if (!kfd->doorbell_kernel_ptr)
    105 		return -ENOMEM;
    106 
    107 	pr_debug("Doorbell initialization:\n");
    108 	pr_debug("doorbell base           == 0x%08lX\n",
    109 			(uintptr_t)kfd->doorbell_base);
    110 
    111 	pr_debug("doorbell_base_dw_offset      == 0x%08lX\n",
    112 			kfd->doorbell_base_dw_offset);
    113 
    114 	pr_debug("doorbell_process_limit  == 0x%08lX\n",
    115 			doorbell_process_limit);
    116 
    117 	pr_debug("doorbell_kernel_offset  == 0x%08lX\n",
    118 			(uintptr_t)kfd->doorbell_base);
    119 
    120 	pr_debug("doorbell aperture size  == 0x%08lX\n",
    121 			kfd->shared_resources.doorbell_aperture_size);
    122 
    123 	pr_debug("doorbell kernel address == %p\n", kfd->doorbell_kernel_ptr);
    124 
    125 	return 0;
    126 }
    127 
    128 void kfd_doorbell_fini(struct kfd_dev *kfd)
    129 {
    130 	if (kfd->doorbell_kernel_ptr)
    131 		iounmap(kfd->doorbell_kernel_ptr);
    132 }
    133 
    134 int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
    135 		      struct vm_area_struct *vma)
    136 {
    137 	phys_addr_t address;
    138 
    139 	/*
    140 	 * For simplicitly we only allow mapping of the entire doorbell
    141 	 * allocation of a single device & process.
    142 	 */
    143 	if (vma->vm_end - vma->vm_start != kfd_doorbell_process_slice(dev))
    144 		return -EINVAL;
    145 
    146 	/* Calculate physical address of doorbell */
    147 	address = kfd_get_process_doorbells(dev, process);
    148 
    149 	vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
    150 				VM_DONTDUMP | VM_PFNMAP;
    151 
    152 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
    153 
    154 	pr_debug("Mapping doorbell page\n"
    155 		 "     target user address == 0x%08llX\n"
    156 		 "     physical address    == 0x%08llX\n"
    157 		 "     vm_flags            == 0x%04lX\n"
    158 		 "     size                == 0x%04lX\n",
    159 		 (unsigned long long) vma->vm_start, address, vma->vm_flags,
    160 		 kfd_doorbell_process_slice(dev));
    161 
    162 
    163 	return io_remap_pfn_range(vma,
    164 				vma->vm_start,
    165 				address >> PAGE_SHIFT,
    166 				kfd_doorbell_process_slice(dev),
    167 				vma->vm_page_prot);
    168 }
    169 
    170 
    171 /* get kernel iomem pointer for a doorbell */
    172 void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
    173 					unsigned int *doorbell_off)
    174 {
    175 	u32 inx;
    176 
    177 	mutex_lock(&kfd->doorbell_mutex);
    178 	inx = find_first_zero_bit(kfd->doorbell_available_index,
    179 					KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
    180 
    181 	__set_bit(inx, kfd->doorbell_available_index);
    182 	mutex_unlock(&kfd->doorbell_mutex);
    183 
    184 	if (inx >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
    185 		return NULL;
    186 
    187 	inx *= kfd->device_info->doorbell_size / sizeof(u32);
    188 
    189 	/*
    190 	 * Calculating the kernel doorbell offset using the first
    191 	 * doorbell page.
    192 	 */
    193 	*doorbell_off = kfd->doorbell_base_dw_offset + inx;
    194 
    195 	pr_debug("Get kernel queue doorbell\n"
    196 			"     doorbell offset   == 0x%08X\n"
    197 			"     doorbell index    == 0x%x\n",
    198 		*doorbell_off, inx);
    199 
    200 	return kfd->doorbell_kernel_ptr + inx;
    201 }
    202 
    203 void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr)
    204 {
    205 	unsigned int inx;
    206 
    207 	inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr)
    208 		* sizeof(u32) / kfd->device_info->doorbell_size;
    209 
    210 	mutex_lock(&kfd->doorbell_mutex);
    211 	__clear_bit(inx, kfd->doorbell_available_index);
    212 	mutex_unlock(&kfd->doorbell_mutex);
    213 }
    214 
    215 void write_kernel_doorbell(void __iomem *db, u32 value)
    216 {
    217 	if (db) {
    218 		writel(value, db);
    219 		pr_debug("Writing %d to doorbell address %p\n", value, db);
    220 	}
    221 }
    222 
    223 void write_kernel_doorbell64(void __iomem *db, u64 value)
    224 {
    225 	if (db) {
    226 		WARN(((unsigned long)db & 7) != 0,
    227 		     "Unaligned 64-bit doorbell");
    228 		writeq(value, (u64 __iomem *)db);
    229 		pr_debug("writing %llu to doorbell address %p\n", value, db);
    230 	}
    231 }
    232 
    233 unsigned int kfd_get_doorbell_dw_offset_in_bar(struct kfd_dev *kfd,
    234 					struct kfd_process *process,
    235 					unsigned int doorbell_id)
    236 {
    237 	/*
    238 	 * doorbell_base_dw_offset accounts for doorbells taken by KGD.
    239 	 * index * kfd_doorbell_process_slice/sizeof(u32) adjusts to
    240 	 * the process's doorbells. The offset returned is in dword
    241 	 * units regardless of the ASIC-dependent doorbell size.
    242 	 */
    243 	return kfd->doorbell_base_dw_offset +
    244 		process->doorbell_index
    245 		* kfd_doorbell_process_slice(kfd) / sizeof(u32) +
    246 		doorbell_id * kfd->device_info->doorbell_size / sizeof(u32);
    247 }
    248 
    249 uint64_t kfd_get_number_elems(struct kfd_dev *kfd)
    250 {
    251 	uint64_t num_of_elems = (kfd->shared_resources.doorbell_aperture_size -
    252 				kfd->shared_resources.doorbell_start_offset) /
    253 					kfd_doorbell_process_slice(kfd) + 1;
    254 
    255 	return num_of_elems;
    256 
    257 }
    258 
    259 phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
    260 					struct kfd_process *process)
    261 {
    262 	return dev->doorbell_base +
    263 		process->doorbell_index * kfd_doorbell_process_slice(dev);
    264 }
    265 
    266 int kfd_alloc_process_doorbells(struct kfd_process *process)
    267 {
    268 	int r = ida_simple_get(&doorbell_ida, 1, max_doorbell_slices,
    269 				GFP_KERNEL);
    270 	if (r > 0)
    271 		process->doorbell_index = r;
    272 
    273 	return r;
    274 }
    275 
    276 void kfd_free_process_doorbells(struct kfd_process *process)
    277 {
    278 	if (process->doorbell_index)
    279 		ida_simple_remove(&doorbell_ida, process->doorbell_index);
    280 }
    281