ttm_bo_vm.c revision 1.14
1 /* $NetBSD: ttm_bo_vm.c,v 1.14 2020/02/23 15:46:40 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: ttm_bo_vm.c,v 1.14 2020/02/23 15:46:40 ad Exp $"); 34 35 #include <sys/types.h> 36 37 #include <uvm/uvm.h> 38 #include <uvm/uvm_extern.h> 39 #include <uvm/uvm_fault.h> 40 41 #include <linux/bitops.h> 42 43 #include <drm/drm_vma_manager.h> 44 45 #include <ttm/ttm_bo_driver.h> 46 47 static int ttm_bo_uvm_fault_idle(struct ttm_buffer_object *, 48 struct uvm_faultinfo *); 49 static int ttm_bo_uvm_lookup(struct ttm_bo_device *, unsigned long, 50 unsigned long, struct ttm_buffer_object **); 51 52 void 53 ttm_bo_uvm_reference(struct uvm_object *uobj) 54 { 55 struct ttm_buffer_object *const bo = container_of(uobj, 56 struct ttm_buffer_object, uvmobj); 57 58 (void)ttm_bo_reference(bo); 59 } 60 61 void 62 ttm_bo_uvm_detach(struct uvm_object *uobj) 63 { 64 struct ttm_buffer_object *bo = container_of(uobj, 65 struct ttm_buffer_object, uvmobj); 66 67 ttm_bo_unref(&bo); 68 KASSERT(bo == NULL); 69 } 70 71 int 72 ttm_bo_uvm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, 73 struct vm_page **pps, int npages, int centeridx, vm_prot_t access_type, 74 int flags) 75 { 76 struct uvm_object *const uobj = ufi->entry->object.uvm_obj; 77 struct ttm_buffer_object *const bo = container_of(uobj, 78 struct ttm_buffer_object, uvmobj); 79 struct ttm_bo_device *const bdev = bo->bdev; 80 struct ttm_mem_type_manager *man = 81 &bdev->man[bo->mem.mem_type]; 82 union { 83 bus_addr_t base; 84 struct ttm_tt *ttm; 85 } u; 86 size_t size __diagused; 87 voff_t uoffset; /* offset in bytes into bo */ 88 unsigned startpage; /* offset in pages into bo */ 89 unsigned i; 90 vm_prot_t vm_prot; /* VM_PROT_* */ 91 pgprot_t pgprot; /* VM_PROT_* | PMAP_* cacheability flags */ 92 int ret; 93 94 /* Thanks, uvm, but we don't need this lock. */ 95 rw_exit(uobj->vmobjlock); 96 97 /* Copy-on-write mappings make no sense for the graphics aperture. */ 98 if (UVM_ET_ISCOPYONWRITE(ufi->entry)) { 99 ret = -EIO; 100 goto out0; 101 } 102 103 /* Try to lock the buffer. */ 104 ret = ttm_bo_reserve(bo, true, true, false, NULL); 105 if (ret) { 106 if (ret != -EBUSY) 107 goto out0; 108 /* 109 * It's currently locked. Unlock the fault, wait for 110 * it, and start over. 111 */ 112 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, NULL); 113 (void)ttm_bo_wait_unreserved(bo); 114 return ERESTART; 115 } 116 117 /* drm prime buffers are not mappable. XXX Catch this earlier? */ 118 if (bo->ttm && ISSET(bo->ttm->page_flags, TTM_PAGE_FLAG_SG)) { 119 ret = -EINVAL; 120 goto out1; 121 } 122 123 /* Notify the driver of a fault if it wants. */ 124 if (bdev->driver->fault_reserve_notify) { 125 ret = (*bdev->driver->fault_reserve_notify)(bo); 126 if (ret) { 127 if (ret == -ERESTART) 128 ret = -EIO; 129 goto out1; 130 } 131 } 132 133 ret = ttm_bo_uvm_fault_idle(bo, ufi); 134 if (ret) { 135 KASSERT(ret == -ERESTART); 136 /* ttm_bo_uvm_fault_idle calls uvmfault_unlockall for us. */ 137 ttm_bo_unreserve(bo); 138 /* XXX errno Linux->NetBSD */ 139 return -ret; 140 } 141 142 ret = ttm_mem_io_lock(man, true); 143 if (ret) { 144 ret = -EIO; 145 goto out1; 146 } 147 ret = ttm_mem_io_reserve_vm(bo); 148 if (ret) { 149 ret = -EIO; 150 goto out2; 151 } 152 153 vm_prot = ufi->entry->protection; 154 if (bo->mem.bus.is_iomem) { 155 u.base = (bo->mem.bus.base + bo->mem.bus.offset); 156 size = bo->mem.bus.size; 157 pgprot = ttm_io_prot(bo->mem.placement, vm_prot); 158 } else { 159 u.ttm = bo->ttm; 160 size = (bo->ttm->num_pages << PAGE_SHIFT); 161 if (ISSET(bo->mem.placement, TTM_PL_FLAG_CACHED)) 162 pgprot = vm_prot; 163 else 164 pgprot = ttm_io_prot(bo->mem.placement, vm_prot); 165 if ((*u.ttm->bdev->driver->ttm_tt_populate)(u.ttm)) { 166 ret = -ENOMEM; 167 goto out2; 168 } 169 } 170 171 KASSERT(ufi->entry->start <= vaddr); 172 KASSERT((ufi->entry->offset & (PAGE_SIZE - 1)) == 0); 173 KASSERT(ufi->entry->offset <= size); 174 KASSERT((vaddr - ufi->entry->start) <= (size - ufi->entry->offset)); 175 KASSERT(npages <= ((size - ufi->entry->offset) - 176 (vaddr - ufi->entry->start))); 177 uoffset = (ufi->entry->offset + (vaddr - ufi->entry->start)); 178 startpage = (uoffset >> PAGE_SHIFT); 179 for (i = 0; i < npages; i++) { 180 paddr_t paddr; 181 182 /* XXX PGO_ALLPAGES? */ 183 if (pps[i] == PGO_DONTCARE) 184 continue; 185 if (bo->mem.bus.is_iomem) { 186 const paddr_t cookie = bus_space_mmap(bdev->memt, 187 u.base, ((startpage + i) << PAGE_SHIFT), vm_prot, 188 0); 189 190 paddr = pmap_phys_address(cookie); 191 } else { 192 paddr = page_to_phys(u.ttm->pages[startpage + i]); 193 } 194 ret = -pmap_enter(ufi->orig_map->pmap, vaddr + i*PAGE_SIZE, 195 paddr, vm_prot, (PMAP_CANFAIL | pgprot)); 196 if (ret) 197 goto out3; 198 } 199 200 out3: pmap_update(ufi->orig_map->pmap); 201 out2: ttm_mem_io_unlock(man); 202 out1: ttm_bo_unreserve(bo); 203 out0: uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, NULL); 204 /* XXX errno Linux->NetBSD */ 205 return -ret; 206 } 207 208 static int 209 ttm_bo_uvm_fault_idle(struct ttm_buffer_object *bo, struct uvm_faultinfo *ufi) 210 { 211 212 if (__predict_true(!test_bit(TTM_BO_PRIV_FLAG_MOVING, 213 &bo->priv_flags))) 214 return 0; 215 if (ttm_bo_wait(bo, false, false, true) == 0) 216 return 0; 217 218 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, NULL); 219 (void)ttm_bo_wait(bo, false, true, false); 220 return -ERESTART; 221 } 222 223 int 224 ttm_bo_mmap_object(struct ttm_bo_device *bdev, off_t offset, size_t size, 225 vm_prot_t prot, struct uvm_object **uobjp, voff_t *uoffsetp, 226 struct file *file) 227 { 228 const unsigned long startpage = (offset >> PAGE_SHIFT); 229 const unsigned long npages = (size >> PAGE_SHIFT); 230 struct ttm_buffer_object *bo; 231 int ret; 232 233 KASSERT(0 == (offset & (PAGE_SIZE - 1))); 234 KASSERT(0 == (size & (PAGE_SIZE - 1))); 235 236 ret = ttm_bo_uvm_lookup(bdev, startpage, npages, &bo); 237 if (ret) 238 goto fail0; 239 KASSERT(drm_vma_node_start(&bo->vma_node) <= offset); 240 /* XXX Just assert this? */ 241 if (__predict_false(bdev->driver->verify_access == NULL)) { 242 ret = -EPERM; 243 goto fail1; 244 } 245 ret = (*bdev->driver->verify_access)(bo, file); 246 if (ret) 247 goto fail1; 248 249 /* Success! */ 250 *uobjp = &bo->uvmobj; 251 *uoffsetp = (offset - 252 (drm_vma_node_start(&bo->vma_node) << PAGE_SHIFT)); 253 return 0; 254 255 fail1: ttm_bo_unref(&bo); 256 fail0: KASSERT(ret); 257 return ret; 258 } 259 260 static int 261 ttm_bo_uvm_lookup(struct ttm_bo_device *bdev, unsigned long startpage, 262 unsigned long npages, struct ttm_buffer_object **bop) 263 { 264 struct ttm_buffer_object *bo = NULL; 265 struct drm_vma_offset_node *node; 266 267 drm_vma_offset_lock_lookup(&bdev->vma_manager); 268 node = drm_vma_offset_lookup_locked(&bdev->vma_manager, startpage, 269 npages); 270 if (node != NULL) { 271 bo = container_of(node, struct ttm_buffer_object, vma_node); 272 if (!kref_get_unless_zero(&bo->kref)) 273 bo = NULL; 274 } 275 drm_vma_offset_unlock_lookup(&bdev->vma_manager); 276 277 if (bo == NULL) 278 return -ENOENT; 279 280 *bop = bo; 281 return 0; 282 } 283
Indexes created Mon Sep 29 03:10:08 GMT 2025