1 /* $NetBSD: intel_ppgtt.c,v 1.4 2021/12/19 12:07:47 riastradh Exp $ */ 2 3 // SPDX-License-Identifier: MIT 4 /* 5 * Copyright 2020 Intel Corporation 6 */ 7 8 #include <sys/cdefs.h> 9 __KERNEL_RCSID(0, "$NetBSD: intel_ppgtt.c,v 1.4 2021/12/19 12:07:47 riastradh Exp $"); 10 11 #include <linux/slab.h> 12 13 #include "i915_trace.h" 14 #include "intel_gtt.h" 15 #include "gen6_ppgtt.h" 16 #include "gen8_ppgtt.h" 17 18 struct i915_page_table *alloc_pt(struct i915_address_space *vm) 19 { 20 struct i915_page_table *pt; 21 22 pt = kmalloc(sizeof(*pt), I915_GFP_ALLOW_FAIL); 23 if (unlikely(!pt)) 24 return ERR_PTR(-ENOMEM); 25 26 if (unlikely(setup_page_dma(vm, &pt->base))) { 27 kfree(pt); 28 return ERR_PTR(-ENOMEM); 29 } 30 31 atomic_set(&pt->used, 0); 32 return pt; 33 } 34 35 struct i915_page_directory *__alloc_pd(size_t sz) 36 { 37 struct i915_page_directory *pd; 38 39 pd = kzalloc(sz, I915_GFP_ALLOW_FAIL); 40 if (unlikely(!pd)) 41 return NULL; 42 43 spin_lock_init(&pd->lock); 44 return pd; 45 } 46 47 struct i915_page_directory *alloc_pd(struct i915_address_space *vm) 48 { 49 struct i915_page_directory *pd; 50 51 pd = __alloc_pd(sizeof(*pd)); 52 if (unlikely(!pd)) 53 return ERR_PTR(-ENOMEM); 54 55 if (unlikely(setup_page_dma(vm, px_base(pd)))) { 56 spin_lock_destroy(&pd->lock); 57 kfree(pd); 58 return ERR_PTR(-ENOMEM); 59 } 60 61 return pd; 62 } 63 64 void free_pd(struct i915_address_space *vm, struct i915_page_dma *pd) 65 { 66 cleanup_page_dma(vm, pd); 67 kfree(pd); 68 } 69 70 static inline void 71 write_dma_entry(struct i915_page_dma * const pdma, 72 const unsigned short idx, 73 const u64 encoded_entry) 74 { 75 u64 * const vaddr = kmap_atomic(pdma->page); 76 77 vaddr[idx] = encoded_entry; 78 kunmap_atomic(vaddr); 79 } 80 81 void 82 __set_pd_entry(struct i915_page_directory * const pd, 83 const unsigned short idx, 84 struct i915_page_dma * const to, 85 u64 (*encode)(const dma_addr_t, const enum i915_cache_level)) 86 { 87 /* Each thread pre-pins the pd, and we may have a thread per pde. */ 88 GEM_BUG_ON(atomic_read(px_used(pd)) > NALLOC * ARRAY_SIZE(pd->entry)); 89 90 atomic_inc(px_used(pd)); 91 pd->entry[idx] = to; 92 #ifdef __NetBSD__ 93 write_dma_entry(px_base(pd), idx, encode(to->map->dm_segs[0].ds_addr, I915_CACHE_LLC)); 94 #else 95 write_dma_entry(px_base(pd), idx, encode(to->daddr, I915_CACHE_LLC)); 96 #endif 97 } 98 99 void 100 clear_pd_entry(struct i915_page_directory * const pd, 101 const unsigned short idx, 102 const struct i915_page_scratch * const scratch) 103 { 104 GEM_BUG_ON(atomic_read(px_used(pd)) == 0); 105 106 write_dma_entry(px_base(pd), idx, scratch->encode); 107 pd->entry[idx] = NULL; 108 atomic_dec(px_used(pd)); 109 } 110 111 bool 112 release_pd_entry(struct i915_page_directory * const pd, 113 const unsigned short idx, 114 struct i915_page_table * const pt, 115 const struct i915_page_scratch * const scratch) 116 { 117 bool free = false; 118 119 if (atomic_add_unless(&pt->used, -1, 1)) 120 return false; 121 122 spin_lock(&pd->lock); 123 if (atomic_dec_and_test(&pt->used)) { 124 clear_pd_entry(pd, idx, scratch); 125 free = true; 126 } 127 spin_unlock(&pd->lock); 128 129 return free; 130 } 131 132 int i915_ppgtt_init_hw(struct intel_gt *gt) 133 { 134 struct drm_i915_private *i915 = gt->i915; 135 136 gtt_write_workarounds(gt); 137 138 if (IS_GEN(i915, 6)) 139 gen6_ppgtt_enable(gt); 140 else if (IS_GEN(i915, 7)) 141 gen7_ppgtt_enable(gt); 142 143 return 0; 144 } 145 146 static struct i915_ppgtt * 147 __ppgtt_create(struct intel_gt *gt) 148 { 149 if (INTEL_GEN(gt->i915) < 8) 150 return gen6_ppgtt_create(gt); 151 else 152 return gen8_ppgtt_create(gt); 153 } 154 155 struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt) 156 { 157 struct i915_ppgtt *ppgtt; 158 159 ppgtt = __ppgtt_create(gt); 160 if (IS_ERR(ppgtt)) 161 return ppgtt; 162 163 trace_i915_ppgtt_create(&ppgtt->vm); 164 165 return ppgtt; 166 } 167 168 static int ppgtt_bind_vma(struct i915_vma *vma, 169 enum i915_cache_level cache_level, 170 u32 flags) 171 { 172 u32 pte_flags; 173 int err; 174 175 if (flags & I915_VMA_ALLOC) { 176 err = vma->vm->allocate_va_range(vma->vm, 177 vma->node.start, vma->size); 178 if (err) 179 return err; 180 181 set_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma)); 182 } 183 184 /* Applicable to VLV, and gen8+ */ 185 pte_flags = 0; 186 if (i915_gem_object_is_readonly(vma->obj)) 187 pte_flags |= PTE_READ_ONLY; 188 189 GEM_BUG_ON(!test_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma))); 190 vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags); 191 wmb(); 192 193 return 0; 194 } 195 196 static void ppgtt_unbind_vma(struct i915_vma *vma) 197 { 198 if (test_and_clear_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma))) 199 vma->vm->clear_range(vma->vm, vma->node.start, vma->size); 200 } 201 202 int ppgtt_set_pages(struct i915_vma *vma) 203 { 204 GEM_BUG_ON(vma->pages); 205 206 vma->pages = vma->obj->mm.pages; 207 208 vma->page_sizes = vma->obj->mm.page_sizes; 209 210 return 0; 211 } 212 213 void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt) 214 { 215 struct drm_i915_private *i915 = gt->i915; 216 217 ppgtt->vm.gt = gt; 218 ppgtt->vm.i915 = i915; 219 #ifdef __NetBSD__ 220 ppgtt->vm.dmat = i915->drm.dmat; 221 #else 222 ppgtt->vm.dma = &i915->drm.pdev->dev; 223 #endif 224 ppgtt->vm.total = BIT_ULL(INTEL_INFO(i915)->ppgtt_size); 225 226 i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT); 227 228 ppgtt->vm.vma_ops.bind_vma = ppgtt_bind_vma; 229 ppgtt->vm.vma_ops.unbind_vma = ppgtt_unbind_vma; 230 ppgtt->vm.vma_ops.set_pages = ppgtt_set_pages; 231 ppgtt->vm.vma_ops.clear_pages = clear_pages; 232 } 233
Indexes created Sat Sep 20 22:09:52 GMT 2025