1 /* $NetBSD: i915_gem_client_blt.c,v 1.4 2021/12/19 11:27:12 riastradh Exp $ */ 2 3 // SPDX-License-Identifier: MIT 4 /* 5 * Copyright 2019 Intel Corporation 6 */ 7 8 #include <sys/cdefs.h> 9 __KERNEL_RCSID(0, "$NetBSD: i915_gem_client_blt.c,v 1.4 2021/12/19 11:27:12 riastradh Exp $"); 10 11 #include "i915_drv.h" 12 #include "gt/intel_context.h" 13 #include "gt/intel_engine_pm.h" 14 #include "gt/intel_engine_pool.h" 15 #include "i915_gem_client_blt.h" 16 #include "i915_gem_object_blt.h" 17 18 struct i915_sleeve { 19 struct i915_vma *vma; 20 struct drm_i915_gem_object *obj; 21 struct sg_table *pages; 22 struct i915_page_sizes page_sizes; 23 }; 24 25 static int vma_set_pages(struct i915_vma *vma) 26 { 27 struct i915_sleeve *sleeve = vma->private; 28 29 vma->pages = sleeve->pages; 30 vma->page_sizes = sleeve->page_sizes; 31 32 return 0; 33 } 34 35 static void vma_clear_pages(struct i915_vma *vma) 36 { 37 GEM_BUG_ON(!vma->pages); 38 vma->pages = NULL; 39 } 40 41 static int vma_bind(struct i915_vma *vma, 42 enum i915_cache_level cache_level, 43 u32 flags) 44 { 45 return vma->vm->vma_ops.bind_vma(vma, cache_level, flags); 46 } 47 48 static void vma_unbind(struct i915_vma *vma) 49 { 50 vma->vm->vma_ops.unbind_vma(vma); 51 } 52 53 static const struct i915_vma_ops proxy_vma_ops = { 54 .set_pages = vma_set_pages, 55 .clear_pages = vma_clear_pages, 56 .bind_vma = vma_bind, 57 .unbind_vma = vma_unbind, 58 }; 59 60 static struct i915_sleeve *create_sleeve(struct i915_address_space *vm, 61 struct drm_i915_gem_object *obj, 62 struct sg_table *pages, 63 struct i915_page_sizes *page_sizes) 64 { 65 struct i915_sleeve *sleeve; 66 struct i915_vma *vma; 67 int err; 68 69 sleeve = kzalloc(sizeof(*sleeve), GFP_KERNEL); 70 if (!sleeve) 71 return ERR_PTR(-ENOMEM); 72 73 vma = i915_vma_instance(obj, vm, NULL); 74 if (IS_ERR(vma)) { 75 err = PTR_ERR(vma); 76 goto err_free; 77 } 78 79 vma->private = sleeve; 80 vma->ops = &proxy_vma_ops; 81 82 sleeve->vma = vma; 83 sleeve->pages = pages; 84 sleeve->page_sizes = *page_sizes; 85 86 return sleeve; 87 88 err_free: 89 kfree(sleeve); 90 return ERR_PTR(err); 91 } 92 93 static void destroy_sleeve(struct i915_sleeve *sleeve) 94 { 95 kfree(sleeve); 96 } 97 98 struct clear_pages_work { 99 struct dma_fence dma; 100 struct dma_fence_cb cb; 101 struct i915_sw_fence wait; 102 struct work_struct work; 103 struct irq_work irq_work; 104 struct i915_sleeve *sleeve; 105 struct intel_context *ce; 106 u32 value; 107 }; 108 109 static const char *clear_pages_work_driver_name(struct dma_fence *fence) 110 { 111 return DRIVER_NAME; 112 } 113 114 static const char *clear_pages_work_timeline_name(struct dma_fence *fence) 115 { 116 return "clear"; 117 } 118 119 static void clear_pages_work_release(struct dma_fence *fence) 120 { 121 struct clear_pages_work *w = container_of(fence, typeof(*w), dma); 122 123 destroy_sleeve(w->sleeve); 124 125 i915_sw_fence_fini(&w->wait); 126 127 BUILD_BUG_ON(offsetof(typeof(*w), dma)); 128 dma_fence_free(&w->dma); 129 } 130 131 static const struct dma_fence_ops clear_pages_work_ops = { 132 .get_driver_name = clear_pages_work_driver_name, 133 .get_timeline_name = clear_pages_work_timeline_name, 134 .release = clear_pages_work_release, 135 }; 136 137 static void clear_pages_signal_irq_worker(struct irq_work *work) 138 { 139 struct clear_pages_work *w = container_of(work, typeof(*w), irq_work); 140 141 dma_fence_signal(&w->dma); 142 dma_fence_put(&w->dma); 143 } 144 145 static void clear_pages_dma_fence_cb(struct dma_fence *fence, 146 struct dma_fence_cb *cb) 147 { 148 struct clear_pages_work *w = container_of(cb, typeof(*w), cb); 149 150 if (fence->error) 151 dma_fence_set_error(&w->dma, fence->error); 152 153 /* 154 * Push the signalling of the fence into yet another worker to avoid 155 * the nightmare locking around the fence spinlock. 156 */ 157 irq_work_queue(&w->irq_work); 158 } 159 160 static void clear_pages_worker(struct work_struct *work) 161 { 162 struct clear_pages_work *w = container_of(work, typeof(*w), work); 163 struct drm_i915_gem_object *obj = w->sleeve->vma->obj; 164 struct i915_vma *vma = w->sleeve->vma; 165 struct i915_request *rq; 166 struct i915_vma *batch; 167 int err = w->dma.error; 168 169 if (unlikely(err)) 170 goto out_signal; 171 172 if (obj->cache_dirty) { 173 if (i915_gem_object_has_struct_page(obj)) 174 drm_clflush_sg(w->sleeve->pages); 175 obj->cache_dirty = false; 176 } 177 obj->read_domains = I915_GEM_GPU_DOMAINS; 178 obj->write_domain = 0; 179 180 err = i915_vma_pin(vma, 0, 0, PIN_USER); 181 if (unlikely(err)) 182 goto out_signal; 183 184 batch = intel_emit_vma_fill_blt(w->ce, vma, w->value); 185 if (IS_ERR(batch)) { 186 err = PTR_ERR(batch); 187 goto out_unpin; 188 } 189 190 rq = intel_context_create_request(w->ce); 191 if (IS_ERR(rq)) { 192 err = PTR_ERR(rq); 193 goto out_batch; 194 } 195 196 /* There's no way the fence has signalled */ 197 if (dma_fence_add_callback(&rq->fence, &w->cb, 198 clear_pages_dma_fence_cb)) 199 GEM_BUG_ON(1); 200 201 err = intel_emit_vma_mark_active(batch, rq); 202 if (unlikely(err)) 203 goto out_request; 204 205 if (w->ce->engine->emit_init_breadcrumb) { 206 err = w->ce->engine->emit_init_breadcrumb(rq); 207 if (unlikely(err)) 208 goto out_request; 209 } 210 211 /* 212 * w->dma is already exported via (vma|obj)->resv we need only 213 * keep track of the GPU activity within this vma/request, and 214 * propagate the signal from the request to w->dma. 215 */ 216 err = __i915_vma_move_to_active(vma, rq); 217 if (err) 218 goto out_request; 219 220 err = w->ce->engine->emit_bb_start(rq, 221 batch->node.start, batch->node.size, 222 0); 223 out_request: 224 if (unlikely(err)) { 225 i915_request_skip(rq, err); 226 err = 0; 227 } 228 229 i915_request_add(rq); 230 out_batch: 231 intel_emit_vma_release(w->ce, batch); 232 out_unpin: 233 i915_vma_unpin(vma); 234 out_signal: 235 if (unlikely(err)) { 236 dma_fence_set_error(&w->dma, err); 237 dma_fence_signal(&w->dma); 238 dma_fence_put(&w->dma); 239 } 240 } 241 242 static int __i915_sw_fence_call 243 clear_pages_work_notify(struct i915_sw_fence *fence, 244 enum i915_sw_fence_notify state) 245 { 246 struct clear_pages_work *w = container_of(fence, typeof(*w), wait); 247 248 switch (state) { 249 case FENCE_COMPLETE: 250 schedule_work(&w->work); 251 break; 252 253 case FENCE_FREE: 254 dma_fence_put(&w->dma); 255 break; 256 } 257 258 return NOTIFY_DONE; 259 } 260 261 static DEFINE_SPINLOCK(fence_lock); 262 263 /* XXX: better name please */ 264 int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj, 265 struct intel_context *ce, 266 struct sg_table *pages, 267 struct i915_page_sizes *page_sizes, 268 u32 value) 269 { 270 struct clear_pages_work *work; 271 struct i915_sleeve *sleeve; 272 int err; 273 274 sleeve = create_sleeve(ce->vm, obj, pages, page_sizes); 275 if (IS_ERR(sleeve)) 276 return PTR_ERR(sleeve); 277 278 work = kmalloc(sizeof(*work), GFP_KERNEL); 279 if (!work) { 280 destroy_sleeve(sleeve); 281 return -ENOMEM; 282 } 283 284 work->value = value; 285 work->sleeve = sleeve; 286 work->ce = ce; 287 288 INIT_WORK(&work->work, clear_pages_worker); 289 290 init_irq_work(&work->irq_work, clear_pages_signal_irq_worker); 291 292 dma_fence_init(&work->dma, &clear_pages_work_ops, &fence_lock, 0, 0); 293 i915_sw_fence_init(&work->wait, clear_pages_work_notify); 294 295 i915_gem_object_lock(obj); 296 err = i915_sw_fence_await_reservation(&work->wait, 297 obj->base.resv, NULL, 298 true, I915_FENCE_TIMEOUT, 299 I915_FENCE_GFP); 300 if (err < 0) { 301 dma_fence_set_error(&work->dma, err); 302 } else { 303 dma_resv_add_excl_fence(obj->base.resv, &work->dma); 304 err = 0; 305 } 306 i915_gem_object_unlock(obj); 307 308 dma_fence_get(&work->dma); 309 i915_sw_fence_commit(&work->wait); 310 311 return err; 312 } 313 314 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 315 #include "selftests/i915_gem_client_blt.c" 316 #endif 317
Indexes created Thu Oct 02 14:10:14 GMT 2025