1/*
2 * © Copyright 2019 Collabora, Ltd.
3 * Copyright 2019 Alyssa Rosenzweig
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 */
25
26#include <fcntl.h>
27#include <xf86drm.h>
28
29#include "drm-uapi/panfrost_drm.h"
30
31#include "util/u_memory.h"
32#include "util/os_time.h"
33#include "os/os_mman.h"
34
35#include "pan_screen.h"
36#include "pan_resource.h"
37#include "pan_context.h"
38#include "pan_drm.h"
39#include "pan_trace.h"
40
41struct panfrost_drm {
42	struct panfrost_driver base;
43	int fd;
44};
45
46static void
47panfrost_drm_allocate_slab(struct panfrost_screen *screen,
48		           struct panfrost_memory *mem,
49		           size_t pages,
50		           bool same_va,
51		           int extra_flags,
52		           int commit_count,
53		           int extent)
54{
55	struct panfrost_drm *drm = (struct panfrost_drm *)screen->driver;
56	struct drm_panfrost_create_bo create_bo = {
57		        .size = pages * 4096,
58		        .flags = 0,  // TODO figure out proper flags..
59	};
60	struct drm_panfrost_mmap_bo mmap_bo = {0,};
61	int ret;
62
63	// TODO cache allocations
64	// TODO properly handle errors
65	// TODO take into account extra_flags
66
67	ret = drmIoctl(drm->fd, DRM_IOCTL_PANFROST_CREATE_BO, &create_bo);
68	if (ret) {
69                fprintf(stderr, "DRM_IOCTL_PANFROST_CREATE_BO failed: %d\n", ret);
70		assert(0);
71	}
72
73	mem->gpu = create_bo.offset;
74	mem->gem_handle = create_bo.handle;
75        mem->stack_bottom = 0;
76        mem->size = create_bo.size;
77
78	// TODO map and unmap on demand?
79	mmap_bo.handle = create_bo.handle;
80	ret = drmIoctl(drm->fd, DRM_IOCTL_PANFROST_MMAP_BO, &mmap_bo);
81	if (ret) {
82                fprintf(stderr, "DRM_IOCTL_PANFROST_MMAP_BO failed: %d\n", ret);
83		assert(0);
84	}
85
86        mem->cpu = os_mmap(NULL, mem->size, PROT_READ | PROT_WRITE, MAP_SHARED,
87                       drm->fd, mmap_bo.offset);
88        if (mem->cpu == MAP_FAILED) {
89                fprintf(stderr, "mmap failed: %p\n", mem->cpu);
90		assert(0);
91	}
92
93        /* Record the mmap if we're tracing */
94        if (!(extra_flags & PAN_ALLOCATE_GROWABLE))
95                pantrace_mmap(mem->gpu, mem->cpu, mem->size, NULL);
96}
97
98static void
99panfrost_drm_free_slab(struct panfrost_screen *screen, struct panfrost_memory *mem)
100{
101	struct panfrost_drm *drm = (struct panfrost_drm *)screen->driver;
102	struct drm_gem_close gem_close = {
103		.handle = mem->gem_handle,
104	};
105	int ret;
106
107        if (os_munmap((void *) (uintptr_t) mem->cpu, mem->size)) {
108                perror("munmap");
109                abort();
110        }
111
112	mem->cpu = NULL;
113
114	ret = drmIoctl(drm->fd, DRM_IOCTL_GEM_CLOSE, &gem_close);
115	if (ret) {
116                fprintf(stderr, "DRM_IOCTL_GEM_CLOSE failed: %d\n", ret);
117		assert(0);
118	}
119
120	mem->gem_handle = -1;
121}
122
123static struct panfrost_bo *
124panfrost_drm_import_bo(struct panfrost_screen *screen, struct winsys_handle *whandle)
125{
126	struct panfrost_bo *bo = CALLOC_STRUCT(panfrost_bo);
127	struct panfrost_drm *drm = (struct panfrost_drm *)screen->driver;
128        struct drm_panfrost_get_bo_offset get_bo_offset = {0,};
129	struct drm_panfrost_mmap_bo mmap_bo = {0,};
130        int ret;
131        unsigned gem_handle;
132
133	ret = drmPrimeFDToHandle(drm->fd, whandle->handle, &gem_handle);
134	assert(!ret);
135
136	get_bo_offset.handle = gem_handle;
137        ret = drmIoctl(drm->fd, DRM_IOCTL_PANFROST_GET_BO_OFFSET, &get_bo_offset);
138        assert(!ret);
139
140	bo->gem_handle = gem_handle;
141        bo->gpu = (mali_ptr) get_bo_offset.offset;
142        pipe_reference_init(&bo->reference, 1);
143
144	// TODO map and unmap on demand?
145	mmap_bo.handle = gem_handle;
146	ret = drmIoctl(drm->fd, DRM_IOCTL_PANFROST_MMAP_BO, &mmap_bo);
147	if (ret) {
148                fprintf(stderr, "DRM_IOCTL_PANFROST_MMAP_BO failed: %d\n", ret);
149		assert(0);
150	}
151
152        bo->size = lseek(whandle->handle, 0, SEEK_END);
153        assert(bo->size > 0);
154        bo->cpu = os_mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
155                       drm->fd, mmap_bo.offset);
156        if (bo->cpu == MAP_FAILED) {
157                fprintf(stderr, "mmap failed: %p\n", bo->cpu);
158		assert(0);
159	}
160
161        /* Record the mmap if we're tracing */
162        pantrace_mmap(bo->gpu, bo->cpu, bo->size, NULL);
163
164        return bo;
165}
166
167static int
168panfrost_drm_export_bo(struct panfrost_screen *screen, int gem_handle, unsigned int stride, struct winsys_handle *whandle)
169{
170	struct panfrost_drm *drm = (struct panfrost_drm *)screen->driver;
171        struct drm_prime_handle args = {
172                .handle = gem_handle,
173                .flags = DRM_CLOEXEC,
174        };
175
176        int ret = drmIoctl(drm->fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &args);
177        if (ret == -1)
178                return FALSE;
179
180        whandle->handle = args.fd;
181        whandle->stride = stride;
182
183        return TRUE;
184}
185
186static void
187panfrost_drm_free_imported_bo(struct panfrost_screen *screen, struct panfrost_bo *bo)
188{
189	struct panfrost_drm *drm = (struct panfrost_drm *)screen->driver;
190	struct drm_gem_close gem_close = {
191		.handle = bo->gem_handle,
192	};
193	int ret;
194
195	ret = drmIoctl(drm->fd, DRM_IOCTL_GEM_CLOSE, &gem_close);
196	if (ret) {
197                fprintf(stderr, "DRM_IOCTL_GEM_CLOSE failed: %d\n", ret);
198		assert(0);
199	}
200
201	bo->gem_handle = -1;
202	bo->gpu = (mali_ptr)NULL;
203}
204
205static int
206panfrost_drm_submit_job(struct panfrost_context *ctx, u64 job_desc, int reqs, struct pipe_surface *surf)
207{
208        struct pipe_context *gallium = (struct pipe_context *) ctx;
209        struct panfrost_screen *screen = pan_screen(gallium->screen);
210	struct panfrost_drm *drm = (struct panfrost_drm *)screen->driver;
211        struct drm_panfrost_submit submit = {0,};
212        int bo_handles[7];
213
214        submit.in_syncs = (u64) (uintptr_t) &ctx->out_sync;
215        submit.in_sync_count = 1;
216
217        submit.out_sync = ctx->out_sync;
218
219	submit.jc = job_desc;
220	submit.requirements = reqs;
221
222	if (surf) {
223		struct panfrost_resource *res = pan_resource(surf->texture);
224		assert(res->bo->gem_handle > 0);
225		bo_handles[submit.bo_handle_count++] = res->bo->gem_handle;
226
227		if (res->bo->checksum_slab.gem_handle)
228			bo_handles[submit.bo_handle_count++] = res->bo->checksum_slab.gem_handle;
229	}
230
231	/* TODO: Add here the transient pools */
232        /* TODO: Add here the BOs listed in the panfrost_job */
233	bo_handles[submit.bo_handle_count++] = ctx->shaders.gem_handle;
234	bo_handles[submit.bo_handle_count++] = ctx->scratchpad.gem_handle;
235	bo_handles[submit.bo_handle_count++] = ctx->tiler_heap.gem_handle;
236	bo_handles[submit.bo_handle_count++] = ctx->varying_mem.gem_handle;
237	bo_handles[submit.bo_handle_count++] = ctx->misc_0.gem_handle;
238	submit.bo_handles = (u64) (uintptr_t) bo_handles;
239
240        /* Dump memory _before_ submitting so we're not corrupted with actual GPU results */
241        pantrace_dump_memory();
242
243	if (drmIoctl(drm->fd, DRM_IOCTL_PANFROST_SUBMIT, &submit)) {
244	        fprintf(stderr, "Error submitting: %m\n");
245	        return errno;
246	}
247
248        /* Trace the job if we're doing that and do a memory dump. We may
249         * want to adjust this logic once we're ready to trace FBOs */
250        pantrace_submit_job(submit.jc, submit.requirements, FALSE);
251
252	return 0;
253}
254
255static int
256panfrost_drm_submit_vs_fs_job(struct panfrost_context *ctx, bool has_draws, bool is_scanout)
257{
258        struct pipe_surface *surf = ctx->pipe_framebuffer.cbufs[0];
259	int ret;
260
261        if (has_draws) {
262		ret = panfrost_drm_submit_job(ctx, ctx->set_value_job, 0, NULL);
263		assert(!ret);
264	}
265
266	ret = panfrost_drm_submit_job(ctx, panfrost_fragment_job(ctx), PANFROST_JD_REQ_FS, surf);
267
268        return ret;
269}
270
271static struct panfrost_fence *
272panfrost_fence_create(struct panfrost_context *ctx)
273{
274        struct pipe_context *gallium = (struct pipe_context *) ctx;
275        struct panfrost_screen *screen = pan_screen(gallium->screen);
276	struct panfrost_drm *drm = (struct panfrost_drm *)screen->driver;
277        struct panfrost_fence *f = calloc(1, sizeof(*f));
278        if (!f)
279                return NULL;
280
281        /* Snapshot the last Panfrost's rendering's out fence.  We'd rather have
282         * another syncobj instead of a sync file, but this is all we get.
283         * (HandleToFD/FDToHandle just gives you another syncobj ID for the
284         * same syncobj).
285         */
286        drmSyncobjExportSyncFile(drm->fd, ctx->out_sync, &f->fd);
287        if (f->fd == -1) {
288                fprintf(stderr, "export failed\n");
289                free(f);
290                return NULL;
291        }
292
293        pipe_reference_init(&f->reference, 1);
294
295        return f;
296}
297
298static void
299panfrost_drm_force_flush_fragment(struct panfrost_context *ctx,
300				  struct pipe_fence_handle **fence)
301{
302        struct pipe_context *gallium = (struct pipe_context *) ctx;
303        struct panfrost_screen *screen = pan_screen(gallium->screen);
304        struct panfrost_drm *drm = (struct panfrost_drm *)screen->driver;
305
306        if (!screen->last_fragment_flushed) {
307		drmSyncobjWait(drm->fd, &ctx->out_sync, 1, INT64_MAX, 0, NULL);
308                screen->last_fragment_flushed = true;
309
310                /* The job finished up, so we're safe to clean it up now */
311                panfrost_free_job(ctx, screen->last_job);
312	}
313
314        if (fence) {
315                struct panfrost_fence *f = panfrost_fence_create(ctx);
316                gallium->screen->fence_reference(gallium->screen, fence, NULL);
317                *fence = (struct pipe_fence_handle *)f;
318        }
319}
320
321static void
322panfrost_drm_enable_counters(struct panfrost_screen *screen)
323{
324	fprintf(stderr, "unimplemented: %s\n", __func__);
325}
326
327static void
328panfrost_drm_dump_counters(struct panfrost_screen *screen)
329{
330	fprintf(stderr, "unimplemented: %s\n", __func__);
331}
332
333static unsigned
334panfrost_drm_query_gpu_version(struct panfrost_screen *screen)
335{
336	struct panfrost_drm *drm = (struct panfrost_drm *)screen->driver;
337        struct drm_panfrost_get_param get_param = {0,};
338        int ret;
339
340	get_param.param = DRM_PANFROST_PARAM_GPU_ID;
341        ret = drmIoctl(drm->fd, DRM_IOCTL_PANFROST_GET_PARAM, &get_param);
342        assert(!ret);
343
344	return get_param.value;
345}
346
347static int
348panfrost_drm_init_context(struct panfrost_context *ctx)
349{
350        struct pipe_context *gallium = (struct pipe_context *) ctx;
351        struct panfrost_screen *screen = pan_screen(gallium->screen);
352	struct panfrost_drm *drm = (struct panfrost_drm *)screen->driver;
353
354        return drmSyncobjCreate(drm->fd, DRM_SYNCOBJ_CREATE_SIGNALED,
355                                &ctx->out_sync);
356}
357
358static void
359panfrost_drm_fence_reference(struct pipe_screen *screen,
360                         struct pipe_fence_handle **ptr,
361                         struct pipe_fence_handle *fence)
362{
363        struct panfrost_fence **p = (struct panfrost_fence **)ptr;
364        struct panfrost_fence *f = (struct panfrost_fence *)fence;
365        struct panfrost_fence *old = *p;
366
367        if (pipe_reference(&(*p)->reference, &f->reference)) {
368                close(old->fd);
369                free(old);
370        }
371        *p = f;
372}
373
374static boolean
375panfrost_drm_fence_finish(struct pipe_screen *pscreen,
376                      struct pipe_context *ctx,
377                      struct pipe_fence_handle *fence,
378                      uint64_t timeout)
379{
380        struct panfrost_screen *screen = pan_screen(pscreen);
381	struct panfrost_drm *drm = (struct panfrost_drm *)screen->driver;
382        struct panfrost_fence *f = (struct panfrost_fence *)fence;
383        int ret;
384
385        unsigned syncobj;
386        ret = drmSyncobjCreate(drm->fd, 0, &syncobj);
387        if (ret) {
388                fprintf(stderr, "Failed to create syncobj to wait on: %m\n");
389                return false;
390        }
391
392        drmSyncobjImportSyncFile(drm->fd, syncobj, f->fd);
393        if (ret) {
394                fprintf(stderr, "Failed to import fence to syncobj: %m\n");
395                return false;
396        }
397
398        uint64_t abs_timeout = os_time_get_absolute_timeout(timeout);
399        if (abs_timeout == OS_TIMEOUT_INFINITE)
400                abs_timeout = INT64_MAX;
401
402        ret = drmSyncobjWait(drm->fd, &syncobj, 1, abs_timeout, 0, NULL);
403
404        drmSyncobjDestroy(drm->fd, syncobj);
405
406        return ret >= 0;
407}
408
409struct panfrost_driver *
410panfrost_create_drm_driver(int fd)
411{
412	struct panfrost_drm *driver = CALLOC_STRUCT(panfrost_drm);
413
414	driver->fd = fd;
415
416	driver->base.import_bo = panfrost_drm_import_bo;
417	driver->base.export_bo = panfrost_drm_export_bo;
418	driver->base.free_imported_bo = panfrost_drm_free_imported_bo;
419	driver->base.submit_vs_fs_job = panfrost_drm_submit_vs_fs_job;
420	driver->base.force_flush_fragment = panfrost_drm_force_flush_fragment;
421	driver->base.allocate_slab = panfrost_drm_allocate_slab;
422	driver->base.free_slab = panfrost_drm_free_slab;
423	driver->base.enable_counters = panfrost_drm_enable_counters;
424	driver->base.query_gpu_version = panfrost_drm_query_gpu_version;
425	driver->base.init_context = panfrost_drm_init_context;
426	driver->base.fence_reference = panfrost_drm_fence_reference;
427	driver->base.fence_finish = panfrost_drm_fence_finish;
428	driver->base.dump_counters = panfrost_drm_dump_counters;
429
430        return &driver->base;
431}
432