1/*
2 * Copyright 2020 Google LLC
3 * SPDX-License-Identifier: MIT
4 */
5
6#include <errno.h>
7#include <fcntl.h>
8#include <poll.h>
9#include <sys/mman.h>
10#include <sys/stat.h>
11#include <sys/types.h>
12#include <unistd.h>
13#include <xf86drm.h>
14
15#include "drm-uapi/virtgpu_drm.h"
16#include "util/sparse_array.h"
17#define VIRGL_RENDERER_UNSTABLE_APIS
18#include "virtio-gpu/virglrenderer_hw.h"
19
20#include "vn_renderer.h"
21
22/* XXX WIP kernel uapi */
23#ifndef VIRTGPU_PARAM_CONTEXT_INIT
24#define VIRTGPU_PARAM_CONTEXT_INIT 6
25#define VIRTGPU_CONTEXT_PARAM_CAPSET_ID 0x0001
26struct drm_virtgpu_context_set_param {
27   __u64 param;
28   __u64 value;
29};
30struct drm_virtgpu_context_init {
31   __u32 num_params;
32   __u32 pad;
33   __u64 ctx_set_params;
34};
35#define DRM_VIRTGPU_CONTEXT_INIT 0xb
36#define DRM_IOCTL_VIRTGPU_CONTEXT_INIT                                       \
37   DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_CONTEXT_INIT,                     \
38            struct drm_virtgpu_context_init)
39#endif /* VIRTGPU_PARAM_CONTEXT_INIT */
40#ifndef VIRTGPU_PARAM_MAX_SYNC_QUEUE_COUNT
41#define VIRTGPU_PARAM_MAX_SYNC_QUEUE_COUNT 100
42#endif /* VIRTGPU_PARAM_MAX_SYNC_QUEUE_COUNT */
43
44/* XXX comment these out to really use kernel uapi */
45#define SIMULATE_BO_SIZE_FIX 1
46//#define SIMULATE_CONTEXT_INIT 1
47#define SIMULATE_SYNCOBJ 1
48#define SIMULATE_SUBMIT 1
49
50#define VIRTGPU_PCI_VENDOR_ID 0x1af4
51#define VIRTGPU_PCI_DEVICE_ID 0x1050
52
53struct virtgpu;
54
55struct virtgpu_shmem {
56   struct vn_renderer_shmem base;
57   uint32_t gem_handle;
58};
59
60struct virtgpu_bo {
61   struct vn_renderer_bo base;
62   uint32_t gem_handle;
63   uint32_t blob_flags;
64};
65
66struct virtgpu_sync {
67   struct vn_renderer_sync base;
68
69   /*
70    * drm_syncobj is in one of these states
71    *
72    *  - value N:      drm_syncobj has a signaled fence chain with seqno N
73    *  - pending N->M: drm_syncobj has an unsignaled fence chain with seqno M
74    *                  (which may point to another unsignaled fence chain with
75    *                   seqno between N and M, and so on)
76    *
77    * TODO Do we want to use binary drm_syncobjs?  They would be
78    *
79    *  - value 0: drm_syncobj has no fence
80    *  - value 1: drm_syncobj has a signaled fence with seqno 0
81    *
82    * They are cheaper but require special care.
83    */
84   uint32_t syncobj_handle;
85};
86
87struct virtgpu {
88   struct vn_renderer base;
89
90   struct vn_instance *instance;
91
92   int fd;
93   int version_minor;
94   drmPciBusInfo bus_info;
95
96   uint32_t max_sync_queue_count;
97
98   struct {
99      enum virgl_renderer_capset id;
100      uint32_t version;
101      struct virgl_renderer_capset_venus data;
102   } capset;
103
104   /* note that we use gem_handle instead of res_id to index because
105    * res_id is monotonically increasing by default (see
106    * virtio_gpu_resource_id_get)
107    */
108   struct util_sparse_array shmem_array;
109   struct util_sparse_array bo_array;
110
111   mtx_t dma_buf_import_mutex;
112};
113
114#ifdef SIMULATE_SYNCOBJ
115
116#include "util/hash_table.h"
117#include "util/u_idalloc.h"
118
119static struct {
120   mtx_t mutex;
121   struct hash_table *syncobjs;
122   struct util_idalloc ida;
123
124   int signaled_fd;
125} sim;
126
127struct sim_syncobj {
128   mtx_t mutex;
129   uint64_t point;
130
131   int pending_fd;
132   uint64_t pending_point;
133   bool pending_cpu;
134};
135
136static uint32_t
137sim_syncobj_create(struct virtgpu *gpu, bool signaled)
138{
139   struct sim_syncobj *syncobj = calloc(1, sizeof(*syncobj));
140   if (!syncobj)
141      return 0;
142
143   mtx_init(&syncobj->mutex, mtx_plain);
144   syncobj->pending_fd = -1;
145
146   mtx_lock(&sim.mutex);
147
148   /* initialize lazily */
149   if (!sim.syncobjs) {
150      sim.syncobjs = _mesa_pointer_hash_table_create(NULL);
151      if (!sim.syncobjs) {
152         mtx_unlock(&sim.mutex);
153         return 0;
154      }
155
156      util_idalloc_init(&sim.ida, 32);
157
158      struct drm_virtgpu_execbuffer args = {
159         .flags = VIRTGPU_EXECBUF_FENCE_FD_OUT,
160      };
161      int ret = drmIoctl(gpu->fd, DRM_IOCTL_VIRTGPU_EXECBUFFER, &args);
162      if (ret || args.fence_fd < 0) {
163         _mesa_hash_table_destroy(sim.syncobjs, NULL);
164         sim.syncobjs = NULL;
165         mtx_unlock(&sim.mutex);
166         return 0;
167      }
168
169      sim.signaled_fd = args.fence_fd;
170   }
171
172   const unsigned syncobj_handle = util_idalloc_alloc(&sim.ida) + 1;
173   _mesa_hash_table_insert(sim.syncobjs,
174                           (const void *)(uintptr_t)syncobj_handle, syncobj);
175
176   mtx_unlock(&sim.mutex);
177
178   return syncobj_handle;
179}
180
181static void
182sim_syncobj_destroy(struct virtgpu *gpu, uint32_t syncobj_handle)
183{
184   struct sim_syncobj *syncobj = NULL;
185
186   mtx_lock(&sim.mutex);
187
188   struct hash_entry *entry = _mesa_hash_table_search(
189      sim.syncobjs, (const void *)(uintptr_t)syncobj_handle);
190   if (entry) {
191      syncobj = entry->data;
192      _mesa_hash_table_remove(sim.syncobjs, entry);
193      util_idalloc_free(&sim.ida, syncobj_handle - 1);
194   }
195
196   mtx_unlock(&sim.mutex);
197
198   if (syncobj) {
199      if (syncobj->pending_fd >= 0)
200         close(syncobj->pending_fd);
201      mtx_destroy(&syncobj->mutex);
202      free(syncobj);
203   }
204}
205
206static VkResult
207sim_syncobj_poll(int fd, int poll_timeout)
208{
209   struct pollfd pollfd = {
210      .fd = fd,
211      .events = POLLIN,
212   };
213   int ret;
214   do {
215      ret = poll(&pollfd, 1, poll_timeout);
216   } while (ret == -1 && (errno == EINTR || errno == EAGAIN));
217
218   if (ret < 0 || (ret > 0 && !(pollfd.revents & POLLIN))) {
219      return (ret < 0 && errno == ENOMEM) ? VK_ERROR_OUT_OF_HOST_MEMORY
220                                          : VK_ERROR_DEVICE_LOST;
221   }
222
223   return ret ? VK_SUCCESS : VK_TIMEOUT;
224}
225
226static void
227sim_syncobj_set_point_locked(struct sim_syncobj *syncobj, uint64_t point)
228{
229   syncobj->point = point;
230
231   if (syncobj->pending_fd >= 0) {
232      close(syncobj->pending_fd);
233      syncobj->pending_fd = -1;
234      syncobj->pending_point = point;
235   }
236}
237
238static void
239sim_syncobj_update_point_locked(struct sim_syncobj *syncobj, int poll_timeout)
240{
241   if (syncobj->pending_fd >= 0) {
242      VkResult result;
243      if (syncobj->pending_cpu) {
244         if (poll_timeout == -1) {
245            const int max_cpu_timeout = 2000;
246            poll_timeout = max_cpu_timeout;
247            result = sim_syncobj_poll(syncobj->pending_fd, poll_timeout);
248            if (result == VK_TIMEOUT) {
249               vn_log(NULL, "cpu sync timed out after %dms; ignoring",
250                      poll_timeout);
251               result = VK_SUCCESS;
252            }
253         } else {
254            result = sim_syncobj_poll(syncobj->pending_fd, poll_timeout);
255         }
256      } else {
257         result = sim_syncobj_poll(syncobj->pending_fd, poll_timeout);
258      }
259      if (result == VK_SUCCESS) {
260         close(syncobj->pending_fd);
261         syncobj->pending_fd = -1;
262         syncobj->point = syncobj->pending_point;
263      }
264   }
265}
266
267static struct sim_syncobj *
268sim_syncobj_lookup(struct virtgpu *gpu, uint32_t syncobj_handle)
269{
270   struct sim_syncobj *syncobj = NULL;
271
272   mtx_lock(&sim.mutex);
273   struct hash_entry *entry = _mesa_hash_table_search(
274      sim.syncobjs, (const void *)(uintptr_t)syncobj_handle);
275   if (entry)
276      syncobj = entry->data;
277   mtx_unlock(&sim.mutex);
278
279   return syncobj;
280}
281
282static int
283sim_syncobj_reset(struct virtgpu *gpu, uint32_t syncobj_handle)
284{
285   struct sim_syncobj *syncobj = sim_syncobj_lookup(gpu, syncobj_handle);
286   if (!syncobj)
287      return -1;
288
289   mtx_lock(&syncobj->mutex);
290   sim_syncobj_set_point_locked(syncobj, 0);
291   mtx_unlock(&syncobj->mutex);
292
293   return 0;
294}
295
296static int
297sim_syncobj_query(struct virtgpu *gpu,
298                  uint32_t syncobj_handle,
299                  uint64_t *point)
300{
301   struct sim_syncobj *syncobj = sim_syncobj_lookup(gpu, syncobj_handle);
302   if (!syncobj)
303      return -1;
304
305   mtx_lock(&syncobj->mutex);
306   sim_syncobj_update_point_locked(syncobj, 0);
307   *point = syncobj->point;
308   mtx_unlock(&syncobj->mutex);
309
310   return 0;
311}
312
313static int
314sim_syncobj_signal(struct virtgpu *gpu,
315                   uint32_t syncobj_handle,
316                   uint64_t point)
317{
318   struct sim_syncobj *syncobj = sim_syncobj_lookup(gpu, syncobj_handle);
319   if (!syncobj)
320      return -1;
321
322   mtx_lock(&syncobj->mutex);
323   sim_syncobj_set_point_locked(syncobj, point);
324   mtx_unlock(&syncobj->mutex);
325
326   return 0;
327}
328
329static int
330sim_syncobj_submit(struct virtgpu *gpu,
331                   uint32_t syncobj_handle,
332                   int sync_fd,
333                   uint64_t point,
334                   bool cpu)
335{
336   struct sim_syncobj *syncobj = sim_syncobj_lookup(gpu, syncobj_handle);
337   if (!syncobj)
338      return -1;
339
340   int pending_fd = dup(sync_fd);
341   if (pending_fd < 0) {
342      vn_log(gpu->instance, "failed to dup sync fd");
343      return -1;
344   }
345
346   mtx_lock(&syncobj->mutex);
347
348   if (syncobj->pending_fd >= 0) {
349      mtx_unlock(&syncobj->mutex);
350
351      /* TODO */
352      vn_log(gpu->instance, "sorry, no simulated timeline semaphore");
353      close(pending_fd);
354      return -1;
355   }
356   if (syncobj->point >= point)
357      vn_log(gpu->instance, "non-monotonic signaling");
358
359   syncobj->pending_fd = pending_fd;
360   syncobj->pending_point = point;
361   syncobj->pending_cpu = cpu;
362
363   mtx_unlock(&syncobj->mutex);
364
365   return 0;
366}
367
368static int
369timeout_to_poll_timeout(uint64_t timeout)
370{
371   const uint64_t ns_per_ms = 1000000;
372   const uint64_t ms = (timeout + ns_per_ms - 1) / ns_per_ms;
373   if (!ms && timeout)
374      return -1;
375   return ms <= INT_MAX ? ms : -1;
376}
377
378static int
379sim_syncobj_wait(struct virtgpu *gpu,
380                 const struct vn_renderer_wait *wait,
381                 bool wait_avail)
382{
383   if (wait_avail)
384      return -1;
385
386   const int poll_timeout = timeout_to_poll_timeout(wait->timeout);
387
388   /* TODO poll all fds at the same time */
389   for (uint32_t i = 0; i < wait->sync_count; i++) {
390      struct virtgpu_sync *sync = (struct virtgpu_sync *)wait->syncs[i];
391      const uint64_t point = wait->sync_values[i];
392
393      struct sim_syncobj *syncobj =
394         sim_syncobj_lookup(gpu, sync->syncobj_handle);
395      if (!syncobj)
396         return -1;
397
398      mtx_lock(&syncobj->mutex);
399
400      if (syncobj->point < point)
401         sim_syncobj_update_point_locked(syncobj, poll_timeout);
402
403      if (syncobj->point < point) {
404         if (wait->wait_any && i < wait->sync_count - 1 &&
405             syncobj->pending_fd < 0) {
406            mtx_unlock(&syncobj->mutex);
407            continue;
408         }
409         errno = ETIME;
410         mtx_unlock(&syncobj->mutex);
411         return -1;
412      }
413
414      mtx_unlock(&syncobj->mutex);
415
416      if (wait->wait_any)
417         break;
418
419      /* TODO adjust poll_timeout */
420   }
421
422   return 0;
423}
424
425static int
426sim_syncobj_export(struct virtgpu *gpu, uint32_t syncobj_handle)
427{
428   struct sim_syncobj *syncobj = sim_syncobj_lookup(gpu, syncobj_handle);
429   if (!syncobj)
430      return -1;
431
432   int fd = -1;
433   mtx_lock(&syncobj->mutex);
434   if (syncobj->pending_fd >= 0)
435      fd = dup(syncobj->pending_fd);
436   else
437      fd = dup(sim.signaled_fd);
438   mtx_unlock(&syncobj->mutex);
439
440   return fd;
441}
442
443static uint32_t
444sim_syncobj_import(struct virtgpu *gpu, uint32_t syncobj_handle, int fd)
445{
446   struct sim_syncobj *syncobj = sim_syncobj_lookup(gpu, syncobj_handle);
447   if (!syncobj)
448      return 0;
449
450   if (sim_syncobj_submit(gpu, syncobj_handle, fd, 1, false))
451      return 0;
452
453   return syncobj_handle;
454}
455
456#endif /* SIMULATE_SYNCOBJ */
457
458#ifdef SIMULATE_SUBMIT
459
460static int
461sim_submit_signal_syncs(struct virtgpu *gpu,
462                        int sync_fd,
463                        struct vn_renderer_sync *const *syncs,
464                        const uint64_t *sync_values,
465                        uint32_t sync_count,
466                        bool cpu)
467{
468   for (uint32_t i = 0; i < sync_count; i++) {
469      struct virtgpu_sync *sync = (struct virtgpu_sync *)syncs[i];
470      const uint64_t pending_point = sync_values[i];
471
472#ifdef SIMULATE_SYNCOBJ
473      int ret = sim_syncobj_submit(gpu, sync->syncobj_handle, sync_fd,
474                                   pending_point, cpu);
475      if (ret)
476         return ret;
477#else
478      /* we can in theory do a DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE followed by a
479       * DRM_IOCTL_SYNCOBJ_TRANSFER
480       */
481      return -1;
482#endif
483   }
484
485   return 0;
486}
487
488static uint32_t *
489sim_submit_alloc_gem_handles(struct vn_renderer_bo *const *bos,
490                             uint32_t bo_count)
491{
492   uint32_t *gem_handles = malloc(sizeof(*gem_handles) * bo_count);
493   if (!gem_handles)
494      return NULL;
495
496   for (uint32_t i = 0; i < bo_count; i++) {
497      struct virtgpu_bo *bo = (struct virtgpu_bo *)bos[i];
498      gem_handles[i] = bo->gem_handle;
499   }
500
501   return gem_handles;
502}
503
504static int
505sim_submit(struct virtgpu *gpu, const struct vn_renderer_submit *submit)
506{
507   /* TODO replace submit->bos by submit->gem_handles to avoid malloc/loop */
508   uint32_t *gem_handles = NULL;
509   if (submit->bo_count) {
510      gem_handles =
511         sim_submit_alloc_gem_handles(submit->bos, submit->bo_count);
512      if (!gem_handles)
513         return -1;
514   }
515
516   int ret = 0;
517   for (uint32_t i = 0; i < submit->batch_count; i++) {
518      const struct vn_renderer_submit_batch *batch = &submit->batches[i];
519
520      struct drm_virtgpu_execbuffer args = {
521         .flags = batch->sync_count ? VIRTGPU_EXECBUF_FENCE_FD_OUT : 0,
522         .size = batch->cs_size,
523         .command = (uintptr_t)batch->cs_data,
524         .bo_handles = (uintptr_t)gem_handles,
525         .num_bo_handles = submit->bo_count,
526      };
527
528      ret = drmIoctl(gpu->fd, DRM_IOCTL_VIRTGPU_EXECBUFFER, &args);
529      if (ret) {
530         vn_log(gpu->instance, "failed to execbuffer: %s", strerror(errno));
531         break;
532      }
533
534      if (batch->sync_count) {
535         ret = sim_submit_signal_syncs(gpu, args.fence_fd, batch->syncs,
536                                       batch->sync_values, batch->sync_count,
537                                       batch->sync_queue_cpu);
538         close(args.fence_fd);
539         if (ret)
540            break;
541      }
542   }
543
544   if (!submit->batch_count && submit->bo_count) {
545      struct drm_virtgpu_execbuffer args = {
546         .bo_handles = (uintptr_t)gem_handles,
547         .num_bo_handles = submit->bo_count,
548      };
549
550      ret = drmIoctl(gpu->fd, DRM_IOCTL_VIRTGPU_EXECBUFFER, &args);
551      if (ret)
552         vn_log(gpu->instance, "failed to execbuffer: %s", strerror(errno));
553   }
554
555   free(gem_handles);
556
557   return ret;
558}
559
560#endif /* SIMULATE_SUBMIT */
561
562static int
563virtgpu_ioctl(struct virtgpu *gpu, unsigned long request, void *args)
564{
565   return drmIoctl(gpu->fd, request, args);
566}
567
568static uint64_t
569virtgpu_ioctl_getparam(struct virtgpu *gpu, uint64_t param)
570{
571#ifdef SIMULATE_CONTEXT_INIT
572   if (param == VIRTGPU_PARAM_CONTEXT_INIT)
573      return 1;
574#endif
575#ifdef SIMULATE_SUBMIT
576   if (param == VIRTGPU_PARAM_MAX_SYNC_QUEUE_COUNT)
577      return 16;
578#endif
579
580   /* val must be zeroed because kernel only writes the lower 32 bits */
581   uint64_t val = 0;
582   struct drm_virtgpu_getparam args = {
583      .param = param,
584      .value = (uintptr_t)&val,
585   };
586
587   const int ret = virtgpu_ioctl(gpu, DRM_IOCTL_VIRTGPU_GETPARAM, &args);
588   return ret ? 0 : val;
589}
590
591static int
592virtgpu_ioctl_get_caps(struct virtgpu *gpu,
593                       enum virgl_renderer_capset id,
594                       uint32_t version,
595                       void *capset,
596                       size_t capset_size)
597{
598#ifdef SIMULATE_CONTEXT_INIT
599   if (id == VIRGL_RENDERER_CAPSET_VENUS && version == 0)
600      return 0;
601#endif
602
603   struct drm_virtgpu_get_caps args = {
604      .cap_set_id = id,
605      .cap_set_ver = version,
606      .addr = (uintptr_t)capset,
607      .size = capset_size,
608   };
609
610   return virtgpu_ioctl(gpu, DRM_IOCTL_VIRTGPU_GET_CAPS, &args);
611}
612
613static int
614virtgpu_ioctl_context_init(struct virtgpu *gpu,
615                           enum virgl_renderer_capset capset_id)
616{
617#ifdef SIMULATE_CONTEXT_INIT
618   if (capset_id == VIRGL_RENDERER_CAPSET_VENUS)
619      return 0;
620#endif
621
622   struct drm_virtgpu_context_init args = {
623      .num_params = 1,
624      .ctx_set_params = (uintptr_t) &
625                        (struct drm_virtgpu_context_set_param){
626                           .param = VIRTGPU_CONTEXT_PARAM_CAPSET_ID,
627                           .value = capset_id,
628                        },
629   };
630
631   return virtgpu_ioctl(gpu, DRM_IOCTL_VIRTGPU_CONTEXT_INIT, &args);
632}
633
634static uint32_t
635virtgpu_ioctl_resource_create_blob(struct virtgpu *gpu,
636                                   uint32_t blob_mem,
637                                   uint32_t blob_flags,
638                                   size_t blob_size,
639                                   uint64_t blob_id,
640                                   uint32_t *res_id)
641{
642#ifdef SIMULATE_BO_SIZE_FIX
643   blob_size = align64(blob_size, 4096);
644#endif
645
646   struct drm_virtgpu_resource_create_blob args = {
647      .blob_mem = blob_mem,
648      .blob_flags = blob_flags,
649      .size = blob_size,
650      .blob_id = blob_id,
651   };
652
653   if (virtgpu_ioctl(gpu, DRM_IOCTL_VIRTGPU_RESOURCE_CREATE_BLOB, &args))
654      return 0;
655
656   *res_id = args.res_handle;
657   return args.bo_handle;
658}
659
660static int
661virtgpu_ioctl_resource_info(struct virtgpu *gpu,
662                            uint32_t gem_handle,
663                            struct drm_virtgpu_resource_info *info)
664{
665   *info = (struct drm_virtgpu_resource_info){
666      .bo_handle = gem_handle,
667   };
668
669   return virtgpu_ioctl(gpu, DRM_IOCTL_VIRTGPU_RESOURCE_INFO, info);
670}
671
672static void
673virtgpu_ioctl_gem_close(struct virtgpu *gpu, uint32_t gem_handle)
674{
675   struct drm_gem_close args = {
676      .handle = gem_handle,
677   };
678
679   ASSERTED const int ret = virtgpu_ioctl(gpu, DRM_IOCTL_GEM_CLOSE, &args);
680   assert(!ret);
681}
682
683static int
684virtgpu_ioctl_prime_handle_to_fd(struct virtgpu *gpu,
685                                 uint32_t gem_handle,
686                                 bool mappable)
687{
688   struct drm_prime_handle args = {
689      .handle = gem_handle,
690      .flags = DRM_CLOEXEC | (mappable ? DRM_RDWR : 0),
691   };
692
693   const int ret = virtgpu_ioctl(gpu, DRM_IOCTL_PRIME_HANDLE_TO_FD, &args);
694   return ret ? -1 : args.fd;
695}
696
697static uint32_t
698virtgpu_ioctl_prime_fd_to_handle(struct virtgpu *gpu, int fd)
699{
700   struct drm_prime_handle args = {
701      .fd = fd,
702   };
703
704   const int ret = virtgpu_ioctl(gpu, DRM_IOCTL_PRIME_FD_TO_HANDLE, &args);
705   return ret ? 0 : args.handle;
706}
707
708static void *
709virtgpu_ioctl_map(struct virtgpu *gpu, uint32_t gem_handle, size_t size)
710{
711   struct drm_virtgpu_map args = {
712      .handle = gem_handle,
713   };
714
715   if (virtgpu_ioctl(gpu, DRM_IOCTL_VIRTGPU_MAP, &args))
716      return NULL;
717
718   void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, gpu->fd,
719                    args.offset);
720   if (ptr == MAP_FAILED)
721      return NULL;
722
723   return ptr;
724}
725
726static uint32_t
727virtgpu_ioctl_syncobj_create(struct virtgpu *gpu, bool signaled)
728{
729#ifdef SIMULATE_SYNCOBJ
730   return sim_syncobj_create(gpu, signaled);
731#endif
732
733   struct drm_syncobj_create args = {
734      .flags = signaled ? DRM_SYNCOBJ_CREATE_SIGNALED : 0,
735   };
736
737   const int ret = virtgpu_ioctl(gpu, DRM_IOCTL_SYNCOBJ_CREATE, &args);
738   return ret ? 0 : args.handle;
739}
740
741static void
742virtgpu_ioctl_syncobj_destroy(struct virtgpu *gpu, uint32_t syncobj_handle)
743{
744#ifdef SIMULATE_SYNCOBJ
745   sim_syncobj_destroy(gpu, syncobj_handle);
746   return;
747#endif
748
749   struct drm_syncobj_destroy args = {
750      .handle = syncobj_handle,
751   };
752
753   ASSERTED const int ret =
754      virtgpu_ioctl(gpu, DRM_IOCTL_SYNCOBJ_DESTROY, &args);
755   assert(!ret);
756}
757
758static int
759virtgpu_ioctl_syncobj_handle_to_fd(struct virtgpu *gpu,
760                                   uint32_t syncobj_handle,
761                                   bool sync_file)
762{
763#ifdef SIMULATE_SYNCOBJ
764   return sync_file ? sim_syncobj_export(gpu, syncobj_handle) : -1;
765#endif
766
767   struct drm_syncobj_handle args = {
768      .handle = syncobj_handle,
769      .flags =
770         sync_file ? DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE : 0,
771   };
772
773   int ret = virtgpu_ioctl(gpu, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &args);
774   if (ret)
775      return -1;
776
777   return args.fd;
778}
779
780static uint32_t
781virtgpu_ioctl_syncobj_fd_to_handle(struct virtgpu *gpu,
782                                   int fd,
783                                   uint32_t syncobj_handle)
784{
785#ifdef SIMULATE_SYNCOBJ
786   return syncobj_handle ? sim_syncobj_import(gpu, syncobj_handle, fd) : 0;
787#endif
788
789   struct drm_syncobj_handle args = {
790      .handle = syncobj_handle,
791      .flags =
792         syncobj_handle ? DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE : 0,
793      .fd = fd,
794   };
795
796   int ret = virtgpu_ioctl(gpu, DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE, &args);
797   if (ret)
798      return 0;
799
800   return args.handle;
801}
802
803static int
804virtgpu_ioctl_syncobj_reset(struct virtgpu *gpu, uint32_t syncobj_handle)
805{
806#ifdef SIMULATE_SYNCOBJ
807   return sim_syncobj_reset(gpu, syncobj_handle);
808#endif
809
810   struct drm_syncobj_array args = {
811      .handles = (uintptr_t)&syncobj_handle,
812      .count_handles = 1,
813   };
814
815   return virtgpu_ioctl(gpu, DRM_IOCTL_SYNCOBJ_RESET, &args);
816}
817
818static int
819virtgpu_ioctl_syncobj_query(struct virtgpu *gpu,
820                            uint32_t syncobj_handle,
821                            uint64_t *point)
822{
823#ifdef SIMULATE_SYNCOBJ
824   return sim_syncobj_query(gpu, syncobj_handle, point);
825#endif
826
827   struct drm_syncobj_timeline_array args = {
828      .handles = (uintptr_t)&syncobj_handle,
829      .points = (uintptr_t)point,
830      .count_handles = 1,
831   };
832
833   return virtgpu_ioctl(gpu, DRM_IOCTL_SYNCOBJ_QUERY, &args);
834}
835
836static int
837virtgpu_ioctl_syncobj_timeline_signal(struct virtgpu *gpu,
838                                      uint32_t syncobj_handle,
839                                      uint64_t point)
840{
841#ifdef SIMULATE_SYNCOBJ
842   return sim_syncobj_signal(gpu, syncobj_handle, point);
843#endif
844
845   struct drm_syncobj_timeline_array args = {
846      .handles = (uintptr_t)&syncobj_handle,
847      .points = (uintptr_t)&point,
848      .count_handles = 1,
849   };
850
851   return virtgpu_ioctl(gpu, DRM_IOCTL_SYNCOBJ_TIMELINE_SIGNAL, &args);
852}
853
854static int
855virtgpu_ioctl_syncobj_timeline_wait(struct virtgpu *gpu,
856                                    const struct vn_renderer_wait *wait,
857                                    bool wait_avail)
858{
859#ifdef SIMULATE_SYNCOBJ
860   return sim_syncobj_wait(gpu, wait, wait_avail);
861#endif
862
863   /* always enable wait-before-submit */
864   uint32_t flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT;
865   if (!wait->wait_any)
866      flags |= DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL;
867   /* wait for fences to appear instead of signaling */
868   if (wait_avail)
869      flags |= DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE;
870
871   /* TODO replace wait->syncs by wait->sync_handles to avoid malloc/loop */
872   uint32_t *syncobj_handles =
873      malloc(sizeof(*syncobj_handles) * wait->sync_count);
874   if (!syncobj_handles)
875      return -1;
876   for (uint32_t i = 0; i < wait->sync_count; i++) {
877      struct virtgpu_sync *sync = (struct virtgpu_sync *)wait->syncs[i];
878      syncobj_handles[i] = sync->syncobj_handle;
879   }
880
881   struct drm_syncobj_timeline_wait args = {
882      .handles = (uintptr_t)syncobj_handles,
883      .points = (uintptr_t)wait->sync_values,
884      .timeout_nsec = os_time_get_absolute_timeout(wait->timeout),
885      .count_handles = wait->sync_count,
886      .flags = flags,
887   };
888
889   const int ret = virtgpu_ioctl(gpu, DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT, &args);
890
891   free(syncobj_handles);
892
893   return ret;
894}
895
896static int
897virtgpu_ioctl_submit(struct virtgpu *gpu,
898                     const struct vn_renderer_submit *submit)
899{
900#ifdef SIMULATE_SUBMIT
901   return sim_submit(gpu, submit);
902#endif
903   return -1;
904}
905
906static VkResult
907virtgpu_sync_write(struct vn_renderer *renderer,
908                   struct vn_renderer_sync *_sync,
909                   uint64_t val)
910{
911   struct virtgpu *gpu = (struct virtgpu *)renderer;
912   struct virtgpu_sync *sync = (struct virtgpu_sync *)_sync;
913
914   const int ret =
915      virtgpu_ioctl_syncobj_timeline_signal(gpu, sync->syncobj_handle, val);
916
917   return ret ? VK_ERROR_OUT_OF_DEVICE_MEMORY : VK_SUCCESS;
918}
919
920static VkResult
921virtgpu_sync_read(struct vn_renderer *renderer,
922                  struct vn_renderer_sync *_sync,
923                  uint64_t *val)
924{
925   struct virtgpu *gpu = (struct virtgpu *)renderer;
926   struct virtgpu_sync *sync = (struct virtgpu_sync *)_sync;
927
928   const int ret =
929      virtgpu_ioctl_syncobj_query(gpu, sync->syncobj_handle, val);
930
931   return ret ? VK_ERROR_OUT_OF_DEVICE_MEMORY : VK_SUCCESS;
932}
933
934static VkResult
935virtgpu_sync_reset(struct vn_renderer *renderer,
936                   struct vn_renderer_sync *_sync,
937                   uint64_t initial_val)
938{
939   struct virtgpu *gpu = (struct virtgpu *)renderer;
940   struct virtgpu_sync *sync = (struct virtgpu_sync *)_sync;
941
942   int ret = virtgpu_ioctl_syncobj_reset(gpu, sync->syncobj_handle);
943   if (!ret) {
944      ret = virtgpu_ioctl_syncobj_timeline_signal(gpu, sync->syncobj_handle,
945                                                  initial_val);
946   }
947
948   return ret ? VK_ERROR_OUT_OF_DEVICE_MEMORY : VK_SUCCESS;
949}
950
951static int
952virtgpu_sync_export_syncobj(struct vn_renderer *renderer,
953                            struct vn_renderer_sync *_sync,
954                            bool sync_file)
955{
956   struct virtgpu *gpu = (struct virtgpu *)renderer;
957   struct virtgpu_sync *sync = (struct virtgpu_sync *)_sync;
958
959   return virtgpu_ioctl_syncobj_handle_to_fd(gpu, sync->syncobj_handle,
960                                             sync_file);
961}
962
963static void
964virtgpu_sync_destroy(struct vn_renderer *renderer,
965                     struct vn_renderer_sync *_sync)
966{
967   struct virtgpu *gpu = (struct virtgpu *)renderer;
968   struct virtgpu_sync *sync = (struct virtgpu_sync *)_sync;
969
970   virtgpu_ioctl_syncobj_destroy(gpu, sync->syncobj_handle);
971
972   free(sync);
973}
974
975static VkResult
976virtgpu_sync_create_from_syncobj(struct vn_renderer *renderer,
977                                 int fd,
978                                 bool sync_file,
979                                 struct vn_renderer_sync **out_sync)
980{
981   struct virtgpu *gpu = (struct virtgpu *)renderer;
982
983   uint32_t syncobj_handle;
984   if (sync_file) {
985      syncobj_handle = virtgpu_ioctl_syncobj_create(gpu, false);
986      if (!syncobj_handle)
987         return VK_ERROR_OUT_OF_HOST_MEMORY;
988      if (!virtgpu_ioctl_syncobj_fd_to_handle(gpu, fd, syncobj_handle)) {
989         virtgpu_ioctl_syncobj_destroy(gpu, syncobj_handle);
990         return VK_ERROR_INVALID_EXTERNAL_HANDLE;
991      }
992   } else {
993      syncobj_handle = virtgpu_ioctl_syncobj_fd_to_handle(gpu, fd, 0);
994      if (!syncobj_handle)
995         return VK_ERROR_INVALID_EXTERNAL_HANDLE;
996   }
997
998   struct virtgpu_sync *sync = calloc(1, sizeof(*sync));
999   if (!sync) {
1000      virtgpu_ioctl_syncobj_destroy(gpu, syncobj_handle);
1001      return VK_ERROR_OUT_OF_HOST_MEMORY;
1002   }
1003
1004   sync->syncobj_handle = syncobj_handle;
1005   sync->base.sync_id = 0; /* TODO */
1006
1007   *out_sync = &sync->base;
1008
1009   return VK_SUCCESS;
1010}
1011
1012static VkResult
1013virtgpu_sync_create(struct vn_renderer *renderer,
1014                    uint64_t initial_val,
1015                    uint32_t flags,
1016                    struct vn_renderer_sync **out_sync)
1017{
1018   struct virtgpu *gpu = (struct virtgpu *)renderer;
1019
1020   /* TODO */
1021   if (flags & VN_RENDERER_SYNC_SHAREABLE)
1022      return VK_ERROR_OUT_OF_DEVICE_MEMORY;
1023
1024   /* always false because we don't use binary drm_syncobjs */
1025   const bool signaled = false;
1026   const uint32_t syncobj_handle =
1027      virtgpu_ioctl_syncobj_create(gpu, signaled);
1028   if (!syncobj_handle)
1029      return VK_ERROR_OUT_OF_DEVICE_MEMORY;
1030
1031   /* add a signaled fence chain with seqno initial_val */
1032   const int ret =
1033      virtgpu_ioctl_syncobj_timeline_signal(gpu, syncobj_handle, initial_val);
1034   if (ret) {
1035      virtgpu_ioctl_syncobj_destroy(gpu, syncobj_handle);
1036      return VK_ERROR_OUT_OF_DEVICE_MEMORY;
1037   }
1038
1039   struct virtgpu_sync *sync = calloc(1, sizeof(*sync));
1040   if (!sync) {
1041      virtgpu_ioctl_syncobj_destroy(gpu, syncobj_handle);
1042      return VK_ERROR_OUT_OF_HOST_MEMORY;
1043   }
1044
1045   sync->syncobj_handle = syncobj_handle;
1046   /* we will have a sync_id when shareable is true and virtio-gpu associates
1047    * a host sync object with guest drm_syncobj
1048    */
1049   sync->base.sync_id = 0;
1050
1051   *out_sync = &sync->base;
1052
1053   return VK_SUCCESS;
1054}
1055
1056static void
1057virtgpu_bo_invalidate(struct vn_renderer *renderer,
1058                      struct vn_renderer_bo *bo,
1059                      VkDeviceSize offset,
1060                      VkDeviceSize size)
1061{
1062   /* nop because kernel makes every mapping coherent */
1063}
1064
1065static void
1066virtgpu_bo_flush(struct vn_renderer *renderer,
1067                 struct vn_renderer_bo *bo,
1068                 VkDeviceSize offset,
1069                 VkDeviceSize size)
1070{
1071   /* nop because kernel makes every mapping coherent */
1072}
1073
1074static void *
1075virtgpu_bo_map(struct vn_renderer *renderer, struct vn_renderer_bo *_bo)
1076{
1077   struct virtgpu *gpu = (struct virtgpu *)renderer;
1078   struct virtgpu_bo *bo = (struct virtgpu_bo *)_bo;
1079   const bool mappable = bo->blob_flags & VIRTGPU_BLOB_FLAG_USE_MAPPABLE;
1080
1081   /* not thread-safe but is fine */
1082   if (!bo->base.mmap_ptr && mappable) {
1083      bo->base.mmap_ptr =
1084         virtgpu_ioctl_map(gpu, bo->gem_handle, bo->base.mmap_size);
1085   }
1086
1087   return bo->base.mmap_ptr;
1088}
1089
1090static int
1091virtgpu_bo_export_dma_buf(struct vn_renderer *renderer,
1092                          struct vn_renderer_bo *_bo)
1093{
1094   struct virtgpu *gpu = (struct virtgpu *)renderer;
1095   struct virtgpu_bo *bo = (struct virtgpu_bo *)_bo;
1096   const bool mappable = bo->blob_flags & VIRTGPU_BLOB_FLAG_USE_MAPPABLE;
1097   const bool shareable = bo->blob_flags & VIRTGPU_BLOB_FLAG_USE_SHAREABLE;
1098
1099   return shareable
1100             ? virtgpu_ioctl_prime_handle_to_fd(gpu, bo->gem_handle, mappable)
1101             : -1;
1102}
1103
1104static bool
1105virtgpu_bo_destroy(struct vn_renderer *renderer, struct vn_renderer_bo *_bo)
1106{
1107   struct virtgpu *gpu = (struct virtgpu *)renderer;
1108   struct virtgpu_bo *bo = (struct virtgpu_bo *)_bo;
1109
1110   mtx_lock(&gpu->dma_buf_import_mutex);
1111
1112   /* Check the refcount again after the import lock is grabbed.  Yes, we use
1113    * the double-checked locking anti-pattern.
1114    */
1115   if (vn_refcount_is_valid(&bo->base.refcount)) {
1116      mtx_unlock(&gpu->dma_buf_import_mutex);
1117      return false;
1118   }
1119
1120   if (bo->base.mmap_ptr)
1121      munmap(bo->base.mmap_ptr, bo->base.mmap_size);
1122   virtgpu_ioctl_gem_close(gpu, bo->gem_handle);
1123
1124   /* set gem_handle to 0 to indicate that the bo is invalid */
1125   bo->gem_handle = 0;
1126
1127   mtx_unlock(&gpu->dma_buf_import_mutex);
1128
1129   return true;
1130}
1131
1132static uint32_t
1133virtgpu_bo_blob_flags(VkMemoryPropertyFlags flags,
1134                      VkExternalMemoryHandleTypeFlags external_handles)
1135{
1136   uint32_t blob_flags = 0;
1137   if (flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
1138      blob_flags |= VIRTGPU_BLOB_FLAG_USE_MAPPABLE;
1139   if (external_handles)
1140      blob_flags |= VIRTGPU_BLOB_FLAG_USE_SHAREABLE;
1141   if (external_handles & VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT)
1142      blob_flags |= VIRTGPU_BLOB_FLAG_USE_CROSS_DEVICE;
1143
1144   return blob_flags;
1145}
1146
1147static VkResult
1148virtgpu_bo_create_from_dma_buf(struct vn_renderer *renderer,
1149                               VkDeviceSize size,
1150                               int fd,
1151                               VkMemoryPropertyFlags flags,
1152                               struct vn_renderer_bo **out_bo)
1153{
1154   struct virtgpu *gpu = (struct virtgpu *)renderer;
1155   struct drm_virtgpu_resource_info info;
1156   uint32_t gem_handle = 0;
1157   struct virtgpu_bo *bo = NULL;
1158
1159   mtx_lock(&gpu->dma_buf_import_mutex);
1160
1161   gem_handle = virtgpu_ioctl_prime_fd_to_handle(gpu, fd);
1162   if (!gem_handle)
1163      goto fail;
1164   bo = util_sparse_array_get(&gpu->bo_array, gem_handle);
1165
1166   if (virtgpu_ioctl_resource_info(gpu, gem_handle, &info))
1167      goto fail;
1168
1169   uint32_t blob_flags;
1170   size_t mmap_size;
1171   if (info.blob_mem) {
1172      /* must be VIRTGPU_BLOB_MEM_HOST3D */
1173      if (info.blob_mem != VIRTGPU_BLOB_MEM_HOST3D)
1174         goto fail;
1175
1176      /* blob_flags is not passed to the kernel and is only for internal use
1177       * on imports.  Set it to what works best for us.
1178       */
1179      blob_flags = virtgpu_bo_blob_flags(flags, 0);
1180      blob_flags |= VIRTGPU_BLOB_FLAG_USE_SHAREABLE;
1181
1182      /* mmap_size is only used when mappable */
1183      mmap_size = 0;
1184      if (blob_flags & VIRTGPU_BLOB_FLAG_USE_MAPPABLE) {
1185         if (info.size < size)
1186            goto fail;
1187
1188         mmap_size = size;
1189      }
1190   } else {
1191      /* must be classic resource here
1192       * set blob_flags to 0 to fail virtgpu_bo_map
1193       * set mmap_size to 0 since mapping is not allowed
1194       */
1195      blob_flags = 0;
1196      mmap_size = 0;
1197   }
1198
1199   /* we check bo->gem_handle instead of bo->refcount because bo->refcount
1200    * might only be memset to 0 and is not considered initialized in theory
1201    */
1202   if (bo->gem_handle == gem_handle) {
1203      if (bo->base.mmap_size < mmap_size)
1204         goto fail;
1205      if (blob_flags & ~bo->blob_flags)
1206         goto fail;
1207
1208      /* we can't use vn_renderer_bo_ref as the refcount may drop to 0
1209       * temporarily before virtgpu_bo_destroy grabs the lock
1210       */
1211      vn_refcount_fetch_add_relaxed(&bo->base.refcount, 1);
1212   } else {
1213      *bo = (struct virtgpu_bo){
1214         .base = {
1215            .refcount = VN_REFCOUNT_INIT(1),
1216            .res_id = info.res_handle,
1217            .mmap_size = mmap_size,
1218         },
1219         .gem_handle = gem_handle,
1220         .blob_flags = blob_flags,
1221      };
1222   }
1223
1224   mtx_unlock(&gpu->dma_buf_import_mutex);
1225
1226   *out_bo = &bo->base;
1227
1228   return VK_SUCCESS;
1229
1230fail:
1231   if (gem_handle && bo->gem_handle != gem_handle)
1232      virtgpu_ioctl_gem_close(gpu, gem_handle);
1233   mtx_unlock(&gpu->dma_buf_import_mutex);
1234   return VK_ERROR_INVALID_EXTERNAL_HANDLE;
1235}
1236
1237static VkResult
1238virtgpu_bo_create_from_device_memory(
1239   struct vn_renderer *renderer,
1240   VkDeviceSize size,
1241   vn_object_id mem_id,
1242   VkMemoryPropertyFlags flags,
1243   VkExternalMemoryHandleTypeFlags external_handles,
1244   struct vn_renderer_bo **out_bo)
1245{
1246   struct virtgpu *gpu = (struct virtgpu *)renderer;
1247   const uint32_t blob_flags = virtgpu_bo_blob_flags(flags, external_handles);
1248
1249   uint32_t res_id;
1250   uint32_t gem_handle = virtgpu_ioctl_resource_create_blob(
1251      gpu, VIRTGPU_BLOB_MEM_HOST3D, blob_flags, size, mem_id, &res_id);
1252   if (!gem_handle)
1253      return VK_ERROR_OUT_OF_DEVICE_MEMORY;
1254
1255   struct virtgpu_bo *bo = util_sparse_array_get(&gpu->bo_array, gem_handle);
1256   *bo = (struct virtgpu_bo){
1257      .base = {
1258         .refcount = VN_REFCOUNT_INIT(1),
1259         .res_id = res_id,
1260         .mmap_size = size,
1261      },
1262      .gem_handle = gem_handle,
1263      .blob_flags = blob_flags,
1264   };
1265
1266   *out_bo = &bo->base;
1267
1268   return VK_SUCCESS;
1269}
1270
1271static void
1272virtgpu_shmem_destroy(struct vn_renderer *renderer,
1273                      struct vn_renderer_shmem *_shmem)
1274{
1275   struct virtgpu *gpu = (struct virtgpu *)renderer;
1276   struct virtgpu_shmem *shmem = (struct virtgpu_shmem *)_shmem;
1277
1278   munmap(shmem->base.mmap_ptr, shmem->base.mmap_size);
1279   virtgpu_ioctl_gem_close(gpu, shmem->gem_handle);
1280}
1281
1282static struct vn_renderer_shmem *
1283virtgpu_shmem_create(struct vn_renderer *renderer, size_t size)
1284{
1285   struct virtgpu *gpu = (struct virtgpu *)renderer;
1286
1287   uint32_t res_id;
1288   uint32_t gem_handle = virtgpu_ioctl_resource_create_blob(
1289      gpu, VIRTGPU_BLOB_MEM_GUEST, VIRTGPU_BLOB_FLAG_USE_MAPPABLE, size, 0,
1290      &res_id);
1291   if (!gem_handle)
1292      return NULL;
1293
1294   void *ptr = virtgpu_ioctl_map(gpu, gem_handle, size);
1295   if (!ptr) {
1296      virtgpu_ioctl_gem_close(gpu, gem_handle);
1297      return NULL;
1298   }
1299
1300   struct virtgpu_shmem *shmem =
1301      util_sparse_array_get(&gpu->shmem_array, gem_handle);
1302   *shmem = (struct virtgpu_shmem){
1303      .base = {
1304         .refcount = VN_REFCOUNT_INIT(1),
1305         .res_id = res_id,
1306         .mmap_size = size,
1307         .mmap_ptr = ptr,
1308      },
1309      .gem_handle = gem_handle,
1310   };
1311
1312   return &shmem->base;
1313}
1314
1315static VkResult
1316virtgpu_wait(struct vn_renderer *renderer,
1317             const struct vn_renderer_wait *wait)
1318{
1319   struct virtgpu *gpu = (struct virtgpu *)renderer;
1320
1321   const int ret = virtgpu_ioctl_syncobj_timeline_wait(gpu, wait, false);
1322   if (ret && errno != ETIME)
1323      return VK_ERROR_DEVICE_LOST;
1324
1325   return ret ? VK_TIMEOUT : VK_SUCCESS;
1326}
1327
1328static VkResult
1329virtgpu_submit(struct vn_renderer *renderer,
1330               const struct vn_renderer_submit *submit)
1331{
1332   struct virtgpu *gpu = (struct virtgpu *)renderer;
1333
1334   const int ret = virtgpu_ioctl_submit(gpu, submit);
1335   return ret ? VK_ERROR_DEVICE_LOST : VK_SUCCESS;
1336}
1337
1338static void
1339virtgpu_get_info(struct vn_renderer *renderer, struct vn_renderer_info *info)
1340{
1341   struct virtgpu *gpu = (struct virtgpu *)renderer;
1342
1343   memset(info, 0, sizeof(*info));
1344
1345   info->pci.vendor_id = VIRTGPU_PCI_VENDOR_ID;
1346   info->pci.device_id = VIRTGPU_PCI_DEVICE_ID;
1347
1348   info->pci.has_bus_info = true;
1349   info->pci.domain = gpu->bus_info.domain;
1350   info->pci.bus = gpu->bus_info.bus;
1351   info->pci.device = gpu->bus_info.dev;
1352   info->pci.function = gpu->bus_info.func;
1353
1354   info->has_dma_buf_import = true;
1355   /* Kernel makes every mapping coherent.  We are better off filtering
1356    * incoherent memory types out than silently making them coherent.
1357    */
1358   info->has_cache_management = false;
1359   /* TODO drm_syncobj */
1360   info->has_external_sync = false;
1361
1362   info->has_implicit_fencing = false;
1363
1364   info->max_sync_queue_count = gpu->max_sync_queue_count;
1365
1366   const struct virgl_renderer_capset_venus *capset = &gpu->capset.data;
1367   info->wire_format_version = capset->wire_format_version;
1368   info->vk_xml_version = capset->vk_xml_version;
1369   info->vk_ext_command_serialization_spec_version =
1370      capset->vk_ext_command_serialization_spec_version;
1371   info->vk_mesa_venus_protocol_spec_version =
1372      capset->vk_mesa_venus_protocol_spec_version;
1373}
1374
1375static void
1376virtgpu_destroy(struct vn_renderer *renderer,
1377                const VkAllocationCallbacks *alloc)
1378{
1379   struct virtgpu *gpu = (struct virtgpu *)renderer;
1380
1381   if (gpu->fd >= 0)
1382      close(gpu->fd);
1383
1384   mtx_destroy(&gpu->dma_buf_import_mutex);
1385
1386   util_sparse_array_finish(&gpu->shmem_array);
1387   util_sparse_array_finish(&gpu->bo_array);
1388
1389   vk_free(alloc, gpu);
1390}
1391
1392static VkResult
1393virtgpu_init_context(struct virtgpu *gpu)
1394{
1395   assert(!gpu->capset.version);
1396   const int ret = virtgpu_ioctl_context_init(gpu, gpu->capset.id);
1397   if (ret) {
1398      if (VN_DEBUG(INIT)) {
1399         vn_log(gpu->instance, "failed to initialize context: %s",
1400                strerror(errno));
1401      }
1402      return VK_ERROR_INITIALIZATION_FAILED;
1403   }
1404
1405   return VK_SUCCESS;
1406}
1407
1408static VkResult
1409virtgpu_init_capset(struct virtgpu *gpu)
1410{
1411   gpu->capset.id = VIRGL_RENDERER_CAPSET_VENUS;
1412   gpu->capset.version = 0;
1413
1414   const int ret =
1415      virtgpu_ioctl_get_caps(gpu, gpu->capset.id, gpu->capset.version,
1416                             &gpu->capset.data, sizeof(gpu->capset.data));
1417   if (ret) {
1418      if (VN_DEBUG(INIT)) {
1419         vn_log(gpu->instance, "failed to get venus v%d capset: %s",
1420                gpu->capset.version, strerror(errno));
1421      }
1422      return VK_ERROR_INITIALIZATION_FAILED;
1423   }
1424
1425   return VK_SUCCESS;
1426}
1427
1428static VkResult
1429virtgpu_init_params(struct virtgpu *gpu)
1430{
1431   const uint64_t required_params[] = {
1432      VIRTGPU_PARAM_3D_FEATURES,   VIRTGPU_PARAM_CAPSET_QUERY_FIX,
1433      VIRTGPU_PARAM_RESOURCE_BLOB, VIRTGPU_PARAM_HOST_VISIBLE,
1434      VIRTGPU_PARAM_CROSS_DEVICE,  VIRTGPU_PARAM_CONTEXT_INIT,
1435   };
1436   uint64_t val;
1437   for (uint32_t i = 0; i < ARRAY_SIZE(required_params); i++) {
1438      val = virtgpu_ioctl_getparam(gpu, required_params[i]);
1439      if (!val) {
1440         if (VN_DEBUG(INIT)) {
1441            vn_log(gpu->instance, "required kernel param %d is missing",
1442                   (int)required_params[i]);
1443         }
1444         return VK_ERROR_INITIALIZATION_FAILED;
1445      }
1446   }
1447
1448   val = virtgpu_ioctl_getparam(gpu, VIRTGPU_PARAM_MAX_SYNC_QUEUE_COUNT);
1449   if (!val) {
1450      if (VN_DEBUG(INIT))
1451         vn_log(gpu->instance, "no sync queue support");
1452      return VK_ERROR_INITIALIZATION_FAILED;
1453   }
1454   gpu->max_sync_queue_count = val;
1455
1456   return VK_SUCCESS;
1457}
1458
1459static VkResult
1460virtgpu_open_device(struct virtgpu *gpu, const drmDevicePtr dev)
1461{
1462   /* skip unless the device has our PCI vendor/device id and a render node */
1463   if (!(dev->available_nodes & (1 << DRM_NODE_RENDER)) ||
1464       dev->bustype != DRM_BUS_PCI ||
1465       dev->deviceinfo.pci->vendor_id != VIRTGPU_PCI_VENDOR_ID ||
1466       dev->deviceinfo.pci->device_id != VIRTGPU_PCI_DEVICE_ID) {
1467      if (VN_DEBUG(INIT)) {
1468         const char *name = "unknown";
1469         for (uint32_t i = 0; i < DRM_NODE_MAX; i++) {
1470            if (dev->available_nodes & (1 << i)) {
1471               name = dev->nodes[i];
1472               break;
1473            }
1474         }
1475         vn_log(gpu->instance, "skipping DRM device %s", name);
1476      }
1477      return VK_ERROR_INITIALIZATION_FAILED;
1478   }
1479
1480   const char *node_path = dev->nodes[DRM_NODE_RENDER];
1481
1482   int fd = open(node_path, O_RDWR | O_CLOEXEC);
1483   if (fd < 0) {
1484      if (VN_DEBUG(INIT))
1485         vn_log(gpu->instance, "failed to open %s", node_path);
1486      return VK_ERROR_INITIALIZATION_FAILED;
1487   }
1488
1489   drmVersionPtr version = drmGetVersion(fd);
1490   if (!version || strcmp(version->name, "virtio_gpu") ||
1491       version->version_major != 0) {
1492      if (VN_DEBUG(INIT)) {
1493         if (version) {
1494            vn_log(gpu->instance, "unknown DRM driver %s version %d",
1495                   version->name, version->version_major);
1496         } else {
1497            vn_log(gpu->instance, "failed to get DRM driver version");
1498         }
1499      }
1500      if (version)
1501         drmFreeVersion(version);
1502      close(fd);
1503      return VK_ERROR_INITIALIZATION_FAILED;
1504   }
1505
1506   gpu->fd = fd;
1507   gpu->version_minor = version->version_minor;
1508   gpu->bus_info = *dev->businfo.pci;
1509
1510   drmFreeVersion(version);
1511
1512   if (VN_DEBUG(INIT))
1513      vn_log(gpu->instance, "using DRM device %s", node_path);
1514
1515   return VK_SUCCESS;
1516}
1517
1518static VkResult
1519virtgpu_open(struct virtgpu *gpu)
1520{
1521   drmDevicePtr devs[8];
1522   int count = drmGetDevices2(0, devs, ARRAY_SIZE(devs));
1523   if (count < 0) {
1524      if (VN_DEBUG(INIT))
1525         vn_log(gpu->instance, "failed to enumerate DRM devices");
1526      return VK_ERROR_INITIALIZATION_FAILED;
1527   }
1528
1529   VkResult result = VK_ERROR_INITIALIZATION_FAILED;
1530   for (int i = 0; i < count; i++) {
1531      result = virtgpu_open_device(gpu, devs[i]);
1532      if (result == VK_SUCCESS)
1533         break;
1534   }
1535
1536   drmFreeDevices(devs, count);
1537
1538   return result;
1539}
1540
1541static VkResult
1542virtgpu_init(struct virtgpu *gpu)
1543{
1544   util_sparse_array_init(&gpu->shmem_array, sizeof(struct virtgpu_shmem),
1545                          1024);
1546   util_sparse_array_init(&gpu->bo_array, sizeof(struct virtgpu_bo), 1024);
1547
1548   mtx_init(&gpu->dma_buf_import_mutex, mtx_plain);
1549
1550   VkResult result = virtgpu_open(gpu);
1551   if (result == VK_SUCCESS)
1552      result = virtgpu_init_params(gpu);
1553   if (result == VK_SUCCESS)
1554      result = virtgpu_init_capset(gpu);
1555   if (result == VK_SUCCESS)
1556      result = virtgpu_init_context(gpu);
1557   if (result != VK_SUCCESS)
1558      return result;
1559
1560   gpu->base.ops.destroy = virtgpu_destroy;
1561   gpu->base.ops.get_info = virtgpu_get_info;
1562   gpu->base.ops.submit = virtgpu_submit;
1563   gpu->base.ops.wait = virtgpu_wait;
1564
1565   gpu->base.shmem_ops.create = virtgpu_shmem_create;
1566   gpu->base.shmem_ops.destroy = virtgpu_shmem_destroy;
1567
1568   gpu->base.bo_ops.create_from_device_memory =
1569      virtgpu_bo_create_from_device_memory;
1570   gpu->base.bo_ops.create_from_dma_buf = virtgpu_bo_create_from_dma_buf;
1571   gpu->base.bo_ops.destroy = virtgpu_bo_destroy;
1572   gpu->base.bo_ops.export_dma_buf = virtgpu_bo_export_dma_buf;
1573   gpu->base.bo_ops.map = virtgpu_bo_map;
1574   gpu->base.bo_ops.flush = virtgpu_bo_flush;
1575   gpu->base.bo_ops.invalidate = virtgpu_bo_invalidate;
1576
1577   gpu->base.sync_ops.create = virtgpu_sync_create;
1578   gpu->base.sync_ops.create_from_syncobj = virtgpu_sync_create_from_syncobj;
1579   gpu->base.sync_ops.destroy = virtgpu_sync_destroy;
1580   gpu->base.sync_ops.export_syncobj = virtgpu_sync_export_syncobj;
1581   gpu->base.sync_ops.reset = virtgpu_sync_reset;
1582   gpu->base.sync_ops.read = virtgpu_sync_read;
1583   gpu->base.sync_ops.write = virtgpu_sync_write;
1584
1585   return VK_SUCCESS;
1586}
1587
1588VkResult
1589vn_renderer_create_virtgpu(struct vn_instance *instance,
1590                           const VkAllocationCallbacks *alloc,
1591                           struct vn_renderer **renderer)
1592{
1593   struct virtgpu *gpu = vk_zalloc(alloc, sizeof(*gpu), VN_DEFAULT_ALIGN,
1594                                   VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
1595   if (!gpu)
1596      return VK_ERROR_OUT_OF_HOST_MEMORY;
1597
1598   gpu->instance = instance;
1599   gpu->fd = -1;
1600
1601   VkResult result = virtgpu_init(gpu);
1602   if (result != VK_SUCCESS) {
1603      virtgpu_destroy(&gpu->base, alloc);
1604      return result;
1605   }
1606
1607   *renderer = &gpu->base;
1608
1609   return VK_SUCCESS;
1610}
1611