1/*
2 * Copyright 2019 Google LLC
3 * SPDX-License-Identifier: MIT
4 *
5 * based in part on anv and radv which are:
6 * Copyright © 2015 Intel Corporation
7 * Copyright © 2016 Red Hat.
8 * Copyright © 2016 Bas Nieuwenhuizen
9 */
10
11#include "vn_device.h"
12
13#include "venus-protocol/vn_protocol_driver_device.h"
14
15#include "vn_android.h"
16#include "vn_instance.h"
17#include "vn_physical_device.h"
18#include "vn_queue.h"
19
20/* device commands */
21
22static void
23vn_queue_fini(struct vn_queue *queue)
24{
25   if (queue->wait_fence != VK_NULL_HANDLE) {
26      vn_DestroyFence(vn_device_to_handle(queue->device), queue->wait_fence,
27                      NULL);
28   }
29   vn_object_base_fini(&queue->base);
30}
31
32static VkResult
33vn_queue_init(struct vn_device *dev,
34              struct vn_queue *queue,
35              const VkDeviceQueueCreateInfo *queue_info,
36              uint32_t queue_index)
37{
38   vn_object_base_init(&queue->base, VK_OBJECT_TYPE_QUEUE, &dev->base);
39
40   VkQueue queue_handle = vn_queue_to_handle(queue);
41   vn_async_vkGetDeviceQueue2(
42      dev->instance, vn_device_to_handle(dev),
43      &(VkDeviceQueueInfo2){
44         .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_INFO_2,
45         .flags = queue_info->flags,
46         .queueFamilyIndex = queue_info->queueFamilyIndex,
47         .queueIndex = queue_index,
48      },
49      &queue_handle);
50
51   queue->device = dev;
52   queue->family = queue_info->queueFamilyIndex;
53   queue->index = queue_index;
54   queue->flags = queue_info->flags;
55
56   const VkExportFenceCreateInfo export_fence_info = {
57      .sType = VK_STRUCTURE_TYPE_EXPORT_FENCE_CREATE_INFO,
58      .pNext = NULL,
59      .handleTypes = VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT,
60   };
61   const VkFenceCreateInfo fence_info = {
62      .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO,
63      .pNext = dev->instance->experimental.globalFencing == VK_TRUE
64                  ? &export_fence_info
65                  : NULL,
66      .flags = 0,
67   };
68   VkResult result = vn_CreateFence(vn_device_to_handle(dev), &fence_info,
69                                    NULL, &queue->wait_fence);
70   if (result != VK_SUCCESS)
71      return result;
72
73   return VK_SUCCESS;
74}
75
76static VkResult
77vn_device_init_queues(struct vn_device *dev,
78                      const VkDeviceCreateInfo *create_info)
79{
80   const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
81
82   uint32_t count = 0;
83   for (uint32_t i = 0; i < create_info->queueCreateInfoCount; i++)
84      count += create_info->pQueueCreateInfos[i].queueCount;
85
86   struct vn_queue *queues =
87      vk_zalloc(alloc, sizeof(*queues) * count, VN_DEFAULT_ALIGN,
88                VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
89   if (!queues)
90      return VK_ERROR_OUT_OF_HOST_MEMORY;
91
92   VkResult result = VK_SUCCESS;
93   count = 0;
94   for (uint32_t i = 0; i < create_info->queueCreateInfoCount; i++) {
95      const VkDeviceQueueCreateInfo *queue_info =
96         &create_info->pQueueCreateInfos[i];
97      for (uint32_t j = 0; j < queue_info->queueCount; j++) {
98         result = vn_queue_init(dev, &queues[count], queue_info, j);
99         if (result != VK_SUCCESS)
100            break;
101
102         count++;
103      }
104   }
105
106   if (result != VK_SUCCESS) {
107      for (uint32_t i = 0; i < count; i++)
108         vn_queue_fini(&queues[i]);
109      vk_free(alloc, queues);
110
111      return result;
112   }
113
114   dev->queues = queues;
115   dev->queue_count = count;
116
117   return VK_SUCCESS;
118}
119
120static bool
121find_extension_names(const char *const *exts,
122                     uint32_t ext_count,
123                     const char *name)
124{
125   for (uint32_t i = 0; i < ext_count; i++) {
126      if (!strcmp(exts[i], name))
127         return true;
128   }
129   return false;
130}
131
132static bool
133merge_extension_names(const char *const *exts,
134                      uint32_t ext_count,
135                      const char *const *extra_exts,
136                      uint32_t extra_count,
137                      const char *const *block_exts,
138                      uint32_t block_count,
139                      const VkAllocationCallbacks *alloc,
140                      const char *const **out_exts,
141                      uint32_t *out_count)
142{
143   const char **merged =
144      vk_alloc(alloc, sizeof(*merged) * (ext_count + extra_count),
145               VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
146   if (!merged)
147      return false;
148
149   uint32_t count = 0;
150   for (uint32_t i = 0; i < ext_count; i++) {
151      if (!find_extension_names(block_exts, block_count, exts[i]))
152         merged[count++] = exts[i];
153   }
154   for (uint32_t i = 0; i < extra_count; i++) {
155      if (!find_extension_names(exts, ext_count, extra_exts[i]))
156         merged[count++] = extra_exts[i];
157   }
158
159   *out_exts = merged;
160   *out_count = count;
161   return true;
162}
163
164static const VkDeviceCreateInfo *
165vn_device_fix_create_info(const struct vn_device *dev,
166                          const VkDeviceCreateInfo *dev_info,
167                          const VkAllocationCallbacks *alloc,
168                          VkDeviceCreateInfo *local_info)
169{
170   const struct vn_physical_device *physical_dev = dev->physical_device;
171   const struct vk_device_extension_table *app_exts =
172      &dev->base.base.enabled_extensions;
173   /* extra_exts and block_exts must not overlap */
174   const char *extra_exts[16];
175   const char *block_exts[16];
176   uint32_t extra_count = 0;
177   uint32_t block_count = 0;
178
179   /* fix for WSI (treat AHB as WSI extension for simplicity) */
180   const bool has_wsi =
181      app_exts->KHR_swapchain || app_exts->ANDROID_native_buffer ||
182      app_exts->ANDROID_external_memory_android_hardware_buffer;
183   if (has_wsi) {
184      /* KHR_swapchain may be advertised without the renderer support for
185       * EXT_image_drm_format_modifier
186       */
187      if (!app_exts->EXT_image_drm_format_modifier &&
188          physical_dev->renderer_extensions.EXT_image_drm_format_modifier) {
189         extra_exts[extra_count++] =
190            VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME;
191
192         if (physical_dev->renderer_version < VK_API_VERSION_1_2 &&
193             !app_exts->KHR_image_format_list) {
194            extra_exts[extra_count++] =
195               VK_KHR_IMAGE_FORMAT_LIST_EXTENSION_NAME;
196         }
197      }
198
199      /* XXX KHR_swapchain may be advertised without the renderer support for
200       * EXT_queue_family_foreign
201       */
202      if (!app_exts->EXT_queue_family_foreign &&
203          physical_dev->renderer_extensions.EXT_queue_family_foreign) {
204         extra_exts[extra_count++] =
205            VK_EXT_QUEUE_FAMILY_FOREIGN_EXTENSION_NAME;
206      }
207
208      if (app_exts->KHR_swapchain) {
209         /* see vn_physical_device_get_native_extensions */
210         block_exts[block_count++] = VK_KHR_SWAPCHAIN_EXTENSION_NAME;
211         block_exts[block_count++] =
212            VK_KHR_SWAPCHAIN_MUTABLE_FORMAT_EXTENSION_NAME;
213         block_exts[block_count++] =
214            VK_KHR_INCREMENTAL_PRESENT_EXTENSION_NAME;
215      }
216
217      if (app_exts->ANDROID_native_buffer)
218         block_exts[block_count++] = VK_ANDROID_NATIVE_BUFFER_EXTENSION_NAME;
219
220      if (app_exts->ANDROID_external_memory_android_hardware_buffer) {
221         block_exts[block_count++] =
222            VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME;
223      }
224   }
225
226   if (app_exts->KHR_external_memory_fd ||
227       app_exts->EXT_external_memory_dma_buf || has_wsi) {
228      switch (physical_dev->external_memory.renderer_handle_type) {
229      case VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT:
230         if (!app_exts->EXT_external_memory_dma_buf) {
231            extra_exts[extra_count++] =
232               VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME;
233         }
234         FALLTHROUGH;
235      case VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT:
236         if (!app_exts->KHR_external_memory_fd) {
237            extra_exts[extra_count++] =
238               VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME;
239         }
240         break;
241      default:
242         /* TODO other handle types */
243         break;
244      }
245   }
246
247   assert(extra_count <= ARRAY_SIZE(extra_exts));
248   assert(block_count <= ARRAY_SIZE(block_exts));
249
250   if (!extra_count && (!block_count || !dev_info->enabledExtensionCount))
251      return dev_info;
252
253   *local_info = *dev_info;
254   if (!merge_extension_names(dev_info->ppEnabledExtensionNames,
255                              dev_info->enabledExtensionCount, extra_exts,
256                              extra_count, block_exts, block_count, alloc,
257                              &local_info->ppEnabledExtensionNames,
258                              &local_info->enabledExtensionCount))
259      return NULL;
260
261   return local_info;
262}
263
264VkResult
265vn_CreateDevice(VkPhysicalDevice physicalDevice,
266                const VkDeviceCreateInfo *pCreateInfo,
267                const VkAllocationCallbacks *pAllocator,
268                VkDevice *pDevice)
269{
270   struct vn_physical_device *physical_dev =
271      vn_physical_device_from_handle(physicalDevice);
272   struct vn_instance *instance = physical_dev->instance;
273   const VkAllocationCallbacks *alloc =
274      pAllocator ? pAllocator : &instance->base.base.alloc;
275   struct vn_device *dev;
276   VkResult result;
277
278   dev = vk_zalloc(alloc, sizeof(*dev), VN_DEFAULT_ALIGN,
279                   VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
280   if (!dev)
281      return vn_error(instance, VK_ERROR_OUT_OF_HOST_MEMORY);
282
283   struct vk_device_dispatch_table dispatch_table;
284   vk_device_dispatch_table_from_entrypoints(&dispatch_table,
285                                             &vn_device_entrypoints, true);
286   vk_device_dispatch_table_from_entrypoints(&dispatch_table,
287                                             &wsi_device_entrypoints, false);
288   result = vn_device_base_init(&dev->base, &physical_dev->base,
289                                &dispatch_table, pCreateInfo, alloc);
290   if (result != VK_SUCCESS) {
291      vk_free(alloc, dev);
292      return vn_error(instance, result);
293   }
294
295   dev->instance = instance;
296   dev->physical_device = physical_dev;
297   dev->renderer = instance->renderer;
298
299   VkDeviceCreateInfo local_create_info;
300   pCreateInfo =
301      vn_device_fix_create_info(dev, pCreateInfo, alloc, &local_create_info);
302   if (!pCreateInfo) {
303      result = VK_ERROR_OUT_OF_HOST_MEMORY;
304      goto fail;
305   }
306
307   VkDevice dev_handle = vn_device_to_handle(dev);
308   result = vn_call_vkCreateDevice(instance, physicalDevice, pCreateInfo,
309                                   NULL, &dev_handle);
310   if (result != VK_SUCCESS)
311      goto fail;
312
313   result = vn_device_init_queues(dev, pCreateInfo);
314   if (result != VK_SUCCESS) {
315      vn_call_vkDestroyDevice(instance, dev_handle, NULL);
316      goto fail;
317   }
318
319   for (uint32_t i = 0; i < ARRAY_SIZE(dev->memory_pools); i++) {
320      struct vn_device_memory_pool *pool = &dev->memory_pools[i];
321      mtx_init(&pool->mutex, mtx_plain);
322   }
323
324   if (dev->base.base.enabled_extensions
325          .ANDROID_external_memory_android_hardware_buffer) {
326      result = vn_android_init_ahb_buffer_memory_type_bits(dev);
327      if (result != VK_SUCCESS) {
328         vn_call_vkDestroyDevice(instance, dev_handle, NULL);
329         goto fail;
330      }
331   }
332
333   *pDevice = dev_handle;
334
335   if (pCreateInfo == &local_create_info)
336      vk_free(alloc, (void *)pCreateInfo->ppEnabledExtensionNames);
337
338   return VK_SUCCESS;
339
340fail:
341   if (pCreateInfo == &local_create_info)
342      vk_free(alloc, (void *)pCreateInfo->ppEnabledExtensionNames);
343   vn_device_base_fini(&dev->base);
344   vk_free(alloc, dev);
345   return vn_error(instance, result);
346}
347
348void
349vn_DestroyDevice(VkDevice device, const VkAllocationCallbacks *pAllocator)
350{
351   struct vn_device *dev = vn_device_from_handle(device);
352   const VkAllocationCallbacks *alloc =
353      pAllocator ? pAllocator : &dev->base.base.alloc;
354
355   if (!dev)
356      return;
357
358   for (uint32_t i = 0; i < ARRAY_SIZE(dev->memory_pools); i++)
359      vn_device_memory_pool_fini(dev, i);
360
361   for (uint32_t i = 0; i < dev->queue_count; i++)
362      vn_queue_fini(&dev->queues[i]);
363
364   /* We must emit vkDestroyDevice before freeing dev->queues.  Otherwise,
365    * another thread might reuse their object ids while they still refer to
366    * the queues in the renderer.
367    */
368   vn_async_vkDestroyDevice(dev->instance, device, NULL);
369
370   vk_free(alloc, dev->queues);
371
372   vn_device_base_fini(&dev->base);
373   vk_free(alloc, dev);
374}
375
376PFN_vkVoidFunction
377vn_GetDeviceProcAddr(VkDevice device, const char *pName)
378{
379   struct vn_device *dev = vn_device_from_handle(device);
380   return vk_device_get_proc_addr(&dev->base.base, pName);
381}
382
383void
384vn_GetDeviceGroupPeerMemoryFeatures(
385   VkDevice device,
386   uint32_t heapIndex,
387   uint32_t localDeviceIndex,
388   uint32_t remoteDeviceIndex,
389   VkPeerMemoryFeatureFlags *pPeerMemoryFeatures)
390{
391   struct vn_device *dev = vn_device_from_handle(device);
392
393   /* TODO get and cache the values in vkCreateDevice */
394   vn_call_vkGetDeviceGroupPeerMemoryFeatures(
395      dev->instance, device, heapIndex, localDeviceIndex, remoteDeviceIndex,
396      pPeerMemoryFeatures);
397}
398
399VkResult
400vn_DeviceWaitIdle(VkDevice device)
401{
402   struct vn_device *dev = vn_device_from_handle(device);
403
404   for (uint32_t i = 0; i < dev->queue_count; i++) {
405      struct vn_queue *queue = &dev->queues[i];
406      VkResult result = vn_QueueWaitIdle(vn_queue_to_handle(queue));
407      if (result != VK_SUCCESS)
408         return vn_error(dev->instance, result);
409   }
410
411   return VK_SUCCESS;
412}
413