1/*
2 * Copyright © 2019 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#include <string.h>
25#include <stdlib.h>
26#include <assert.h>
27#include <stdio.h>
28
29#include <vulkan/vulkan.h>
30#include <vulkan/vk_layer.h>
31
32#include "util/debug.h"
33#include "util/hash_table.h"
34#include "util/macros.h"
35#include "util/simple_mtx.h"
36
37#include "vk_dispatch_table.h"
38#include "vk_enum_to_str.h"
39#include "vk_util.h"
40
41struct instance_data {
42   struct vk_instance_dispatch_table vtable;
43   VkInstance instance;
44};
45
46struct device_data {
47   struct instance_data *instance;
48
49   PFN_vkSetDeviceLoaderData set_device_loader_data;
50
51   struct vk_device_dispatch_table vtable;
52   VkPhysicalDevice physical_device;
53   VkDevice device;
54};
55
56static struct hash_table_u64 *vk_object_to_data = NULL;
57static simple_mtx_t vk_object_to_data_mutex = _SIMPLE_MTX_INITIALIZER_NP;
58
59static inline void ensure_vk_object_map(void)
60{
61   if (!vk_object_to_data)
62      vk_object_to_data = _mesa_hash_table_u64_create(NULL);
63}
64
65#define HKEY(obj) ((uint64_t)(obj))
66#define FIND(type, obj) ((type *)find_object_data(HKEY(obj)))
67
68static void *find_object_data(uint64_t obj)
69{
70   simple_mtx_lock(&vk_object_to_data_mutex);
71   ensure_vk_object_map();
72   void *data = _mesa_hash_table_u64_search(vk_object_to_data, obj);
73   simple_mtx_unlock(&vk_object_to_data_mutex);
74   return data;
75}
76
77static void map_object(uint64_t obj, void *data)
78{
79   simple_mtx_lock(&vk_object_to_data_mutex);
80   ensure_vk_object_map();
81   _mesa_hash_table_u64_insert(vk_object_to_data, obj, data);
82   simple_mtx_unlock(&vk_object_to_data_mutex);
83}
84
85static void unmap_object(uint64_t obj)
86{
87   simple_mtx_lock(&vk_object_to_data_mutex);
88   _mesa_hash_table_u64_remove(vk_object_to_data, obj);
89   simple_mtx_unlock(&vk_object_to_data_mutex);
90}
91
92/**/
93
94#define VK_CHECK(expr) \
95   do { \
96      VkResult __result = (expr); \
97      if (__result != VK_SUCCESS) { \
98         fprintf(stderr, "'%s' line %i failed with %s\n", \
99                 #expr, __LINE__, vk_Result_to_str(__result)); \
100      } \
101   } while (0)
102
103/**/
104
105static void override_queue(struct device_data *device_data,
106                           VkDevice device,
107                           uint32_t queue_family_index,
108                           VkQueue queue)
109{
110   VkCommandPoolCreateInfo cmd_buffer_pool_info = {
111      .sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
112      .queueFamilyIndex = queue_family_index,
113   };
114   VkCommandPool cmd_pool;
115   VK_CHECK(device_data->vtable.CreateCommandPool(device,
116                                                  &cmd_buffer_pool_info,
117                                                  NULL, &cmd_pool));
118
119
120   VkCommandBufferAllocateInfo cmd_buffer_info = {
121      .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
122      .commandPool = cmd_pool,
123      .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY,
124      .commandBufferCount = 1,
125   };
126   VkCommandBuffer cmd_buffer;
127   VK_CHECK(device_data->vtable.AllocateCommandBuffers(device,
128                                                       &cmd_buffer_info,
129                                                       &cmd_buffer));
130   VK_CHECK(device_data->set_device_loader_data(device, cmd_buffer));
131
132   VkCommandBufferBeginInfo buffer_begin_info = {
133      .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
134   };
135   device_data->vtable.BeginCommandBuffer(cmd_buffer, &buffer_begin_info);
136
137   VkPerformanceOverrideInfoINTEL override_info = {
138      .sType = VK_STRUCTURE_TYPE_PERFORMANCE_OVERRIDE_INFO_INTEL,
139      .type = VK_PERFORMANCE_OVERRIDE_TYPE_NULL_HARDWARE_INTEL,
140      .enable = VK_TRUE,
141   };
142   device_data->vtable.CmdSetPerformanceOverrideINTEL(cmd_buffer, &override_info);
143
144   device_data->vtable.EndCommandBuffer(cmd_buffer);
145
146   VkSubmitInfo submit_info = {
147      .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
148      .commandBufferCount = 1,
149      .pCommandBuffers = &cmd_buffer,
150   };
151   VK_CHECK(device_data->vtable.QueueSubmit(queue, 1, &submit_info, VK_NULL_HANDLE));
152
153   VK_CHECK(device_data->vtable.QueueWaitIdle(queue));
154
155   device_data->vtable.DestroyCommandPool(device, cmd_pool, NULL);
156}
157
158static void device_override_queues(struct device_data *device_data,
159                                   const VkDeviceCreateInfo *pCreateInfo)
160{
161   for (uint32_t i = 0; i < pCreateInfo->queueCreateInfoCount; i++) {
162      for (uint32_t j = 0; j < pCreateInfo->pQueueCreateInfos[i].queueCount; j++) {
163         VkQueue queue;
164         device_data->vtable.GetDeviceQueue(device_data->device,
165                                            pCreateInfo->pQueueCreateInfos[i].queueFamilyIndex,
166                                            j, &queue);
167
168         VK_CHECK(device_data->set_device_loader_data(device_data->device, queue));
169
170         override_queue(device_data, device_data->device,
171                        pCreateInfo->pQueueCreateInfos[i].queueFamilyIndex, queue);
172      }
173   }
174}
175
176static VkLayerDeviceCreateInfo *get_device_chain_info(const VkDeviceCreateInfo *pCreateInfo,
177                                                      VkLayerFunction func)
178{
179   vk_foreach_struct(item, pCreateInfo->pNext) {
180      if (item->sType == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO &&
181          ((VkLayerDeviceCreateInfo *) item)->function == func)
182         return (VkLayerDeviceCreateInfo *)item;
183   }
184   unreachable("device chain info not found");
185   return NULL;
186}
187
188static struct device_data *new_device_data(VkDevice device, struct instance_data *instance)
189{
190   struct device_data *data = calloc(1, sizeof(*data));
191   data->instance = instance;
192   data->device = device;
193   map_object(HKEY(data->device), data);
194   return data;
195}
196
197static void destroy_device_data(struct device_data *data)
198{
199   unmap_object(HKEY(data->device));
200   free(data);
201}
202
203static VkResult nullhw_CreateDevice(
204    VkPhysicalDevice                            physicalDevice,
205    const VkDeviceCreateInfo*                   pCreateInfo,
206    const VkAllocationCallbacks*                pAllocator,
207    VkDevice*                                   pDevice)
208{
209   VkLayerDeviceCreateInfo *chain_info =
210      get_device_chain_info(pCreateInfo, VK_LAYER_LINK_INFO);
211
212   assert(chain_info->u.pLayerInfo);
213   PFN_vkGetInstanceProcAddr fpGetInstanceProcAddr = chain_info->u.pLayerInfo->pfnNextGetInstanceProcAddr;
214   PFN_vkGetDeviceProcAddr fpGetDeviceProcAddr = chain_info->u.pLayerInfo->pfnNextGetDeviceProcAddr;
215   PFN_vkCreateDevice fpCreateDevice = (PFN_vkCreateDevice)fpGetInstanceProcAddr(NULL, "vkCreateDevice");
216   if (fpCreateDevice == NULL) {
217      return VK_ERROR_INITIALIZATION_FAILED;
218   }
219
220   // Advance the link info for the next element on the chain
221   chain_info->u.pLayerInfo = chain_info->u.pLayerInfo->pNext;
222
223   VkDeviceCreateInfo device_info = *pCreateInfo;
224   const char **extensions = calloc(device_info.enabledExtensionCount + 1, sizeof(*extensions));
225   bool found = false;
226   for (uint32_t i = 0; i < device_info.enabledExtensionCount; i++) {
227      if (!strcmp(device_info.ppEnabledExtensionNames[i], "VK_INTEL_performance_query")) {
228         found = true;
229         break;
230      }
231   }
232   if (!found) {
233      memcpy(extensions, device_info.ppEnabledExtensionNames,
234             sizeof(*extensions) * device_info.enabledExtensionCount);
235      extensions[device_info.enabledExtensionCount++] = "VK_INTEL_performance_query";
236      device_info.ppEnabledExtensionNames = extensions;
237   }
238
239   VkResult result = fpCreateDevice(physicalDevice, &device_info, pAllocator, pDevice);
240   free(extensions);
241   if (result != VK_SUCCESS) return result;
242
243   struct instance_data *instance_data = FIND(struct instance_data, physicalDevice);
244   struct device_data *device_data = new_device_data(*pDevice, instance_data);
245   device_data->physical_device = physicalDevice;
246   vk_device_dispatch_table_load(&device_data->vtable, fpGetDeviceProcAddr, *pDevice);
247
248   VkLayerDeviceCreateInfo *load_data_info =
249      get_device_chain_info(pCreateInfo, VK_LOADER_DATA_CALLBACK);
250   device_data->set_device_loader_data = load_data_info->u.pfnSetDeviceLoaderData;
251
252   device_override_queues(device_data, pCreateInfo);
253
254   return result;
255}
256
257static void nullhw_DestroyDevice(
258    VkDevice                                    device,
259    const VkAllocationCallbacks*                pAllocator)
260{
261   struct device_data *device_data = FIND(struct device_data, device);
262   device_data->vtable.DestroyDevice(device, pAllocator);
263   destroy_device_data(device_data);
264}
265
266static struct instance_data *new_instance_data(VkInstance instance)
267{
268   struct instance_data *data = calloc(1, sizeof(*data));
269   data->instance = instance;
270   map_object(HKEY(data->instance), data);
271   return data;
272}
273
274static void destroy_instance_data(struct instance_data *data)
275{
276   unmap_object(HKEY(data->instance));
277   free(data);
278}
279
280static VkLayerInstanceCreateInfo *get_instance_chain_info(const VkInstanceCreateInfo *pCreateInfo,
281                                                          VkLayerFunction func)
282{
283   vk_foreach_struct(item, pCreateInfo->pNext) {
284      if (item->sType == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO &&
285          ((VkLayerInstanceCreateInfo *) item)->function == func)
286         return (VkLayerInstanceCreateInfo *) item;
287   }
288   unreachable("instance chain info not found");
289   return NULL;
290}
291
292static VkResult nullhw_CreateInstance(
293    const VkInstanceCreateInfo*                 pCreateInfo,
294    const VkAllocationCallbacks*                pAllocator,
295    VkInstance*                                 pInstance)
296{
297   VkLayerInstanceCreateInfo *chain_info =
298      get_instance_chain_info(pCreateInfo, VK_LAYER_LINK_INFO);
299
300   assert(chain_info->u.pLayerInfo);
301   PFN_vkGetInstanceProcAddr fpGetInstanceProcAddr =
302      chain_info->u.pLayerInfo->pfnNextGetInstanceProcAddr;
303   PFN_vkCreateInstance fpCreateInstance =
304      (PFN_vkCreateInstance)fpGetInstanceProcAddr(NULL, "vkCreateInstance");
305   if (fpCreateInstance == NULL) {
306      return VK_ERROR_INITIALIZATION_FAILED;
307   }
308
309   // Advance the link info for the next element on the chain
310   chain_info->u.pLayerInfo = chain_info->u.pLayerInfo->pNext;
311
312   VkResult result = fpCreateInstance(pCreateInfo, pAllocator, pInstance);
313   if (result != VK_SUCCESS) return result;
314
315   struct instance_data *instance_data = new_instance_data(*pInstance);
316   vk_instance_dispatch_table_load(&instance_data->vtable,
317                                   fpGetInstanceProcAddr,
318                                   instance_data->instance);
319
320   return result;
321}
322
323static void nullhw_DestroyInstance(
324    VkInstance                                  instance,
325    const VkAllocationCallbacks*                pAllocator)
326{
327   struct instance_data *instance_data = FIND(struct instance_data, instance);
328   instance_data->vtable.DestroyInstance(instance, pAllocator);
329   destroy_instance_data(instance_data);
330}
331
332static const struct {
333   const char *name;
334   void *ptr;
335} name_to_funcptr_map[] = {
336   { "vkGetDeviceProcAddr", (void *) vkGetDeviceProcAddr },
337#define ADD_HOOK(fn) { "vk" # fn, (void *) nullhw_ ## fn }
338   ADD_HOOK(CreateInstance),
339   ADD_HOOK(DestroyInstance),
340   ADD_HOOK(CreateDevice),
341   ADD_HOOK(DestroyDevice),
342};
343
344static void *find_ptr(const char *name)
345{
346   for (uint32_t i = 0; i < ARRAY_SIZE(name_to_funcptr_map); i++) {
347      if (strcmp(name, name_to_funcptr_map[i].name) == 0)
348         return name_to_funcptr_map[i].ptr;
349   }
350
351   return NULL;
352}
353
354VK_LAYER_EXPORT VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetDeviceProcAddr(VkDevice dev,
355                                                                             const char *funcName)
356{
357   void *ptr = find_ptr(funcName);
358   if (ptr) return (PFN_vkVoidFunction)(ptr);
359
360   if (dev == NULL) return NULL;
361
362   struct device_data *device_data = FIND(struct device_data, dev);
363   if (device_data->vtable.GetDeviceProcAddr == NULL) return NULL;
364   return device_data->vtable.GetDeviceProcAddr(dev, funcName);
365}
366
367VK_LAYER_EXPORT VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,
368                                                                               const char *funcName)
369{
370   void *ptr = find_ptr(funcName);
371   if (ptr) return (PFN_vkVoidFunction) ptr;
372
373   struct instance_data *instance_data = FIND(struct instance_data, instance);
374   if (instance_data->vtable.GetInstanceProcAddr == NULL) return NULL;
375   return instance_data->vtable.GetInstanceProcAddr(instance, funcName);
376}
377