1/*
2 * Copyright © 2020 Raspberry Pi
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#include "v3dv_private.h"
25
26VKAPI_ATTR VkResult VKAPI_CALL
27v3dv_CreateQueryPool(VkDevice _device,
28                     const VkQueryPoolCreateInfo *pCreateInfo,
29                     const VkAllocationCallbacks *pAllocator,
30                     VkQueryPool *pQueryPool)
31{
32   V3DV_FROM_HANDLE(v3dv_device, device, _device);
33
34   assert(pCreateInfo->queryType == VK_QUERY_TYPE_OCCLUSION ||
35          pCreateInfo->queryType == VK_QUERY_TYPE_TIMESTAMP);
36   assert(pCreateInfo->queryCount > 0);
37
38   struct v3dv_query_pool *pool =
39      vk_object_zalloc(&device->vk, pAllocator, sizeof(*pool),
40                       VK_OBJECT_TYPE_QUERY_POOL);
41   if (pool == NULL)
42      return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
43
44   pool->query_type = pCreateInfo->queryType;
45   pool->query_count = pCreateInfo->queryCount;
46
47   VkResult result;
48
49   const uint32_t pool_bytes = sizeof(struct v3dv_query) * pool->query_count;
50   pool->queries = vk_alloc2(&device->vk.alloc, pAllocator, pool_bytes, 8,
51                             VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
52   if (pool->queries == NULL) {
53      result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
54      goto fail;
55   }
56
57   if (pool->query_type == VK_QUERY_TYPE_OCCLUSION) {
58      /* The hardware allows us to setup groups of 16 queries in consecutive
59       * 4-byte addresses, requiring only that each group of 16 queries is
60       * aligned to a 1024 byte boundary.
61       */
62      const uint32_t query_groups = DIV_ROUND_UP(pool->query_count, 16);
63      const uint32_t bo_size = query_groups * 1024;
64      pool->bo = v3dv_bo_alloc(device, bo_size, "query", true);
65      if (!pool->bo) {
66         result = vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
67         goto fail;
68      }
69      if (!v3dv_bo_map(device, pool->bo, bo_size)) {
70         result = vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
71         goto fail;
72      }
73   }
74
75   uint32_t i;
76   for (i = 0; i < pool->query_count; i++) {
77      pool->queries[i].maybe_available = false;
78      switch (pool->query_type) {
79      case VK_QUERY_TYPE_OCCLUSION: {
80         const uint32_t query_group = i / 16;
81         const uint32_t query_offset = query_group * 1024 + (i % 16) * 4;
82         pool->queries[i].bo = pool->bo;
83         pool->queries[i].offset = query_offset;
84         break;
85         }
86      case VK_QUERY_TYPE_TIMESTAMP:
87         pool->queries[i].value = 0;
88         break;
89      default:
90         unreachable("Unsupported query type");
91      }
92   }
93
94   *pQueryPool = v3dv_query_pool_to_handle(pool);
95
96   return VK_SUCCESS;
97
98fail:
99   if (pool->bo)
100      v3dv_bo_free(device, pool->bo);
101   if (pool->queries)
102      vk_free2(&device->vk.alloc, pAllocator, pool->queries);
103   vk_object_free(&device->vk, pAllocator, pool);
104
105   return result;
106}
107
108VKAPI_ATTR void VKAPI_CALL
109v3dv_DestroyQueryPool(VkDevice _device,
110                      VkQueryPool queryPool,
111                      const VkAllocationCallbacks *pAllocator)
112{
113   V3DV_FROM_HANDLE(v3dv_device, device, _device);
114   V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool);
115
116   if (!pool)
117      return;
118
119   if (pool->bo)
120      v3dv_bo_free(device, pool->bo);
121
122   if (pool->queries)
123      vk_free2(&device->vk.alloc, pAllocator, pool->queries);
124
125   vk_object_free(&device->vk, pAllocator, pool);
126}
127
128static void
129write_query_result(void *dst, uint32_t idx, bool do_64bit, uint64_t value)
130{
131   if (do_64bit) {
132      uint64_t *dst64 = (uint64_t *) dst;
133      dst64[idx] = value;
134   } else {
135      uint32_t *dst32 = (uint32_t *) dst;
136      dst32[idx] = (uint32_t) value;
137   }
138}
139
140static VkResult
141get_occlusion_query_result(struct v3dv_device *device,
142                           struct v3dv_query_pool *pool,
143                           uint32_t query,
144                           bool do_wait,
145                           bool *available,
146                           uint64_t *value)
147{
148   assert(pool && pool->query_type == VK_QUERY_TYPE_OCCLUSION);
149
150   struct v3dv_query *q = &pool->queries[query];
151   assert(q->bo && q->bo->map);
152
153   if (do_wait) {
154      /* From the Vulkan 1.0 spec:
155       *
156       *    "If VK_QUERY_RESULT_WAIT_BIT is set, (...) If the query does not
157       *     become available in a finite amount of time (e.g. due to not
158       *     issuing a query since the last reset), a VK_ERROR_DEVICE_LOST
159       *     error may occur."
160       */
161      if (!q->maybe_available)
162         return vk_error(device, VK_ERROR_DEVICE_LOST);
163
164      if (!v3dv_bo_wait(device, q->bo, 0xffffffffffffffffull))
165         return vk_error(device, VK_ERROR_DEVICE_LOST);
166
167      *available = true;
168   } else {
169      *available = q->maybe_available && v3dv_bo_wait(device, q->bo, 0);
170   }
171
172   const uint8_t *query_addr = ((uint8_t *) q->bo->map) + q->offset;
173   *value = (uint64_t) *((uint32_t *)query_addr);
174   return VK_SUCCESS;
175}
176
177static VkResult
178get_timestamp_query_result(struct v3dv_device *device,
179                           struct v3dv_query_pool *pool,
180                           uint32_t query,
181                           bool do_wait,
182                           bool *available,
183                           uint64_t *value)
184{
185   assert(pool && pool->query_type == VK_QUERY_TYPE_TIMESTAMP);
186
187   struct v3dv_query *q = &pool->queries[query];
188
189   if (do_wait) {
190      /* From the Vulkan 1.0 spec:
191       *
192       *    "If VK_QUERY_RESULT_WAIT_BIT is set, (...) If the query does not
193       *     become available in a finite amount of time (e.g. due to not
194       *     issuing a query since the last reset), a VK_ERROR_DEVICE_LOST
195       *     error may occur."
196       */
197      if (!q->maybe_available)
198         return vk_error(device, VK_ERROR_DEVICE_LOST);
199
200      *available = true;
201   } else {
202      *available = q->maybe_available;
203   }
204
205   *value = q->value;
206   return VK_SUCCESS;
207}
208
209static VkResult
210get_query_result(struct v3dv_device *device,
211                 struct v3dv_query_pool *pool,
212                 uint32_t query,
213                 bool do_wait,
214                 bool *available,
215                 uint64_t *value)
216{
217   switch (pool->query_type) {
218   case VK_QUERY_TYPE_OCCLUSION:
219      return get_occlusion_query_result(device, pool, query, do_wait,
220                                        available, value);
221   case VK_QUERY_TYPE_TIMESTAMP:
222      return get_timestamp_query_result(device, pool, query, do_wait,
223                                        available, value);
224   default:
225      unreachable("Unsupported query type");
226   }
227}
228
229VkResult
230v3dv_get_query_pool_results_cpu(struct v3dv_device *device,
231                                struct v3dv_query_pool *pool,
232                                uint32_t first,
233                                uint32_t count,
234                                void *data,
235                                VkDeviceSize stride,
236                                VkQueryResultFlags flags)
237{
238   assert(first < pool->query_count);
239   assert(first + count <= pool->query_count);
240   assert(data);
241
242   const bool do_64bit = flags & VK_QUERY_RESULT_64_BIT;
243   const bool do_wait = flags & VK_QUERY_RESULT_WAIT_BIT;
244   const bool do_partial = flags & VK_QUERY_RESULT_PARTIAL_BIT;
245
246   VkResult result = VK_SUCCESS;
247   for (uint32_t i = first; i < first + count; i++) {
248      bool available = false;
249      uint64_t value = 0;
250      VkResult query_result =
251         get_query_result(device, pool, i, do_wait, &available, &value);
252      if (query_result == VK_ERROR_DEVICE_LOST)
253         result = VK_ERROR_DEVICE_LOST;
254
255      /**
256       * From the Vulkan 1.0 spec:
257       *
258       *    "If VK_QUERY_RESULT_WAIT_BIT and VK_QUERY_RESULT_PARTIAL_BIT are
259       *     both not set then no result values are written to pData for queries
260       *     that are in the unavailable state at the time of the call, and
261       *     vkGetQueryPoolResults returns VK_NOT_READY. However, availability
262       *     state is still written to pData for those queries if
263       *     VK_QUERY_RESULT_WITH_AVAILABILITY_BIT is set."
264       */
265      uint32_t slot = 0;
266
267      const bool write_result = available || do_partial;
268      if (write_result)
269         write_query_result(data, slot, do_64bit, value);
270      slot++;
271
272      if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT)
273         write_query_result(data, slot++, do_64bit, available ? 1u : 0u);
274
275      if (!write_result && result != VK_ERROR_DEVICE_LOST)
276         result = VK_NOT_READY;
277
278      data += stride;
279   }
280
281   return result;
282}
283
284VKAPI_ATTR VkResult VKAPI_CALL
285v3dv_GetQueryPoolResults(VkDevice _device,
286                         VkQueryPool queryPool,
287                         uint32_t firstQuery,
288                         uint32_t queryCount,
289                         size_t dataSize,
290                         void *pData,
291                         VkDeviceSize stride,
292                         VkQueryResultFlags flags)
293{
294   V3DV_FROM_HANDLE(v3dv_device, device, _device);
295   V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool);
296
297   return v3dv_get_query_pool_results_cpu(device, pool, firstQuery, queryCount,
298                                          pData, stride, flags);
299}
300
301VKAPI_ATTR void VKAPI_CALL
302v3dv_CmdResetQueryPool(VkCommandBuffer commandBuffer,
303                       VkQueryPool queryPool,
304                       uint32_t firstQuery,
305                       uint32_t queryCount)
306{
307   V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer);
308   V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool);
309
310   v3dv_cmd_buffer_reset_queries(cmd_buffer, pool, firstQuery, queryCount);
311}
312
313VKAPI_ATTR void VKAPI_CALL
314v3dv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer,
315                             VkQueryPool queryPool,
316                             uint32_t firstQuery,
317                             uint32_t queryCount,
318                             VkBuffer dstBuffer,
319                             VkDeviceSize dstOffset,
320                             VkDeviceSize stride,
321                             VkQueryResultFlags flags)
322{
323   V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer);
324   V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool);
325   V3DV_FROM_HANDLE(v3dv_buffer, dst, dstBuffer);
326
327   v3dv_cmd_buffer_copy_query_results(cmd_buffer, pool,
328                                      firstQuery, queryCount,
329                                      dst, dstOffset, stride, flags);
330}
331
332VKAPI_ATTR void VKAPI_CALL
333v3dv_CmdBeginQuery(VkCommandBuffer commandBuffer,
334                   VkQueryPool queryPool,
335                   uint32_t query,
336                   VkQueryControlFlags flags)
337{
338   V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer);
339   V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool);
340
341   v3dv_cmd_buffer_begin_query(cmd_buffer, pool, query, flags);
342}
343
344VKAPI_ATTR void VKAPI_CALL
345v3dv_CmdEndQuery(VkCommandBuffer commandBuffer,
346                 VkQueryPool queryPool,
347                 uint32_t query)
348{
349   V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer);
350   V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool);
351
352   v3dv_cmd_buffer_end_query(cmd_buffer, pool, query);
353}
354