1/*
2 * Copyright © 2021 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 *    Mike Blumenkrantz <michael.blumenkrantz@gmail.com>
25 */
26
27#ifndef ZINK_BO_H
28#define ZINK_BO_H
29#include <vulkan/vulkan.h>
30#include "pipebuffer/pb_cache.h"
31#include "pipebuffer/pb_slab.h"
32#include "zink_batch.h"
33
34#define VK_VIS_VRAM (VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
35#define VK_LAZY_VRAM (VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
36enum zink_resource_access {
37   ZINK_RESOURCE_ACCESS_READ = 1,
38   ZINK_RESOURCE_ACCESS_WRITE = 32,
39   ZINK_RESOURCE_ACCESS_RW = ZINK_RESOURCE_ACCESS_READ | ZINK_RESOURCE_ACCESS_WRITE,
40};
41
42
43enum zink_heap {
44   ZINK_HEAP_DEVICE_LOCAL,
45   ZINK_HEAP_DEVICE_LOCAL_SPARSE,
46   ZINK_HEAP_DEVICE_LOCAL_LAZY,
47   ZINK_HEAP_DEVICE_LOCAL_VISIBLE,
48   ZINK_HEAP_HOST_VISIBLE_COHERENT,
49   ZINK_HEAP_HOST_VISIBLE_CACHED,
50   ZINK_HEAP_MAX,
51};
52
53enum zink_alloc_flag {
54   ZINK_ALLOC_SPARSE = 1<<0,
55   ZINK_ALLOC_NO_SUBALLOC = 1<<1,
56};
57
58
59struct zink_bo {
60   struct pb_buffer base;
61
62   union {
63      struct {
64         void *cpu_ptr; /* for user_ptr and permanent maps */
65         int map_count;
66
67         bool is_user_ptr;
68         bool use_reusable_pool;
69
70         /* Whether buffer_get_handle or buffer_from_handle has been called,
71          * it can only transition from false to true. Protected by lock.
72          */
73         bool is_shared;
74      } real;
75      struct {
76         struct pb_slab_entry entry;
77         struct zink_bo *real;
78      } slab;
79      struct {
80         uint32_t num_va_pages;
81         uint32_t num_backing_pages;
82
83         struct list_head backing;
84
85         /* Commitment information for each page of the virtual memory area. */
86         struct zink_sparse_commitment *commitments;
87      } sparse;
88   } u;
89
90   VkDeviceMemory mem;
91   uint64_t offset;
92
93   uint32_t unique_id;
94
95   simple_mtx_t lock;
96
97   struct zink_batch_usage *reads;
98   struct zink_batch_usage *writes;
99
100   struct pb_cache_entry cache_entry[];
101};
102
103static inline struct zink_bo *
104zink_bo(struct pb_buffer *pbuf)
105{
106   return (struct zink_bo*)pbuf;
107}
108
109static inline enum zink_alloc_flag
110zink_alloc_flags_from_heap(enum zink_heap heap)
111{
112   enum zink_alloc_flag flags = 0;
113   switch (heap) {
114   case ZINK_HEAP_DEVICE_LOCAL_SPARSE:
115      flags |= ZINK_ALLOC_SPARSE;
116      break;
117   default:
118      break;
119   }
120   return flags;
121}
122
123static inline VkMemoryPropertyFlags
124vk_domain_from_heap(enum zink_heap heap)
125{
126   VkMemoryPropertyFlags domains = 0;
127
128   switch (heap) {
129   case ZINK_HEAP_DEVICE_LOCAL:
130   case ZINK_HEAP_DEVICE_LOCAL_SPARSE:
131      domains = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
132      break;
133   case ZINK_HEAP_DEVICE_LOCAL_LAZY:
134      domains = VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
135      break;
136   case ZINK_HEAP_DEVICE_LOCAL_VISIBLE:
137      domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
138      break;
139   case ZINK_HEAP_HOST_VISIBLE_COHERENT:
140      domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
141      break;
142   case ZINK_HEAP_HOST_VISIBLE_CACHED:
143      domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
144      break;
145   default:
146      break;
147   }
148   return domains;
149}
150
151static inline enum zink_heap
152zink_heap_from_domain_flags(VkMemoryPropertyFlags domains, enum zink_alloc_flag flags)
153{
154   if (flags & ZINK_ALLOC_SPARSE)
155      return ZINK_HEAP_DEVICE_LOCAL_SPARSE;
156
157   if ((domains & VK_VIS_VRAM) == VK_VIS_VRAM)
158      return ZINK_HEAP_DEVICE_LOCAL_VISIBLE;
159
160   if (domains & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
161      return ZINK_HEAP_DEVICE_LOCAL;
162
163   if (domains & VK_MEMORY_PROPERTY_HOST_CACHED_BIT)
164      return ZINK_HEAP_HOST_VISIBLE_CACHED;
165
166   return ZINK_HEAP_HOST_VISIBLE_COHERENT;
167}
168
169bool
170zink_bo_init(struct zink_screen *screen);
171
172void
173zink_bo_deinit(struct zink_screen *screen);
174
175struct pb_buffer *
176zink_bo_create(struct zink_screen *screen, uint64_t size, unsigned alignment, enum zink_heap heap, enum zink_alloc_flag flags, const void *pNext);
177
178static inline uint64_t
179zink_bo_get_offset(const struct zink_bo *bo)
180{
181   return bo->offset;
182}
183
184static inline VkDeviceMemory
185zink_bo_get_mem(const struct zink_bo *bo)
186{
187   return bo->mem ? bo->mem : bo->u.slab.real->mem;
188}
189
190static inline VkDeviceSize
191zink_bo_get_size(const struct zink_bo *bo)
192{
193   return bo->mem ? bo->base.size : bo->u.slab.real->base.size;
194}
195
196void *
197zink_bo_map(struct zink_screen *screen, struct zink_bo *bo);
198void
199zink_bo_unmap(struct zink_screen *screen, struct zink_bo *bo);
200
201bool
202zink_bo_commit(struct zink_screen *screen, struct zink_resource *res, uint32_t offset, uint32_t size, bool commit);
203
204static inline bool
205zink_bo_has_unflushed_usage(const struct zink_bo *bo)
206{
207   return zink_batch_usage_is_unflushed(bo->reads) ||
208          zink_batch_usage_is_unflushed(bo->writes);
209}
210
211static inline bool
212zink_bo_has_usage(const struct zink_bo *bo)
213{
214   return zink_batch_usage_exists(bo->reads) ||
215          zink_batch_usage_exists(bo->writes);
216}
217
218static inline bool
219zink_bo_usage_matches(const struct zink_bo *bo, const struct zink_batch_state *bs)
220{
221   return zink_batch_usage_matches(bo->reads, bs) ||
222          zink_batch_usage_matches(bo->writes, bs);
223}
224
225static inline bool
226zink_bo_usage_check_completion(struct zink_screen *screen, struct zink_bo *bo, enum zink_resource_access access)
227{
228   if (access & ZINK_RESOURCE_ACCESS_READ && !zink_screen_usage_check_completion(screen, bo->reads))
229      return false;
230   if (access & ZINK_RESOURCE_ACCESS_WRITE && !zink_screen_usage_check_completion(screen, bo->writes))
231      return false;
232   return true;
233}
234
235static inline void
236zink_bo_usage_wait(struct zink_context *ctx, struct zink_bo *bo, enum zink_resource_access access)
237{
238   if (access & ZINK_RESOURCE_ACCESS_READ)
239      zink_batch_usage_wait(ctx, bo->reads);
240   if (access & ZINK_RESOURCE_ACCESS_WRITE)
241      zink_batch_usage_wait(ctx, bo->writes);
242}
243
244static inline void
245zink_bo_usage_set(struct zink_bo *bo, struct zink_batch_state *bs, bool write)
246{
247   if (write)
248      zink_batch_usage_set(&bo->writes, bs);
249   else
250      zink_batch_usage_set(&bo->reads, bs);
251}
252
253static inline bool
254zink_bo_usage_unset(struct zink_bo *bo, struct zink_batch_state *bs)
255{
256   zink_batch_usage_unset(&bo->reads, bs);
257   zink_batch_usage_unset(&bo->writes, bs);
258   return bo->reads || bo->writes;
259}
260
261
262static inline void
263zink_bo_unref(struct zink_screen *screen, struct zink_bo *bo)
264{
265   struct pb_buffer *pbuf = &bo->base;
266   pb_reference_with_winsys(screen, &pbuf, NULL);
267}
268
269#endif
270