1/*
2 * Copyright 2021 Google LLC
3 * SPDX-License-Identifier: MIT
4 */
5
6#include "vn_ring.h"
7
8#include "vn_cs.h"
9#include "vn_renderer.h"
10
11enum vn_ring_status_flag {
12   VN_RING_STATUS_IDLE = 1u << 0,
13};
14
15static uint32_t
16vn_ring_load_head(const struct vn_ring *ring)
17{
18   /* the renderer is expected to store the head with memory_order_release,
19    * forming a release-acquire ordering
20    */
21   return atomic_load_explicit(ring->shared.head, memory_order_acquire);
22}
23
24static void
25vn_ring_store_tail(struct vn_ring *ring)
26{
27   /* the renderer is expected to load the tail with memory_order_acquire,
28    * forming a release-acquire ordering
29    */
30   return atomic_store_explicit(ring->shared.tail, ring->cur,
31                                memory_order_release);
32}
33
34static uint32_t
35vn_ring_load_status(const struct vn_ring *ring)
36{
37   /* this must be called and ordered after vn_ring_store_tail */
38   return atomic_load_explicit(ring->shared.status, memory_order_seq_cst);
39}
40
41static void
42vn_ring_write_buffer(struct vn_ring *ring, const void *data, uint32_t size)
43{
44   assert(ring->cur + size - vn_ring_load_head(ring) <= ring->buffer_size);
45
46   const uint32_t offset = ring->cur & ring->buffer_mask;
47   if (offset + size <= ring->buffer_size) {
48      memcpy(ring->shared.buffer + offset, data, size);
49   } else {
50      const uint32_t s = ring->buffer_size - offset;
51      memcpy(ring->shared.buffer + offset, data, s);
52      memcpy(ring->shared.buffer, data + s, size - s);
53   }
54
55   ring->cur += size;
56}
57
58static bool
59vn_ring_ge_seqno(const struct vn_ring *ring, uint32_t a, uint32_t b)
60{
61   /* this can return false negative when not called fast enough (e.g., when
62    * called once every couple hours), but following calls with larger a's
63    * will correct itself
64    *
65    * TODO use real seqnos?
66    */
67   if (a >= b)
68      return ring->cur >= a || ring->cur < b;
69   else
70      return ring->cur >= a && ring->cur < b;
71}
72
73static void
74vn_ring_retire_submits(struct vn_ring *ring, uint32_t seqno)
75{
76   list_for_each_entry_safe(struct vn_ring_submit, submit, &ring->submits,
77                            head) {
78      if (!vn_ring_ge_seqno(ring, seqno, submit->seqno))
79         break;
80
81      for (uint32_t i = 0; i < submit->shmem_count; i++)
82         vn_renderer_shmem_unref(ring->renderer, submit->shmems[i]);
83
84      list_del(&submit->head);
85      list_add(&submit->head, &ring->free_submits);
86   }
87}
88
89static uint32_t
90vn_ring_wait_seqno(const struct vn_ring *ring, uint32_t seqno)
91{
92   /* A renderer wait incurs several hops and the renderer might poll
93    * repeatedly anyway.  Let's just poll here.
94    */
95   uint32_t iter = 0;
96   do {
97      const uint32_t head = vn_ring_load_head(ring);
98      if (vn_ring_ge_seqno(ring, head, seqno))
99         return head;
100      vn_relax(&iter, "ring seqno");
101   } while (true);
102}
103
104static uint32_t
105vn_ring_wait_space(const struct vn_ring *ring, uint32_t size)
106{
107   assert(size <= ring->buffer_size);
108
109   /* see the reasoning in vn_ring_wait_seqno */
110   uint32_t iter = 0;
111   do {
112      const uint32_t head = vn_ring_load_head(ring);
113      if (ring->cur + size - head <= ring->buffer_size)
114         return head;
115      vn_relax(&iter, "ring space");
116   } while (true);
117}
118
119void
120vn_ring_get_layout(size_t buf_size,
121                   size_t extra_size,
122                   struct vn_ring_layout *layout)
123{
124   /* this can be changed/extended quite freely */
125   struct layout {
126      uint32_t head __attribute__((aligned(64)));
127      uint32_t tail __attribute__((aligned(64)));
128      uint32_t status __attribute__((aligned(64)));
129
130      uint8_t buffer[] __attribute__((aligned(64)));
131   };
132
133   assert(buf_size && util_is_power_of_two_or_zero(buf_size));
134
135   layout->head_offset = offsetof(struct layout, head);
136   layout->tail_offset = offsetof(struct layout, tail);
137   layout->status_offset = offsetof(struct layout, status);
138
139   layout->buffer_offset = offsetof(struct layout, buffer);
140   layout->buffer_size = buf_size;
141
142   layout->extra_offset = layout->buffer_offset + layout->buffer_size;
143   layout->extra_size = extra_size;
144
145   layout->shmem_size = layout->extra_offset + layout->extra_size;
146}
147
148void
149vn_ring_init(struct vn_ring *ring,
150             struct vn_renderer *renderer,
151             const struct vn_ring_layout *layout,
152             void *shared)
153{
154   memset(ring, 0, sizeof(*ring));
155   memset(shared, 0, layout->shmem_size);
156
157   ring->renderer = renderer;
158
159   assert(layout->buffer_size &&
160          util_is_power_of_two_or_zero(layout->buffer_size));
161   ring->buffer_size = layout->buffer_size;
162   ring->buffer_mask = ring->buffer_size - 1;
163
164   ring->shared.head = shared + layout->head_offset;
165   ring->shared.tail = shared + layout->tail_offset;
166   ring->shared.status = shared + layout->status_offset;
167   ring->shared.buffer = shared + layout->buffer_offset;
168   ring->shared.extra = shared + layout->extra_offset;
169
170   list_inithead(&ring->submits);
171   list_inithead(&ring->free_submits);
172}
173
174void
175vn_ring_fini(struct vn_ring *ring)
176{
177   vn_ring_retire_submits(ring, ring->cur);
178   assert(list_is_empty(&ring->submits));
179
180   list_for_each_entry_safe(struct vn_ring_submit, submit,
181                            &ring->free_submits, head)
182      free(submit);
183}
184
185struct vn_ring_submit *
186vn_ring_get_submit(struct vn_ring *ring, uint32_t shmem_count)
187{
188   const uint32_t min_shmem_count = 2;
189   struct vn_ring_submit *submit;
190
191   /* TODO this could be simplified if we could omit shmem_count */
192   if (shmem_count <= min_shmem_count &&
193       !list_is_empty(&ring->free_submits)) {
194      submit =
195         list_first_entry(&ring->free_submits, struct vn_ring_submit, head);
196      list_del(&submit->head);
197   } else {
198      shmem_count = MAX2(shmem_count, min_shmem_count);
199      submit =
200         malloc(sizeof(*submit) + sizeof(submit->shmems[0]) * shmem_count);
201   }
202
203   return submit;
204}
205
206bool
207vn_ring_submit(struct vn_ring *ring,
208               struct vn_ring_submit *submit,
209               const struct vn_cs_encoder *cs,
210               uint32_t *seqno)
211{
212   /* write cs to the ring */
213   assert(!vn_cs_encoder_is_empty(cs));
214   uint32_t cur_seqno;
215   for (uint32_t i = 0; i < cs->buffer_count; i++) {
216      const struct vn_cs_encoder_buffer *buf = &cs->buffers[i];
217      cur_seqno = vn_ring_wait_space(ring, buf->committed_size);
218      vn_ring_write_buffer(ring, buf->base, buf->committed_size);
219   }
220
221   vn_ring_store_tail(ring);
222   const bool notify = vn_ring_load_status(ring) & VN_RING_STATUS_IDLE;
223
224   vn_ring_retire_submits(ring, cur_seqno);
225
226   submit->seqno = ring->cur;
227   list_addtail(&submit->head, &ring->submits);
228
229   *seqno = submit->seqno;
230   return notify;
231}
232
233/**
234 * This is thread-safe.
235 */
236void
237vn_ring_wait(const struct vn_ring *ring, uint32_t seqno)
238{
239   vn_ring_wait_seqno(ring, seqno);
240}
241