nouveau_mm.c revision 3464ebd5 
1
2#include "util/u_inlines.h"
3#include "util/u_memory.h"
4#include "util/u_double_list.h"
5
6#include "nouveau_screen.h"
7#include "nouveau_mm.h"
8
9#include "nouveau/nouveau_bo.h"
10
11#define MM_MIN_ORDER 7
12#define MM_MAX_ORDER 20
13
14#define MM_NUM_BUCKETS (MM_MAX_ORDER - MM_MIN_ORDER + 1)
15
16#define MM_MIN_SIZE (1 << MM_MIN_ORDER)
17#define MM_MAX_SIZE (1 << MM_MAX_ORDER)
18
19struct mm_bucket {
20   struct list_head free;
21   struct list_head used;
22   struct list_head full;
23   int num_free;
24};
25
26struct nouveau_mman {
27   struct nouveau_device *dev;
28   struct mm_bucket bucket[MM_NUM_BUCKETS];
29   uint32_t storage_type;
30   uint32_t domain;
31   uint64_t allocated;
32};
33
34struct mm_slab {
35   struct list_head head;
36   struct nouveau_bo *bo;
37   struct nouveau_mman *cache;
38   int order;
39   int count;
40   int free;
41   uint32_t bits[0];
42};
43
44static int
45mm_slab_alloc(struct mm_slab *slab)
46{
47   int i, n, b;
48
49   if (slab->free == 0)
50      return -1;
51
52   for (i = 0; i < (slab->count + 31) / 32; ++i) {
53      b = ffs(slab->bits[i]) - 1;
54      if (b >= 0) {
55         n = i * 32 + b;
56         assert(n < slab->count);
57         slab->free--;
58         slab->bits[i] &= ~(1 << b);
59         return n;
60      }
61   }
62   return -1;
63}
64
65static INLINE void
66mm_slab_free(struct mm_slab *slab, int i)
67{
68   assert(i < slab->count);
69   slab->bits[i / 32] |= 1 << (i % 32);
70   slab->free++;
71   assert(slab->free <= slab->count);
72}
73
74static INLINE int
75mm_get_order(uint32_t size)
76{
77   int s = __builtin_clz(size) ^ 31;
78
79   if (size > (1 << s))
80      s += 1;
81   return s;
82}
83
84static struct mm_bucket *
85mm_bucket_by_order(struct nouveau_mman *cache, int order)
86{
87   if (order > MM_MAX_ORDER)
88      return NULL;
89   return &cache->bucket[MAX2(order, MM_MIN_ORDER) - MM_MIN_ORDER];
90}
91
92static struct mm_bucket *
93mm_bucket_by_size(struct nouveau_mman *cache, unsigned size)
94{
95   return mm_bucket_by_order(cache, mm_get_order(size));
96}
97
98/* size of bo allocation for slab with chunks of (1 << chunk_order) bytes */
99static INLINE uint32_t
100mm_default_slab_size(unsigned chunk_order)
101{
102   static const int8_t slab_order[MM_MAX_ORDER - MM_MIN_ORDER + 1] =
103   {
104      12, 12, 13, 14, 14, 17, 17, 17, 17, 19, 19, 20, 21, 22
105   };
106
107   assert(chunk_order <= MM_MAX_ORDER && chunk_order >= MM_MIN_ORDER);
108
109   return 1 << slab_order[chunk_order - MM_MIN_ORDER];
110}
111
112static int
113mm_slab_new(struct nouveau_mman *cache, int chunk_order)
114{
115   struct mm_slab *slab;
116   int words, ret;
117   const uint32_t size = mm_default_slab_size(chunk_order);
118
119   words = ((size >> chunk_order) + 31) / 32;
120   assert(words);
121
122   slab = MALLOC(sizeof(struct mm_slab) + words * 4);
123   if (!slab)
124      return PIPE_ERROR_OUT_OF_MEMORY;
125
126   memset(&slab->bits[0], ~0, words * 4);
127
128   slab->bo = NULL;
129   ret = nouveau_bo_new_tile(cache->dev, cache->domain, 0, size,
130                             0, cache->storage_type, &slab->bo);
131   if (ret) {
132      FREE(slab);
133      return PIPE_ERROR_OUT_OF_MEMORY;
134   }
135
136   LIST_INITHEAD(&slab->head);
137
138   slab->cache = cache;
139   slab->order = chunk_order;
140   slab->count = slab->free = size >> chunk_order;
141
142   LIST_ADD(&slab->head, &mm_bucket_by_order(cache, chunk_order)->free);
143
144   cache->allocated += size;
145
146   debug_printf("MM: new slab, total memory = %llu KiB\n",
147                cache->allocated / 1024);
148
149   return PIPE_OK;
150}
151
152/* @return token to identify slab or NULL if we just allocated a new bo */
153struct nouveau_mm_allocation *
154nouveau_mm_allocate(struct nouveau_mman *cache,
155                 uint32_t size, struct nouveau_bo **bo, uint32_t *offset)
156{
157   struct mm_bucket *bucket;
158   struct mm_slab *slab;
159   struct nouveau_mm_allocation *alloc;
160   int ret;
161
162   bucket = mm_bucket_by_size(cache, size);
163   if (!bucket) {
164      ret = nouveau_bo_new_tile(cache->dev, cache->domain, 0, size,
165                                0, cache->storage_type, bo);
166      if (ret)
167         debug_printf("bo_new(%x, %x): %i\n", size, cache->storage_type, ret);
168
169      *offset = 0;
170      return NULL;
171   }
172
173   if (!LIST_IS_EMPTY(&bucket->used)) {
174      slab = LIST_ENTRY(struct mm_slab, bucket->used.next, head);
175   } else {
176      if (LIST_IS_EMPTY(&bucket->free)) {
177         mm_slab_new(cache, MAX2(mm_get_order(size), MM_MIN_ORDER));
178      }
179      slab = LIST_ENTRY(struct mm_slab, bucket->free.next, head);
180
181      LIST_DEL(&slab->head);
182      LIST_ADD(&slab->head, &bucket->used);
183   }
184
185   *offset = mm_slab_alloc(slab) << slab->order;
186
187   alloc = MALLOC_STRUCT(nouveau_mm_allocation);
188   if (!alloc)
189      return NULL;
190
191   nouveau_bo_ref(slab->bo, bo);
192
193   if (slab->free == 0) {
194      LIST_DEL(&slab->head);
195      LIST_ADD(&slab->head, &bucket->full);
196   }
197
198   alloc->next = NULL;
199   alloc->offset = *offset;
200   alloc->priv = (void *)slab;
201
202   return alloc;
203}
204
205void
206nouveau_mm_free(struct nouveau_mm_allocation *alloc)
207{
208   struct mm_slab *slab = (struct mm_slab *)alloc->priv;
209   struct mm_bucket *bucket = mm_bucket_by_order(slab->cache, slab->order);
210
211   mm_slab_free(slab, alloc->offset >> slab->order);
212
213   if (slab->free == 1) {
214      LIST_DEL(&slab->head);
215
216      if (slab->count > 1)
217         LIST_ADDTAIL(&slab->head, &bucket->used);
218      else
219         LIST_ADDTAIL(&slab->head, &bucket->free);
220   }
221
222   FREE(alloc);
223}
224
225void
226nouveau_mm_free_work(void *data)
227{
228   nouveau_mm_free(data);
229}
230
231struct nouveau_mman *
232nouveau_mm_create(struct nouveau_device *dev, uint32_t domain,
233               uint32_t storage_type)
234{
235   struct nouveau_mman *cache = MALLOC_STRUCT(nouveau_mman);
236   int i;
237
238   if (!cache)
239      return NULL;
240
241   cache->dev = dev;
242   cache->domain = domain;
243   cache->storage_type = storage_type;
244   cache->allocated = 0;
245
246   for (i = 0; i < MM_NUM_BUCKETS; ++i) {
247      LIST_INITHEAD(&cache->bucket[i].free);
248      LIST_INITHEAD(&cache->bucket[i].used);
249      LIST_INITHEAD(&cache->bucket[i].full);
250   }
251
252   return cache;
253}
254
255static INLINE void
256nouveau_mm_free_slabs(struct list_head *head)
257{
258   struct mm_slab *slab, *next;
259
260   LIST_FOR_EACH_ENTRY_SAFE(slab, next, head, head) {
261      LIST_DEL(&slab->head);
262      nouveau_bo_ref(NULL, &slab->bo);
263      FREE(slab);
264   }
265}
266
267void
268nouveau_mm_destroy(struct nouveau_mman *cache)
269{
270   int i;
271
272   if (!cache)
273      return;
274
275   for (i = 0; i < MM_NUM_BUCKETS; ++i) {
276      if (!LIST_IS_EMPTY(&cache->bucket[i].used) ||
277          !LIST_IS_EMPTY(&cache->bucket[i].full))
278         debug_printf("WARNING: destroying GPU memory cache "
279                      "with some buffers still in use\n");
280
281      nouveau_mm_free_slabs(&cache->bucket[i].free);
282      nouveau_mm_free_slabs(&cache->bucket[i].used);
283      nouveau_mm_free_slabs(&cache->bucket[i].full);
284   }
285
286   FREE(cache);
287}
288
289