nouveau_buffer.c revision 3464ebd5 
1
2#include "util/u_inlines.h"
3#include "util/u_memory.h"
4#include "util/u_math.h"
5
6#include "nouveau_screen.h"
7#include "nouveau_context.h"
8#include "nouveau_winsys.h"
9#include "nouveau_fence.h"
10#include "nouveau_buffer.h"
11#include "nouveau_mm.h"
12
13struct nouveau_transfer {
14   struct pipe_transfer base;
15};
16
17static INLINE struct nouveau_transfer *
18nouveau_transfer(struct pipe_transfer *transfer)
19{
20   return (struct nouveau_transfer *)transfer;
21}
22
23static INLINE boolean
24nouveau_buffer_allocate(struct nouveau_screen *screen,
25                        struct nv04_resource *buf, unsigned domain)
26{
27   if (domain == NOUVEAU_BO_VRAM) {
28      buf->mm = nouveau_mm_allocate(screen->mm_VRAM, buf->base.width0,
29                                    &buf->bo, &buf->offset);
30      if (!buf->bo)
31         return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
32   } else
33   if (domain == NOUVEAU_BO_GART) {
34      buf->mm = nouveau_mm_allocate(screen->mm_GART, buf->base.width0,
35                                    &buf->bo, &buf->offset);
36      if (!buf->bo)
37         return FALSE;
38   }
39   if (domain != NOUVEAU_BO_GART) {
40      if (!buf->data) {
41         buf->data = MALLOC(buf->base.width0);
42         if (!buf->data)
43            return FALSE;
44      }
45   }
46   buf->domain = domain;
47   return TRUE;
48}
49
50static INLINE void
51release_allocation(struct nouveau_mm_allocation **mm,
52                   struct nouveau_fence *fence)
53{
54   nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
55   (*mm) = NULL;
56}
57
58INLINE void
59nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
60{
61   nouveau_bo_ref(NULL, &buf->bo);
62
63   if (buf->mm)
64      release_allocation(&buf->mm, buf->fence);
65
66   buf->domain = 0;
67}
68
69static INLINE boolean
70nouveau_buffer_reallocate(struct nouveau_screen *screen,
71                          struct nv04_resource *buf, unsigned domain)
72{
73   nouveau_buffer_release_gpu_storage(buf);
74
75   return nouveau_buffer_allocate(screen, buf, domain);
76}
77
78static void
79nouveau_buffer_destroy(struct pipe_screen *pscreen,
80                       struct pipe_resource *presource)
81{
82   struct nv04_resource *res = nv04_resource(presource);
83
84   nouveau_buffer_release_gpu_storage(res);
85
86   if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
87      FREE(res->data);
88
89   FREE(res);
90}
91
92/* Maybe just migrate to GART right away if we actually need to do this. */
93boolean
94nouveau_buffer_download(struct nouveau_context *nv, struct nv04_resource *buf,
95                        unsigned start, unsigned size)
96{
97   struct nouveau_mm_allocation *mm;
98   struct nouveau_bo *bounce = NULL;
99   uint32_t offset;
100
101   assert(buf->domain == NOUVEAU_BO_VRAM);
102
103   mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
104   if (!bounce)
105      return FALSE;
106
107   nv->copy_data(nv, bounce, offset, NOUVEAU_BO_GART,
108                 buf->bo, buf->offset + start, NOUVEAU_BO_VRAM, size);
109
110   if (nouveau_bo_map_range(bounce, offset, size, NOUVEAU_BO_RD))
111      return FALSE;
112   memcpy(buf->data + start, bounce->map, size);
113   nouveau_bo_unmap(bounce);
114
115   buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
116
117   nouveau_bo_ref(NULL, &bounce);
118   if (mm)
119      nouveau_mm_free(mm);
120   return TRUE;
121}
122
123static boolean
124nouveau_buffer_upload(struct nouveau_context *nv, struct nv04_resource *buf,
125                      unsigned start, unsigned size)
126{
127   struct nouveau_mm_allocation *mm;
128   struct nouveau_bo *bounce = NULL;
129   uint32_t offset;
130
131   if (size <= 192) {
132      nv->push_data(nv, buf->bo, buf->offset + start, buf->domain,
133                    size, buf->data + start);
134      return TRUE;
135   }
136
137   mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
138   if (!bounce)
139      return FALSE;
140
141   nouveau_bo_map_range(bounce, offset, size,
142                        NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
143   memcpy(bounce->map, buf->data + start, size);
144   nouveau_bo_unmap(bounce);
145
146   nv->copy_data(nv, buf->bo, buf->offset + start, NOUVEAU_BO_VRAM,
147                 bounce, offset, NOUVEAU_BO_GART, size);
148
149   nouveau_bo_ref(NULL, &bounce);
150   if (mm)
151      release_allocation(&mm, nv->screen->fence.current);
152
153   if (start == 0 && size == buf->base.width0)
154      buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
155   return TRUE;
156}
157
158static struct pipe_transfer *
159nouveau_buffer_transfer_get(struct pipe_context *pipe,
160                            struct pipe_resource *resource,
161                            unsigned level, unsigned usage,
162                            const struct pipe_box *box)
163{
164   struct nv04_resource *buf = nv04_resource(resource);
165   struct nouveau_context *nv = nouveau_context(pipe);
166   struct nouveau_transfer *xfr = CALLOC_STRUCT(nouveau_transfer);
167   if (!xfr)
168      return NULL;
169
170   xfr->base.resource = resource;
171   xfr->base.box.x = box->x;
172   xfr->base.box.width = box->width;
173   xfr->base.usage = usage;
174
175   if (buf->domain == NOUVEAU_BO_VRAM) {
176      if (usage & PIPE_TRANSFER_READ) {
177         if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING)
178            nouveau_buffer_download(nv, buf, 0, buf->base.width0);
179      }
180   }
181
182   return &xfr->base;
183}
184
185static void
186nouveau_buffer_transfer_destroy(struct pipe_context *pipe,
187                                struct pipe_transfer *transfer)
188{
189   struct nv04_resource *buf = nv04_resource(transfer->resource);
190   struct nouveau_transfer *xfr = nouveau_transfer(transfer);
191   struct nouveau_context *nv = nouveau_context(pipe);
192
193   if (xfr->base.usage & PIPE_TRANSFER_WRITE) {
194      /* writing is worse */
195      nouveau_buffer_adjust_score(nv, buf, -5000);
196
197      if (buf->domain == NOUVEAU_BO_VRAM) {
198         nouveau_buffer_upload(nv, buf, transfer->box.x, transfer->box.width);
199      }
200
201      if (buf->domain != 0 && (buf->base.bind & (PIPE_BIND_VERTEX_BUFFER |
202                                                 PIPE_BIND_INDEX_BUFFER)))
203         nouveau_context(pipe)->vbo_dirty = TRUE;
204   }
205
206   FREE(xfr);
207}
208
209static INLINE boolean
210nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
211{
212   if (rw == PIPE_TRANSFER_READ) {
213      if (!buf->fence_wr)
214         return TRUE;
215      if (!nouveau_fence_wait(buf->fence_wr))
216         return FALSE;
217   } else {
218      if (!buf->fence)
219         return TRUE;
220      if (!nouveau_fence_wait(buf->fence))
221         return FALSE;
222
223      nouveau_fence_ref(NULL, &buf->fence);
224   }
225   nouveau_fence_ref(NULL, &buf->fence_wr);
226
227   return TRUE;
228}
229
230static INLINE boolean
231nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
232{
233   if (rw == PIPE_TRANSFER_READ)
234      return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
235   else
236      return (buf->fence && !nouveau_fence_signalled(buf->fence));
237}
238
239static void *
240nouveau_buffer_transfer_map(struct pipe_context *pipe,
241                            struct pipe_transfer *transfer)
242{
243   struct nouveau_transfer *xfr = nouveau_transfer(transfer);
244   struct nv04_resource *buf = nv04_resource(transfer->resource);
245   struct nouveau_bo *bo = buf->bo;
246   uint8_t *map;
247   int ret;
248   uint32_t offset = xfr->base.box.x;
249   uint32_t flags;
250
251   nouveau_buffer_adjust_score(nouveau_context(pipe), buf, -250);
252
253   if (buf->domain != NOUVEAU_BO_GART)
254      return buf->data + offset;
255
256   if (buf->mm)
257      flags = NOUVEAU_BO_NOSYNC | NOUVEAU_BO_RDWR;
258   else
259      flags = nouveau_screen_transfer_flags(xfr->base.usage);
260
261   offset += buf->offset;
262
263   ret = nouveau_bo_map_range(buf->bo, offset, xfr->base.box.width, flags);
264   if (ret)
265      return NULL;
266   map = bo->map;
267
268   /* Unmap right now. Since multiple buffers can share a single nouveau_bo,
269    * not doing so might make future maps fail or trigger "reloc while mapped"
270    * errors. For now, mappings to userspace are guaranteed to be persistent.
271    */
272   nouveau_bo_unmap(bo);
273
274   if (buf->mm) {
275      if (xfr->base.usage & PIPE_TRANSFER_DONTBLOCK) {
276         if (nouveau_buffer_busy(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE))
277            return NULL;
278      } else
279      if (!(xfr->base.usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
280         nouveau_buffer_sync(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE);
281      }
282   }
283   return map;
284}
285
286
287
288static void
289nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
290                                     struct pipe_transfer *transfer,
291                                     const struct pipe_box *box)
292{
293   struct nv04_resource *res = nv04_resource(transfer->resource);
294   struct nouveau_bo *bo = res->bo;
295   unsigned offset = res->offset + transfer->box.x + box->x;
296
297   /* not using non-snoop system memory yet, no need for cflush */
298   if (1)
299      return;
300
301   /* XXX: maybe need to upload for VRAM buffers here */
302
303   nouveau_screen_bo_map_flush_range(pipe->screen, bo, offset, box->width);
304}
305
306static void
307nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
308                              struct pipe_transfer *transfer)
309{
310   /* we've called nouveau_bo_unmap right after map */
311}
312
313const struct u_resource_vtbl nouveau_buffer_vtbl =
314{
315   u_default_resource_get_handle,     /* get_handle */
316   nouveau_buffer_destroy,               /* resource_destroy */
317   nouveau_buffer_transfer_get,          /* get_transfer */
318   nouveau_buffer_transfer_destroy,      /* transfer_destroy */
319   nouveau_buffer_transfer_map,          /* transfer_map */
320   nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
321   nouveau_buffer_transfer_unmap,        /* transfer_unmap */
322   u_default_transfer_inline_write    /* transfer_inline_write */
323};
324
325struct pipe_resource *
326nouveau_buffer_create(struct pipe_screen *pscreen,
327                      const struct pipe_resource *templ)
328{
329   struct nouveau_screen *screen = nouveau_screen(pscreen);
330   struct nv04_resource *buffer;
331   boolean ret;
332
333   buffer = CALLOC_STRUCT(nv04_resource);
334   if (!buffer)
335      return NULL;
336
337   buffer->base = *templ;
338   buffer->vtbl = &nouveau_buffer_vtbl;
339   pipe_reference_init(&buffer->base.reference, 1);
340   buffer->base.screen = pscreen;
341
342   if ((buffer->base.bind & screen->sysmem_bindings) == screen->sysmem_bindings)
343      ret = nouveau_buffer_allocate(screen, buffer, 0);
344   else
345      ret = nouveau_buffer_allocate(screen, buffer, NOUVEAU_BO_GART);
346
347   if (ret == FALSE)
348      goto fail;
349
350   return &buffer->base;
351
352fail:
353   FREE(buffer);
354   return NULL;
355}
356
357
358struct pipe_resource *
359nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
360                           unsigned bytes, unsigned bind)
361{
362   struct nv04_resource *buffer;
363
364   buffer = CALLOC_STRUCT(nv04_resource);
365   if (!buffer)
366      return NULL;
367
368   pipe_reference_init(&buffer->base.reference, 1);
369   buffer->vtbl = &nouveau_buffer_vtbl;
370   buffer->base.screen = pscreen;
371   buffer->base.format = PIPE_FORMAT_R8_UNORM;
372   buffer->base.usage = PIPE_USAGE_IMMUTABLE;
373   buffer->base.bind = bind;
374   buffer->base.width0 = bytes;
375   buffer->base.height0 = 1;
376   buffer->base.depth0 = 1;
377
378   buffer->data = ptr;
379   buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
380
381   return &buffer->base;
382}
383
384/* Like download, but for GART buffers. Merge ? */
385static INLINE boolean
386nouveau_buffer_data_fetch(struct nv04_resource *buf, struct nouveau_bo *bo,
387                          unsigned offset, unsigned size)
388{
389   if (!buf->data) {
390      buf->data = MALLOC(size);
391      if (!buf->data)
392         return FALSE;
393   }
394   if (nouveau_bo_map_range(bo, offset, size, NOUVEAU_BO_RD))
395      return FALSE;
396   memcpy(buf->data, bo->map, size);
397   nouveau_bo_unmap(bo);
398
399   return TRUE;
400}
401
402/* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
403boolean
404nouveau_buffer_migrate(struct nouveau_context *nv,
405                       struct nv04_resource *buf, const unsigned new_domain)
406{
407   struct nouveau_screen *screen = nv->screen;
408   struct nouveau_bo *bo;
409   const unsigned old_domain = buf->domain;
410   unsigned size = buf->base.width0;
411   unsigned offset;
412   int ret;
413
414   assert(new_domain != old_domain);
415
416   if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
417      if (!nouveau_buffer_allocate(screen, buf, new_domain))
418         return FALSE;
419      ret = nouveau_bo_map_range(buf->bo, buf->offset, size, NOUVEAU_BO_WR |
420                                 NOUVEAU_BO_NOSYNC);
421      if (ret)
422         return ret;
423      memcpy(buf->bo->map, buf->data, size);
424      nouveau_bo_unmap(buf->bo);
425      FREE(buf->data);
426   } else
427   if (old_domain != 0 && new_domain != 0) {
428      struct nouveau_mm_allocation *mm = buf->mm;
429
430      if (new_domain == NOUVEAU_BO_VRAM) {
431         /* keep a system memory copy of our data in case we hit a fallback */
432         if (!nouveau_buffer_data_fetch(buf, buf->bo, buf->offset, size))
433            return FALSE;
434         debug_printf("migrating %u KiB to VRAM\n", size / 1024);
435      }
436
437      offset = buf->offset;
438      bo = buf->bo;
439      buf->bo = NULL;
440      buf->mm = NULL;
441      nouveau_buffer_allocate(screen, buf, new_domain);
442
443      nv->copy_data(nv, buf->bo, buf->offset, new_domain,
444                    bo, offset, old_domain, buf->base.width0);
445
446      nouveau_bo_ref(NULL, &bo);
447      if (mm)
448         release_allocation(&mm, screen->fence.current);
449   } else
450   if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
451      if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
452         return FALSE;
453      if (!nouveau_buffer_upload(nv, buf, 0, buf->base.width0))
454         return FALSE;
455   } else
456      return FALSE;
457
458   assert(buf->domain == new_domain);
459   return TRUE;
460}
461
462/* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
463 * We'd like to only allocate @size bytes here, but then we'd have to rebase
464 * the vertex indices ...
465 */
466boolean
467nouveau_user_buffer_upload(struct nv04_resource *buf,
468                           unsigned base, unsigned size)
469{
470   struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
471   int ret;
472
473   assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
474
475   buf->base.width0 = base + size;
476   if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
477      return FALSE;
478
479   ret = nouveau_bo_map_range(buf->bo, buf->offset + base, size,
480                              NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
481   if (ret)
482      return FALSE;
483   memcpy(buf->bo->map, buf->data + base, size);
484   nouveau_bo_unmap(buf->bo);
485
486   return TRUE;
487}
488