nouveau_buffer.c revision af69d88d
1 2#include "util/u_inlines.h" 3#include "util/u_memory.h" 4#include "util/u_math.h" 5#include "util/u_surface.h" 6 7#include "nouveau_screen.h" 8#include "nouveau_context.h" 9#include "nouveau_winsys.h" 10#include "nouveau_fence.h" 11#include "nouveau_buffer.h" 12#include "nouveau_mm.h" 13 14#define NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD 192 15 16struct nouveau_transfer { 17 struct pipe_transfer base; 18 19 uint8_t *map; 20 struct nouveau_bo *bo; 21 struct nouveau_mm_allocation *mm; 22 uint32_t offset; 23}; 24 25static INLINE struct nouveau_transfer * 26nouveau_transfer(struct pipe_transfer *transfer) 27{ 28 return (struct nouveau_transfer *)transfer; 29} 30 31static INLINE boolean 32nouveau_buffer_malloc(struct nv04_resource *buf) 33{ 34 if (!buf->data) 35 buf->data = align_malloc(buf->base.width0, NOUVEAU_MIN_BUFFER_MAP_ALIGN); 36 return !!buf->data; 37} 38 39static INLINE boolean 40nouveau_buffer_allocate(struct nouveau_screen *screen, 41 struct nv04_resource *buf, unsigned domain) 42{ 43 uint32_t size = buf->base.width0; 44 45 if (buf->base.bind & (PIPE_BIND_CONSTANT_BUFFER | 46 PIPE_BIND_COMPUTE_RESOURCE | 47 PIPE_BIND_SHADER_RESOURCE)) 48 size = align(size, 0x100); 49 50 if (domain == NOUVEAU_BO_VRAM) { 51 buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size, 52 &buf->bo, &buf->offset); 53 if (!buf->bo) 54 return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART); 55 NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_vid, buf->base.width0); 56 } else 57 if (domain == NOUVEAU_BO_GART) { 58 buf->mm = nouveau_mm_allocate(screen->mm_GART, size, 59 &buf->bo, &buf->offset); 60 if (!buf->bo) 61 return FALSE; 62 NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_sys, buf->base.width0); 63 } else { 64 assert(domain == 0); 65 if (!nouveau_buffer_malloc(buf)) 66 return FALSE; 67 } 68 buf->domain = domain; 69 if (buf->bo) 70 buf->address = buf->bo->offset + buf->offset; 71 72 util_range_set_empty(&buf->valid_buffer_range); 73 74 return TRUE; 75} 76 77static INLINE void 78release_allocation(struct nouveau_mm_allocation **mm, 79 struct nouveau_fence *fence) 80{ 81 nouveau_fence_work(fence, nouveau_mm_free_work, *mm); 82 (*mm) = NULL; 83} 84 85INLINE void 86nouveau_buffer_release_gpu_storage(struct nv04_resource *buf) 87{ 88 nouveau_bo_ref(NULL, &buf->bo); 89 90 if (buf->mm) 91 release_allocation(&buf->mm, buf->fence); 92 93 if (buf->domain == NOUVEAU_BO_VRAM) 94 NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_vid, -(uint64_t)buf->base.width0); 95 if (buf->domain == NOUVEAU_BO_GART) 96 NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_sys, -(uint64_t)buf->base.width0); 97 98 buf->domain = 0; 99} 100 101static INLINE boolean 102nouveau_buffer_reallocate(struct nouveau_screen *screen, 103 struct nv04_resource *buf, unsigned domain) 104{ 105 nouveau_buffer_release_gpu_storage(buf); 106 107 nouveau_fence_ref(NULL, &buf->fence); 108 nouveau_fence_ref(NULL, &buf->fence_wr); 109 110 buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK; 111 112 return nouveau_buffer_allocate(screen, buf, domain); 113} 114 115static void 116nouveau_buffer_destroy(struct pipe_screen *pscreen, 117 struct pipe_resource *presource) 118{ 119 struct nv04_resource *res = nv04_resource(presource); 120 121 nouveau_buffer_release_gpu_storage(res); 122 123 if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY)) 124 align_free(res->data); 125 126 nouveau_fence_ref(NULL, &res->fence); 127 nouveau_fence_ref(NULL, &res->fence_wr); 128 129 util_range_destroy(&res->valid_buffer_range); 130 131 FREE(res); 132 133 NOUVEAU_DRV_STAT(nouveau_screen(pscreen), buf_obj_current_count, -1); 134} 135 136/* Set up a staging area for the transfer. This is either done in "regular" 137 * system memory if the driver supports push_data (nv50+) and the data is 138 * small enough (and permit_pb == true), or in GART memory. 139 */ 140static uint8_t * 141nouveau_transfer_staging(struct nouveau_context *nv, 142 struct nouveau_transfer *tx, boolean permit_pb) 143{ 144 const unsigned adj = tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK; 145 const unsigned size = align(tx->base.box.width, 4) + adj; 146 147 if (!nv->push_data) 148 permit_pb = FALSE; 149 150 if ((size <= NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD) && permit_pb) { 151 tx->map = align_malloc(size, NOUVEAU_MIN_BUFFER_MAP_ALIGN); 152 if (tx->map) 153 tx->map += adj; 154 } else { 155 tx->mm = 156 nouveau_mm_allocate(nv->screen->mm_GART, size, &tx->bo, &tx->offset); 157 if (tx->bo) { 158 tx->offset += adj; 159 if (!nouveau_bo_map(tx->bo, 0, NULL)) 160 tx->map = (uint8_t *)tx->bo->map + tx->offset; 161 } 162 } 163 return tx->map; 164} 165 166/* Copies data from the resource into the the transfer's temporary GART 167 * buffer. Also updates buf->data if present. 168 * 169 * Maybe just migrate to GART right away if we actually need to do this. */ 170static boolean 171nouveau_transfer_read(struct nouveau_context *nv, struct nouveau_transfer *tx) 172{ 173 struct nv04_resource *buf = nv04_resource(tx->base.resource); 174 const unsigned base = tx->base.box.x; 175 const unsigned size = tx->base.box.width; 176 177 NOUVEAU_DRV_STAT(nv->screen, buf_read_bytes_staging_vid, size); 178 179 nv->copy_data(nv, tx->bo, tx->offset, NOUVEAU_BO_GART, 180 buf->bo, buf->offset + base, buf->domain, size); 181 182 if (nouveau_bo_wait(tx->bo, NOUVEAU_BO_RD, nv->client)) 183 return FALSE; 184 185 if (buf->data) 186 memcpy(buf->data + base, tx->map, size); 187 188 return TRUE; 189} 190 191static void 192nouveau_transfer_write(struct nouveau_context *nv, struct nouveau_transfer *tx, 193 unsigned offset, unsigned size) 194{ 195 struct nv04_resource *buf = nv04_resource(tx->base.resource); 196 uint8_t *data = tx->map + offset; 197 const unsigned base = tx->base.box.x + offset; 198 const boolean can_cb = !((base | size) & 3); 199 200 if (buf->data) 201 memcpy(data, buf->data + base, size); 202 else 203 buf->status |= NOUVEAU_BUFFER_STATUS_DIRTY; 204 205 if (buf->domain == NOUVEAU_BO_VRAM) 206 NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_vid, size); 207 if (buf->domain == NOUVEAU_BO_GART) 208 NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_sys, size); 209 210 if (tx->bo) 211 nv->copy_data(nv, buf->bo, buf->offset + base, buf->domain, 212 tx->bo, tx->offset + offset, NOUVEAU_BO_GART, size); 213 else 214 if ((buf->base.bind & PIPE_BIND_CONSTANT_BUFFER) && nv->push_cb && can_cb) 215 nv->push_cb(nv, buf->bo, buf->domain, buf->offset, buf->base.width0, 216 base, size / 4, (const uint32_t *)data); 217 else 218 nv->push_data(nv, buf->bo, buf->offset + base, buf->domain, size, data); 219 220 nouveau_fence_ref(nv->screen->fence.current, &buf->fence); 221 nouveau_fence_ref(nv->screen->fence.current, &buf->fence_wr); 222} 223 224/* Does a CPU wait for the buffer's backing data to become reliably accessible 225 * for write/read by waiting on the buffer's relevant fences. 226 */ 227static INLINE boolean 228nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw) 229{ 230 if (rw == PIPE_TRANSFER_READ) { 231 if (!buf->fence_wr) 232 return TRUE; 233 NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count, 234 !nouveau_fence_signalled(buf->fence_wr)); 235 if (!nouveau_fence_wait(buf->fence_wr)) 236 return FALSE; 237 } else { 238 if (!buf->fence) 239 return TRUE; 240 NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count, 241 !nouveau_fence_signalled(buf->fence)); 242 if (!nouveau_fence_wait(buf->fence)) 243 return FALSE; 244 245 nouveau_fence_ref(NULL, &buf->fence); 246 } 247 nouveau_fence_ref(NULL, &buf->fence_wr); 248 249 return TRUE; 250} 251 252static INLINE boolean 253nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw) 254{ 255 if (rw == PIPE_TRANSFER_READ) 256 return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr)); 257 else 258 return (buf->fence && !nouveau_fence_signalled(buf->fence)); 259} 260 261static INLINE void 262nouveau_buffer_transfer_init(struct nouveau_transfer *tx, 263 struct pipe_resource *resource, 264 const struct pipe_box *box, 265 unsigned usage) 266{ 267 tx->base.resource = resource; 268 tx->base.level = 0; 269 tx->base.usage = usage; 270 tx->base.box.x = box->x; 271 tx->base.box.y = 0; 272 tx->base.box.z = 0; 273 tx->base.box.width = box->width; 274 tx->base.box.height = 1; 275 tx->base.box.depth = 1; 276 tx->base.stride = 0; 277 tx->base.layer_stride = 0; 278 279 tx->bo = NULL; 280 tx->map = NULL; 281} 282 283static INLINE void 284nouveau_buffer_transfer_del(struct nouveau_context *nv, 285 struct nouveau_transfer *tx) 286{ 287 if (tx->map) { 288 if (likely(tx->bo)) { 289 nouveau_bo_ref(NULL, &tx->bo); 290 if (tx->mm) 291 release_allocation(&tx->mm, nv->screen->fence.current); 292 } else { 293 align_free(tx->map - 294 (tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK)); 295 } 296 } 297} 298 299/* Creates a cache in system memory of the buffer data. */ 300static boolean 301nouveau_buffer_cache(struct nouveau_context *nv, struct nv04_resource *buf) 302{ 303 struct nouveau_transfer tx; 304 boolean ret; 305 tx.base.resource = &buf->base; 306 tx.base.box.x = 0; 307 tx.base.box.width = buf->base.width0; 308 tx.bo = NULL; 309 tx.map = NULL; 310 311 if (!buf->data) 312 if (!nouveau_buffer_malloc(buf)) 313 return FALSE; 314 if (!(buf->status & NOUVEAU_BUFFER_STATUS_DIRTY)) 315 return TRUE; 316 nv->stats.buf_cache_count++; 317 318 if (!nouveau_transfer_staging(nv, &tx, FALSE)) 319 return FALSE; 320 321 ret = nouveau_transfer_read(nv, &tx); 322 if (ret) { 323 buf->status &= ~NOUVEAU_BUFFER_STATUS_DIRTY; 324 memcpy(buf->data, tx.map, buf->base.width0); 325 } 326 nouveau_buffer_transfer_del(nv, &tx); 327 return ret; 328} 329 330 331#define NOUVEAU_TRANSFER_DISCARD \ 332 (PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) 333 334/* Checks whether it is possible to completely discard the memory backing this 335 * resource. This can be useful if we would otherwise have to wait for a read 336 * operation to complete on this data. 337 */ 338static INLINE boolean 339nouveau_buffer_should_discard(struct nv04_resource *buf, unsigned usage) 340{ 341 if (!(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)) 342 return FALSE; 343 if (unlikely(buf->base.bind & PIPE_BIND_SHARED)) 344 return FALSE; 345 if (unlikely(usage & PIPE_TRANSFER_PERSISTENT)) 346 return FALSE; 347 return buf->mm && nouveau_buffer_busy(buf, PIPE_TRANSFER_WRITE); 348} 349 350/* Returns a pointer to a memory area representing a window into the 351 * resource's data. 352 * 353 * This may or may not be the _actual_ memory area of the resource. However 354 * when calling nouveau_buffer_transfer_unmap, if it wasn't the actual memory 355 * area, the contents of the returned map are copied over to the resource. 356 * 357 * The usage indicates what the caller plans to do with the map: 358 * 359 * WRITE means that the user plans to write to it 360 * 361 * READ means that the user plans on reading from it 362 * 363 * DISCARD_WHOLE_RESOURCE means that the whole resource is going to be 364 * potentially overwritten, and even if it isn't, the bits that aren't don't 365 * need to be maintained. 366 * 367 * DISCARD_RANGE means that all the data in the specified range is going to 368 * be overwritten. 369 * 370 * The strategy for determining what kind of memory area to return is complex, 371 * see comments inside of the function. 372 */ 373static void * 374nouveau_buffer_transfer_map(struct pipe_context *pipe, 375 struct pipe_resource *resource, 376 unsigned level, unsigned usage, 377 const struct pipe_box *box, 378 struct pipe_transfer **ptransfer) 379{ 380 struct nouveau_context *nv = nouveau_context(pipe); 381 struct nv04_resource *buf = nv04_resource(resource); 382 struct nouveau_transfer *tx = MALLOC_STRUCT(nouveau_transfer); 383 uint8_t *map; 384 int ret; 385 386 if (!tx) 387 return NULL; 388 nouveau_buffer_transfer_init(tx, resource, box, usage); 389 *ptransfer = &tx->base; 390 391 if (usage & PIPE_TRANSFER_READ) 392 NOUVEAU_DRV_STAT(nv->screen, buf_transfers_rd, 1); 393 if (usage & PIPE_TRANSFER_WRITE) 394 NOUVEAU_DRV_STAT(nv->screen, buf_transfers_wr, 1); 395 396 /* If we are trying to write to an uninitialized range, the user shouldn't 397 * care what was there before. So we can treat the write as if the target 398 * range were being discarded. Furthermore, since we know that even if this 399 * buffer is busy due to GPU activity, because the contents were 400 * uninitialized, the GPU can't care what was there, and so we can treat 401 * the write as being unsynchronized. 402 */ 403 if ((usage & PIPE_TRANSFER_WRITE) && 404 !util_ranges_intersect(&buf->valid_buffer_range, box->x, box->x + box->width)) 405 usage |= PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_UNSYNCHRONIZED; 406 407 if (usage & PIPE_TRANSFER_PERSISTENT) 408 usage |= PIPE_TRANSFER_UNSYNCHRONIZED; 409 410 if (buf->domain == NOUVEAU_BO_VRAM) { 411 if (usage & NOUVEAU_TRANSFER_DISCARD) { 412 /* Set up a staging area for the user to write to. It will be copied 413 * back into VRAM on unmap. */ 414 if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) 415 buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK; 416 nouveau_transfer_staging(nv, tx, TRUE); 417 } else { 418 if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) { 419 /* The GPU is currently writing to this buffer. Copy its current 420 * contents to a staging area in the GART. This is necessary since 421 * not the whole area being mapped is being discarded. 422 */ 423 if (buf->data) { 424 align_free(buf->data); 425 buf->data = NULL; 426 } 427 nouveau_transfer_staging(nv, tx, FALSE); 428 nouveau_transfer_read(nv, tx); 429 } else { 430 /* The buffer is currently idle. Create a staging area for writes, 431 * and make sure that the cached data is up-to-date. */ 432 if (usage & PIPE_TRANSFER_WRITE) 433 nouveau_transfer_staging(nv, tx, TRUE); 434 if (!buf->data) 435 nouveau_buffer_cache(nv, buf); 436 } 437 } 438 return buf->data ? (buf->data + box->x) : tx->map; 439 } else 440 if (unlikely(buf->domain == 0)) { 441 return buf->data + box->x; 442 } 443 444 /* At this point, buf->domain == GART */ 445 446 if (nouveau_buffer_should_discard(buf, usage)) { 447 int ref = buf->base.reference.count - 1; 448 nouveau_buffer_reallocate(nv->screen, buf, buf->domain); 449 if (ref > 0) /* any references inside context possible ? */ 450 nv->invalidate_resource_storage(nv, &buf->base, ref); 451 } 452 453 /* Note that nouveau_bo_map ends up doing a nouveau_bo_wait with the 454 * relevant flags. If buf->mm is set, that means this resource is part of a 455 * larger slab bo that holds multiple resources. So in that case, don't 456 * wait on the whole slab and instead use the logic below to return a 457 * reasonable buffer for that case. 458 */ 459 ret = nouveau_bo_map(buf->bo, 460 buf->mm ? 0 : nouveau_screen_transfer_flags(usage), 461 nv->client); 462 if (ret) { 463 FREE(tx); 464 return NULL; 465 } 466 map = (uint8_t *)buf->bo->map + buf->offset + box->x; 467 468 /* using kernel fences only if !buf->mm */ 469 if ((usage & PIPE_TRANSFER_UNSYNCHRONIZED) || !buf->mm) 470 return map; 471 472 /* If the GPU is currently reading/writing this buffer, we shouldn't 473 * interfere with its progress. So instead we either wait for the GPU to 474 * complete its operation, or set up a staging area to perform our work in. 475 */ 476 if (nouveau_buffer_busy(buf, usage & PIPE_TRANSFER_READ_WRITE)) { 477 if (unlikely(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)) { 478 /* Discarding was not possible, must sync because 479 * subsequent transfers might use UNSYNCHRONIZED. */ 480 nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE); 481 } else 482 if (usage & PIPE_TRANSFER_DISCARD_RANGE) { 483 /* The whole range is being discarded, so it doesn't matter what was 484 * there before. No need to copy anything over. */ 485 nouveau_transfer_staging(nv, tx, TRUE); 486 map = tx->map; 487 } else 488 if (nouveau_buffer_busy(buf, PIPE_TRANSFER_READ)) { 489 if (usage & PIPE_TRANSFER_DONTBLOCK) 490 map = NULL; 491 else 492 nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE); 493 } else { 494 /* It is expected that the returned buffer be a representation of the 495 * data in question, so we must copy it over from the buffer. */ 496 nouveau_transfer_staging(nv, tx, TRUE); 497 if (tx->map) 498 memcpy(tx->map, map, box->width); 499 map = tx->map; 500 } 501 } 502 if (!map) 503 FREE(tx); 504 return map; 505} 506 507 508 509static void 510nouveau_buffer_transfer_flush_region(struct pipe_context *pipe, 511 struct pipe_transfer *transfer, 512 const struct pipe_box *box) 513{ 514 struct nouveau_transfer *tx = nouveau_transfer(transfer); 515 struct nv04_resource *buf = nv04_resource(transfer->resource); 516 517 if (tx->map) 518 nouveau_transfer_write(nouveau_context(pipe), tx, box->x, box->width); 519 520 util_range_add(&buf->valid_buffer_range, 521 tx->base.box.x + box->x, 522 tx->base.box.x + box->x + box->width); 523} 524 525/* Unmap stage of the transfer. If it was a WRITE transfer and the map that 526 * was returned was not the real resource's data, this needs to transfer the 527 * data back to the resource. 528 * 529 * Also marks vbo dirty based on the buffer's binding 530 */ 531static void 532nouveau_buffer_transfer_unmap(struct pipe_context *pipe, 533 struct pipe_transfer *transfer) 534{ 535 struct nouveau_context *nv = nouveau_context(pipe); 536 struct nouveau_transfer *tx = nouveau_transfer(transfer); 537 struct nv04_resource *buf = nv04_resource(transfer->resource); 538 539 if (tx->base.usage & PIPE_TRANSFER_WRITE) { 540 if (!(tx->base.usage & PIPE_TRANSFER_FLUSH_EXPLICIT) && tx->map) 541 nouveau_transfer_write(nv, tx, 0, tx->base.box.width); 542 543 if (likely(buf->domain)) { 544 const uint8_t bind = buf->base.bind; 545 /* make sure we invalidate dedicated caches */ 546 if (bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER)) 547 nv->vbo_dirty = TRUE; 548 } 549 550 util_range_add(&buf->valid_buffer_range, 551 tx->base.box.x, tx->base.box.x + tx->base.box.width); 552 } 553 554 if (!tx->bo && (tx->base.usage & PIPE_TRANSFER_WRITE)) 555 NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_direct, tx->base.box.width); 556 557 nouveau_buffer_transfer_del(nv, tx); 558 FREE(tx); 559} 560 561 562void 563nouveau_copy_buffer(struct nouveau_context *nv, 564 struct nv04_resource *dst, unsigned dstx, 565 struct nv04_resource *src, unsigned srcx, unsigned size) 566{ 567 assert(dst->base.target == PIPE_BUFFER && src->base.target == PIPE_BUFFER); 568 569 if (likely(dst->domain) && likely(src->domain)) { 570 nv->copy_data(nv, 571 dst->bo, dst->offset + dstx, dst->domain, 572 src->bo, src->offset + srcx, src->domain, size); 573 574 dst->status |= NOUVEAU_BUFFER_STATUS_GPU_WRITING; 575 nouveau_fence_ref(nv->screen->fence.current, &dst->fence); 576 nouveau_fence_ref(nv->screen->fence.current, &dst->fence_wr); 577 578 src->status |= NOUVEAU_BUFFER_STATUS_GPU_READING; 579 nouveau_fence_ref(nv->screen->fence.current, &src->fence); 580 } else { 581 struct pipe_box src_box; 582 src_box.x = srcx; 583 src_box.y = 0; 584 src_box.z = 0; 585 src_box.width = size; 586 src_box.height = 1; 587 src_box.depth = 1; 588 util_resource_copy_region(&nv->pipe, 589 &dst->base, 0, dstx, 0, 0, 590 &src->base, 0, &src_box); 591 } 592 593 util_range_add(&dst->valid_buffer_range, dstx, dstx + size); 594} 595 596 597void * 598nouveau_resource_map_offset(struct nouveau_context *nv, 599 struct nv04_resource *res, uint32_t offset, 600 uint32_t flags) 601{ 602 if (unlikely(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY)) 603 return res->data + offset; 604 605 if (res->domain == NOUVEAU_BO_VRAM) { 606 if (!res->data || (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING)) 607 nouveau_buffer_cache(nv, res); 608 } 609 if (res->domain != NOUVEAU_BO_GART) 610 return res->data + offset; 611 612 if (res->mm) { 613 unsigned rw; 614 rw = (flags & NOUVEAU_BO_WR) ? PIPE_TRANSFER_WRITE : PIPE_TRANSFER_READ; 615 nouveau_buffer_sync(res, rw); 616 if (nouveau_bo_map(res->bo, 0, NULL)) 617 return NULL; 618 } else { 619 if (nouveau_bo_map(res->bo, flags, nv->client)) 620 return NULL; 621 } 622 return (uint8_t *)res->bo->map + res->offset + offset; 623} 624 625 626const struct u_resource_vtbl nouveau_buffer_vtbl = 627{ 628 u_default_resource_get_handle, /* get_handle */ 629 nouveau_buffer_destroy, /* resource_destroy */ 630 nouveau_buffer_transfer_map, /* transfer_map */ 631 nouveau_buffer_transfer_flush_region, /* transfer_flush_region */ 632 nouveau_buffer_transfer_unmap, /* transfer_unmap */ 633 u_default_transfer_inline_write /* transfer_inline_write */ 634}; 635 636struct pipe_resource * 637nouveau_buffer_create(struct pipe_screen *pscreen, 638 const struct pipe_resource *templ) 639{ 640 struct nouveau_screen *screen = nouveau_screen(pscreen); 641 struct nv04_resource *buffer; 642 boolean ret; 643 644 buffer = CALLOC_STRUCT(nv04_resource); 645 if (!buffer) 646 return NULL; 647 648 buffer->base = *templ; 649 buffer->vtbl = &nouveau_buffer_vtbl; 650 pipe_reference_init(&buffer->base.reference, 1); 651 buffer->base.screen = pscreen; 652 653 if (buffer->base.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT | 654 PIPE_RESOURCE_FLAG_MAP_COHERENT)) { 655 buffer->domain = NOUVEAU_BO_GART; 656 } else if (buffer->base.bind & 657 (screen->vidmem_bindings & screen->sysmem_bindings)) { 658 switch (buffer->base.usage) { 659 case PIPE_USAGE_DEFAULT: 660 case PIPE_USAGE_IMMUTABLE: 661 buffer->domain = NOUVEAU_BO_VRAM; 662 break; 663 case PIPE_USAGE_DYNAMIC: 664 /* For most apps, we'd have to do staging transfers to avoid sync 665 * with this usage, and GART -> GART copies would be suboptimal. 666 */ 667 buffer->domain = NOUVEAU_BO_VRAM; 668 break; 669 case PIPE_USAGE_STAGING: 670 case PIPE_USAGE_STREAM: 671 buffer->domain = NOUVEAU_BO_GART; 672 break; 673 default: 674 assert(0); 675 break; 676 } 677 } else { 678 if (buffer->base.bind & screen->vidmem_bindings) 679 buffer->domain = NOUVEAU_BO_VRAM; 680 else 681 if (buffer->base.bind & screen->sysmem_bindings) 682 buffer->domain = NOUVEAU_BO_GART; 683 } 684 ret = nouveau_buffer_allocate(screen, buffer, buffer->domain); 685 686 if (ret == FALSE) 687 goto fail; 688 689 if (buffer->domain == NOUVEAU_BO_VRAM && screen->hint_buf_keep_sysmem_copy) 690 nouveau_buffer_cache(NULL, buffer); 691 692 NOUVEAU_DRV_STAT(screen, buf_obj_current_count, 1); 693 694 util_range_init(&buffer->valid_buffer_range); 695 696 return &buffer->base; 697 698fail: 699 FREE(buffer); 700 return NULL; 701} 702 703 704struct pipe_resource * 705nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr, 706 unsigned bytes, unsigned bind) 707{ 708 struct nv04_resource *buffer; 709 710 buffer = CALLOC_STRUCT(nv04_resource); 711 if (!buffer) 712 return NULL; 713 714 pipe_reference_init(&buffer->base.reference, 1); 715 buffer->vtbl = &nouveau_buffer_vtbl; 716 buffer->base.screen = pscreen; 717 buffer->base.format = PIPE_FORMAT_R8_UNORM; 718 buffer->base.usage = PIPE_USAGE_IMMUTABLE; 719 buffer->base.bind = bind; 720 buffer->base.width0 = bytes; 721 buffer->base.height0 = 1; 722 buffer->base.depth0 = 1; 723 724 buffer->data = ptr; 725 buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY; 726 727 util_range_init(&buffer->valid_buffer_range); 728 util_range_add(&buffer->valid_buffer_range, 0, bytes); 729 730 return &buffer->base; 731} 732 733static INLINE boolean 734nouveau_buffer_data_fetch(struct nouveau_context *nv, struct nv04_resource *buf, 735 struct nouveau_bo *bo, unsigned offset, unsigned size) 736{ 737 if (!nouveau_buffer_malloc(buf)) 738 return FALSE; 739 if (nouveau_bo_map(bo, NOUVEAU_BO_RD, nv->client)) 740 return FALSE; 741 memcpy(buf->data, (uint8_t *)bo->map + offset, size); 742 return TRUE; 743} 744 745/* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */ 746boolean 747nouveau_buffer_migrate(struct nouveau_context *nv, 748 struct nv04_resource *buf, const unsigned new_domain) 749{ 750 struct nouveau_screen *screen = nv->screen; 751 struct nouveau_bo *bo; 752 const unsigned old_domain = buf->domain; 753 unsigned size = buf->base.width0; 754 unsigned offset; 755 int ret; 756 757 assert(new_domain != old_domain); 758 759 if (new_domain == NOUVEAU_BO_GART && old_domain == 0) { 760 if (!nouveau_buffer_allocate(screen, buf, new_domain)) 761 return FALSE; 762 ret = nouveau_bo_map(buf->bo, 0, nv->client); 763 if (ret) 764 return ret; 765 memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size); 766 align_free(buf->data); 767 } else 768 if (old_domain != 0 && new_domain != 0) { 769 struct nouveau_mm_allocation *mm = buf->mm; 770 771 if (new_domain == NOUVEAU_BO_VRAM) { 772 /* keep a system memory copy of our data in case we hit a fallback */ 773 if (!nouveau_buffer_data_fetch(nv, buf, buf->bo, buf->offset, size)) 774 return FALSE; 775 if (nouveau_mesa_debug) 776 debug_printf("migrating %u KiB to VRAM\n", size / 1024); 777 } 778 779 offset = buf->offset; 780 bo = buf->bo; 781 buf->bo = NULL; 782 buf->mm = NULL; 783 nouveau_buffer_allocate(screen, buf, new_domain); 784 785 nv->copy_data(nv, buf->bo, buf->offset, new_domain, 786 bo, offset, old_domain, buf->base.width0); 787 788 nouveau_bo_ref(NULL, &bo); 789 if (mm) 790 release_allocation(&mm, screen->fence.current); 791 } else 792 if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) { 793 struct nouveau_transfer tx; 794 if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM)) 795 return FALSE; 796 tx.base.resource = &buf->base; 797 tx.base.box.x = 0; 798 tx.base.box.width = buf->base.width0; 799 tx.bo = NULL; 800 tx.map = NULL; 801 if (!nouveau_transfer_staging(nv, &tx, FALSE)) 802 return FALSE; 803 nouveau_transfer_write(nv, &tx, 0, tx.base.box.width); 804 nouveau_buffer_transfer_del(nv, &tx); 805 } else 806 return FALSE; 807 808 assert(buf->domain == new_domain); 809 return TRUE; 810} 811 812/* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART. 813 * We'd like to only allocate @size bytes here, but then we'd have to rebase 814 * the vertex indices ... 815 */ 816boolean 817nouveau_user_buffer_upload(struct nouveau_context *nv, 818 struct nv04_resource *buf, 819 unsigned base, unsigned size) 820{ 821 struct nouveau_screen *screen = nouveau_screen(buf->base.screen); 822 int ret; 823 824 assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY); 825 826 buf->base.width0 = base + size; 827 if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART)) 828 return FALSE; 829 830 ret = nouveau_bo_map(buf->bo, 0, nv->client); 831 if (ret) 832 return FALSE; 833 memcpy((uint8_t *)buf->bo->map + buf->offset + base, buf->data + base, size); 834 835 return TRUE; 836} 837 838 839/* Scratch data allocation. */ 840 841static INLINE int 842nouveau_scratch_bo_alloc(struct nouveau_context *nv, struct nouveau_bo **pbo, 843 unsigned size) 844{ 845 return nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 846 4096, size, NULL, pbo); 847} 848 849void 850nouveau_scratch_runout_release(struct nouveau_context *nv) 851{ 852 if (!nv->scratch.nr_runout) 853 return; 854 do { 855 --nv->scratch.nr_runout; 856 nouveau_bo_ref(NULL, &nv->scratch.runout[nv->scratch.nr_runout]); 857 } while (nv->scratch.nr_runout); 858 859 FREE(nv->scratch.runout); 860 nv->scratch.end = 0; 861 nv->scratch.runout = NULL; 862} 863 864/* Allocate an extra bo if we can't fit everything we need simultaneously. 865 * (Could happen for very large user arrays.) 866 */ 867static INLINE boolean 868nouveau_scratch_runout(struct nouveau_context *nv, unsigned size) 869{ 870 int ret; 871 const unsigned n = nv->scratch.nr_runout++; 872 873 nv->scratch.runout = REALLOC(nv->scratch.runout, 874 (n + 0) * sizeof(*nv->scratch.runout), 875 (n + 1) * sizeof(*nv->scratch.runout)); 876 nv->scratch.runout[n] = NULL; 877 878 ret = nouveau_scratch_bo_alloc(nv, &nv->scratch.runout[n], size); 879 if (!ret) { 880 ret = nouveau_bo_map(nv->scratch.runout[n], 0, NULL); 881 if (ret) 882 nouveau_bo_ref(NULL, &nv->scratch.runout[--nv->scratch.nr_runout]); 883 } 884 if (!ret) { 885 nv->scratch.current = nv->scratch.runout[n]; 886 nv->scratch.offset = 0; 887 nv->scratch.end = size; 888 nv->scratch.map = nv->scratch.current->map; 889 } 890 return !ret; 891} 892 893/* Continue to next scratch buffer, if available (no wrapping, large enough). 894 * Allocate it if it has not yet been created. 895 */ 896static INLINE boolean 897nouveau_scratch_next(struct nouveau_context *nv, unsigned size) 898{ 899 struct nouveau_bo *bo; 900 int ret; 901 const unsigned i = (nv->scratch.id + 1) % NOUVEAU_MAX_SCRATCH_BUFS; 902 903 if ((size > nv->scratch.bo_size) || (i == nv->scratch.wrap)) 904 return FALSE; 905 nv->scratch.id = i; 906 907 bo = nv->scratch.bo[i]; 908 if (!bo) { 909 ret = nouveau_scratch_bo_alloc(nv, &bo, nv->scratch.bo_size); 910 if (ret) 911 return FALSE; 912 nv->scratch.bo[i] = bo; 913 } 914 nv->scratch.current = bo; 915 nv->scratch.offset = 0; 916 nv->scratch.end = nv->scratch.bo_size; 917 918 ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, nv->client); 919 if (!ret) 920 nv->scratch.map = bo->map; 921 return !ret; 922} 923 924static boolean 925nouveau_scratch_more(struct nouveau_context *nv, unsigned min_size) 926{ 927 boolean ret; 928 929 ret = nouveau_scratch_next(nv, min_size); 930 if (!ret) 931 ret = nouveau_scratch_runout(nv, min_size); 932 return ret; 933} 934 935 936/* Copy data to a scratch buffer and return address & bo the data resides in. */ 937uint64_t 938nouveau_scratch_data(struct nouveau_context *nv, 939 const void *data, unsigned base, unsigned size, 940 struct nouveau_bo **bo) 941{ 942 unsigned bgn = MAX2(base, nv->scratch.offset); 943 unsigned end = bgn + size; 944 945 if (end >= nv->scratch.end) { 946 end = base + size; 947 if (!nouveau_scratch_more(nv, end)) 948 return 0; 949 bgn = base; 950 } 951 nv->scratch.offset = align(end, 4); 952 953 memcpy(nv->scratch.map + bgn, (const uint8_t *)data + base, size); 954 955 *bo = nv->scratch.current; 956 return (*bo)->offset + (bgn - base); 957} 958 959void * 960nouveau_scratch_get(struct nouveau_context *nv, 961 unsigned size, uint64_t *gpu_addr, struct nouveau_bo **pbo) 962{ 963 unsigned bgn = nv->scratch.offset; 964 unsigned end = nv->scratch.offset + size; 965 966 if (end >= nv->scratch.end) { 967 end = size; 968 if (!nouveau_scratch_more(nv, end)) 969 return NULL; 970 bgn = 0; 971 } 972 nv->scratch.offset = align(end, 4); 973 974 *pbo = nv->scratch.current; 975 *gpu_addr = nv->scratch.current->offset + bgn; 976 return nv->scratch.map + bgn; 977} 978