pushbuf.c revision 7cdc0497
1/* 2 * Copyright 2012 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Ben Skeggs 23 */ 24 25#include <stdio.h> 26#include <stdlib.h> 27#include <stdint.h> 28#include <stdbool.h> 29#include <string.h> 30#include <assert.h> 31#include <errno.h> 32 33#include <xf86drm.h> 34#include <xf86atomic.h> 35#include "libdrm_lists.h" 36#include "nouveau_drm.h" 37 38#include "nouveau.h" 39#include "private.h" 40 41struct nouveau_pushbuf_krec { 42 struct nouveau_pushbuf_krec *next; 43 struct drm_nouveau_gem_pushbuf_bo buffer[NOUVEAU_GEM_MAX_BUFFERS]; 44 struct drm_nouveau_gem_pushbuf_reloc reloc[NOUVEAU_GEM_MAX_RELOCS]; 45 struct drm_nouveau_gem_pushbuf_push push[NOUVEAU_GEM_MAX_PUSH]; 46 int nr_buffer; 47 int nr_reloc; 48 int nr_push; 49 uint64_t vram_used; 50 uint64_t gart_used; 51}; 52 53struct nouveau_pushbuf_priv { 54 struct nouveau_pushbuf base; 55 struct nouveau_pushbuf_krec *list; 56 struct nouveau_pushbuf_krec *krec; 57 struct nouveau_list bctx_list; 58 struct nouveau_bo *bo; 59 uint32_t type; 60 uint32_t suffix0; 61 uint32_t suffix1; 62 uint32_t *ptr; 63 uint32_t *bgn; 64 int bo_next; 65 int bo_nr; 66 struct nouveau_bo *bos[]; 67}; 68 69static inline struct nouveau_pushbuf_priv * 70nouveau_pushbuf(struct nouveau_pushbuf *push) 71{ 72 return (struct nouveau_pushbuf_priv *)push; 73} 74 75static int pushbuf_validate(struct nouveau_pushbuf *, bool); 76static int pushbuf_flush(struct nouveau_pushbuf *); 77 78static bool 79pushbuf_kref_fits(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 80 uint32_t *domains) 81{ 82 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 83 struct nouveau_pushbuf_krec *krec = nvpb->krec; 84 struct nouveau_device *dev = push->client->device; 85 struct nouveau_bo *kbo; 86 struct drm_nouveau_gem_pushbuf_bo *kref; 87 int i; 88 89 /* VRAM is the only valid domain. GART and VRAM|GART buffers 90 * are all accounted to GART, so if this doesn't fit in VRAM 91 * straight up, a flush is needed. 92 */ 93 if (*domains == NOUVEAU_GEM_DOMAIN_VRAM) { 94 if (krec->vram_used + bo->size > dev->vram_limit) 95 return false; 96 krec->vram_used += bo->size; 97 return true; 98 } 99 100 /* GART or VRAM|GART buffer. Account both of these buffer types 101 * to GART only for the moment, which simplifies things. If the 102 * buffer can fit already, we're done here. 103 */ 104 if (krec->gart_used + bo->size <= dev->gart_limit) { 105 krec->gart_used += bo->size; 106 return true; 107 } 108 109 /* Ran out of GART space, if it's a VRAM|GART buffer and it'll 110 * fit into available VRAM, turn it into a VRAM buffer 111 */ 112 if ((*domains & NOUVEAU_GEM_DOMAIN_VRAM) && 113 krec->vram_used + bo->size <= dev->vram_limit) { 114 *domains &= NOUVEAU_GEM_DOMAIN_VRAM; 115 krec->vram_used += bo->size; 116 return true; 117 } 118 119 /* Still couldn't fit the buffer in anywhere, so as a last resort; 120 * scan the buffer list for VRAM|GART buffers and turn them into 121 * VRAM buffers until we have enough space in GART for this one 122 */ 123 kref = krec->buffer; 124 for (i = 0; i < krec->nr_buffer; i++, kref++) { 125 if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 126 continue; 127 128 kbo = (void *)(unsigned long)kref->user_priv; 129 if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) || 130 krec->vram_used + kbo->size > dev->vram_limit) 131 continue; 132 133 kref->valid_domains &= NOUVEAU_GEM_DOMAIN_VRAM; 134 krec->gart_used -= kbo->size; 135 krec->vram_used += kbo->size; 136 if (krec->gart_used + bo->size <= dev->gart_limit) { 137 krec->gart_used += bo->size; 138 return true; 139 } 140 } 141 142 /* Couldn't resolve a placement, need to force a flush */ 143 return false; 144} 145 146static struct drm_nouveau_gem_pushbuf_bo * 147pushbuf_kref(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 148 uint32_t flags) 149{ 150 struct nouveau_device *dev = push->client->device; 151 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 152 struct nouveau_pushbuf_krec *krec = nvpb->krec; 153 struct nouveau_pushbuf *fpush; 154 struct drm_nouveau_gem_pushbuf_bo *kref; 155 uint32_t domains, domains_wr, domains_rd; 156 157 domains = 0; 158 if (flags & NOUVEAU_BO_VRAM) 159 domains |= NOUVEAU_GEM_DOMAIN_VRAM; 160 if (flags & NOUVEAU_BO_GART) 161 domains |= NOUVEAU_GEM_DOMAIN_GART; 162 domains_wr = domains * !!(flags & NOUVEAU_BO_WR); 163 domains_rd = domains * !!(flags & NOUVEAU_BO_RD); 164 165 /* if buffer is referenced on another pushbuf that is owned by the 166 * same client, we need to flush the other pushbuf first to ensure 167 * the correct ordering of commands 168 */ 169 fpush = cli_push_get(push->client, bo); 170 if (fpush && fpush != push) 171 pushbuf_flush(fpush); 172 173 kref = cli_kref_get(push->client, bo); 174 if (kref) { 175 /* possible conflict in memory types - flush and retry */ 176 if (!(kref->valid_domains & domains)) 177 return NULL; 178 179 /* VRAM|GART buffer turning into a VRAM buffer. Make sure 180 * it'll fit in VRAM and force a flush if not. 181 */ 182 if ((kref->valid_domains & NOUVEAU_GEM_DOMAIN_GART) && 183 ( domains == NOUVEAU_GEM_DOMAIN_VRAM)) { 184 if (krec->vram_used + bo->size > dev->vram_limit) 185 return NULL; 186 krec->vram_used += bo->size; 187 krec->gart_used -= bo->size; 188 } 189 190 kref->valid_domains &= domains; 191 kref->write_domains |= domains_wr; 192 kref->read_domains |= domains_rd; 193 } else { 194 if (krec->nr_buffer == NOUVEAU_GEM_MAX_BUFFERS || 195 !pushbuf_kref_fits(push, bo, &domains)) 196 return NULL; 197 198 kref = &krec->buffer[krec->nr_buffer++]; 199 kref->user_priv = (unsigned long)bo; 200 kref->handle = bo->handle; 201 kref->valid_domains = domains; 202 kref->write_domains = domains_wr; 203 kref->read_domains = domains_rd; 204 kref->presumed.valid = 1; 205 kref->presumed.offset = bo->offset; 206 if (bo->flags & NOUVEAU_BO_VRAM) 207 kref->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 208 else 209 kref->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 210 211 cli_kref_set(push->client, bo, kref, push); 212 atomic_inc(&nouveau_bo(bo)->refcnt); 213 } 214 215 return kref; 216} 217 218static uint32_t 219pushbuf_krel(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 220 uint32_t data, uint32_t flags, uint32_t vor, uint32_t tor) 221{ 222 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 223 struct nouveau_pushbuf_krec *krec = nvpb->krec; 224 struct drm_nouveau_gem_pushbuf_reloc *krel; 225 struct drm_nouveau_gem_pushbuf_bo *pkref; 226 struct drm_nouveau_gem_pushbuf_bo *bkref; 227 uint32_t reloc = data; 228 229 pkref = cli_kref_get(push->client, nvpb->bo); 230 bkref = cli_kref_get(push->client, bo); 231 krel = &krec->reloc[krec->nr_reloc++]; 232 233 assert(pkref); 234 assert(bkref); 235 krel->reloc_bo_index = pkref - krec->buffer; 236 krel->reloc_bo_offset = (push->cur - nvpb->ptr) * 4; 237 krel->bo_index = bkref - krec->buffer; 238 krel->flags = 0; 239 krel->data = data; 240 krel->vor = vor; 241 krel->tor = tor; 242 243 if (flags & NOUVEAU_BO_LOW) { 244 reloc = (bkref->presumed.offset + data); 245 krel->flags |= NOUVEAU_GEM_RELOC_LOW; 246 } else 247 if (flags & NOUVEAU_BO_HIGH) { 248 reloc = (bkref->presumed.offset + data) >> 32; 249 krel->flags |= NOUVEAU_GEM_RELOC_HIGH; 250 } 251 if (flags & NOUVEAU_BO_OR) { 252 if (bkref->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) 253 reloc |= vor; 254 else 255 reloc |= tor; 256 krel->flags |= NOUVEAU_GEM_RELOC_OR; 257 } 258 259 return reloc; 260} 261 262static void 263pushbuf_dump(struct nouveau_pushbuf_krec *krec, int krec_id, int chid) 264{ 265 struct drm_nouveau_gem_pushbuf_reloc *krel; 266 struct drm_nouveau_gem_pushbuf_push *kpsh; 267 struct drm_nouveau_gem_pushbuf_bo *kref; 268 struct nouveau_bo *bo; 269 uint32_t *bgn, *end; 270 int i; 271 272 err("ch%d: krec %d pushes %d bufs %d relocs %d\n", chid, 273 krec_id, krec->nr_push, krec->nr_buffer, krec->nr_reloc); 274 275 kref = krec->buffer; 276 for (i = 0; i < krec->nr_buffer; i++, kref++) { 277 err("ch%d: buf %08x %08x %08x %08x %08x\n", chid, i, 278 kref->handle, kref->valid_domains, 279 kref->read_domains, kref->write_domains); 280 } 281 282 krel = krec->reloc; 283 for (i = 0; i < krec->nr_reloc; i++, krel++) { 284 err("ch%d: rel %08x %08x %08x %08x %08x %08x %08x\n", 285 chid, krel->reloc_bo_index, krel->reloc_bo_offset, 286 krel->bo_index, krel->flags, krel->data, 287 krel->vor, krel->tor); 288 } 289 290 kpsh = krec->push; 291 for (i = 0; i < krec->nr_push; i++, kpsh++) { 292 kref = krec->buffer + kpsh->bo_index; 293 bo = (void *)(unsigned long)kref->user_priv; 294 bgn = (uint32_t *)((char *)bo->map + kpsh->offset); 295 end = bgn + (kpsh->length /4); 296 297 err("ch%d: psh %08x %010llx %010llx\n", chid, kpsh->bo_index, 298 (unsigned long long)kpsh->offset, 299 (unsigned long long)(kpsh->offset + kpsh->length)); 300 while (bgn < end) 301 err("\t0x%08x\n", *bgn++); 302 } 303} 304 305static int 306pushbuf_submit(struct nouveau_pushbuf *push, struct nouveau_object *chan) 307{ 308 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 309 struct nouveau_pushbuf_krec *krec = nvpb->list; 310 struct nouveau_device *dev = push->client->device; 311 struct nouveau_drm *drm = nouveau_drm(&dev->object); 312 struct drm_nouveau_gem_pushbuf_bo_presumed *info; 313 struct drm_nouveau_gem_pushbuf_bo *kref; 314 struct drm_nouveau_gem_pushbuf req; 315 struct nouveau_fifo *fifo = chan->data; 316 struct nouveau_bo *bo; 317 int krec_id = 0; 318 int ret = 0, i; 319 320 if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS) 321 return -EINVAL; 322 323 if (push->kick_notify) 324 push->kick_notify(push); 325 326 nouveau_pushbuf_data(push, NULL, 0, 0); 327 328 while (krec && krec->nr_push) { 329 req.channel = fifo->channel; 330 req.nr_buffers = krec->nr_buffer; 331 req.buffers = (uint64_t)(unsigned long)krec->buffer; 332 req.nr_relocs = krec->nr_reloc; 333 req.nr_push = krec->nr_push; 334 req.relocs = (uint64_t)(unsigned long)krec->reloc; 335 req.push = (uint64_t)(unsigned long)krec->push; 336 req.suffix0 = nvpb->suffix0; 337 req.suffix1 = nvpb->suffix1; 338 req.vram_available = 0; /* for valgrind */ 339 req.gart_available = 0; 340 341 if (dbg_on(0)) 342 pushbuf_dump(krec, krec_id++, fifo->channel); 343 344#ifndef SIMULATE 345 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_PUSHBUF, 346 &req, sizeof(req)); 347 nvpb->suffix0 = req.suffix0; 348 nvpb->suffix1 = req.suffix1; 349 dev->vram_limit = (req.vram_available * 350 nouveau_device(dev)->vram_limit_percent) / 100; 351 dev->gart_limit = (req.gart_available * 352 nouveau_device(dev)->gart_limit_percent) / 100; 353#else 354 if (dbg_on(31)) 355 ret = -EINVAL; 356#endif 357 358 if (ret) { 359 err("kernel rejected pushbuf: %s\n", strerror(-ret)); 360 pushbuf_dump(krec, krec_id++, fifo->channel); 361 break; 362 } 363 364 kref = krec->buffer; 365 for (i = 0; i < krec->nr_buffer; i++, kref++) { 366 bo = (void *)(unsigned long)kref->user_priv; 367 368 info = &kref->presumed; 369 if (!info->valid) { 370 bo->flags &= ~NOUVEAU_BO_APER; 371 if (info->domain == NOUVEAU_GEM_DOMAIN_VRAM) 372 bo->flags |= NOUVEAU_BO_VRAM; 373 else 374 bo->flags |= NOUVEAU_BO_GART; 375 bo->offset = info->offset; 376 } 377 378 if (kref->write_domains) 379 nouveau_bo(bo)->access |= NOUVEAU_BO_WR; 380 if (kref->read_domains) 381 nouveau_bo(bo)->access |= NOUVEAU_BO_RD; 382 } 383 384 krec = krec->next; 385 } 386 387 return ret; 388} 389 390static int 391pushbuf_flush(struct nouveau_pushbuf *push) 392{ 393 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 394 struct nouveau_pushbuf_krec *krec = nvpb->krec; 395 struct drm_nouveau_gem_pushbuf_bo *kref; 396 struct nouveau_bufctx *bctx, *btmp; 397 struct nouveau_bo *bo; 398 int ret = 0, i; 399 400 if (push->channel) { 401 ret = pushbuf_submit(push, push->channel); 402 } else { 403 nouveau_pushbuf_data(push, NULL, 0, 0); 404 krec->next = malloc(sizeof(*krec)); 405 nvpb->krec = krec->next; 406 } 407 408 kref = krec->buffer; 409 for (i = 0; i < krec->nr_buffer; i++, kref++) { 410 bo = (void *)(unsigned long)kref->user_priv; 411 cli_kref_set(push->client, bo, NULL, NULL); 412 if (push->channel) 413 nouveau_bo_ref(NULL, &bo); 414 } 415 416 krec = nvpb->krec; 417 krec->vram_used = 0; 418 krec->gart_used = 0; 419 krec->nr_buffer = 0; 420 krec->nr_reloc = 0; 421 krec->nr_push = 0; 422 423 DRMLISTFOREACHENTRYSAFE(bctx, btmp, &nvpb->bctx_list, head) { 424 DRMLISTJOIN(&bctx->current, &bctx->pending); 425 DRMINITLISTHEAD(&bctx->current); 426 DRMLISTDELINIT(&bctx->head); 427 } 428 429 return ret; 430} 431 432static void 433pushbuf_refn_fail(struct nouveau_pushbuf *push, int sref, int srel) 434{ 435 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 436 struct nouveau_pushbuf_krec *krec = nvpb->krec; 437 struct drm_nouveau_gem_pushbuf_bo *kref; 438 439 kref = krec->buffer + sref; 440 while (krec->nr_buffer-- > sref) { 441 struct nouveau_bo *bo = (void *)(unsigned long)kref->user_priv; 442 cli_kref_set(push->client, bo, NULL, NULL); 443 nouveau_bo_ref(NULL, &bo); 444 kref++; 445 } 446 krec->nr_buffer = sref; 447 krec->nr_reloc = srel; 448} 449 450static int 451pushbuf_refn(struct nouveau_pushbuf *push, bool retry, 452 struct nouveau_pushbuf_refn *refs, int nr) 453{ 454 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 455 struct nouveau_pushbuf_krec *krec = nvpb->krec; 456 struct drm_nouveau_gem_pushbuf_bo *kref; 457 int sref = krec->nr_buffer; 458 int ret = 0, i; 459 460 for (i = 0; i < nr; i++) { 461 kref = pushbuf_kref(push, refs[i].bo, refs[i].flags); 462 if (!kref) { 463 ret = -ENOSPC; 464 break; 465 } 466 } 467 468 if (ret) { 469 pushbuf_refn_fail(push, sref, krec->nr_reloc); 470 if (retry) { 471 pushbuf_flush(push); 472 nouveau_pushbuf_space(push, 0, 0, 0); 473 return pushbuf_refn(push, false, refs, nr); 474 } 475 } 476 477 return ret; 478} 479 480static int 481pushbuf_validate(struct nouveau_pushbuf *push, bool retry) 482{ 483 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 484 struct nouveau_pushbuf_krec *krec = nvpb->krec; 485 struct drm_nouveau_gem_pushbuf_bo *kref; 486 struct nouveau_bufctx *bctx = push->bufctx; 487 struct nouveau_bufref *bref; 488 int relocs = bctx ? bctx->relocs * 2: 0; 489 int sref, srel, ret; 490 491 ret = nouveau_pushbuf_space(push, relocs, relocs, 0); 492 if (ret || bctx == NULL) 493 return ret; 494 495 sref = krec->nr_buffer; 496 srel = krec->nr_reloc; 497 498 DRMLISTDEL(&bctx->head); 499 DRMLISTADD(&bctx->head, &nvpb->bctx_list); 500 501 DRMLISTFOREACHENTRY(bref, &bctx->pending, thead) { 502 kref = pushbuf_kref(push, bref->bo, bref->flags); 503 if (!kref) { 504 ret = -ENOSPC; 505 break; 506 } 507 508 if (bref->packet) { 509 pushbuf_krel(push, bref->bo, bref->packet, 0, 0, 0); 510 *push->cur++ = 0; 511 pushbuf_krel(push, bref->bo, bref->data, bref->flags, 512 bref->vor, bref->tor); 513 *push->cur++ = 0; 514 } 515 } 516 517 DRMLISTJOIN(&bctx->pending, &bctx->current); 518 DRMINITLISTHEAD(&bctx->pending); 519 520 if (ret) { 521 pushbuf_refn_fail(push, sref, srel); 522 if (retry) { 523 pushbuf_flush(push); 524 return pushbuf_validate(push, false); 525 } 526 } 527 528 return ret; 529} 530 531drm_public int 532nouveau_pushbuf_new(struct nouveau_client *client, struct nouveau_object *chan, 533 int nr, uint32_t size, bool immediate, 534 struct nouveau_pushbuf **ppush) 535{ 536 struct nouveau_drm *drm = nouveau_drm(&client->device->object); 537 struct nouveau_fifo *fifo = chan->data; 538 struct nouveau_pushbuf_priv *nvpb; 539 struct nouveau_pushbuf *push; 540 struct drm_nouveau_gem_pushbuf req = {}; 541 int ret; 542 543 if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS) 544 return -EINVAL; 545 546 /* nop pushbuf call, to get the current "return to main" sequence 547 * we need to append to the pushbuf on early chipsets 548 */ 549 req.channel = fifo->channel; 550 req.nr_push = 0; 551 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_PUSHBUF, 552 &req, sizeof(req)); 553 if (ret) 554 return ret; 555 556 nvpb = calloc(1, sizeof(*nvpb) + nr * sizeof(*nvpb->bos)); 557 if (!nvpb) 558 return -ENOMEM; 559 560#ifndef SIMULATE 561 nvpb->suffix0 = req.suffix0; 562 nvpb->suffix1 = req.suffix1; 563#else 564 nvpb->suffix0 = 0xffffffff; 565 nvpb->suffix1 = 0xffffffff; 566#endif 567 nvpb->krec = calloc(1, sizeof(*nvpb->krec)); 568 nvpb->list = nvpb->krec; 569 if (!nvpb->krec) { 570 free(nvpb); 571 return -ENOMEM; 572 } 573 574 push = &nvpb->base; 575 push->client = client; 576 push->channel = immediate ? chan : NULL; 577 push->flags = NOUVEAU_BO_RD; 578 if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_GART) { 579 push->flags |= NOUVEAU_BO_GART; 580 nvpb->type = NOUVEAU_BO_GART; 581 } else 582 if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_VRAM) { 583 push->flags |= NOUVEAU_BO_VRAM; 584 nvpb->type = NOUVEAU_BO_VRAM; 585 } 586 nvpb->type |= NOUVEAU_BO_MAP; 587 588 for (nvpb->bo_nr = 0; nvpb->bo_nr < nr; nvpb->bo_nr++) { 589 ret = nouveau_bo_new(client->device, nvpb->type, 0, size, 590 NULL, &nvpb->bos[nvpb->bo_nr]); 591 if (ret) { 592 nouveau_pushbuf_del(&push); 593 return ret; 594 } 595 } 596 597 DRMINITLISTHEAD(&nvpb->bctx_list); 598 *ppush = push; 599 return 0; 600} 601 602drm_public void 603nouveau_pushbuf_del(struct nouveau_pushbuf **ppush) 604{ 605 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(*ppush); 606 if (nvpb) { 607 struct drm_nouveau_gem_pushbuf_bo *kref; 608 struct nouveau_pushbuf_krec *krec; 609 while ((krec = nvpb->list)) { 610 kref = krec->buffer; 611 while (krec->nr_buffer--) { 612 unsigned long priv = kref++->user_priv; 613 struct nouveau_bo *bo = (void *)priv; 614 cli_kref_set(nvpb->base.client, bo, NULL, NULL); 615 nouveau_bo_ref(NULL, &bo); 616 } 617 nvpb->list = krec->next; 618 free(krec); 619 } 620 while (nvpb->bo_nr--) 621 nouveau_bo_ref(NULL, &nvpb->bos[nvpb->bo_nr]); 622 nouveau_bo_ref(NULL, &nvpb->bo); 623 free(nvpb); 624 } 625 *ppush = NULL; 626} 627 628drm_public struct nouveau_bufctx * 629nouveau_pushbuf_bufctx(struct nouveau_pushbuf *push, struct nouveau_bufctx *ctx) 630{ 631 struct nouveau_bufctx *prev = push->bufctx; 632 push->bufctx = ctx; 633 return prev; 634} 635 636drm_public int 637nouveau_pushbuf_space(struct nouveau_pushbuf *push, 638 uint32_t dwords, uint32_t relocs, uint32_t pushes) 639{ 640 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 641 struct nouveau_pushbuf_krec *krec = nvpb->krec; 642 struct nouveau_client *client = push->client; 643 struct nouveau_bo *bo = NULL; 644 bool flushed = false; 645 int ret = 0; 646 647 /* switch to next buffer if insufficient space in the current one */ 648 if (push->cur + dwords >= push->end) { 649 if (nvpb->bo_next < nvpb->bo_nr) { 650 nouveau_bo_ref(nvpb->bos[nvpb->bo_next++], &bo); 651 if (nvpb->bo_next == nvpb->bo_nr && push->channel) 652 nvpb->bo_next = 0; 653 } else { 654 ret = nouveau_bo_new(client->device, nvpb->type, 0, 655 nvpb->bos[0]->size, NULL, &bo); 656 if (ret) 657 return ret; 658 } 659 } 660 661 /* make sure there's always enough space to queue up the pending 662 * data in the pushbuf proper 663 */ 664 pushes++; 665 666 /* need to flush if we've run out of space on an immediate pushbuf, 667 * if the new buffer won't fit, or if the kernel push/reloc limits 668 * have been hit 669 */ 670 if ((bo && ( push->channel || 671 !pushbuf_kref(push, bo, push->flags))) || 672 krec->nr_reloc + relocs >= NOUVEAU_GEM_MAX_RELOCS || 673 krec->nr_push + pushes >= NOUVEAU_GEM_MAX_PUSH) { 674 if (nvpb->bo && krec->nr_buffer) 675 pushbuf_flush(push); 676 flushed = true; 677 } 678 679 /* if necessary, switch to new buffer */ 680 if (bo) { 681 ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, push->client); 682 if (ret) 683 return ret; 684 685 nouveau_pushbuf_data(push, NULL, 0, 0); 686 nouveau_bo_ref(bo, &nvpb->bo); 687 nouveau_bo_ref(NULL, &bo); 688 689 nvpb->bgn = nvpb->bo->map; 690 nvpb->ptr = nvpb->bgn; 691 push->cur = nvpb->bgn; 692 push->end = push->cur + (nvpb->bo->size / 4); 693 push->end -= 2 + push->rsvd_kick; /* space for suffix */ 694 } 695 696 pushbuf_kref(push, nvpb->bo, push->flags); 697 return flushed ? pushbuf_validate(push, false) : 0; 698} 699 700drm_public void 701nouveau_pushbuf_data(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 702 uint64_t offset, uint64_t length) 703{ 704 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 705 struct nouveau_pushbuf_krec *krec = nvpb->krec; 706 struct drm_nouveau_gem_pushbuf_push *kpsh; 707 struct drm_nouveau_gem_pushbuf_bo *kref; 708 709 if (bo != nvpb->bo && nvpb->bgn != push->cur) { 710 if (nvpb->suffix0 || nvpb->suffix1) { 711 *push->cur++ = nvpb->suffix0; 712 *push->cur++ = nvpb->suffix1; 713 } 714 715 nouveau_pushbuf_data(push, nvpb->bo, 716 (nvpb->bgn - nvpb->ptr) * 4, 717 (push->cur - nvpb->bgn) * 4); 718 nvpb->bgn = push->cur; 719 } 720 721 if (bo) { 722 kref = cli_kref_get(push->client, bo); 723 assert(kref); 724 kpsh = &krec->push[krec->nr_push++]; 725 kpsh->bo_index = kref - krec->buffer; 726 kpsh->offset = offset; 727 kpsh->length = length; 728 } 729} 730 731drm_public int 732nouveau_pushbuf_refn(struct nouveau_pushbuf *push, 733 struct nouveau_pushbuf_refn *refs, int nr) 734{ 735 return pushbuf_refn(push, true, refs, nr); 736} 737 738drm_public void 739nouveau_pushbuf_reloc(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 740 uint32_t data, uint32_t flags, uint32_t vor, uint32_t tor) 741{ 742 *push->cur = pushbuf_krel(push, bo, data, flags, vor, tor); 743 push->cur++; 744} 745 746drm_public int 747nouveau_pushbuf_validate(struct nouveau_pushbuf *push) 748{ 749 return pushbuf_validate(push, true); 750} 751 752drm_public uint32_t 753nouveau_pushbuf_refd(struct nouveau_pushbuf *push, struct nouveau_bo *bo) 754{ 755 struct drm_nouveau_gem_pushbuf_bo *kref; 756 uint32_t flags = 0; 757 758 if (cli_push_get(push->client, bo) == push) { 759 kref = cli_kref_get(push->client, bo); 760 assert(kref); 761 if (kref->read_domains) 762 flags |= NOUVEAU_BO_RD; 763 if (kref->write_domains) 764 flags |= NOUVEAU_BO_WR; 765 } 766 767 return flags; 768} 769 770drm_public int 771nouveau_pushbuf_kick(struct nouveau_pushbuf *push, struct nouveau_object *chan) 772{ 773 if (!push->channel) 774 return pushbuf_submit(push, chan); 775 pushbuf_flush(push); 776 return pushbuf_validate(push, false); 777} 778