1/* 2 * Copyright 2012 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Ben Skeggs 23 */ 24 25#include <stdio.h> 26#include <stdlib.h> 27#include <stdint.h> 28#include <stdbool.h> 29#include <string.h> 30#include <assert.h> 31#include <errno.h> 32#include <inttypes.h> 33 34#include <xf86drm.h> 35#include <xf86atomic.h> 36#include "libdrm_lists.h" 37#include "nouveau_drm.h" 38 39#include "nouveau.h" 40#include "private.h" 41 42struct nouveau_pushbuf_krec { 43 struct nouveau_pushbuf_krec *next; 44 struct drm_nouveau_gem_pushbuf_bo buffer[NOUVEAU_GEM_MAX_BUFFERS]; 45 struct drm_nouveau_gem_pushbuf_reloc reloc[NOUVEAU_GEM_MAX_RELOCS]; 46 struct drm_nouveau_gem_pushbuf_push push[NOUVEAU_GEM_MAX_PUSH]; 47 int nr_buffer; 48 int nr_reloc; 49 int nr_push; 50 uint64_t vram_used; 51 uint64_t gart_used; 52}; 53 54struct nouveau_pushbuf_priv { 55 struct nouveau_pushbuf base; 56 struct nouveau_pushbuf_krec *list; 57 struct nouveau_pushbuf_krec *krec; 58 struct nouveau_list bctx_list; 59 struct nouveau_bo *bo; 60 uint32_t type; 61 uint32_t suffix0; 62 uint32_t suffix1; 63 uint32_t *ptr; 64 uint32_t *bgn; 65 int bo_next; 66 int bo_nr; 67 struct nouveau_bo *bos[]; 68}; 69 70static inline struct nouveau_pushbuf_priv * 71nouveau_pushbuf(struct nouveau_pushbuf *push) 72{ 73 return (struct nouveau_pushbuf_priv *)push; 74} 75 76static int pushbuf_validate(struct nouveau_pushbuf *, bool); 77static int pushbuf_flush(struct nouveau_pushbuf *); 78 79static bool 80pushbuf_kref_fits(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 81 uint32_t *domains) 82{ 83 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 84 struct nouveau_pushbuf_krec *krec = nvpb->krec; 85 struct nouveau_device *dev = push->client->device; 86 struct nouveau_bo *kbo; 87 struct drm_nouveau_gem_pushbuf_bo *kref; 88 int i; 89 90 /* VRAM is the only valid domain. GART and VRAM|GART buffers 91 * are all accounted to GART, so if this doesn't fit in VRAM 92 * straight up, a flush is needed. 93 */ 94 if (*domains == NOUVEAU_GEM_DOMAIN_VRAM) { 95 if (krec->vram_used + bo->size > dev->vram_limit) 96 return false; 97 krec->vram_used += bo->size; 98 return true; 99 } 100 101 /* GART or VRAM|GART buffer. Account both of these buffer types 102 * to GART only for the moment, which simplifies things. If the 103 * buffer can fit already, we're done here. 104 */ 105 if (krec->gart_used + bo->size <= dev->gart_limit) { 106 krec->gart_used += bo->size; 107 return true; 108 } 109 110 /* Ran out of GART space, if it's a VRAM|GART buffer and it'll 111 * fit into available VRAM, turn it into a VRAM buffer 112 */ 113 if ((*domains & NOUVEAU_GEM_DOMAIN_VRAM) && 114 krec->vram_used + bo->size <= dev->vram_limit) { 115 *domains &= NOUVEAU_GEM_DOMAIN_VRAM; 116 krec->vram_used += bo->size; 117 return true; 118 } 119 120 /* Still couldn't fit the buffer in anywhere, so as a last resort; 121 * scan the buffer list for VRAM|GART buffers and turn them into 122 * VRAM buffers until we have enough space in GART for this one 123 */ 124 kref = krec->buffer; 125 for (i = 0; i < krec->nr_buffer; i++, kref++) { 126 if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 127 continue; 128 129 kbo = (void *)(unsigned long)kref->user_priv; 130 if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) || 131 krec->vram_used + kbo->size > dev->vram_limit) 132 continue; 133 134 kref->valid_domains &= NOUVEAU_GEM_DOMAIN_VRAM; 135 krec->gart_used -= kbo->size; 136 krec->vram_used += kbo->size; 137 if (krec->gart_used + bo->size <= dev->gart_limit) { 138 krec->gart_used += bo->size; 139 return true; 140 } 141 } 142 143 /* Couldn't resolve a placement, need to force a flush */ 144 return false; 145} 146 147static struct drm_nouveau_gem_pushbuf_bo * 148pushbuf_kref(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 149 uint32_t flags) 150{ 151 struct nouveau_device *dev = push->client->device; 152 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 153 struct nouveau_pushbuf_krec *krec = nvpb->krec; 154 struct nouveau_pushbuf *fpush; 155 struct drm_nouveau_gem_pushbuf_bo *kref; 156 uint32_t domains, domains_wr, domains_rd; 157 158 domains = 0; 159 if (flags & NOUVEAU_BO_VRAM) 160 domains |= NOUVEAU_GEM_DOMAIN_VRAM; 161 if (flags & NOUVEAU_BO_GART) 162 domains |= NOUVEAU_GEM_DOMAIN_GART; 163 domains_wr = domains * !!(flags & NOUVEAU_BO_WR); 164 domains_rd = domains * !!(flags & NOUVEAU_BO_RD); 165 166 /* if buffer is referenced on another pushbuf that is owned by the 167 * same client, we need to flush the other pushbuf first to ensure 168 * the correct ordering of commands 169 */ 170 fpush = cli_push_get(push->client, bo); 171 if (fpush && fpush != push) 172 pushbuf_flush(fpush); 173 174 kref = cli_kref_get(push->client, bo); 175 if (kref) { 176 /* possible conflict in memory types - flush and retry */ 177 if (!(kref->valid_domains & domains)) 178 return NULL; 179 180 /* VRAM|GART buffer turning into a VRAM buffer. Make sure 181 * it'll fit in VRAM and force a flush if not. 182 */ 183 if ((kref->valid_domains & NOUVEAU_GEM_DOMAIN_GART) && 184 ( domains == NOUVEAU_GEM_DOMAIN_VRAM)) { 185 if (krec->vram_used + bo->size > dev->vram_limit) 186 return NULL; 187 krec->vram_used += bo->size; 188 krec->gart_used -= bo->size; 189 } 190 191 kref->valid_domains &= domains; 192 kref->write_domains |= domains_wr; 193 kref->read_domains |= domains_rd; 194 } else { 195 if (krec->nr_buffer == NOUVEAU_GEM_MAX_BUFFERS || 196 !pushbuf_kref_fits(push, bo, &domains)) 197 return NULL; 198 199 kref = &krec->buffer[krec->nr_buffer++]; 200 kref->user_priv = (unsigned long)bo; 201 kref->handle = bo->handle; 202 kref->valid_domains = domains; 203 kref->write_domains = domains_wr; 204 kref->read_domains = domains_rd; 205 kref->presumed.valid = 1; 206 kref->presumed.offset = bo->offset; 207 if (bo->flags & NOUVEAU_BO_VRAM) 208 kref->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 209 else 210 kref->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 211 212 cli_kref_set(push->client, bo, kref, push); 213 atomic_inc(&nouveau_bo(bo)->refcnt); 214 } 215 216 return kref; 217} 218 219static uint32_t 220pushbuf_krel(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 221 uint32_t data, uint32_t flags, uint32_t vor, uint32_t tor) 222{ 223 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 224 struct nouveau_pushbuf_krec *krec = nvpb->krec; 225 struct drm_nouveau_gem_pushbuf_reloc *krel; 226 struct drm_nouveau_gem_pushbuf_bo *pkref; 227 struct drm_nouveau_gem_pushbuf_bo *bkref; 228 uint32_t reloc = data; 229 230 pkref = cli_kref_get(push->client, nvpb->bo); 231 bkref = cli_kref_get(push->client, bo); 232 krel = &krec->reloc[krec->nr_reloc++]; 233 234 assert(pkref); 235 assert(bkref); 236 krel->reloc_bo_index = pkref - krec->buffer; 237 krel->reloc_bo_offset = (push->cur - nvpb->ptr) * 4; 238 krel->bo_index = bkref - krec->buffer; 239 krel->flags = 0; 240 krel->data = data; 241 krel->vor = vor; 242 krel->tor = tor; 243 244 if (flags & NOUVEAU_BO_LOW) { 245 reloc = (bkref->presumed.offset + data); 246 krel->flags |= NOUVEAU_GEM_RELOC_LOW; 247 } else 248 if (flags & NOUVEAU_BO_HIGH) { 249 reloc = (bkref->presumed.offset + data) >> 32; 250 krel->flags |= NOUVEAU_GEM_RELOC_HIGH; 251 } 252 if (flags & NOUVEAU_BO_OR) { 253 if (bkref->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) 254 reloc |= vor; 255 else 256 reloc |= tor; 257 krel->flags |= NOUVEAU_GEM_RELOC_OR; 258 } 259 260 return reloc; 261} 262 263static void 264pushbuf_dump(struct nouveau_pushbuf_krec *krec, int krec_id, int chid) 265{ 266 struct drm_nouveau_gem_pushbuf_reloc *krel; 267 struct drm_nouveau_gem_pushbuf_push *kpsh; 268 struct drm_nouveau_gem_pushbuf_bo *kref; 269 struct nouveau_bo *bo; 270 uint32_t *bgn, *end; 271 int i; 272 273 err("ch%d: krec %d pushes %d bufs %d relocs %d\n", chid, 274 krec_id, krec->nr_push, krec->nr_buffer, krec->nr_reloc); 275 276 kref = krec->buffer; 277 for (i = 0; i < krec->nr_buffer; i++, kref++) { 278 bo = (void *)(uintptr_t)kref->user_priv; 279 err("ch%d: buf %08x %08x %08x %08x %08x %p 0x%"PRIx64" 0x%"PRIx64"\n", chid, i, 280 kref->handle, kref->valid_domains, 281 kref->read_domains, kref->write_domains, bo->map, bo->offset, bo->size); 282 } 283 284 krel = krec->reloc; 285 for (i = 0; i < krec->nr_reloc; i++, krel++) { 286 err("ch%d: rel %08x %08x %08x %08x %08x %08x %08x\n", 287 chid, krel->reloc_bo_index, krel->reloc_bo_offset, 288 krel->bo_index, krel->flags, krel->data, 289 krel->vor, krel->tor); 290 } 291 292 kpsh = krec->push; 293 for (i = 0; i < krec->nr_push; i++, kpsh++) { 294 kref = krec->buffer + kpsh->bo_index; 295 bo = (void *)(unsigned long)kref->user_priv; 296 bgn = (uint32_t *)((char *)bo->map + kpsh->offset); 297 end = bgn + ((kpsh->length & 0x7fffff) /4); 298 299 err("ch%d: psh %s%08x %010llx %010llx\n", chid, 300 bo->map ? "" : "(unmapped) ", kpsh->bo_index, 301 (unsigned long long)kpsh->offset, 302 (unsigned long long)(kpsh->offset + kpsh->length)); 303 if (!bo->map) 304 continue; 305 while (bgn < end) 306 err("\t0x%08x\n", *bgn++); 307 } 308} 309 310static int 311pushbuf_submit(struct nouveau_pushbuf *push, struct nouveau_object *chan) 312{ 313 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 314 struct nouveau_pushbuf_krec *krec = nvpb->list; 315 struct nouveau_device *dev = push->client->device; 316 struct nouveau_drm *drm = nouveau_drm(&dev->object); 317 struct drm_nouveau_gem_pushbuf_bo_presumed *info; 318 struct drm_nouveau_gem_pushbuf_bo *kref; 319 struct drm_nouveau_gem_pushbuf req; 320 struct nouveau_fifo *fifo = chan->data; 321 struct nouveau_bo *bo; 322 int krec_id = 0; 323 int ret = 0, i; 324 325 if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS) 326 return -EINVAL; 327 328 if (push->kick_notify) 329 push->kick_notify(push); 330 331 nouveau_pushbuf_data(push, NULL, 0, 0); 332 333 while (krec && krec->nr_push) { 334 req.channel = fifo->channel; 335 req.nr_buffers = krec->nr_buffer; 336 req.buffers = (uint64_t)(unsigned long)krec->buffer; 337 req.nr_relocs = krec->nr_reloc; 338 req.nr_push = krec->nr_push; 339 req.relocs = (uint64_t)(unsigned long)krec->reloc; 340 req.push = (uint64_t)(unsigned long)krec->push; 341 req.suffix0 = nvpb->suffix0; 342 req.suffix1 = nvpb->suffix1; 343 req.vram_available = 0; /* for valgrind */ 344 if (dbg_on(1)) 345 req.vram_available |= NOUVEAU_GEM_PUSHBUF_SYNC; 346 req.gart_available = 0; 347 348 if (dbg_on(0)) 349 pushbuf_dump(krec, krec_id++, fifo->channel); 350 351#ifndef SIMULATE 352 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_PUSHBUF, 353 &req, sizeof(req)); 354 nvpb->suffix0 = req.suffix0; 355 nvpb->suffix1 = req.suffix1; 356 dev->vram_limit = (req.vram_available * 357 nouveau_device(dev)->vram_limit_percent) / 100; 358 dev->gart_limit = (req.gart_available * 359 nouveau_device(dev)->gart_limit_percent) / 100; 360#else 361 if (dbg_on(31)) 362 ret = -EINVAL; 363#endif 364 365 if (ret) { 366 err("kernel rejected pushbuf: %s\n", strerror(-ret)); 367 pushbuf_dump(krec, krec_id++, fifo->channel); 368 break; 369 } 370 371 kref = krec->buffer; 372 for (i = 0; i < krec->nr_buffer; i++, kref++) { 373 bo = (void *)(unsigned long)kref->user_priv; 374 375 info = &kref->presumed; 376 if (!info->valid) { 377 bo->flags &= ~NOUVEAU_BO_APER; 378 if (info->domain == NOUVEAU_GEM_DOMAIN_VRAM) 379 bo->flags |= NOUVEAU_BO_VRAM; 380 else 381 bo->flags |= NOUVEAU_BO_GART; 382 bo->offset = info->offset; 383 } 384 385 if (kref->write_domains) 386 nouveau_bo(bo)->access |= NOUVEAU_BO_WR; 387 if (kref->read_domains) 388 nouveau_bo(bo)->access |= NOUVEAU_BO_RD; 389 } 390 391 krec = krec->next; 392 } 393 394 return ret; 395} 396 397static int 398pushbuf_flush(struct nouveau_pushbuf *push) 399{ 400 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 401 struct nouveau_pushbuf_krec *krec = nvpb->krec; 402 struct drm_nouveau_gem_pushbuf_bo *kref; 403 struct nouveau_bufctx *bctx, *btmp; 404 struct nouveau_bo *bo; 405 int ret = 0, i; 406 407 if (push->channel) { 408 ret = pushbuf_submit(push, push->channel); 409 } else { 410 nouveau_pushbuf_data(push, NULL, 0, 0); 411 krec->next = malloc(sizeof(*krec)); 412 nvpb->krec = krec->next; 413 } 414 415 kref = krec->buffer; 416 for (i = 0; i < krec->nr_buffer; i++, kref++) { 417 bo = (void *)(unsigned long)kref->user_priv; 418 cli_kref_set(push->client, bo, NULL, NULL); 419 if (push->channel) 420 nouveau_bo_ref(NULL, &bo); 421 } 422 423 krec = nvpb->krec; 424 krec->vram_used = 0; 425 krec->gart_used = 0; 426 krec->nr_buffer = 0; 427 krec->nr_reloc = 0; 428 krec->nr_push = 0; 429 430 DRMLISTFOREACHENTRYSAFE(bctx, btmp, &nvpb->bctx_list, head) { 431 DRMLISTJOIN(&bctx->current, &bctx->pending); 432 DRMINITLISTHEAD(&bctx->current); 433 DRMLISTDELINIT(&bctx->head); 434 } 435 436 return ret; 437} 438 439static void 440pushbuf_refn_fail(struct nouveau_pushbuf *push, int sref, int srel) 441{ 442 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 443 struct nouveau_pushbuf_krec *krec = nvpb->krec; 444 struct drm_nouveau_gem_pushbuf_bo *kref; 445 446 kref = krec->buffer + sref; 447 while (krec->nr_buffer-- > sref) { 448 struct nouveau_bo *bo = (void *)(unsigned long)kref->user_priv; 449 cli_kref_set(push->client, bo, NULL, NULL); 450 nouveau_bo_ref(NULL, &bo); 451 kref++; 452 } 453 krec->nr_buffer = sref; 454 krec->nr_reloc = srel; 455} 456 457static int 458pushbuf_refn(struct nouveau_pushbuf *push, bool retry, 459 struct nouveau_pushbuf_refn *refs, int nr) 460{ 461 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 462 struct nouveau_pushbuf_krec *krec = nvpb->krec; 463 struct drm_nouveau_gem_pushbuf_bo *kref; 464 int sref = krec->nr_buffer; 465 int ret = 0, i; 466 467 for (i = 0; i < nr; i++) { 468 kref = pushbuf_kref(push, refs[i].bo, refs[i].flags); 469 if (!kref) { 470 ret = -ENOSPC; 471 break; 472 } 473 } 474 475 if (ret) { 476 pushbuf_refn_fail(push, sref, krec->nr_reloc); 477 if (retry) { 478 pushbuf_flush(push); 479 nouveau_pushbuf_space(push, 0, 0, 0); 480 return pushbuf_refn(push, false, refs, nr); 481 } 482 } 483 484 return ret; 485} 486 487static int 488pushbuf_validate(struct nouveau_pushbuf *push, bool retry) 489{ 490 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 491 struct nouveau_pushbuf_krec *krec = nvpb->krec; 492 struct drm_nouveau_gem_pushbuf_bo *kref; 493 struct nouveau_bufctx *bctx = push->bufctx; 494 struct nouveau_bufref *bref; 495 int relocs = bctx ? bctx->relocs * 2: 0; 496 int sref, srel, ret; 497 498 ret = nouveau_pushbuf_space(push, relocs, relocs, 0); 499 if (ret || bctx == NULL) 500 return ret; 501 502 sref = krec->nr_buffer; 503 srel = krec->nr_reloc; 504 505 DRMLISTDEL(&bctx->head); 506 DRMLISTADD(&bctx->head, &nvpb->bctx_list); 507 508 DRMLISTFOREACHENTRY(bref, &bctx->pending, thead) { 509 kref = pushbuf_kref(push, bref->bo, bref->flags); 510 if (!kref) { 511 ret = -ENOSPC; 512 break; 513 } 514 515 if (bref->packet) { 516 pushbuf_krel(push, bref->bo, bref->packet, 0, 0, 0); 517 *push->cur++ = 0; 518 pushbuf_krel(push, bref->bo, bref->data, bref->flags, 519 bref->vor, bref->tor); 520 *push->cur++ = 0; 521 } 522 } 523 524 DRMLISTJOIN(&bctx->pending, &bctx->current); 525 DRMINITLISTHEAD(&bctx->pending); 526 527 if (ret) { 528 pushbuf_refn_fail(push, sref, srel); 529 if (retry) { 530 pushbuf_flush(push); 531 return pushbuf_validate(push, false); 532 } 533 } 534 535 return ret; 536} 537 538drm_public int 539nouveau_pushbuf_new(struct nouveau_client *client, struct nouveau_object *chan, 540 int nr, uint32_t size, bool immediate, 541 struct nouveau_pushbuf **ppush) 542{ 543 struct nouveau_drm *drm = nouveau_drm(&client->device->object); 544 struct nouveau_fifo *fifo = chan->data; 545 struct nouveau_pushbuf_priv *nvpb; 546 struct nouveau_pushbuf *push; 547 struct drm_nouveau_gem_pushbuf req = {}; 548 int ret; 549 550 if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS) 551 return -EINVAL; 552 553 /* nop pushbuf call, to get the current "return to main" sequence 554 * we need to append to the pushbuf on early chipsets 555 */ 556 req.channel = fifo->channel; 557 req.nr_push = 0; 558 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_PUSHBUF, 559 &req, sizeof(req)); 560 if (ret) 561 return ret; 562 563 nvpb = calloc(1, sizeof(*nvpb) + nr * sizeof(*nvpb->bos)); 564 if (!nvpb) 565 return -ENOMEM; 566 567#ifndef SIMULATE 568 nvpb->suffix0 = req.suffix0; 569 nvpb->suffix1 = req.suffix1; 570#else 571 nvpb->suffix0 = 0xffffffff; 572 nvpb->suffix1 = 0xffffffff; 573#endif 574 nvpb->krec = calloc(1, sizeof(*nvpb->krec)); 575 nvpb->list = nvpb->krec; 576 if (!nvpb->krec) { 577 free(nvpb); 578 return -ENOMEM; 579 } 580 581 push = &nvpb->base; 582 push->client = client; 583 push->channel = immediate ? chan : NULL; 584 push->flags = NOUVEAU_BO_RD; 585 if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_GART) { 586 push->flags |= NOUVEAU_BO_GART; 587 nvpb->type = NOUVEAU_BO_GART; 588 } else 589 if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_VRAM) { 590 push->flags |= NOUVEAU_BO_VRAM; 591 nvpb->type = NOUVEAU_BO_VRAM; 592 } 593 nvpb->type |= NOUVEAU_BO_MAP; 594 595 for (nvpb->bo_nr = 0; nvpb->bo_nr < nr; nvpb->bo_nr++) { 596 ret = nouveau_bo_new(client->device, nvpb->type, 0, size, 597 NULL, &nvpb->bos[nvpb->bo_nr]); 598 if (ret) { 599 nouveau_pushbuf_del(&push); 600 return ret; 601 } 602 } 603 604 DRMINITLISTHEAD(&nvpb->bctx_list); 605 *ppush = push; 606 return 0; 607} 608 609drm_public void 610nouveau_pushbuf_del(struct nouveau_pushbuf **ppush) 611{ 612 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(*ppush); 613 if (nvpb) { 614 struct drm_nouveau_gem_pushbuf_bo *kref; 615 struct nouveau_pushbuf_krec *krec; 616 while ((krec = nvpb->list)) { 617 kref = krec->buffer; 618 while (krec->nr_buffer--) { 619 unsigned long priv = kref++->user_priv; 620 struct nouveau_bo *bo = (void *)priv; 621 cli_kref_set(nvpb->base.client, bo, NULL, NULL); 622 nouveau_bo_ref(NULL, &bo); 623 } 624 nvpb->list = krec->next; 625 free(krec); 626 } 627 while (nvpb->bo_nr--) 628 nouveau_bo_ref(NULL, &nvpb->bos[nvpb->bo_nr]); 629 nouveau_bo_ref(NULL, &nvpb->bo); 630 free(nvpb); 631 } 632 *ppush = NULL; 633} 634 635drm_public struct nouveau_bufctx * 636nouveau_pushbuf_bufctx(struct nouveau_pushbuf *push, struct nouveau_bufctx *ctx) 637{ 638 struct nouveau_bufctx *prev = push->bufctx; 639 push->bufctx = ctx; 640 return prev; 641} 642 643drm_public int 644nouveau_pushbuf_space(struct nouveau_pushbuf *push, 645 uint32_t dwords, uint32_t relocs, uint32_t pushes) 646{ 647 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 648 struct nouveau_pushbuf_krec *krec = nvpb->krec; 649 struct nouveau_client *client = push->client; 650 struct nouveau_bo *bo = NULL; 651 bool flushed = false; 652 int ret = 0; 653 654 /* switch to next buffer if insufficient space in the current one */ 655 if (push->cur + dwords >= push->end) { 656 if (nvpb->bo_next < nvpb->bo_nr) { 657 nouveau_bo_ref(nvpb->bos[nvpb->bo_next++], &bo); 658 if (nvpb->bo_next == nvpb->bo_nr && push->channel) 659 nvpb->bo_next = 0; 660 } else { 661 ret = nouveau_bo_new(client->device, nvpb->type, 0, 662 nvpb->bos[0]->size, NULL, &bo); 663 if (ret) 664 return ret; 665 } 666 } 667 668 /* make sure there's always enough space to queue up the pending 669 * data in the pushbuf proper 670 */ 671 pushes++; 672 673 /* need to flush if we've run out of space on an immediate pushbuf, 674 * if the new buffer won't fit, or if the kernel push/reloc limits 675 * have been hit 676 */ 677 if ((bo && ( push->channel || 678 !pushbuf_kref(push, bo, push->flags))) || 679 krec->nr_reloc + relocs >= NOUVEAU_GEM_MAX_RELOCS || 680 krec->nr_push + pushes >= NOUVEAU_GEM_MAX_PUSH) { 681 if (nvpb->bo && krec->nr_buffer) 682 pushbuf_flush(push); 683 flushed = true; 684 } 685 686 /* if necessary, switch to new buffer */ 687 if (bo) { 688 ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, push->client); 689 if (ret) 690 return ret; 691 692 nouveau_pushbuf_data(push, NULL, 0, 0); 693 nouveau_bo_ref(bo, &nvpb->bo); 694 nouveau_bo_ref(NULL, &bo); 695 696 nvpb->bgn = nvpb->bo->map; 697 nvpb->ptr = nvpb->bgn; 698 push->cur = nvpb->bgn; 699 push->end = push->cur + (nvpb->bo->size / 4); 700 push->end -= 2 + push->rsvd_kick; /* space for suffix */ 701 } 702 703 pushbuf_kref(push, nvpb->bo, push->flags); 704 return flushed ? pushbuf_validate(push, false) : 0; 705} 706 707drm_public void 708nouveau_pushbuf_data(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 709 uint64_t offset, uint64_t length) 710{ 711 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 712 struct nouveau_pushbuf_krec *krec = nvpb->krec; 713 struct drm_nouveau_gem_pushbuf_push *kpsh; 714 struct drm_nouveau_gem_pushbuf_bo *kref; 715 716 if (bo != nvpb->bo && nvpb->bgn != push->cur) { 717 if (nvpb->suffix0 || nvpb->suffix1) { 718 *push->cur++ = nvpb->suffix0; 719 *push->cur++ = nvpb->suffix1; 720 } 721 722 nouveau_pushbuf_data(push, nvpb->bo, 723 (nvpb->bgn - nvpb->ptr) * 4, 724 (push->cur - nvpb->bgn) * 4); 725 nvpb->bgn = push->cur; 726 } 727 728 if (bo) { 729 kref = cli_kref_get(push->client, bo); 730 assert(kref); 731 kpsh = &krec->push[krec->nr_push++]; 732 kpsh->bo_index = kref - krec->buffer; 733 kpsh->offset = offset; 734 kpsh->length = length; 735 } 736} 737 738drm_public int 739nouveau_pushbuf_refn(struct nouveau_pushbuf *push, 740 struct nouveau_pushbuf_refn *refs, int nr) 741{ 742 return pushbuf_refn(push, true, refs, nr); 743} 744 745drm_public void 746nouveau_pushbuf_reloc(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 747 uint32_t data, uint32_t flags, uint32_t vor, uint32_t tor) 748{ 749 *push->cur = pushbuf_krel(push, bo, data, flags, vor, tor); 750 push->cur++; 751} 752 753drm_public int 754nouveau_pushbuf_validate(struct nouveau_pushbuf *push) 755{ 756 return pushbuf_validate(push, true); 757} 758 759drm_public uint32_t 760nouveau_pushbuf_refd(struct nouveau_pushbuf *push, struct nouveau_bo *bo) 761{ 762 struct drm_nouveau_gem_pushbuf_bo *kref; 763 uint32_t flags = 0; 764 765 if (cli_push_get(push->client, bo) == push) { 766 kref = cli_kref_get(push->client, bo); 767 assert(kref); 768 if (kref->read_domains) 769 flags |= NOUVEAU_BO_RD; 770 if (kref->write_domains) 771 flags |= NOUVEAU_BO_WR; 772 } 773 774 return flags; 775} 776 777drm_public int 778nouveau_pushbuf_kick(struct nouveau_pushbuf *push, struct nouveau_object *chan) 779{ 780 if (!push->channel) 781 return pushbuf_submit(push, chan); 782 pushbuf_flush(push); 783 return pushbuf_validate(push, false); 784} 785 786drm_public bool 787nouveau_check_dead_channel(struct nouveau_drm *drm, struct nouveau_object *chan) 788{ 789 struct drm_nouveau_gem_pushbuf req = {}; 790 struct nouveau_fifo *fifo = chan->data; 791 int ret; 792 793 req.channel = fifo->channel; 794 req.nr_push = 0; 795 796 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_PUSHBUF, 797 &req, sizeof(req)); 798 /* nouveau returns ENODEV once the channel was killed */ 799 return ret == -ENODEV; 800} 801