i915_gem_gtt.c revision 1.16.2.1
1 /* $NetBSD: i915_gem_gtt.c,v 1.16.2.1 2020/02/29 20:20:14 ad Exp $ */ 2 3 /* 4 * Copyright 2010 Daniel Vetter 5 * Copyright 2011-2014 Intel Corporation 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the "Software"), 9 * to deal in the Software without restriction, including without limitation 10 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 11 * and/or sell copies of the Software, and to permit persons to whom the 12 * Software is furnished to do so, subject to the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the next 15 * paragraph) shall be included in all copies or substantial portions of the 16 * Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 24 * IN THE SOFTWARE. 25 * 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: i915_gem_gtt.c,v 1.16.2.1 2020/02/29 20:20:14 ad Exp $"); 30 31 #include <linux/bitmap.h> 32 #include <linux/seq_file.h> 33 #include <drm/drmP.h> 34 #include <drm/i915_drm.h> 35 #include "i915_drv.h" 36 #include "i915_vgpu.h" 37 #include "i915_trace.h" 38 #include "intel_drv.h" 39 40 #ifdef __NetBSD__ 41 #include <drm/bus_dma_hacks.h> 42 #include <x86/machdep.h> 43 #include <x86/pte.h> 44 #define _PAGE_PRESENT PG_V /* 0x01 PTE is present / valid */ 45 #define _PAGE_RW PG_RW /* 0x02 read/write */ 46 #define _PAGE_PWT PG_WT /* 0x08 write-through */ 47 #define _PAGE_PCD PG_N /* 0x10 page cache disabled / non-cacheable */ 48 #define _PAGE_PAT PG_PAT /* 0x80 page attribute table on PTE */ 49 #endif 50 51 /** 52 * DOC: Global GTT views 53 * 54 * Background and previous state 55 * 56 * Historically objects could exists (be bound) in global GTT space only as 57 * singular instances with a view representing all of the object's backing pages 58 * in a linear fashion. This view will be called a normal view. 59 * 60 * To support multiple views of the same object, where the number of mapped 61 * pages is not equal to the backing store, or where the layout of the pages 62 * is not linear, concept of a GGTT view was added. 63 * 64 * One example of an alternative view is a stereo display driven by a single 65 * image. In this case we would have a framebuffer looking like this 66 * (2x2 pages): 67 * 68 * 12 69 * 34 70 * 71 * Above would represent a normal GGTT view as normally mapped for GPU or CPU 72 * rendering. In contrast, fed to the display engine would be an alternative 73 * view which could look something like this: 74 * 75 * 1212 76 * 3434 77 * 78 * In this example both the size and layout of pages in the alternative view is 79 * different from the normal view. 80 * 81 * Implementation and usage 82 * 83 * GGTT views are implemented using VMAs and are distinguished via enum 84 * i915_ggtt_view_type and struct i915_ggtt_view. 85 * 86 * A new flavour of core GEM functions which work with GGTT bound objects were 87 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid 88 * renaming in large amounts of code. They take the struct i915_ggtt_view 89 * parameter encapsulating all metadata required to implement a view. 90 * 91 * As a helper for callers which are only interested in the normal view, 92 * globally const i915_ggtt_view_normal singleton instance exists. All old core 93 * GEM API functions, the ones not taking the view parameter, are operating on, 94 * or with the normal GGTT view. 95 * 96 * Code wanting to add or use a new GGTT view needs to: 97 * 98 * 1. Add a new enum with a suitable name. 99 * 2. Extend the metadata in the i915_ggtt_view structure if required. 100 * 3. Add support to i915_get_vma_pages(). 101 * 102 * New views are required to build a scatter-gather table from within the 103 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and 104 * exists for the lifetime of an VMA. 105 * 106 * Core API is designed to have copy semantics which means that passed in 107 * struct i915_ggtt_view does not need to be persistent (left around after 108 * calling the core API functions). 109 * 110 */ 111 112 static int 113 i915_get_ggtt_vma_pages(struct i915_vma *vma); 114 115 const struct i915_ggtt_view i915_ggtt_view_normal; 116 const struct i915_ggtt_view i915_ggtt_view_rotated = { 117 .type = I915_GGTT_VIEW_ROTATED 118 }; 119 120 static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) 121 { 122 bool has_aliasing_ppgtt; 123 bool has_full_ppgtt; 124 125 has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6; 126 has_full_ppgtt = INTEL_INFO(dev)->gen >= 7; 127 128 if (intel_vgpu_active(dev)) 129 has_full_ppgtt = false; /* emulation is too hard */ 130 131 /* 132 * We don't allow disabling PPGTT for gen9+ as it's a requirement for 133 * execlists, the sole mechanism available to submit work. 134 */ 135 if (INTEL_INFO(dev)->gen < 9 && 136 (enable_ppgtt == 0 || !has_aliasing_ppgtt)) 137 return 0; 138 139 /* Full PPGTT is required by the Gen9 cmdparser */ 140 if (enable_ppgtt == 1 && INTEL_INFO(dev)->gen != 9) 141 return 1; 142 143 if (enable_ppgtt == 2 && has_full_ppgtt) 144 return 2; 145 146 #ifdef CONFIG_INTEL_IOMMU 147 /* Disable ppgtt on SNB if VT-d is on. */ 148 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) { 149 DRM_INFO("Disabling PPGTT because VT-d is on\n"); 150 return 0; 151 } 152 #endif 153 154 /* Early VLV doesn't have this */ 155 if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && 156 dev->pdev->revision < 0xb) { 157 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n"); 158 return 0; 159 } 160 161 if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists) 162 return 2; 163 else 164 return has_aliasing_ppgtt ? 1 : 0; 165 } 166 167 static int ppgtt_bind_vma(struct i915_vma *vma, 168 enum i915_cache_level cache_level, 169 u32 unused) 170 { 171 u32 pte_flags = 0; 172 173 /* Applicable to VLV, and gen8+ */ 174 pte_flags = 0; 175 if (vma->obj->gt_ro) 176 pte_flags |= PTE_READ_ONLY; 177 178 vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start, 179 cache_level, pte_flags); 180 181 return 0; 182 } 183 184 static void ppgtt_unbind_vma(struct i915_vma *vma) 185 { 186 vma->vm->clear_range(vma->vm, 187 vma->node.start, 188 vma->obj->base.size, 189 true); 190 } 191 192 static gen8_pte_t gen8_pte_encode(dma_addr_t addr, 193 enum i915_cache_level level, 194 bool valid, u32 flags) 195 { 196 gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0; 197 pte |= addr; 198 199 if (unlikely(flags & PTE_READ_ONLY)) 200 pte &= ~_PAGE_RW; 201 202 switch (level) { 203 case I915_CACHE_NONE: 204 pte |= PPAT_UNCACHED_INDEX; 205 break; 206 case I915_CACHE_WT: 207 pte |= PPAT_DISPLAY_ELLC_INDEX; 208 break; 209 default: 210 pte |= PPAT_CACHED_INDEX; 211 break; 212 } 213 214 return pte; 215 } 216 217 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, 218 const enum i915_cache_level level) 219 { 220 gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW; 221 pde |= addr; 222 if (level != I915_CACHE_NONE) 223 pde |= PPAT_CACHED_PDE_INDEX; 224 else 225 pde |= PPAT_UNCACHED_INDEX; 226 return pde; 227 } 228 229 #define gen8_pdpe_encode gen8_pde_encode 230 #define gen8_pml4e_encode gen8_pde_encode 231 232 static gen6_pte_t snb_pte_encode(dma_addr_t addr, 233 enum i915_cache_level level, 234 bool valid, u32 unused) 235 { 236 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 237 pte |= GEN6_PTE_ADDR_ENCODE(addr); 238 239 switch (level) { 240 case I915_CACHE_L3_LLC: 241 case I915_CACHE_LLC: 242 pte |= GEN6_PTE_CACHE_LLC; 243 break; 244 case I915_CACHE_NONE: 245 pte |= GEN6_PTE_UNCACHED; 246 break; 247 default: 248 MISSING_CASE(level); 249 } 250 251 return pte; 252 } 253 254 static gen6_pte_t ivb_pte_encode(dma_addr_t addr, 255 enum i915_cache_level level, 256 bool valid, u32 unused) 257 { 258 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 259 pte |= GEN6_PTE_ADDR_ENCODE(addr); 260 261 switch (level) { 262 case I915_CACHE_L3_LLC: 263 pte |= GEN7_PTE_CACHE_L3_LLC; 264 break; 265 case I915_CACHE_LLC: 266 pte |= GEN6_PTE_CACHE_LLC; 267 break; 268 case I915_CACHE_NONE: 269 pte |= GEN6_PTE_UNCACHED; 270 break; 271 default: 272 MISSING_CASE(level); 273 } 274 275 return pte; 276 } 277 278 static gen6_pte_t byt_pte_encode(dma_addr_t addr, 279 enum i915_cache_level level, 280 bool valid, u32 flags) 281 { 282 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 283 pte |= GEN6_PTE_ADDR_ENCODE(addr); 284 285 if (!(flags & PTE_READ_ONLY)) 286 pte |= BYT_PTE_WRITEABLE; 287 288 if (level != I915_CACHE_NONE) 289 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 290 291 return pte; 292 } 293 294 static gen6_pte_t hsw_pte_encode(dma_addr_t addr, 295 enum i915_cache_level level, 296 bool valid, u32 unused) 297 { 298 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 299 pte |= HSW_PTE_ADDR_ENCODE(addr); 300 301 if (level != I915_CACHE_NONE) 302 pte |= HSW_WB_LLC_AGE3; 303 304 return pte; 305 } 306 307 static gen6_pte_t iris_pte_encode(dma_addr_t addr, 308 enum i915_cache_level level, 309 bool valid, u32 unused) 310 { 311 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 312 pte |= HSW_PTE_ADDR_ENCODE(addr); 313 314 switch (level) { 315 case I915_CACHE_NONE: 316 break; 317 case I915_CACHE_WT: 318 pte |= HSW_WT_ELLC_LLC_AGE3; 319 break; 320 default: 321 pte |= HSW_WB_ELLC_LLC_AGE3; 322 break; 323 } 324 325 return pte; 326 } 327 328 static void *kmap_page_dma(struct i915_page_dma *); 329 static void kunmap_page_dma(struct drm_device *, void *); 330 331 static int __setup_page_dma(struct drm_device *dev, 332 struct i915_page_dma *p, gfp_t flags) 333 { 334 #ifdef __NetBSD__ 335 int busdmaflags = 0; 336 int error; 337 int nseg = 1; 338 339 if (flags & __GFP_WAIT) 340 busdmaflags |= BUS_DMA_WAITOK; 341 else 342 busdmaflags |= BUS_DMA_NOWAIT; 343 344 error = bus_dmamem_alloc(dev->dmat, PAGE_SIZE, PAGE_SIZE, 0, &p->seg, 345 nseg, &nseg, busdmaflags); 346 if (error) { 347 fail0: p->map = NULL; 348 return -error; /* XXX errno NetBSD->Linux */ 349 } 350 KASSERT(nseg == 1); 351 error = bus_dmamap_create(dev->dmat, PAGE_SIZE, 1, PAGE_SIZE, 0, 352 busdmaflags, &p->map); 353 if (error) { 354 fail1: bus_dmamem_free(dev->dmat, &p->seg, 1); 355 goto fail0; 356 } 357 error = bus_dmamap_load_raw(dev->dmat, p->map, &p->seg, 1, PAGE_SIZE, 358 busdmaflags); 359 if (error) { 360 fail2: __unused 361 bus_dmamap_destroy(dev->dmat, p->map); 362 goto fail1; 363 } 364 365 if (flags & __GFP_ZERO) { 366 void *va = kmap_page_dma(p); 367 memset(va, 0, PAGE_SIZE); 368 kunmap_page_dma(dev, va); 369 } 370 #else 371 struct device *device = &dev->pdev->dev; 372 373 p->page = alloc_page(flags); 374 if (!p->page) 375 return -ENOMEM; 376 377 p->daddr = dma_map_page(device, 378 p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL); 379 380 if (dma_mapping_error(device, p->daddr)) { 381 __free_page(p->page); 382 return -EINVAL; 383 } 384 #endif 385 386 return 0; 387 } 388 389 static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p) 390 { 391 return __setup_page_dma(dev, p, GFP_KERNEL); 392 } 393 394 static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p) 395 { 396 #ifdef __NetBSD__ 397 if (WARN_ON(!p->map)) 398 return; 399 400 bus_dmamap_unload(dev->dmat, p->map); 401 bus_dmamap_destroy(dev->dmat, p->map); 402 bus_dmamem_free(dev->dmat, &p->seg, 1); 403 p->map = NULL; 404 #else 405 if (WARN_ON(!p->page)) 406 return; 407 408 dma_unmap_page(&dev->pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL); 409 __free_page(p->page); 410 memset(p, 0, sizeof(*p)); 411 #endif 412 } 413 414 static void *kmap_page_dma(struct i915_page_dma *p) 415 { 416 #ifdef __NetBSD__ 417 return kmap_atomic(container_of(PHYS_TO_VM_PAGE(p->seg.ds_addr), 418 struct page, p_vmp)); 419 #else 420 return kmap_atomic(p->page); 421 #endif 422 } 423 424 /* We use the flushing unmap only with ppgtt structures: 425 * page directories, page tables and scratch pages. 426 */ 427 static void kunmap_page_dma(struct drm_device *dev, void *vaddr) 428 { 429 /* There are only few exceptions for gen >=6. chv and bxt. 430 * And we are not sure about the latter so play safe for now. 431 */ 432 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 433 drm_clflush_virt_range(vaddr, PAGE_SIZE); 434 435 kunmap_atomic(vaddr); 436 } 437 438 #define kmap_px(px) kmap_page_dma(px_base(px)) 439 #define kunmap_px(ppgtt, vaddr) kunmap_page_dma((ppgtt)->base.dev, (vaddr)) 440 441 #define setup_px(dev, px) setup_page_dma((dev), px_base(px)) 442 #define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px)) 443 #define fill_px(dev, px, v) fill_page_dma((dev), px_base(px), (v)) 444 #define fill32_px(dev, px, v) fill_page_dma_32((dev), px_base(px), (v)) 445 446 static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p, 447 const uint64_t val) 448 { 449 int i; 450 uint64_t * const vaddr = kmap_page_dma(p); 451 452 for (i = 0; i < 512; i++) 453 vaddr[i] = val; 454 455 kunmap_page_dma(dev, vaddr); 456 } 457 458 static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p, 459 const uint32_t val32) 460 { 461 uint64_t v = val32; 462 463 v = v << 32 | val32; 464 465 fill_page_dma(dev, p, v); 466 } 467 468 static struct i915_page_scratch *alloc_scratch_page(struct drm_device *dev) 469 { 470 struct i915_page_scratch *sp; 471 int ret; 472 473 sp = kzalloc(sizeof(*sp), GFP_KERNEL); 474 if (sp == NULL) 475 return ERR_PTR(-ENOMEM); 476 477 ret = __setup_page_dma(dev, px_base(sp), GFP_DMA32 | __GFP_ZERO); 478 if (ret) { 479 kfree(sp); 480 return ERR_PTR(ret); 481 } 482 483 #ifndef __NetBSD__ /* XXX ??? */ 484 set_pages_uc(px_page(sp), 1); 485 #endif 486 487 return sp; 488 } 489 490 static void free_scratch_page(struct drm_device *dev, 491 struct i915_page_scratch *sp) 492 { 493 #ifndef __NetBSD__ /* XXX ??? */ 494 set_pages_wb(px_page(sp), 1); 495 #endif 496 497 cleanup_px(dev, sp); 498 kfree(sp); 499 } 500 501 static struct i915_page_table *alloc_pt(struct drm_device *dev) 502 { 503 struct i915_page_table *pt; 504 const size_t count = INTEL_INFO(dev)->gen >= 8 ? 505 GEN8_PTES : GEN6_PTES; 506 int ret = -ENOMEM; 507 508 pt = kzalloc(sizeof(*pt), GFP_KERNEL); 509 if (!pt) 510 return ERR_PTR(-ENOMEM); 511 512 pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes), 513 GFP_KERNEL); 514 515 if (!pt->used_ptes) 516 goto fail_bitmap; 517 518 ret = setup_px(dev, pt); 519 if (ret) 520 goto fail_page_m; 521 522 return pt; 523 524 fail_page_m: 525 kfree(pt->used_ptes); 526 fail_bitmap: 527 kfree(pt); 528 529 return ERR_PTR(ret); 530 } 531 532 static void free_pt(struct drm_device *dev, struct i915_page_table *pt) 533 { 534 cleanup_px(dev, pt); 535 kfree(pt->used_ptes); 536 kfree(pt); 537 } 538 539 static void gen8_initialize_pt(struct i915_address_space *vm, 540 struct i915_page_table *pt) 541 { 542 gen8_pte_t scratch_pte; 543 544 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 545 I915_CACHE_LLC, true, 0); 546 547 fill_px(vm->dev, pt, scratch_pte); 548 } 549 550 static void gen6_initialize_pt(struct i915_address_space *vm, 551 struct i915_page_table *pt) 552 { 553 gen6_pte_t scratch_pte; 554 555 WARN_ON(px_dma(vm->scratch_page) == 0); 556 557 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 558 I915_CACHE_LLC, true, 0); 559 560 fill32_px(vm->dev, pt, scratch_pte); 561 } 562 563 static struct i915_page_directory *alloc_pd(struct drm_device *dev) 564 { 565 struct i915_page_directory *pd; 566 int ret = -ENOMEM; 567 568 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 569 if (!pd) 570 return ERR_PTR(-ENOMEM); 571 572 pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES), 573 sizeof(*pd->used_pdes), GFP_KERNEL); 574 if (!pd->used_pdes) 575 goto fail_bitmap; 576 577 ret = setup_px(dev, pd); 578 if (ret) 579 goto fail_page_m; 580 581 return pd; 582 583 fail_page_m: 584 kfree(pd->used_pdes); 585 fail_bitmap: 586 kfree(pd); 587 588 return ERR_PTR(ret); 589 } 590 591 static void free_pd(struct drm_device *dev, struct i915_page_directory *pd) 592 { 593 if (px_page(pd)) { 594 cleanup_px(dev, pd); 595 kfree(pd->used_pdes); 596 kfree(pd); 597 } 598 } 599 600 static void gen8_initialize_pd(struct i915_address_space *vm, 601 struct i915_page_directory *pd) 602 { 603 gen8_pde_t scratch_pde; 604 605 scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC); 606 607 fill_px(vm->dev, pd, scratch_pde); 608 } 609 610 static int __pdp_init(struct drm_device *dev, 611 struct i915_page_directory_pointer *pdp) 612 { 613 size_t pdpes = I915_PDPES_PER_PDP(dev); 614 615 pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes), 616 sizeof(unsigned long), 617 GFP_KERNEL); 618 if (!pdp->used_pdpes) 619 return -ENOMEM; 620 621 pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory), 622 GFP_KERNEL); 623 if (!pdp->page_directory) { 624 kfree(pdp->used_pdpes); 625 /* the PDP might be the statically allocated top level. Keep it 626 * as clean as possible */ 627 pdp->used_pdpes = NULL; 628 return -ENOMEM; 629 } 630 631 return 0; 632 } 633 634 static void __pdp_fini(struct i915_page_directory_pointer *pdp) 635 { 636 kfree(pdp->used_pdpes); 637 kfree(pdp->page_directory); 638 pdp->page_directory = NULL; 639 } 640 641 static struct 642 i915_page_directory_pointer *alloc_pdp(struct drm_device *dev) 643 { 644 struct i915_page_directory_pointer *pdp; 645 int ret = -ENOMEM; 646 647 WARN_ON(!USES_FULL_48BIT_PPGTT(dev)); 648 649 pdp = kzalloc(sizeof(*pdp), GFP_KERNEL); 650 if (!pdp) 651 return ERR_PTR(-ENOMEM); 652 653 ret = __pdp_init(dev, pdp); 654 if (ret) 655 goto fail_bitmap; 656 657 ret = setup_px(dev, pdp); 658 if (ret) 659 goto fail_page_m; 660 661 return pdp; 662 663 fail_page_m: 664 __pdp_fini(pdp); 665 fail_bitmap: 666 kfree(pdp); 667 668 return ERR_PTR(ret); 669 } 670 671 static void free_pdp(struct drm_device *dev, 672 struct i915_page_directory_pointer *pdp) 673 { 674 __pdp_fini(pdp); 675 if (USES_FULL_48BIT_PPGTT(dev)) { 676 cleanup_px(dev, pdp); 677 kfree(pdp); 678 } 679 } 680 681 static void gen8_initialize_pdp(struct i915_address_space *vm, 682 struct i915_page_directory_pointer *pdp) 683 { 684 gen8_ppgtt_pdpe_t scratch_pdpe; 685 686 scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC); 687 688 fill_px(vm->dev, pdp, scratch_pdpe); 689 } 690 691 static void gen8_initialize_pml4(struct i915_address_space *vm, 692 struct i915_pml4 *pml4) 693 { 694 gen8_ppgtt_pml4e_t scratch_pml4e; 695 696 scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp), 697 I915_CACHE_LLC); 698 699 fill_px(vm->dev, pml4, scratch_pml4e); 700 } 701 702 static void 703 gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt, 704 struct i915_page_directory_pointer *pdp, 705 struct i915_page_directory *pd, 706 int index) 707 { 708 gen8_ppgtt_pdpe_t *page_directorypo; 709 710 if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) 711 return; 712 713 page_directorypo = kmap_px(pdp); 714 page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC); 715 kunmap_px(ppgtt, page_directorypo); 716 } 717 718 static void 719 gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt, 720 struct i915_pml4 *pml4, 721 struct i915_page_directory_pointer *pdp, 722 int index) 723 { 724 gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4); 725 726 WARN_ON(!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)); 727 pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC); 728 kunmap_px(ppgtt, pagemap); 729 } 730 731 /* Broadwell Page Directory Pointer Descriptors */ 732 static int gen8_write_pdp(struct drm_i915_gem_request *req, 733 unsigned entry, 734 dma_addr_t addr) 735 { 736 struct intel_engine_cs *ring = req->ring; 737 int ret; 738 739 BUG_ON(entry >= 4); 740 741 ret = intel_ring_begin(req, 6); 742 if (ret) 743 return ret; 744 745 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 746 intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry)); 747 intel_ring_emit(ring, upper_32_bits(addr)); 748 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 749 intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry)); 750 intel_ring_emit(ring, lower_32_bits(addr)); 751 intel_ring_advance(ring); 752 753 return 0; 754 } 755 756 static int gen8_legacy_mm_switch(struct i915_hw_ppgtt *ppgtt, 757 struct drm_i915_gem_request *req) 758 { 759 int i, ret; 760 761 for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) { 762 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 763 764 ret = gen8_write_pdp(req, i, pd_daddr); 765 if (ret) 766 return ret; 767 } 768 769 return 0; 770 } 771 772 static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt, 773 struct drm_i915_gem_request *req) 774 { 775 return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4)); 776 } 777 778 static void gen8_ppgtt_clear_pte_range(struct i915_address_space *vm, 779 struct i915_page_directory_pointer *pdp, 780 uint64_t start, 781 uint64_t length, 782 gen8_pte_t scratch_pte) 783 { 784 struct i915_hw_ppgtt *ppgtt = 785 container_of(vm, struct i915_hw_ppgtt, base); 786 gen8_pte_t *pt_vaddr; 787 unsigned pdpe = gen8_pdpe_index(start); 788 unsigned pde = gen8_pde_index(start); 789 unsigned pte = gen8_pte_index(start); 790 unsigned num_entries = length >> PAGE_SHIFT; 791 unsigned last_pte, i; 792 793 if (WARN_ON(!pdp)) 794 return; 795 796 while (num_entries) { 797 struct i915_page_directory *pd; 798 struct i915_page_table *pt; 799 800 if (WARN_ON(!pdp->page_directory[pdpe])) 801 break; 802 803 pd = pdp->page_directory[pdpe]; 804 805 if (WARN_ON(!pd->page_table[pde])) 806 break; 807 808 pt = pd->page_table[pde]; 809 810 if (WARN_ON(!px_page(pt))) 811 break; 812 813 last_pte = pte + num_entries; 814 if (last_pte > GEN8_PTES) 815 last_pte = GEN8_PTES; 816 817 pt_vaddr = kmap_px(pt); 818 819 for (i = pte; i < last_pte; i++) { 820 pt_vaddr[i] = scratch_pte; 821 num_entries--; 822 } 823 824 kunmap_px(ppgtt, pt_vaddr); 825 826 pte = 0; 827 if (++pde == I915_PDES) { 828 if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) 829 break; 830 pde = 0; 831 } 832 } 833 } 834 835 static void gen8_ppgtt_clear_range(struct i915_address_space *vm, 836 uint64_t start, 837 uint64_t length, 838 bool use_scratch) 839 { 840 struct i915_hw_ppgtt *ppgtt = 841 container_of(vm, struct i915_hw_ppgtt, base); 842 gen8_pte_t scratch_pte = 843 gen8_pte_encode(px_dma(vm->scratch_page), 844 I915_CACHE_LLC, use_scratch, 0); 845 846 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 847 gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length, 848 scratch_pte); 849 } else { 850 uint64_t templ4, pml4e; 851 struct i915_page_directory_pointer *pdp; 852 853 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) { 854 gen8_ppgtt_clear_pte_range(vm, pdp, start, length, 855 scratch_pte); 856 } 857 } 858 } 859 860 #ifdef __NetBSD__ 861 static void 862 gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm, 863 struct i915_page_directory_pointer *pdp, bus_dmamap_t dmamap, 864 unsigned *segp, uint64_t start, enum i915_cache_level cache_level, 865 u32 flags) 866 { 867 struct i915_hw_ppgtt *ppgtt = 868 container_of(vm, struct i915_hw_ppgtt, base); 869 gen8_pte_t *pt_vaddr; 870 unsigned pdpe = gen8_pdpe_index(start); 871 unsigned pde = gen8_pde_index(start); 872 unsigned pte = gen8_pte_index(start); 873 874 pt_vaddr = NULL; 875 for (; *segp < dmamap->dm_nsegs; (*segp)++) { 876 KASSERT(dmamap->dm_segs[*segp].ds_len == PAGE_SIZE); 877 if (pt_vaddr == NULL) { 878 struct i915_page_directory *pd = 879 pdp->page_directory[pdpe]; 880 struct i915_page_table *pt = pd->page_table[pde]; 881 pt_vaddr = kmap_px(pt); 882 } 883 pt_vaddr[pte] = gen8_pte_encode(dmamap->dm_segs[*segp].ds_addr, 884 cache_level, true, flags); 885 if (++pte == GEN8_PTES) { 886 kunmap_px(ppgtt, pt_vaddr); 887 pt_vaddr = NULL; 888 if (++pde == I915_PDES) { 889 if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) 890 break; 891 pde = 0; 892 } 893 pte = 0; 894 } 895 } 896 if (pt_vaddr) 897 kunmap_px(ppgtt, pt_vaddr); 898 } 899 #else 900 static void 901 gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm, 902 struct i915_page_directory_pointer *pdp, 903 struct sg_page_iter *sg_iter, 904 uint64_t start, 905 enum i915_cache_level cache_level, 906 u32 flags) 907 { 908 struct i915_hw_ppgtt *ppgtt = 909 container_of(vm, struct i915_hw_ppgtt, base); 910 gen8_pte_t *pt_vaddr; 911 unsigned pdpe = gen8_pdpe_index(start); 912 unsigned pde = gen8_pde_index(start); 913 unsigned pte = gen8_pte_index(start); 914 915 pt_vaddr = NULL; 916 917 while (__sg_page_iter_next(sg_iter)) { 918 if (pt_vaddr == NULL) { 919 struct i915_page_directory *pd = pdp->page_directory[pdpe]; 920 struct i915_page_table *pt = pd->page_table[pde]; 921 pt_vaddr = kmap_px(pt); 922 } 923 924 pt_vaddr[pte] = 925 gen8_pte_encode(sg_page_iter_dma_address(sg_iter), 926 cache_level, true, flags); 927 if (++pte == GEN8_PTES) { 928 kunmap_px(ppgtt, pt_vaddr); 929 pt_vaddr = NULL; 930 if (++pde == I915_PDES) { 931 if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) 932 break; 933 pde = 0; 934 } 935 pte = 0; 936 } 937 } 938 939 if (pt_vaddr) 940 kunmap_px(ppgtt, pt_vaddr); 941 } 942 #endif 943 944 #ifdef __NetBSD__ 945 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, 946 bus_dmamap_t dmamap, uint64_t start, enum i915_cache_level cache_level, 947 u32 flags) 948 { 949 struct i915_hw_ppgtt *ppgtt = 950 container_of(vm, struct i915_hw_ppgtt, base); 951 unsigned seg = 0; 952 953 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 954 gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, dmamap, &seg, 955 start, cache_level, flags); 956 } else { 957 struct i915_page_directory_pointer *pdp; 958 uint64_t templ4, pml4e; 959 uint64_t length = dmamap->dm_mapsize; 960 961 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, 962 pml4e) { 963 gen8_ppgtt_insert_pte_entries(vm, pdp, dmamap, &seg, 964 start, cache_level, flags); 965 } 966 } 967 } 968 #else 969 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, 970 struct sg_table *pages, 971 uint64_t start, 972 enum i915_cache_level cache_level, 973 u32 flags) 974 { 975 struct i915_hw_ppgtt *ppgtt = 976 container_of(vm, struct i915_hw_ppgtt, base); 977 struct sg_page_iter sg_iter; 978 979 __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0); 980 981 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 982 gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start, 983 cache_level, flags); 984 } else { 985 struct i915_page_directory_pointer *pdp; 986 uint64_t templ4, pml4e; 987 uint64_t length = (uint64_t)pages->orig_nents << PAGE_SHIFT; 988 989 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) { 990 gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter, 991 start, cache_level, flags); 992 } 993 } 994 } 995 #endif 996 997 static void gen8_free_page_tables(struct drm_device *dev, 998 struct i915_page_directory *pd) 999 { 1000 int i; 1001 1002 if (!px_page(pd)) 1003 return; 1004 1005 for_each_set_bit(i, pd->used_pdes, I915_PDES) { 1006 if (WARN_ON(!pd->page_table[i])) 1007 continue; 1008 1009 free_pt(dev, pd->page_table[i]); 1010 pd->page_table[i] = NULL; 1011 } 1012 } 1013 1014 static int gen8_init_scratch(struct i915_address_space *vm) 1015 { 1016 struct drm_device *dev = vm->dev; 1017 1018 vm->scratch_page = alloc_scratch_page(dev); 1019 if (IS_ERR(vm->scratch_page)) 1020 return PTR_ERR(vm->scratch_page); 1021 1022 vm->scratch_pt = alloc_pt(dev); 1023 if (IS_ERR(vm->scratch_pt)) { 1024 free_scratch_page(dev, vm->scratch_page); 1025 return PTR_ERR(vm->scratch_pt); 1026 } 1027 1028 vm->scratch_pd = alloc_pd(dev); 1029 if (IS_ERR(vm->scratch_pd)) { 1030 free_pt(dev, vm->scratch_pt); 1031 free_scratch_page(dev, vm->scratch_page); 1032 return PTR_ERR(vm->scratch_pd); 1033 } 1034 1035 if (USES_FULL_48BIT_PPGTT(dev)) { 1036 vm->scratch_pdp = alloc_pdp(dev); 1037 if (IS_ERR(vm->scratch_pdp)) { 1038 free_pd(dev, vm->scratch_pd); 1039 free_pt(dev, vm->scratch_pt); 1040 free_scratch_page(dev, vm->scratch_page); 1041 return PTR_ERR(vm->scratch_pdp); 1042 } 1043 } 1044 1045 gen8_initialize_pt(vm, vm->scratch_pt); 1046 gen8_initialize_pd(vm, vm->scratch_pd); 1047 if (USES_FULL_48BIT_PPGTT(dev)) 1048 gen8_initialize_pdp(vm, vm->scratch_pdp); 1049 1050 return 0; 1051 } 1052 1053 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create) 1054 { 1055 enum vgt_g2v_type msg; 1056 struct drm_device *dev = ppgtt->base.dev; 1057 struct drm_i915_private *dev_priv = dev->dev_private; 1058 unsigned int offset = vgtif_reg(pdp0_lo); 1059 int i; 1060 1061 if (USES_FULL_48BIT_PPGTT(dev)) { 1062 u64 daddr = px_dma(&ppgtt->pml4); 1063 1064 I915_WRITE(offset, lower_32_bits(daddr)); 1065 I915_WRITE(offset + 4, upper_32_bits(daddr)); 1066 1067 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE : 1068 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY); 1069 } else { 1070 for (i = 0; i < GEN8_LEGACY_PDPES; i++) { 1071 u64 daddr = i915_page_dir_dma_addr(ppgtt, i); 1072 1073 I915_WRITE(offset, lower_32_bits(daddr)); 1074 I915_WRITE(offset + 4, upper_32_bits(daddr)); 1075 1076 offset += 8; 1077 } 1078 1079 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE : 1080 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY); 1081 } 1082 1083 I915_WRITE(vgtif_reg(g2v_notify), msg); 1084 1085 return 0; 1086 } 1087 1088 static void gen8_free_scratch(struct i915_address_space *vm) 1089 { 1090 struct drm_device *dev = vm->dev; 1091 1092 if (USES_FULL_48BIT_PPGTT(dev)) 1093 free_pdp(dev, vm->scratch_pdp); 1094 free_pd(dev, vm->scratch_pd); 1095 free_pt(dev, vm->scratch_pt); 1096 free_scratch_page(dev, vm->scratch_page); 1097 } 1098 1099 static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev, 1100 struct i915_page_directory_pointer *pdp) 1101 { 1102 int i; 1103 1104 for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) { 1105 if (WARN_ON(!pdp->page_directory[i])) 1106 continue; 1107 1108 gen8_free_page_tables(dev, pdp->page_directory[i]); 1109 free_pd(dev, pdp->page_directory[i]); 1110 } 1111 1112 free_pdp(dev, pdp); 1113 } 1114 1115 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt) 1116 { 1117 int i; 1118 1119 for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) { 1120 if (WARN_ON(!ppgtt->pml4.pdps[i])) 1121 continue; 1122 1123 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, ppgtt->pml4.pdps[i]); 1124 } 1125 1126 cleanup_px(ppgtt->base.dev, &ppgtt->pml4); 1127 } 1128 1129 static void gen8_ppgtt_cleanup(struct i915_address_space *vm) 1130 { 1131 struct i915_hw_ppgtt *ppgtt = 1132 container_of(vm, struct i915_hw_ppgtt, base); 1133 1134 if (intel_vgpu_active(vm->dev)) 1135 gen8_ppgtt_notify_vgt(ppgtt, false); 1136 1137 if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) 1138 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, &ppgtt->pdp); 1139 else 1140 gen8_ppgtt_cleanup_4lvl(ppgtt); 1141 1142 gen8_free_scratch(vm); 1143 } 1144 1145 /** 1146 * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range. 1147 * @vm: Master vm structure. 1148 * @pd: Page directory for this address range. 1149 * @start: Starting virtual address to begin allocations. 1150 * @length: Size of the allocations. 1151 * @new_pts: Bitmap set by function with new allocations. Likely used by the 1152 * caller to free on error. 1153 * 1154 * Allocate the required number of page tables. Extremely similar to 1155 * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by 1156 * the page directory boundary (instead of the page directory pointer). That 1157 * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is 1158 * possible, and likely that the caller will need to use multiple calls of this 1159 * function to achieve the appropriate allocation. 1160 * 1161 * Return: 0 if success; negative error code otherwise. 1162 */ 1163 static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm, 1164 struct i915_page_directory *pd, 1165 uint64_t start, 1166 uint64_t length, 1167 unsigned long *new_pts) 1168 { 1169 struct drm_device *dev = vm->dev; 1170 struct i915_page_table *pt; 1171 uint64_t temp; 1172 uint32_t pde; 1173 1174 gen8_for_each_pde(pt, pd, start, length, temp, pde) { 1175 /* Don't reallocate page tables */ 1176 if (test_bit(pde, pd->used_pdes)) { 1177 /* Scratch is never allocated this way */ 1178 WARN_ON(pt == vm->scratch_pt); 1179 continue; 1180 } 1181 1182 pt = alloc_pt(dev); 1183 if (IS_ERR(pt)) 1184 goto unwind_out; 1185 1186 gen8_initialize_pt(vm, pt); 1187 pd->page_table[pde] = pt; 1188 __set_bit(pde, new_pts); 1189 trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT); 1190 } 1191 1192 return 0; 1193 1194 unwind_out: 1195 for_each_set_bit(pde, new_pts, I915_PDES) 1196 free_pt(dev, pd->page_table[pde]); 1197 1198 return -ENOMEM; 1199 } 1200 1201 /** 1202 * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range. 1203 * @vm: Master vm structure. 1204 * @pdp: Page directory pointer for this address range. 1205 * @start: Starting virtual address to begin allocations. 1206 * @length: Size of the allocations. 1207 * @new_pds: Bitmap set by function with new allocations. Likely used by the 1208 * caller to free on error. 1209 * 1210 * Allocate the required number of page directories starting at the pde index of 1211 * @start, and ending at the pde index @start + @length. This function will skip 1212 * over already allocated page directories within the range, and only allocate 1213 * new ones, setting the appropriate pointer within the pdp as well as the 1214 * correct position in the bitmap @new_pds. 1215 * 1216 * The function will only allocate the pages within the range for a give page 1217 * directory pointer. In other words, if @start + @length straddles a virtually 1218 * addressed PDP boundary (512GB for 4k pages), there will be more allocations 1219 * required by the caller, This is not currently possible, and the BUG in the 1220 * code will prevent it. 1221 * 1222 * Return: 0 if success; negative error code otherwise. 1223 */ 1224 static int 1225 gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm, 1226 struct i915_page_directory_pointer *pdp, 1227 uint64_t start, 1228 uint64_t length, 1229 unsigned long *new_pds) 1230 { 1231 struct drm_device *dev = vm->dev; 1232 struct i915_page_directory *pd; 1233 uint64_t temp; 1234 uint32_t pdpe; 1235 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1236 1237 WARN_ON(!bitmap_empty(new_pds, pdpes)); 1238 1239 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { 1240 if (test_bit(pdpe, pdp->used_pdpes)) 1241 continue; 1242 1243 pd = alloc_pd(dev); 1244 if (IS_ERR(pd)) 1245 goto unwind_out; 1246 1247 gen8_initialize_pd(vm, pd); 1248 pdp->page_directory[pdpe] = pd; 1249 __set_bit(pdpe, new_pds); 1250 trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT); 1251 } 1252 1253 return 0; 1254 1255 unwind_out: 1256 for_each_set_bit(pdpe, new_pds, pdpes) 1257 free_pd(dev, pdp->page_directory[pdpe]); 1258 1259 return -ENOMEM; 1260 } 1261 1262 /** 1263 * gen8_ppgtt_alloc_page_dirpointers() - Allocate pdps for VA range. 1264 * @vm: Master vm structure. 1265 * @pml4: Page map level 4 for this address range. 1266 * @start: Starting virtual address to begin allocations. 1267 * @length: Size of the allocations. 1268 * @new_pdps: Bitmap set by function with new allocations. Likely used by the 1269 * caller to free on error. 1270 * 1271 * Allocate the required number of page directory pointers. Extremely similar to 1272 * gen8_ppgtt_alloc_page_directories() and gen8_ppgtt_alloc_pagetabs(). 1273 * The main difference is here we are limited by the pml4 boundary (instead of 1274 * the page directory pointer). 1275 * 1276 * Return: 0 if success; negative error code otherwise. 1277 */ 1278 static int 1279 gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm, 1280 struct i915_pml4 *pml4, 1281 uint64_t start, 1282 uint64_t length, 1283 unsigned long *new_pdps) 1284 { 1285 struct drm_device *dev = vm->dev; 1286 struct i915_page_directory_pointer *pdp; 1287 uint64_t temp; 1288 uint32_t pml4e; 1289 1290 WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4)); 1291 1292 gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) { 1293 if (!test_bit(pml4e, pml4->used_pml4es)) { 1294 pdp = alloc_pdp(dev); 1295 if (IS_ERR(pdp)) 1296 goto unwind_out; 1297 1298 gen8_initialize_pdp(vm, pdp); 1299 pml4->pdps[pml4e] = pdp; 1300 __set_bit(pml4e, new_pdps); 1301 trace_i915_page_directory_pointer_entry_alloc(vm, 1302 pml4e, 1303 start, 1304 GEN8_PML4E_SHIFT); 1305 } 1306 } 1307 1308 return 0; 1309 1310 unwind_out: 1311 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) 1312 free_pdp(dev, pml4->pdps[pml4e]); 1313 1314 return -ENOMEM; 1315 } 1316 1317 static void 1318 free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long *new_pts) 1319 { 1320 kfree(new_pts); 1321 kfree(new_pds); 1322 } 1323 1324 /* Fills in the page directory bitmap, and the array of page tables bitmap. Both 1325 * of these are based on the number of PDPEs in the system. 1326 */ 1327 static 1328 int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds, 1329 unsigned long **new_pts, 1330 uint32_t pdpes) 1331 { 1332 unsigned long *pds; 1333 unsigned long *pts; 1334 1335 pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_TEMPORARY); 1336 if (!pds) 1337 return -ENOMEM; 1338 1339 pts = kcalloc(pdpes, BITS_TO_LONGS(I915_PDES) * sizeof(unsigned long), 1340 GFP_TEMPORARY); 1341 if (!pts) 1342 goto err_out; 1343 1344 *new_pds = pds; 1345 *new_pts = pts; 1346 1347 return 0; 1348 1349 err_out: 1350 free_gen8_temp_bitmaps(pds, pts); 1351 return -ENOMEM; 1352 } 1353 1354 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify 1355 * the page table structures, we mark them dirty so that 1356 * context switching/execlist queuing code takes extra steps 1357 * to ensure that tlbs are flushed. 1358 */ 1359 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) 1360 { 1361 ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask; 1362 } 1363 1364 static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm, 1365 struct i915_page_directory_pointer *pdp, 1366 uint64_t start, 1367 uint64_t length) 1368 { 1369 struct i915_hw_ppgtt *ppgtt = 1370 container_of(vm, struct i915_hw_ppgtt, base); 1371 unsigned long *new_page_dirs, *new_page_tables; 1372 struct drm_device *dev = vm->dev; 1373 struct i915_page_directory *pd; 1374 const uint64_t orig_start = start; 1375 const uint64_t orig_length = length; 1376 uint64_t temp; 1377 uint32_t pdpe; 1378 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1379 int ret; 1380 1381 /* Wrap is never okay since we can only represent 48b, and we don't 1382 * actually use the other side of the canonical address space. 1383 */ 1384 if (WARN_ON(start + length < start)) 1385 return -ENODEV; 1386 1387 if (WARN_ON(start + length > vm->total)) 1388 return -ENODEV; 1389 1390 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes); 1391 if (ret) 1392 return ret; 1393 1394 /* Do the allocations first so we can easily bail out */ 1395 ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length, 1396 new_page_dirs); 1397 if (ret) { 1398 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1399 return ret; 1400 } 1401 1402 /* For every page directory referenced, allocate page tables */ 1403 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { 1404 ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length, 1405 new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES)); 1406 if (ret) 1407 goto err_out; 1408 } 1409 1410 start = orig_start; 1411 length = orig_length; 1412 1413 /* Allocations have completed successfully, so set the bitmaps, and do 1414 * the mappings. */ 1415 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { 1416 gen8_pde_t *const page_directory = kmap_px(pd); 1417 struct i915_page_table *pt; 1418 uint64_t pd_len = length; 1419 uint64_t pd_start = start; 1420 uint32_t pde; 1421 1422 /* Every pd should be allocated, we just did that above. */ 1423 WARN_ON(!pd); 1424 1425 gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) { 1426 /* Same reasoning as pd */ 1427 WARN_ON(!pt); 1428 WARN_ON(!pd_len); 1429 WARN_ON(!gen8_pte_count(pd_start, pd_len)); 1430 1431 /* Set our used ptes within the page table */ 1432 bitmap_set(pt->used_ptes, 1433 gen8_pte_index(pd_start), 1434 gen8_pte_count(pd_start, pd_len)); 1435 1436 /* Our pde is now pointing to the pagetable, pt */ 1437 __set_bit(pde, pd->used_pdes); 1438 1439 /* Map the PDE to the page table */ 1440 page_directory[pde] = gen8_pde_encode(px_dma(pt), 1441 I915_CACHE_LLC); 1442 trace_i915_page_table_entry_map(&ppgtt->base, pde, pt, 1443 gen8_pte_index(start), 1444 gen8_pte_count(start, length), 1445 GEN8_PTES); 1446 1447 /* NB: We haven't yet mapped ptes to pages. At this 1448 * point we're still relying on insert_entries() */ 1449 } 1450 1451 kunmap_px(ppgtt, page_directory); 1452 __set_bit(pdpe, pdp->used_pdpes); 1453 gen8_setup_page_directory(ppgtt, pdp, pd, pdpe); 1454 } 1455 1456 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1457 mark_tlbs_dirty(ppgtt); 1458 return 0; 1459 1460 err_out: 1461 while (pdpe--) { 1462 for_each_set_bit(temp, new_page_tables + pdpe * 1463 BITS_TO_LONGS(I915_PDES), I915_PDES) 1464 free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]); 1465 } 1466 1467 for_each_set_bit(pdpe, new_page_dirs, pdpes) 1468 free_pd(dev, pdp->page_directory[pdpe]); 1469 1470 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1471 mark_tlbs_dirty(ppgtt); 1472 return ret; 1473 } 1474 1475 static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm, 1476 struct i915_pml4 *pml4, 1477 uint64_t start, 1478 uint64_t length) 1479 { 1480 DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4); 1481 struct i915_hw_ppgtt *ppgtt = 1482 container_of(vm, struct i915_hw_ppgtt, base); 1483 struct i915_page_directory_pointer *pdp; 1484 uint64_t temp, pml4e; 1485 int ret = 0; 1486 1487 /* Do the pml4 allocations first, so we don't need to track the newly 1488 * allocated tables below the pdp */ 1489 bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4); 1490 1491 /* The pagedirectory and pagetable allocations are done in the shared 3 1492 * and 4 level code. Just allocate the pdps. 1493 */ 1494 ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length, 1495 new_pdps); 1496 if (ret) 1497 return ret; 1498 1499 WARN(bitmap_weight(new_pdps, GEN8_PML4ES_PER_PML4) > 2, 1500 "The allocation has spanned more than 512GB. " 1501 "It is highly likely this is incorrect."); 1502 1503 gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) { 1504 WARN_ON(!pdp); 1505 1506 ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length); 1507 if (ret) 1508 goto err_out; 1509 1510 gen8_setup_page_directory_pointer(ppgtt, pml4, pdp, pml4e); 1511 } 1512 1513 bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es, 1514 GEN8_PML4ES_PER_PML4); 1515 1516 return 0; 1517 1518 err_out: 1519 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) 1520 gen8_ppgtt_cleanup_3lvl(vm->dev, pml4->pdps[pml4e]); 1521 1522 return ret; 1523 } 1524 1525 static int gen8_alloc_va_range(struct i915_address_space *vm, 1526 uint64_t start, uint64_t length) 1527 { 1528 struct i915_hw_ppgtt *ppgtt = 1529 container_of(vm, struct i915_hw_ppgtt, base); 1530 1531 if (USES_FULL_48BIT_PPGTT(vm->dev)) 1532 return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length); 1533 else 1534 return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length); 1535 } 1536 1537 #ifndef __NetBSD__ /* XXX debugfs */ 1538 static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp, 1539 uint64_t start, uint64_t length, 1540 gen8_pte_t scratch_pte, 1541 struct seq_file *m) 1542 { 1543 struct i915_page_directory *pd; 1544 uint64_t temp; 1545 uint32_t pdpe; 1546 1547 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { 1548 struct i915_page_table *pt; 1549 uint64_t pd_len = length; 1550 uint64_t pd_start = start; 1551 uint32_t pde; 1552 1553 if (!test_bit(pdpe, pdp->used_pdpes)) 1554 continue; 1555 1556 seq_printf(m, "\tPDPE #%d\n", pdpe); 1557 gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) { 1558 uint32_t pte; 1559 gen8_pte_t *pt_vaddr; 1560 1561 if (!test_bit(pde, pd->used_pdes)) 1562 continue; 1563 1564 pt_vaddr = kmap_px(pt); 1565 for (pte = 0; pte < GEN8_PTES; pte += 4) { 1566 uint64_t va = 1567 (pdpe << GEN8_PDPE_SHIFT) | 1568 (pde << GEN8_PDE_SHIFT) | 1569 (pte << GEN8_PTE_SHIFT); 1570 int i; 1571 bool found = false; 1572 1573 for (i = 0; i < 4; i++) 1574 if (pt_vaddr[pte + i] != scratch_pte) 1575 found = true; 1576 if (!found) 1577 continue; 1578 1579 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte); 1580 for (i = 0; i < 4; i++) { 1581 if (pt_vaddr[pte + i] != scratch_pte) 1582 seq_printf(m, " %llx", pt_vaddr[pte + i]); 1583 else 1584 seq_puts(m, " SCRATCH "); 1585 } 1586 seq_puts(m, "\n"); 1587 } 1588 /* don't use kunmap_px, it could trigger 1589 * an unnecessary flush. 1590 */ 1591 kunmap_atomic(pt_vaddr); 1592 } 1593 } 1594 } 1595 1596 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) 1597 { 1598 struct i915_address_space *vm = &ppgtt->base; 1599 uint64_t start = ppgtt->base.start; 1600 uint64_t length = ppgtt->base.total; 1601 gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 1602 I915_CACHE_LLC, true, 0); 1603 1604 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 1605 gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m); 1606 } else { 1607 uint64_t templ4, pml4e; 1608 struct i915_pml4 *pml4 = &ppgtt->pml4; 1609 struct i915_page_directory_pointer *pdp; 1610 1611 gen8_for_each_pml4e(pdp, pml4, start, length, templ4, pml4e) { 1612 if (!test_bit(pml4e, pml4->used_pml4es)) 1613 continue; 1614 1615 seq_printf(m, " PML4E #%llu\n", pml4e); 1616 gen8_dump_pdp(pdp, start, length, scratch_pte, m); 1617 } 1618 } 1619 } 1620 #endif 1621 1622 static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt) 1623 { 1624 unsigned long *new_page_dirs, *new_page_tables; 1625 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1626 int ret; 1627 1628 /* We allocate temp bitmap for page tables for no gain 1629 * but as this is for init only, lets keep the things simple 1630 */ 1631 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes); 1632 if (ret) 1633 return ret; 1634 1635 /* Allocate for all pdps regardless of how the ppgtt 1636 * was defined. 1637 */ 1638 ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp, 1639 0, 1ULL << 32, 1640 new_page_dirs); 1641 if (!ret) 1642 *ppgtt->pdp.used_pdpes = *new_page_dirs; 1643 1644 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1645 1646 return ret; 1647 } 1648 1649 /* 1650 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers 1651 * with a net effect resembling a 2-level page table in normal x86 terms. Each 1652 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address 1653 * space. 1654 * 1655 */ 1656 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 1657 { 1658 int ret; 1659 1660 ret = gen8_init_scratch(&ppgtt->base); 1661 if (ret) 1662 return ret; 1663 1664 ppgtt->base.start = 0; 1665 ppgtt->base.cleanup = gen8_ppgtt_cleanup; 1666 ppgtt->base.allocate_va_range = gen8_alloc_va_range; 1667 ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; 1668 ppgtt->base.clear_range = gen8_ppgtt_clear_range; 1669 ppgtt->base.unbind_vma = ppgtt_unbind_vma; 1670 ppgtt->base.bind_vma = ppgtt_bind_vma; 1671 1672 /* 1673 * From bdw, there is support for read-only pages in the PPGTT. 1674 * 1675 * XXX GVT is not honouring the lack of RW in the PTE bits. 1676 */ 1677 ppgtt->base.has_read_only = !intel_vgpu_active(ppgtt->base.dev); 1678 1679 #ifndef __NetBSD__ /* XXX debugfs */ 1680 ppgtt->debug_dump = gen8_dump_ppgtt; 1681 #endif 1682 1683 if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) { 1684 ret = setup_px(ppgtt->base.dev, &ppgtt->pml4); 1685 if (ret) 1686 goto free_scratch; 1687 1688 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4); 1689 1690 ppgtt->base.total = 1ULL << 48; 1691 ppgtt->switch_mm = gen8_48b_mm_switch; 1692 } else { 1693 ret = __pdp_init(ppgtt->base.dev, &ppgtt->pdp); 1694 if (ret) 1695 goto free_scratch; 1696 1697 ppgtt->base.total = 1ULL << 32; 1698 ppgtt->switch_mm = gen8_legacy_mm_switch; 1699 trace_i915_page_directory_pointer_entry_alloc(&ppgtt->base, 1700 0, 0, 1701 GEN8_PML4E_SHIFT); 1702 1703 if (intel_vgpu_active(ppgtt->base.dev)) { 1704 ret = gen8_preallocate_top_level_pdps(ppgtt); 1705 if (ret) 1706 goto free_scratch; 1707 } 1708 } 1709 1710 if (intel_vgpu_active(ppgtt->base.dev)) 1711 gen8_ppgtt_notify_vgt(ppgtt, true); 1712 1713 return 0; 1714 1715 free_scratch: 1716 gen8_free_scratch(&ppgtt->base); 1717 return ret; 1718 } 1719 1720 #ifndef __NetBSD__ /* XXX debugfs */ 1721 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) 1722 { 1723 struct i915_address_space *vm = &ppgtt->base; 1724 struct i915_page_table *unused; 1725 gen6_pte_t scratch_pte; 1726 uint32_t pd_entry; 1727 uint32_t pte, pde, temp; 1728 uint32_t start = ppgtt->base.start, length = ppgtt->base.total; 1729 1730 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 1731 I915_CACHE_LLC, true, 0); 1732 1733 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) { 1734 u32 expected; 1735 gen6_pte_t *pt_vaddr; 1736 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]); 1737 pd_entry = readl(ppgtt->pd_addr + pde); 1738 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID); 1739 1740 if (pd_entry != expected) 1741 seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n", 1742 pde, 1743 pd_entry, 1744 expected); 1745 seq_printf(m, "\tPDE: %x\n", pd_entry); 1746 1747 pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]); 1748 1749 for (pte = 0; pte < GEN6_PTES; pte+=4) { 1750 unsigned long va = 1751 (pde * PAGE_SIZE * GEN6_PTES) + 1752 (pte * PAGE_SIZE); 1753 int i; 1754 bool found = false; 1755 for (i = 0; i < 4; i++) 1756 if (pt_vaddr[pte + i] != scratch_pte) 1757 found = true; 1758 if (!found) 1759 continue; 1760 1761 seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte); 1762 for (i = 0; i < 4; i++) { 1763 if (pt_vaddr[pte + i] != scratch_pte) 1764 seq_printf(m, " %08x", pt_vaddr[pte + i]); 1765 else 1766 seq_puts(m, " SCRATCH "); 1767 } 1768 seq_puts(m, "\n"); 1769 } 1770 kunmap_px(ppgtt, pt_vaddr); 1771 } 1772 } 1773 #endif 1774 1775 /* Write pde (index) from the page directory @pd to the page table @pt */ 1776 static void gen6_write_pde(struct i915_page_directory *pd, 1777 const int pde, struct i915_page_table *pt) 1778 { 1779 /* Caller needs to make sure the write completes if necessary */ 1780 struct i915_hw_ppgtt *ppgtt = 1781 container_of(pd, struct i915_hw_ppgtt, pd); 1782 #ifdef __NetBSD__ 1783 struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private; 1784 const bus_space_tag_t bst = dev_priv->gtt.bst; 1785 const bus_space_handle_t bsh = dev_priv->gtt.bsh; 1786 const bus_addr_t pd_base = ppgtt->pd.base.ggtt_offset; 1787 #endif 1788 u32 pd_entry; 1789 1790 pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt)); 1791 pd_entry |= GEN6_PDE_VALID; 1792 1793 #ifdef __NetBSD__ 1794 CTASSERT(sizeof(gen6_pte_t) == 4); 1795 bus_space_write_4(bst, bsh, pd_base + 4*pde, pd_entry); 1796 #else 1797 writel(pd_entry, ppgtt->pd_addr + pde); 1798 #endif 1799 } 1800 1801 /* Write all the page tables found in the ppgtt structure to incrementing page 1802 * directories. */ 1803 static void gen6_write_page_range(struct drm_i915_private *dev_priv, 1804 struct i915_page_directory *pd, 1805 uint32_t start, uint32_t length) 1806 { 1807 struct i915_page_table *pt; 1808 uint32_t pde, temp; 1809 1810 gen6_for_each_pde(pt, pd, start, length, temp, pde) 1811 gen6_write_pde(pd, pde, pt); 1812 1813 /* Make sure write is complete before other code can use this page 1814 * table. Also require for WC mapped PTEs */ 1815 #ifdef __NetBSD__ 1816 bus_space_read_4(dev_priv->gtt.bst, dev_priv->gtt.bsh, 0); 1817 #else 1818 readl(dev_priv->gtt.gsm); 1819 #endif 1820 } 1821 1822 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt) 1823 { 1824 BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f); 1825 1826 return (ppgtt->pd.base.ggtt_offset / 64) << 16; 1827 } 1828 1829 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, 1830 struct drm_i915_gem_request *req) 1831 { 1832 struct intel_engine_cs *ring = req->ring; 1833 int ret; 1834 1835 /* NB: TLBs must be flushed and invalidated before a switch */ 1836 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1837 if (ret) 1838 return ret; 1839 1840 ret = intel_ring_begin(req, 6); 1841 if (ret) 1842 return ret; 1843 1844 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); 1845 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); 1846 intel_ring_emit(ring, PP_DIR_DCLV_2G); 1847 intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); 1848 intel_ring_emit(ring, get_pd_offset(ppgtt)); 1849 intel_ring_emit(ring, MI_NOOP); 1850 intel_ring_advance(ring); 1851 1852 return 0; 1853 } 1854 1855 static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt, 1856 struct drm_i915_gem_request *req) 1857 { 1858 struct intel_engine_cs *ring = req->ring; 1859 struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev); 1860 1861 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 1862 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); 1863 return 0; 1864 } 1865 1866 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, 1867 struct drm_i915_gem_request *req) 1868 { 1869 struct intel_engine_cs *ring = req->ring; 1870 int ret; 1871 1872 /* NB: TLBs must be flushed and invalidated before a switch */ 1873 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1874 if (ret) 1875 return ret; 1876 1877 ret = intel_ring_begin(req, 6); 1878 if (ret) 1879 return ret; 1880 1881 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); 1882 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); 1883 intel_ring_emit(ring, PP_DIR_DCLV_2G); 1884 intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); 1885 intel_ring_emit(ring, get_pd_offset(ppgtt)); 1886 intel_ring_emit(ring, MI_NOOP); 1887 intel_ring_advance(ring); 1888 1889 /* XXX: RCS is the only one to auto invalidate the TLBs? */ 1890 if (ring->id != RCS) { 1891 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1892 if (ret) 1893 return ret; 1894 } 1895 1896 return 0; 1897 } 1898 1899 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, 1900 struct drm_i915_gem_request *req) 1901 { 1902 struct intel_engine_cs *ring = req->ring; 1903 struct drm_device *dev = ppgtt->base.dev; 1904 struct drm_i915_private *dev_priv = dev->dev_private; 1905 1906 1907 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 1908 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); 1909 1910 POSTING_READ(RING_PP_DIR_DCLV(ring)); 1911 1912 return 0; 1913 } 1914 1915 static void gen8_ppgtt_enable(struct drm_device *dev) 1916 { 1917 struct drm_i915_private *dev_priv = dev->dev_private; 1918 struct intel_engine_cs *ring; 1919 int j; 1920 1921 for_each_ring(ring, dev_priv, j) { 1922 u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0; 1923 I915_WRITE(RING_MODE_GEN7(ring), 1924 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level)); 1925 } 1926 } 1927 1928 static void gen7_ppgtt_enable(struct drm_device *dev) 1929 { 1930 struct drm_i915_private *dev_priv = dev->dev_private; 1931 struct intel_engine_cs *ring; 1932 uint32_t ecochk, ecobits; 1933 int i; 1934 1935 ecobits = I915_READ(GAC_ECO_BITS); 1936 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); 1937 1938 ecochk = I915_READ(GAM_ECOCHK); 1939 if (IS_HASWELL(dev)) { 1940 ecochk |= ECOCHK_PPGTT_WB_HSW; 1941 } else { 1942 ecochk |= ECOCHK_PPGTT_LLC_IVB; 1943 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; 1944 } 1945 I915_WRITE(GAM_ECOCHK, ecochk); 1946 1947 for_each_ring(ring, dev_priv, i) { 1948 /* GFX_MODE is per-ring on gen7+ */ 1949 I915_WRITE(RING_MODE_GEN7(ring), 1950 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 1951 } 1952 } 1953 1954 static void gen6_ppgtt_enable(struct drm_device *dev) 1955 { 1956 struct drm_i915_private *dev_priv = dev->dev_private; 1957 uint32_t ecochk, gab_ctl, ecobits; 1958 1959 ecobits = I915_READ(GAC_ECO_BITS); 1960 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | 1961 ECOBITS_PPGTT_CACHE64B); 1962 1963 gab_ctl = I915_READ(GAB_CTL); 1964 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); 1965 1966 ecochk = I915_READ(GAM_ECOCHK); 1967 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B); 1968 1969 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 1970 } 1971 1972 /* PPGTT support for Sandybdrige/Gen6 and later */ 1973 static void gen6_ppgtt_clear_range(struct i915_address_space *vm, 1974 uint64_t start, 1975 uint64_t length, 1976 bool use_scratch) 1977 { 1978 struct i915_hw_ppgtt *ppgtt = 1979 container_of(vm, struct i915_hw_ppgtt, base); 1980 gen6_pte_t *pt_vaddr, scratch_pte; 1981 unsigned first_entry = start >> PAGE_SHIFT; 1982 unsigned num_entries = length >> PAGE_SHIFT; 1983 unsigned act_pt = first_entry / GEN6_PTES; 1984 unsigned first_pte = first_entry % GEN6_PTES; 1985 unsigned last_pte, i; 1986 1987 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 1988 I915_CACHE_LLC, true, 0); 1989 1990 while (num_entries) { 1991 last_pte = first_pte + num_entries; 1992 if (last_pte > GEN6_PTES) 1993 last_pte = GEN6_PTES; 1994 1995 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); 1996 1997 for (i = first_pte; i < last_pte; i++) 1998 pt_vaddr[i] = scratch_pte; 1999 2000 kunmap_px(ppgtt, pt_vaddr); 2001 2002 num_entries -= last_pte - first_pte; 2003 first_pte = 0; 2004 act_pt++; 2005 } 2006 } 2007 2008 #ifdef __NetBSD__ 2009 static void 2010 gen6_ppgtt_insert_entries(struct i915_address_space *vm, bus_dmamap_t dmamap, 2011 uint64_t start, enum i915_cache_level cache_level, uint32_t flags) 2012 { 2013 struct i915_hw_ppgtt *ppgtt = 2014 container_of(vm, struct i915_hw_ppgtt, base); 2015 gen6_pte_t *pt_vaddr; 2016 unsigned first_entry = start >> PAGE_SHIFT; 2017 unsigned act_pt = first_entry / GEN6_PTES; 2018 unsigned act_pte = first_entry % GEN6_PTES; 2019 unsigned seg; 2020 2021 pt_vaddr = NULL; 2022 KASSERT(0 < dmamap->dm_nsegs); 2023 for (seg = 0; seg < dmamap->dm_nsegs; seg++) { 2024 KASSERT(dmamap->dm_segs[seg].ds_len == PAGE_SIZE); 2025 if (pt_vaddr == NULL) 2026 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); 2027 pt_vaddr[act_pte] = 2028 vm->pte_encode(dmamap->dm_segs[seg].ds_addr, cache_level, 2029 true, flags); 2030 if (++act_pte == GEN6_PTES) { 2031 kunmap_px(ppgtt, pt_vaddr); 2032 pt_vaddr = NULL; 2033 act_pt++; 2034 act_pte = 0; 2035 } 2036 } 2037 if (pt_vaddr) 2038 kunmap_px(ppgtt, pt_vaddr); 2039 } 2040 #else 2041 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, 2042 struct sg_table *pages, 2043 uint64_t start, 2044 enum i915_cache_level cache_level, u32 flags) 2045 { 2046 struct i915_hw_ppgtt *ppgtt = 2047 container_of(vm, struct i915_hw_ppgtt, base); 2048 gen6_pte_t *pt_vaddr; 2049 unsigned first_entry = start >> PAGE_SHIFT; 2050 unsigned act_pt = first_entry / GEN6_PTES; 2051 unsigned act_pte = first_entry % GEN6_PTES; 2052 struct sg_page_iter sg_iter; 2053 2054 pt_vaddr = NULL; 2055 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { 2056 if (pt_vaddr == NULL) 2057 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); 2058 2059 pt_vaddr[act_pte] = 2060 vm->pte_encode(sg_page_iter_dma_address(&sg_iter), 2061 cache_level, true, flags); 2062 2063 if (++act_pte == GEN6_PTES) { 2064 kunmap_px(ppgtt, pt_vaddr); 2065 pt_vaddr = NULL; 2066 act_pt++; 2067 act_pte = 0; 2068 } 2069 } 2070 if (pt_vaddr) 2071 kunmap_px(ppgtt, pt_vaddr); 2072 } 2073 #endif 2074 2075 static int gen6_alloc_va_range(struct i915_address_space *vm, 2076 uint64_t start_in, uint64_t length_in) 2077 { 2078 DECLARE_BITMAP(new_page_tables, I915_PDES); 2079 struct drm_device *dev = vm->dev; 2080 struct drm_i915_private *dev_priv = dev->dev_private; 2081 struct i915_hw_ppgtt *ppgtt = 2082 container_of(vm, struct i915_hw_ppgtt, base); 2083 struct i915_page_table *pt; 2084 uint32_t start, length, start_save, length_save; 2085 uint32_t pde, temp; 2086 int ret; 2087 2088 if (WARN_ON(start_in + length_in > ppgtt->base.total)) 2089 return -ENODEV; 2090 2091 start = start_save = start_in; 2092 length = length_save = length_in; 2093 2094 bitmap_zero(new_page_tables, I915_PDES); 2095 2096 /* The allocation is done in two stages so that we can bail out with 2097 * minimal amount of pain. The first stage finds new page tables that 2098 * need allocation. The second stage marks use ptes within the page 2099 * tables. 2100 */ 2101 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { 2102 if (pt != vm->scratch_pt) { 2103 WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES)); 2104 continue; 2105 } 2106 2107 /* We've already allocated a page table */ 2108 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES)); 2109 2110 pt = alloc_pt(dev); 2111 if (IS_ERR(pt)) { 2112 ret = PTR_ERR(pt); 2113 goto unwind_out; 2114 } 2115 2116 gen6_initialize_pt(vm, pt); 2117 2118 ppgtt->pd.page_table[pde] = pt; 2119 __set_bit(pde, new_page_tables); 2120 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT); 2121 } 2122 2123 start = start_save; 2124 length = length_save; 2125 2126 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { 2127 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES); 2128 2129 bitmap_zero(tmp_bitmap, GEN6_PTES); 2130 bitmap_set(tmp_bitmap, gen6_pte_index(start), 2131 gen6_pte_count(start, length)); 2132 2133 if (__test_and_clear_bit(pde, new_page_tables)) 2134 gen6_write_pde(&ppgtt->pd, pde, pt); 2135 2136 trace_i915_page_table_entry_map(vm, pde, pt, 2137 gen6_pte_index(start), 2138 gen6_pte_count(start, length), 2139 GEN6_PTES); 2140 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes, 2141 GEN6_PTES); 2142 } 2143 2144 WARN_ON(!bitmap_empty(new_page_tables, I915_PDES)); 2145 2146 /* Make sure write is complete before other code can use this page 2147 * table. Also require for WC mapped PTEs */ 2148 #ifdef __NetBSD__ 2149 bus_space_read_4(dev_priv->gtt.bst, dev_priv->gtt.bsh, 0); 2150 #else 2151 readl(dev_priv->gtt.gsm); 2152 #endif 2153 2154 mark_tlbs_dirty(ppgtt); 2155 return 0; 2156 2157 unwind_out: 2158 for_each_set_bit(pde, new_page_tables, I915_PDES) { 2159 struct i915_page_table *pt = ppgtt->pd.page_table[pde]; 2160 2161 ppgtt->pd.page_table[pde] = vm->scratch_pt; 2162 free_pt(vm->dev, pt); 2163 } 2164 2165 mark_tlbs_dirty(ppgtt); 2166 return ret; 2167 } 2168 2169 static int gen6_init_scratch(struct i915_address_space *vm) 2170 { 2171 struct drm_device *dev = vm->dev; 2172 2173 vm->scratch_page = alloc_scratch_page(dev); 2174 if (IS_ERR(vm->scratch_page)) 2175 return PTR_ERR(vm->scratch_page); 2176 2177 vm->scratch_pt = alloc_pt(dev); 2178 if (IS_ERR(vm->scratch_pt)) { 2179 free_scratch_page(dev, vm->scratch_page); 2180 return PTR_ERR(vm->scratch_pt); 2181 } 2182 2183 gen6_initialize_pt(vm, vm->scratch_pt); 2184 2185 return 0; 2186 } 2187 2188 static void gen6_free_scratch(struct i915_address_space *vm) 2189 { 2190 struct drm_device *dev = vm->dev; 2191 2192 free_pt(dev, vm->scratch_pt); 2193 free_scratch_page(dev, vm->scratch_page); 2194 } 2195 2196 static void gen6_ppgtt_cleanup(struct i915_address_space *vm) 2197 { 2198 struct i915_hw_ppgtt *ppgtt = 2199 container_of(vm, struct i915_hw_ppgtt, base); 2200 struct i915_page_table *pt; 2201 uint32_t pde; 2202 2203 drm_mm_remove_node(&ppgtt->node); 2204 2205 gen6_for_all_pdes(pt, ppgtt, pde) { 2206 if (pt != vm->scratch_pt) 2207 free_pt(ppgtt->base.dev, pt); 2208 } 2209 2210 gen6_free_scratch(vm); 2211 } 2212 2213 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) 2214 { 2215 struct i915_address_space *vm = &ppgtt->base; 2216 struct drm_device *dev = ppgtt->base.dev; 2217 struct drm_i915_private *dev_priv = dev->dev_private; 2218 bool retried = false; 2219 int ret; 2220 2221 /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The 2222 * allocator works in address space sizes, so it's multiplied by page 2223 * size. We allocate at the top of the GTT to avoid fragmentation. 2224 */ 2225 BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm)); 2226 2227 ret = gen6_init_scratch(vm); 2228 if (ret) 2229 return ret; 2230 2231 alloc: 2232 ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm, 2233 &ppgtt->node, GEN6_PD_SIZE, 2234 GEN6_PD_ALIGN, 0, 2235 0, dev_priv->gtt.base.total, 2236 DRM_MM_TOPDOWN); 2237 if (ret == -ENOSPC && !retried) { 2238 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base, 2239 GEN6_PD_SIZE, GEN6_PD_ALIGN, 2240 I915_CACHE_NONE, 2241 0, dev_priv->gtt.base.total, 2242 0); 2243 if (ret) 2244 goto err_out; 2245 2246 retried = true; 2247 goto alloc; 2248 } 2249 2250 if (ret) 2251 goto err_out; 2252 2253 2254 if (ppgtt->node.start < dev_priv->gtt.mappable_end) 2255 DRM_DEBUG("Forced to use aperture for PDEs\n"); 2256 2257 return 0; 2258 2259 err_out: 2260 gen6_free_scratch(vm); 2261 return ret; 2262 } 2263 2264 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt) 2265 { 2266 return gen6_ppgtt_allocate_page_directories(ppgtt); 2267 } 2268 2269 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt, 2270 uint64_t start, uint64_t length) 2271 { 2272 struct i915_page_table *unused __unused; 2273 uint32_t pde, temp; 2274 2275 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) 2276 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt; 2277 } 2278 2279 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 2280 { 2281 struct drm_device *dev = ppgtt->base.dev; 2282 struct drm_i915_private *dev_priv = dev->dev_private; 2283 int ret; 2284 2285 ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode; 2286 if (IS_GEN6(dev)) { 2287 ppgtt->switch_mm = gen6_mm_switch; 2288 } else if (IS_HASWELL(dev)) { 2289 ppgtt->switch_mm = hsw_mm_switch; 2290 } else if (IS_GEN7(dev)) { 2291 ppgtt->switch_mm = gen7_mm_switch; 2292 } else 2293 BUG(); 2294 2295 if (intel_vgpu_active(dev)) 2296 ppgtt->switch_mm = vgpu_mm_switch; 2297 2298 ret = gen6_ppgtt_alloc(ppgtt); 2299 if (ret) 2300 return ret; 2301 2302 ppgtt->base.allocate_va_range = gen6_alloc_va_range; 2303 ppgtt->base.clear_range = gen6_ppgtt_clear_range; 2304 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; 2305 ppgtt->base.unbind_vma = ppgtt_unbind_vma; 2306 ppgtt->base.bind_vma = ppgtt_bind_vma; 2307 ppgtt->base.cleanup = gen6_ppgtt_cleanup; 2308 ppgtt->base.start = 0; 2309 ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE; 2310 #ifndef __NetBSD__ 2311 ppgtt->debug_dump = gen6_dump_ppgtt; 2312 #endif 2313 2314 ppgtt->pd.base.ggtt_offset = 2315 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t); 2316 2317 #ifndef __NetBSD__ 2318 ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm + 2319 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t); 2320 #endif 2321 2322 gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total); 2323 2324 gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total); 2325 2326 DRM_DEBUG_DRIVER("Allocated pde space (%"PRId64"M) at GTT entry: %"PRIx64"\n", 2327 ppgtt->node.size >> 20, 2328 ppgtt->node.start / PAGE_SIZE); 2329 2330 DRM_DEBUG("Adding PPGTT at offset %x\n", 2331 ppgtt->pd.base.ggtt_offset << 10); 2332 2333 return 0; 2334 } 2335 2336 static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) 2337 { 2338 ppgtt->base.dev = dev; 2339 2340 if (INTEL_INFO(dev)->gen < 8) 2341 return gen6_ppgtt_init(ppgtt); 2342 else 2343 return gen8_ppgtt_init(ppgtt); 2344 } 2345 2346 static void i915_address_space_init(struct i915_address_space *vm, 2347 struct drm_i915_private *dev_priv) 2348 { 2349 drm_mm_init(&vm->mm, vm->start, vm->total); 2350 vm->dev = dev_priv->dev; 2351 INIT_LIST_HEAD(&vm->active_list); 2352 INIT_LIST_HEAD(&vm->inactive_list); 2353 list_add_tail(&vm->global_link, &dev_priv->vm_list); 2354 } 2355 2356 int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) 2357 { 2358 struct drm_i915_private *dev_priv = dev->dev_private; 2359 int ret = 0; 2360 2361 ret = __hw_ppgtt_init(dev, ppgtt); 2362 if (ret == 0) { 2363 kref_init(&ppgtt->ref); 2364 i915_address_space_init(&ppgtt->base, dev_priv); 2365 } 2366 2367 return ret; 2368 } 2369 2370 int i915_ppgtt_init_hw(struct drm_device *dev) 2371 { 2372 /* In the case of execlists, PPGTT is enabled by the context descriptor 2373 * and the PDPs are contained within the context itself. We don't 2374 * need to do anything here. */ 2375 if (i915.enable_execlists) 2376 return 0; 2377 2378 if (!USES_PPGTT(dev)) 2379 return 0; 2380 2381 if (IS_GEN6(dev)) 2382 gen6_ppgtt_enable(dev); 2383 else if (IS_GEN7(dev)) 2384 gen7_ppgtt_enable(dev); 2385 else if (INTEL_INFO(dev)->gen >= 8) 2386 gen8_ppgtt_enable(dev); 2387 else 2388 MISSING_CASE(INTEL_INFO(dev)->gen); 2389 2390 return 0; 2391 } 2392 2393 int i915_ppgtt_init_ring(struct drm_i915_gem_request *req) 2394 { 2395 struct drm_i915_private *dev_priv = req->ring->dev->dev_private; 2396 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 2397 2398 if (i915.enable_execlists) 2399 return 0; 2400 2401 if (!ppgtt) 2402 return 0; 2403 2404 return ppgtt->switch_mm(ppgtt, req); 2405 } 2406 2407 struct i915_hw_ppgtt * 2408 i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv) 2409 { 2410 struct i915_hw_ppgtt *ppgtt; 2411 int ret; 2412 2413 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 2414 if (!ppgtt) 2415 return ERR_PTR(-ENOMEM); 2416 2417 ret = i915_ppgtt_init(dev, ppgtt); 2418 if (ret) { 2419 kfree(ppgtt); 2420 return ERR_PTR(ret); 2421 } 2422 2423 ppgtt->file_priv = fpriv; 2424 2425 trace_i915_ppgtt_create(&ppgtt->base); 2426 2427 return ppgtt; 2428 } 2429 2430 void i915_ppgtt_release(struct kref *kref) 2431 { 2432 struct i915_hw_ppgtt *ppgtt = 2433 container_of(kref, struct i915_hw_ppgtt, ref); 2434 2435 trace_i915_ppgtt_release(&ppgtt->base); 2436 2437 /* vmas should already be unbound */ 2438 WARN_ON(!list_empty(&ppgtt->base.active_list)); 2439 WARN_ON(!list_empty(&ppgtt->base.inactive_list)); 2440 2441 list_del(&ppgtt->base.global_link); 2442 drm_mm_takedown(&ppgtt->base.mm); 2443 2444 ppgtt->base.cleanup(&ppgtt->base); 2445 kfree(ppgtt); 2446 } 2447 2448 extern int intel_iommu_gfx_mapped; 2449 /* Certain Gen5 chipsets require require idling the GPU before 2450 * unmapping anything from the GTT when VT-d is enabled. 2451 */ 2452 static bool needs_idle_maps(struct drm_device *dev) 2453 { 2454 #ifdef CONFIG_INTEL_IOMMU 2455 /* Query intel_iommu to see if we need the workaround. Presumably that 2456 * was loaded first. 2457 */ 2458 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped) 2459 return true; 2460 #endif 2461 return false; 2462 } 2463 2464 static bool do_idling(struct drm_i915_private *dev_priv) 2465 { 2466 bool ret = dev_priv->mm.interruptible; 2467 2468 if (unlikely(dev_priv->gtt.do_idle_maps)) { 2469 dev_priv->mm.interruptible = false; 2470 if (i915_gpu_idle(dev_priv->dev)) { 2471 DRM_ERROR("Couldn't idle GPU\n"); 2472 /* Wait a bit, in hopes it avoids the hang */ 2473 udelay(10); 2474 } 2475 } 2476 2477 return ret; 2478 } 2479 2480 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) 2481 { 2482 if (unlikely(dev_priv->gtt.do_idle_maps)) 2483 dev_priv->mm.interruptible = interruptible; 2484 } 2485 2486 void i915_check_and_clear_faults(struct drm_device *dev) 2487 { 2488 struct drm_i915_private *dev_priv = dev->dev_private; 2489 struct intel_engine_cs *ring; 2490 int i; 2491 2492 if (INTEL_INFO(dev)->gen < 6) 2493 return; 2494 2495 for_each_ring(ring, dev_priv, i) { 2496 u32 fault_reg; 2497 fault_reg = I915_READ(RING_FAULT_REG(ring)); 2498 if (fault_reg & RING_FAULT_VALID) { 2499 DRM_DEBUG_DRIVER("Unexpected fault\n" 2500 "\tAddr: 0x%08"PRIx32"\n" 2501 "\tAddress space: %s\n" 2502 "\tSource ID: %d\n" 2503 "\tType: %d\n", 2504 fault_reg & PAGE_MASK, 2505 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT", 2506 RING_FAULT_SRCID(fault_reg), 2507 RING_FAULT_FAULT_TYPE(fault_reg)); 2508 I915_WRITE(RING_FAULT_REG(ring), 2509 fault_reg & ~RING_FAULT_VALID); 2510 } 2511 } 2512 POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS])); 2513 } 2514 2515 static void i915_ggtt_flush(struct drm_i915_private *dev_priv) 2516 { 2517 if (INTEL_INFO(dev_priv->dev)->gen < 6) { 2518 intel_gtt_chipset_flush(); 2519 } else { 2520 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2521 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2522 } 2523 } 2524 2525 void i915_gem_suspend_gtt_mappings(struct drm_device *dev) 2526 { 2527 struct drm_i915_private *dev_priv = dev->dev_private; 2528 2529 /* Don't bother messing with faults pre GEN6 as we have little 2530 * documentation supporting that it's a good idea. 2531 */ 2532 if (INTEL_INFO(dev)->gen < 6) 2533 return; 2534 2535 i915_check_and_clear_faults(dev); 2536 2537 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, 2538 dev_priv->gtt.base.start, 2539 dev_priv->gtt.base.total, 2540 true); 2541 2542 i915_ggtt_flush(dev_priv); 2543 } 2544 2545 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) 2546 { 2547 #ifdef __NetBSD__ 2548 KASSERT(0 < obj->base.size); 2549 /* XXX errno NetBSD->Linux */ 2550 return -bus_dmamap_load_pglist(obj->base.dev->dmat, obj->pages, 2551 &obj->pageq, obj->base.size, BUS_DMA_NOWAIT); 2552 #else 2553 if (!dma_map_sg(&obj->base.dev->pdev->dev, 2554 obj->pages->sgl, obj->pages->nents, 2555 PCI_DMA_BIDIRECTIONAL)) 2556 return -ENOSPC; 2557 2558 return 0; 2559 #endif 2560 } 2561 2562 #ifdef __NetBSD__ 2563 static gen8_pte_t 2564 gen8_get_pte(bus_space_tag_t bst, bus_space_handle_t bsh, unsigned i) 2565 { 2566 CTASSERT(_BYTE_ORDER == _LITTLE_ENDIAN); /* x86 */ 2567 CTASSERT(sizeof(gen8_pte_t) == 8); 2568 #ifdef _LP64 /* XXX How to detect bus_space_read_8? */ 2569 return bus_space_read_8(bst, bsh, 8*i); 2570 #else 2571 /* 2572 * XXX I'm not sure this case can actually happen in practice: 2573 * 32-bit gen8 chipsets? 2574 */ 2575 return bus_space_read_4(bst, bsh, 8*i) | 2576 ((uint64_t)bus_space_read_4(bst, bsh, 8*i + 4) << 32); 2577 #endif 2578 } 2579 2580 static inline void 2581 gen8_set_pte(bus_space_tag_t bst, bus_space_handle_t bsh, unsigned i, 2582 gen8_pte_t pte) 2583 { 2584 CTASSERT(_BYTE_ORDER == _LITTLE_ENDIAN); /* x86 */ 2585 CTASSERT(sizeof(gen8_pte_t) == 8); 2586 #ifdef _LP64 /* XXX How to detect bus_space_write_8? */ 2587 bus_space_write_8(bst, bsh, 8*i, pte); 2588 #else 2589 bus_space_write_4(bst, bsh, 8*i, (uint32_t)pte); 2590 bus_space_write_4(bst, bsh, 8*i + 4, (uint32_t)(pte >> 32)); 2591 #endif 2592 } 2593 #else 2594 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 2595 { 2596 #ifdef writeq 2597 writeq(pte, addr); 2598 #else 2599 iowrite32((u32)pte, addr); 2600 iowrite32(pte >> 32, addr + 4); 2601 #endif 2602 } 2603 #endif 2604 2605 #ifdef __NetBSD__ 2606 static void 2607 gen8_ggtt_insert_entries(struct i915_address_space *vm, bus_dmamap_t dmamap, 2608 uint64_t start, enum i915_cache_level level, uint32_t flags) 2609 { 2610 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2611 unsigned first_entry = start >> PAGE_SHIFT; 2612 const bus_space_tag_t bst = dev_priv->gtt.bst; 2613 const bus_space_handle_t bsh = dev_priv->gtt.bsh; 2614 unsigned i; 2615 2616 KASSERT(0 < dmamap->dm_nsegs); 2617 for (i = 0; i < dmamap->dm_nsegs; i++) { 2618 KASSERT(dmamap->dm_segs[i].ds_len == PAGE_SIZE); 2619 gen8_set_pte(bst, bsh, first_entry + i, 2620 gen8_pte_encode(dmamap->dm_segs[i].ds_addr, level, true, flags)); 2621 } 2622 if (0 < i) { 2623 /* Posting read. */ 2624 WARN_ON(gen8_get_pte(bst, bsh, (first_entry + i - 1)) 2625 != gen8_pte_encode(dmamap->dm_segs[i - 1].ds_addr, level, 2626 true, flags)); 2627 } 2628 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2629 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2630 } 2631 #else 2632 static void gen8_ggtt_insert_entries(struct i915_address_space *vm, 2633 struct sg_table *st, 2634 uint64_t start, 2635 enum i915_cache_level level, u32 flags) 2636 { 2637 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2638 unsigned first_entry = start >> PAGE_SHIFT; 2639 gen8_pte_t __iomem *gtt_entries = 2640 (gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; 2641 int i = 0; 2642 struct sg_page_iter sg_iter; 2643 dma_addr_t addr = 0; /* shut up gcc */ 2644 2645 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { 2646 addr = sg_dma_address(sg_iter.sg) + 2647 (sg_iter.sg_pgoffset << PAGE_SHIFT); 2648 gen8_set_pte(>t_entries[i], 2649 gen8_pte_encode(addr, level, true, flags)); 2650 i++; 2651 } 2652 2653 /* 2654 * XXX: This serves as a posting read to make sure that the PTE has 2655 * actually been updated. There is some concern that even though 2656 * registers and PTEs are within the same BAR that they are potentially 2657 * of NUMA access patterns. Therefore, even with the way we assume 2658 * hardware should work, we must keep this posting read for paranoia. 2659 */ 2660 if (i != 0) 2661 WARN_ON(readq(>t_entries[i-1]) 2662 != gen8_pte_encode(addr, level, true, flags)); 2663 2664 /* This next bit makes the above posting read even more important. We 2665 * want to flush the TLBs only after we're certain all the PTE updates 2666 * have finished. 2667 */ 2668 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2669 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2670 } 2671 #endif 2672 2673 /* 2674 * Binds an object into the global gtt with the specified cache level. The object 2675 * will be accessible to the GPU via commands whose operands reference offsets 2676 * within the global GTT as well as accessible by the GPU through the GMADR 2677 * mapped BAR (dev_priv->mm.gtt->gtt). 2678 */ 2679 #ifdef __NetBSD__ 2680 static void 2681 gen6_ggtt_insert_entries(struct i915_address_space *vm, bus_dmamap_t dmamap, 2682 uint64_t start, enum i915_cache_level level, uint32_t flags) 2683 { 2684 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2685 unsigned first_entry = start >> PAGE_SHIFT; 2686 const bus_space_tag_t bst = dev_priv->gtt.bst; 2687 const bus_space_handle_t bsh = dev_priv->gtt.bsh; 2688 unsigned i; 2689 2690 KASSERT(0 < dmamap->dm_nsegs); 2691 for (i = 0; i < dmamap->dm_nsegs; i++) { 2692 KASSERT(dmamap->dm_segs[i].ds_len == PAGE_SIZE); 2693 CTASSERT(sizeof(gen6_pte_t) == 4); 2694 bus_space_write_4(bst, bsh, 4*(first_entry + i), 2695 vm->pte_encode(dmamap->dm_segs[i].ds_addr, level, true, 2696 flags)); 2697 } 2698 if (0 < i) { 2699 /* Posting read. */ 2700 WARN_ON(bus_space_read_4(bst, bsh, 4*(first_entry + i - 1)) 2701 != vm->pte_encode(dmamap->dm_segs[i - 1].ds_addr, level, 2702 true, flags)); 2703 } 2704 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2705 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2706 } 2707 #else 2708 static void gen6_ggtt_insert_entries(struct i915_address_space *vm, 2709 struct sg_table *st, 2710 uint64_t start, 2711 enum i915_cache_level level, u32 flags) 2712 { 2713 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2714 unsigned first_entry = start >> PAGE_SHIFT; 2715 gen6_pte_t __iomem *gtt_entries = 2716 (gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; 2717 int i = 0; 2718 struct sg_page_iter sg_iter; 2719 dma_addr_t addr = 0; 2720 2721 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { 2722 addr = sg_page_iter_dma_address(&sg_iter); 2723 iowrite32(vm->pte_encode(addr, level, true, flags), >t_entries[i]); 2724 i++; 2725 } 2726 2727 /* XXX: This serves as a posting read to make sure that the PTE has 2728 * actually been updated. There is some concern that even though 2729 * registers and PTEs are within the same BAR that they are potentially 2730 * of NUMA access patterns. Therefore, even with the way we assume 2731 * hardware should work, we must keep this posting read for paranoia. 2732 */ 2733 if (i != 0) { 2734 unsigned long gtt = readl(>t_entries[i-1]); 2735 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags)); 2736 } 2737 2738 /* This next bit makes the above posting read even more important. We 2739 * want to flush the TLBs only after we're certain all the PTE updates 2740 * have finished. 2741 */ 2742 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2743 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2744 } 2745 #endif 2746 2747 static void gen8_ggtt_clear_range(struct i915_address_space *vm, 2748 uint64_t start, 2749 uint64_t length, 2750 bool use_scratch) 2751 { 2752 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2753 unsigned first_entry = start >> PAGE_SHIFT; 2754 unsigned num_entries = length >> PAGE_SHIFT; 2755 #ifdef __NetBSD__ 2756 const bus_space_tag_t bst = dev_priv->gtt.bst; 2757 const bus_space_handle_t bsh = dev_priv->gtt.bsh; 2758 gen8_pte_t scratch_pte; 2759 #else 2760 gen8_pte_t scratch_pte, __iomem *gtt_base = 2761 (gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; 2762 #endif 2763 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; 2764 int i; 2765 2766 if (WARN(num_entries > max_entries, 2767 "First entry = %d; Num entries = %d (max=%d)\n", 2768 first_entry, num_entries, max_entries)) 2769 num_entries = max_entries; 2770 2771 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 2772 I915_CACHE_LLC, 2773 use_scratch, 0); 2774 #ifdef __NetBSD__ 2775 for (i = 0; i < num_entries; i++) 2776 gen8_set_pte(bst, bsh, first_entry + i, scratch_pte); 2777 (void)gen8_get_pte(bst, bsh, first_entry); 2778 #else 2779 for (i = 0; i < num_entries; i++) 2780 gen8_set_pte(>t_base[i], scratch_pte); 2781 readl(gtt_base); 2782 #endif 2783 } 2784 2785 static void gen6_ggtt_clear_range(struct i915_address_space *vm, 2786 uint64_t start, 2787 uint64_t length, 2788 bool use_scratch) 2789 { 2790 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2791 unsigned first_entry = start >> PAGE_SHIFT; 2792 unsigned num_entries = length >> PAGE_SHIFT; 2793 #ifdef __NetBSD__ 2794 const bus_space_tag_t bst = dev_priv->gtt.bst; 2795 const bus_space_handle_t bsh = dev_priv->gtt.bsh; 2796 gen8_pte_t scratch_pte; 2797 #else 2798 gen6_pte_t scratch_pte, __iomem *gtt_base = 2799 (gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; 2800 #endif 2801 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; 2802 int i; 2803 2804 if (WARN(num_entries > max_entries, 2805 "First entry = %d; Num entries = %d (max=%d)\n", 2806 first_entry, num_entries, max_entries)) 2807 num_entries = max_entries; 2808 2809 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 2810 I915_CACHE_LLC, use_scratch, 0); 2811 2812 #ifdef __NetBSD__ 2813 CTASSERT(sizeof(gen6_pte_t) == 4); 2814 for (i = 0; i < num_entries; i++) 2815 bus_space_write_4(bst, bsh, 4*(first_entry + i), scratch_pte); 2816 (void)bus_space_read_4(bst, bsh, 4*first_entry); 2817 #else 2818 for (i = 0; i < num_entries; i++) 2819 iowrite32(scratch_pte, >t_base[i]); 2820 readl(gtt_base); 2821 #endif 2822 } 2823 2824 static void i915_ggtt_insert_entries(struct i915_address_space *vm, 2825 #ifdef __NetBSD__ 2826 bus_dmamap_t pages, 2827 #else 2828 struct sg_table *pages, 2829 #endif 2830 uint64_t start, 2831 enum i915_cache_level cache_level, u32 unused) 2832 { 2833 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 2834 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 2835 2836 intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags); 2837 } 2838 2839 static void i915_ggtt_clear_range(struct i915_address_space *vm, 2840 uint64_t start, 2841 uint64_t length, 2842 bool unused) 2843 { 2844 unsigned first_entry = start >> PAGE_SHIFT; 2845 unsigned num_entries = length >> PAGE_SHIFT; 2846 intel_gtt_clear_range(first_entry, num_entries); 2847 } 2848 2849 static int ggtt_bind_vma(struct i915_vma *vma, 2850 enum i915_cache_level cache_level, 2851 u32 flags) 2852 { 2853 struct drm_i915_gem_object *obj = vma->obj; 2854 u32 pte_flags = 0; 2855 int ret; 2856 2857 ret = i915_get_ggtt_vma_pages(vma); 2858 if (ret) 2859 return ret; 2860 2861 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ 2862 pte_flags = 0; 2863 if (obj->gt_ro) 2864 pte_flags |= PTE_READ_ONLY; 2865 2866 vma->vm->insert_entries(vma->vm, vma->ggtt_view.pages, 2867 vma->node.start, 2868 cache_level, pte_flags); 2869 2870 /* 2871 * Without aliasing PPGTT there's no difference between 2872 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally 2873 * upgrade to both bound if we bind either to avoid double-binding. 2874 */ 2875 vma->bound |= GLOBAL_BIND | LOCAL_BIND; 2876 2877 return 0; 2878 } 2879 2880 static int aliasing_gtt_bind_vma(struct i915_vma *vma, 2881 enum i915_cache_level cache_level, 2882 u32 flags) 2883 { 2884 struct drm_device *dev = vma->vm->dev; 2885 struct drm_i915_private *dev_priv = dev->dev_private; 2886 struct drm_i915_gem_object *obj = vma->obj; 2887 #ifdef __NetBSD__ 2888 bus_dmamap_t pages = obj->pages; 2889 #else 2890 struct sg_table *pages = obj->pages; 2891 #endif 2892 u32 pte_flags = 0; 2893 int ret; 2894 2895 ret = i915_get_ggtt_vma_pages(vma); 2896 if (ret) 2897 return ret; 2898 pages = vma->ggtt_view.pages; 2899 2900 /* Currently applicable only to VLV */ 2901 if (obj->gt_ro) 2902 pte_flags |= PTE_READ_ONLY; 2903 2904 2905 if (flags & GLOBAL_BIND) { 2906 vma->vm->insert_entries(vma->vm, pages, 2907 vma->node.start, 2908 cache_level, pte_flags); 2909 } 2910 2911 if (flags & LOCAL_BIND) { 2912 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; 2913 appgtt->base.insert_entries(&appgtt->base, pages, 2914 vma->node.start, 2915 cache_level, pte_flags); 2916 } 2917 2918 return 0; 2919 } 2920 2921 static void ggtt_unbind_vma(struct i915_vma *vma) 2922 { 2923 struct drm_device *dev = vma->vm->dev; 2924 struct drm_i915_private *dev_priv = dev->dev_private; 2925 struct drm_i915_gem_object *obj = vma->obj; 2926 const uint64_t size = min_t(uint64_t, 2927 obj->base.size, 2928 vma->node.size); 2929 2930 if (vma->bound & GLOBAL_BIND) { 2931 vma->vm->clear_range(vma->vm, 2932 vma->node.start, 2933 size, 2934 true); 2935 } 2936 2937 if (dev_priv->mm.aliasing_ppgtt && vma->bound & LOCAL_BIND) { 2938 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; 2939 2940 appgtt->base.clear_range(&appgtt->base, 2941 vma->node.start, 2942 size, 2943 true); 2944 } 2945 } 2946 2947 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) 2948 { 2949 struct drm_device *dev = obj->base.dev; 2950 struct drm_i915_private *dev_priv = dev->dev_private; 2951 bool interruptible; 2952 2953 interruptible = do_idling(dev_priv); 2954 2955 #ifdef __NetBSD__ 2956 bus_dmamap_unload(dev->dmat, obj->pages); 2957 #else 2958 dma_unmap_sg(&dev->pdev->dev, obj->pages->sgl, obj->pages->nents, 2959 PCI_DMA_BIDIRECTIONAL); 2960 #endif 2961 2962 undo_idling(dev_priv, interruptible); 2963 } 2964 2965 static void i915_gtt_color_adjust(struct drm_mm_node *node, 2966 unsigned long color, 2967 u64 *start, 2968 u64 *end) 2969 { 2970 if (node->color != color) 2971 *start += 4096; 2972 2973 if (!list_empty(&node->node_list)) { 2974 node = list_entry(node->node_list.next, 2975 struct drm_mm_node, 2976 node_list); 2977 if (node->allocated && node->color != color) 2978 *end -= 4096; 2979 } 2980 } 2981 2982 static int i915_gem_setup_global_gtt(struct drm_device *dev, 2983 u64 start, 2984 u64 mappable_end, 2985 u64 end) 2986 { 2987 /* Let GEM Manage all of the aperture. 2988 * 2989 * However, leave one page at the end still bound to the scratch page. 2990 * There are a number of places where the hardware apparently prefetches 2991 * past the end of the object, and we've seen multiple hangs with the 2992 * GPU head pointer stuck in a batchbuffer bound at the last page of the 2993 * aperture. One page should be enough to keep any prefetching inside 2994 * of the aperture. 2995 */ 2996 struct drm_i915_private *dev_priv = dev->dev_private; 2997 struct i915_address_space *ggtt_vm = &dev_priv->gtt.base; 2998 struct drm_mm_node *entry; 2999 struct drm_i915_gem_object *obj; 3000 unsigned long hole_start, hole_end; 3001 int ret; 3002 3003 BUG_ON(mappable_end > end); 3004 3005 ggtt_vm->start = start; 3006 3007 /* Subtract the guard page before address space initialization to 3008 * shrink the range used by drm_mm */ 3009 ggtt_vm->total = end - start - PAGE_SIZE; 3010 i915_address_space_init(ggtt_vm, dev_priv); 3011 ggtt_vm->total += PAGE_SIZE; 3012 3013 /* Only VLV supports read-only GGTT mappings */ 3014 ggtt_vm->has_read_only = IS_VALLEYVIEW(dev_priv); 3015 3016 if (intel_vgpu_active(dev)) { 3017 ret = intel_vgt_balloon(dev); 3018 if (ret) 3019 return ret; 3020 } 3021 3022 if (!HAS_LLC(dev)) 3023 ggtt_vm->mm.color_adjust = i915_gtt_color_adjust; 3024 3025 /* Mark any preallocated objects as occupied */ 3026 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 3027 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); 3028 3029 DRM_DEBUG_KMS("reserving preallocated space: %"PRIx64" + %zx\n", 3030 i915_gem_obj_ggtt_offset(obj), obj->base.size); 3031 3032 WARN_ON(i915_gem_obj_ggtt_bound(obj)); 3033 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node); 3034 if (ret) { 3035 DRM_DEBUG_KMS("Reservation failed: %i\n", ret); 3036 return ret; 3037 } 3038 vma->bound |= GLOBAL_BIND; 3039 __i915_vma_set_map_and_fenceable(vma); 3040 list_add_tail(&vma->mm_list, &ggtt_vm->inactive_list); 3041 } 3042 3043 /* Clear any non-preallocated blocks */ 3044 drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) { 3045 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", 3046 hole_start, hole_end); 3047 ggtt_vm->clear_range(ggtt_vm, hole_start, 3048 hole_end - hole_start, true); 3049 } 3050 3051 /* And finally clear the reserved guard page */ 3052 ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true); 3053 3054 if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) { 3055 struct i915_hw_ppgtt *ppgtt; 3056 3057 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 3058 if (!ppgtt) 3059 return -ENOMEM; 3060 3061 ret = __hw_ppgtt_init(dev, ppgtt); 3062 if (ret) { 3063 ppgtt->base.cleanup(&ppgtt->base); 3064 kfree(ppgtt); 3065 return ret; 3066 } 3067 3068 if (ppgtt->base.allocate_va_range) 3069 ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0, 3070 ppgtt->base.total); 3071 if (ret) { 3072 ppgtt->base.cleanup(&ppgtt->base); 3073 kfree(ppgtt); 3074 return ret; 3075 } 3076 3077 ppgtt->base.clear_range(&ppgtt->base, 3078 ppgtt->base.start, 3079 ppgtt->base.total, 3080 true); 3081 3082 dev_priv->mm.aliasing_ppgtt = ppgtt; 3083 WARN_ON(dev_priv->gtt.base.bind_vma != ggtt_bind_vma); 3084 dev_priv->gtt.base.bind_vma = aliasing_gtt_bind_vma; 3085 } 3086 3087 return 0; 3088 } 3089 3090 void i915_gem_init_global_gtt(struct drm_device *dev) 3091 { 3092 struct drm_i915_private *dev_priv = dev->dev_private; 3093 u64 gtt_size, mappable_size; 3094 3095 gtt_size = dev_priv->gtt.base.total; 3096 mappable_size = dev_priv->gtt.mappable_end; 3097 3098 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size); 3099 } 3100 3101 void i915_global_gtt_cleanup(struct drm_device *dev) 3102 { 3103 struct drm_i915_private *dev_priv = dev->dev_private; 3104 struct i915_address_space *vm = &dev_priv->gtt.base; 3105 3106 if (dev_priv->mm.aliasing_ppgtt) { 3107 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 3108 3109 ppgtt->base.cleanup(&ppgtt->base); 3110 kfree(ppgtt); 3111 } 3112 3113 if (drm_mm_initialized(&vm->mm)) { 3114 if (intel_vgpu_active(dev)) 3115 intel_vgt_deballoon(); 3116 3117 drm_mm_takedown(&vm->mm); 3118 list_del(&vm->global_link); 3119 } 3120 3121 vm->cleanup(vm); 3122 } 3123 3124 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 3125 { 3126 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 3127 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 3128 return snb_gmch_ctl << 20; 3129 } 3130 3131 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) 3132 { 3133 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; 3134 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; 3135 if (bdw_gmch_ctl) 3136 bdw_gmch_ctl = 1 << bdw_gmch_ctl; 3137 3138 #ifdef CONFIG_X86_32 3139 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */ 3140 if (bdw_gmch_ctl > 4) 3141 bdw_gmch_ctl = 4; 3142 #endif 3143 3144 return bdw_gmch_ctl << 20; 3145 } 3146 3147 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) 3148 { 3149 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; 3150 gmch_ctrl &= SNB_GMCH_GGMS_MASK; 3151 3152 if (gmch_ctrl) 3153 return 1 << (20 + gmch_ctrl); 3154 3155 return 0; 3156 } 3157 3158 static size_t gen6_get_stolen_size(u16 snb_gmch_ctl) 3159 { 3160 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; 3161 snb_gmch_ctl &= SNB_GMCH_GMS_MASK; 3162 return snb_gmch_ctl << 25; /* 32 MB units */ 3163 } 3164 3165 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl) 3166 { 3167 bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; 3168 bdw_gmch_ctl &= BDW_GMCH_GMS_MASK; 3169 return bdw_gmch_ctl << 25; /* 32 MB units */ 3170 } 3171 3172 static size_t chv_get_stolen_size(u16 gmch_ctrl) 3173 { 3174 gmch_ctrl >>= SNB_GMCH_GMS_SHIFT; 3175 gmch_ctrl &= SNB_GMCH_GMS_MASK; 3176 3177 /* 3178 * 0x0 to 0x10: 32MB increments starting at 0MB 3179 * 0x11 to 0x16: 4MB increments starting at 8MB 3180 * 0x17 to 0x1d: 4MB increments start at 36MB 3181 */ 3182 if (gmch_ctrl < 0x11) 3183 return gmch_ctrl << 25; 3184 else if (gmch_ctrl < 0x17) 3185 return (gmch_ctrl - 0x11 + 2) << 22; 3186 else 3187 return (gmch_ctrl - 0x17 + 9) << 22; 3188 } 3189 3190 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl) 3191 { 3192 gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; 3193 gen9_gmch_ctl &= BDW_GMCH_GMS_MASK; 3194 3195 if (gen9_gmch_ctl < 0xf0) 3196 return gen9_gmch_ctl << 25; /* 32 MB units */ 3197 else 3198 /* 4MB increments starting at 0xf0 for 4MB */ 3199 return (gen9_gmch_ctl - 0xf0 + 1) << 22; 3200 } 3201 3202 static int ggtt_probe_common(struct drm_device *dev, 3203 size_t gtt_size) 3204 { 3205 struct drm_i915_private *dev_priv = dev->dev_private; 3206 struct i915_page_scratch *scratch_page; 3207 phys_addr_t gtt_phys_addr; 3208 3209 /* For Modern GENs the PTEs and register space are split in the BAR */ 3210 gtt_phys_addr = pci_resource_start(dev->pdev, 0) + 3211 (pci_resource_len(dev->pdev, 0) / 2); 3212 3213 #ifdef __NetBSD__ 3214 int ret; 3215 dev_priv->gtt.bst = dev->pdev->pd_pa.pa_memt; 3216 /* XXX errno NetBSD->Linux */ 3217 ret = -bus_space_map(dev_priv->gtt.bst, gtt_phys_addr, gtt_size, 3218 IS_BROXTON(dev) ? 0 : BUS_SPACE_MAP_PREFETCHABLE, 3219 &dev_priv->gtt.bsh); 3220 if (ret) { 3221 DRM_ERROR("Failed to map the graphics translation table: %d\n", 3222 ret); 3223 return ret; 3224 } 3225 dev_priv->gtt.size = gtt_size; 3226 #else 3227 /* 3228 * On BXT writes larger than 64 bit to the GTT pagetable range will be 3229 * dropped. For WC mappings in general we have 64 byte burst writes 3230 * when the WC buffer is flushed, so we can't use it, but have to 3231 * resort to an uncached mapping. The WC issue is easily caught by the 3232 * readback check when writing GTT PTE entries. 3233 */ 3234 if (IS_BROXTON(dev)) 3235 dev_priv->gtt.gsm = ioremap_nocache(gtt_phys_addr, gtt_size); 3236 else 3237 dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size); 3238 if (!dev_priv->gtt.gsm) { 3239 DRM_ERROR("Failed to map the gtt page table\n"); 3240 return -ENOMEM; 3241 } 3242 #endif 3243 3244 scratch_page = alloc_scratch_page(dev); 3245 if (IS_ERR(scratch_page)) { 3246 DRM_ERROR("Scratch setup failed\n"); 3247 /* iounmap will also get called at remove, but meh */ 3248 #ifdef __NetBSD__ 3249 bus_space_unmap(dev_priv->gtt.bst, dev_priv->gtt.bsh, 3250 dev_priv->gtt.size); 3251 #else 3252 iounmap(dev_priv->gtt.gsm); 3253 #endif 3254 return PTR_ERR(scratch_page); 3255 } 3256 3257 dev_priv->gtt.base.scratch_page = scratch_page; 3258 3259 return 0; 3260 } 3261 3262 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability 3263 * bits. When using advanced contexts each context stores its own PAT, but 3264 * writing this data shouldn't be harmful even in those cases. */ 3265 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv) 3266 { 3267 uint64_t pat; 3268 3269 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */ 3270 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */ 3271 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */ 3272 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */ 3273 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) | 3274 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) | 3275 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) | 3276 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3)); 3277 3278 if (!USES_PPGTT(dev_priv->dev)) 3279 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry, 3280 * so RTL will always use the value corresponding to 3281 * pat_sel = 000". 3282 * So let's disable cache for GGTT to avoid screen corruptions. 3283 * MOCS still can be used though. 3284 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work 3285 * before this patch, i.e. the same uncached + snooping access 3286 * like on gen6/7 seems to be in effect. 3287 * - So this just fixes blitter/render access. Again it looks 3288 * like it's not just uncached access, but uncached + snooping. 3289 * So we can still hold onto all our assumptions wrt cpu 3290 * clflushing on LLC machines. 3291 */ 3292 pat = GEN8_PPAT(0, GEN8_PPAT_UC); 3293 3294 /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b 3295 * write would work. */ 3296 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); 3297 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); 3298 } 3299 3300 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv) 3301 { 3302 uint64_t pat; 3303 3304 /* 3305 * Map WB on BDW to snooped on CHV. 3306 * 3307 * Only the snoop bit has meaning for CHV, the rest is 3308 * ignored. 3309 * 3310 * The hardware will never snoop for certain types of accesses: 3311 * - CPU GTT (GMADR->GGTT->no snoop->memory) 3312 * - PPGTT page tables 3313 * - some other special cycles 3314 * 3315 * As with BDW, we also need to consider the following for GT accesses: 3316 * "For GGTT, there is NO pat_sel[2:0] from the entry, 3317 * so RTL will always use the value corresponding to 3318 * pat_sel = 000". 3319 * Which means we must set the snoop bit in PAT entry 0 3320 * in order to keep the global status page working. 3321 */ 3322 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) | 3323 GEN8_PPAT(1, 0) | 3324 GEN8_PPAT(2, 0) | 3325 GEN8_PPAT(3, 0) | 3326 GEN8_PPAT(4, CHV_PPAT_SNOOP) | 3327 GEN8_PPAT(5, CHV_PPAT_SNOOP) | 3328 GEN8_PPAT(6, CHV_PPAT_SNOOP) | 3329 GEN8_PPAT(7, CHV_PPAT_SNOOP); 3330 3331 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); 3332 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); 3333 } 3334 3335 static int gen8_gmch_probe(struct drm_device *dev, 3336 u64 *gtt_total, 3337 size_t *stolen, 3338 phys_addr_t *mappable_base, 3339 u64 *mappable_end) 3340 { 3341 struct drm_i915_private *dev_priv = dev->dev_private; 3342 u64 gtt_size; 3343 u16 snb_gmch_ctl; 3344 int ret; 3345 3346 /* TODO: We're not aware of mappable constraints on gen8 yet */ 3347 *mappable_base = pci_resource_start(dev->pdev, 2); 3348 *mappable_end = pci_resource_len(dev->pdev, 2); 3349 3350 #ifndef __NetBSD__ 3351 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39))) 3352 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39)); 3353 #endif 3354 3355 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 3356 3357 if (INTEL_INFO(dev)->gen >= 9) { 3358 *stolen = gen9_get_stolen_size(snb_gmch_ctl); 3359 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); 3360 } else if (IS_CHERRYVIEW(dev)) { 3361 *stolen = chv_get_stolen_size(snb_gmch_ctl); 3362 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl); 3363 } else { 3364 *stolen = gen8_get_stolen_size(snb_gmch_ctl); 3365 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); 3366 } 3367 3368 *gtt_total = (gtt_size / sizeof(gen8_pte_t)) << PAGE_SHIFT; 3369 3370 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 3371 chv_setup_private_ppat(dev_priv); 3372 else 3373 bdw_setup_private_ppat(dev_priv); 3374 3375 ret = ggtt_probe_common(dev, gtt_size); 3376 3377 dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range; 3378 dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries; 3379 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 3380 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 3381 3382 /* XXX 39-bit addresses? Really? See pci_set_dma_mask above... */ 3383 dev_priv->gtt.max_paddr = __BITS(38, 0); 3384 3385 return ret; 3386 } 3387 3388 static int gen6_gmch_probe(struct drm_device *dev, 3389 u64 *gtt_total, 3390 size_t *stolen, 3391 phys_addr_t *mappable_base, 3392 u64 *mappable_end) 3393 { 3394 struct drm_i915_private *dev_priv = dev->dev_private; 3395 unsigned int gtt_size; 3396 u16 snb_gmch_ctl; 3397 int ret; 3398 3399 *mappable_base = pci_resource_start(dev->pdev, 2); 3400 *mappable_end = pci_resource_len(dev->pdev, 2); 3401 3402 /* 64/512MB is the current min/max we actually know of, but this is just 3403 * a coarse sanity check. 3404 */ 3405 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { 3406 DRM_ERROR("Unknown GMADR size (%"PRIx64")\n", 3407 dev_priv->gtt.mappable_end); 3408 return -ENXIO; 3409 } 3410 3411 #ifndef __NetBSD__ 3412 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) 3413 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); 3414 #endif 3415 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 3416 3417 *stolen = gen6_get_stolen_size(snb_gmch_ctl); 3418 3419 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); 3420 *gtt_total = (gtt_size / sizeof(gen6_pte_t)) << PAGE_SHIFT; 3421 3422 ret = ggtt_probe_common(dev, gtt_size); 3423 3424 dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range; 3425 dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries; 3426 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 3427 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 3428 3429 dev_priv->gtt.max_paddr = __BITS(39, 0); 3430 3431 return ret; 3432 } 3433 3434 static void gen6_gmch_remove(struct i915_address_space *vm) 3435 { 3436 struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); 3437 3438 #ifdef __NetBSD__ 3439 bus_space_unmap(gtt->bst, gtt->bsh, gtt->size); 3440 #else 3441 iounmap(gtt->gsm); 3442 #endif 3443 free_scratch_page(vm->dev, vm->scratch_page); 3444 } 3445 3446 static int i915_gmch_probe(struct drm_device *dev, 3447 u64 *gtt_total, 3448 size_t *stolen, 3449 phys_addr_t *mappable_base, 3450 u64 *mappable_end) 3451 { 3452 struct drm_i915_private *dev_priv = dev->dev_private; 3453 int ret; 3454 3455 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL); 3456 if (!ret) { 3457 DRM_ERROR("failed to set up gmch\n"); 3458 return -EIO; 3459 } 3460 3461 intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); 3462 3463 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); 3464 dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries; 3465 dev_priv->gtt.base.clear_range = i915_ggtt_clear_range; 3466 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 3467 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 3468 3469 if (unlikely(dev_priv->gtt.do_idle_maps)) 3470 DRM_INFO("applying Ironlake quirks for intel_iommu\n"); 3471 3472 if (INTEL_INFO(dev)->gen <= 2) 3473 dev_priv->gtt.max_paddr = __BITS(29, 0); 3474 else if ((INTEL_INFO(dev)->gen <= 3) || 3475 IS_BROADWATER(dev) || IS_CRESTLINE(dev)) 3476 dev_priv->gtt.max_paddr = __BITS(31, 0); 3477 else if (INTEL_INFO(dev)->gen <= 5) 3478 dev_priv->gtt.max_paddr = __BITS(35, 0); 3479 else 3480 dev_priv->gtt.max_paddr = __BITS(39, 0); 3481 3482 return 0; 3483 } 3484 3485 static void i915_gmch_remove(struct i915_address_space *vm) 3486 { 3487 intel_gmch_remove(); 3488 } 3489 3490 int i915_gem_gtt_init(struct drm_device *dev) 3491 { 3492 struct drm_i915_private *dev_priv = dev->dev_private; 3493 struct i915_gtt *gtt = &dev_priv->gtt; 3494 int ret; 3495 3496 if (INTEL_INFO(dev)->gen <= 5) { 3497 gtt->gtt_probe = i915_gmch_probe; 3498 gtt->base.cleanup = i915_gmch_remove; 3499 } else if (INTEL_INFO(dev)->gen < 8) { 3500 gtt->gtt_probe = gen6_gmch_probe; 3501 gtt->base.cleanup = gen6_gmch_remove; 3502 if (IS_HASWELL(dev) && dev_priv->ellc_size) 3503 gtt->base.pte_encode = iris_pte_encode; 3504 else if (IS_HASWELL(dev)) 3505 gtt->base.pte_encode = hsw_pte_encode; 3506 else if (IS_VALLEYVIEW(dev)) 3507 gtt->base.pte_encode = byt_pte_encode; 3508 else if (INTEL_INFO(dev)->gen >= 7) 3509 gtt->base.pte_encode = ivb_pte_encode; 3510 else 3511 gtt->base.pte_encode = snb_pte_encode; 3512 } else { 3513 dev_priv->gtt.gtt_probe = gen8_gmch_probe; 3514 dev_priv->gtt.base.cleanup = gen6_gmch_remove; 3515 } 3516 3517 gtt->base.dev = dev; 3518 3519 ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size, 3520 >t->mappable_base, >t->mappable_end); 3521 if (ret) 3522 return ret; 3523 3524 #ifdef __NetBSD__ 3525 dev_priv->gtt.pgfl = x86_select_freelist(dev_priv->gtt.max_paddr); 3526 ret = drm_limit_dma_space(dev, 0, dev_priv->gtt.max_paddr); 3527 if (ret) { 3528 DRM_ERROR("Unable to limit DMA paddr allocations: %d!\n", ret); 3529 gtt->base.cleanup(>t->base); 3530 return ret; 3531 } 3532 #endif 3533 3534 /* GMADR is the PCI mmio aperture into the global GTT. */ 3535 DRM_INFO("Memory usable by graphics device = %"PRIu64"M\n", 3536 gtt->base.total >> 20); 3537 DRM_DEBUG_DRIVER("GMADR size = %"PRId64"M\n", gtt->mappable_end >> 20); 3538 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20); 3539 #ifdef CONFIG_INTEL_IOMMU 3540 if (intel_iommu_gfx_mapped) 3541 DRM_INFO("VT-d active for gfx access\n"); 3542 #endif 3543 /* 3544 * i915.enable_ppgtt is read-only, so do an early pass to validate the 3545 * user's requested state against the hardware/driver capabilities. We 3546 * do this now so that we can print out any log messages once rather 3547 * than every time we check intel_enable_ppgtt(). 3548 */ 3549 i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt); 3550 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt); 3551 3552 return 0; 3553 } 3554 3555 void i915_gem_restore_gtt_mappings(struct drm_device *dev) 3556 { 3557 struct drm_i915_private *dev_priv = dev->dev_private; 3558 struct drm_i915_gem_object *obj; 3559 struct i915_address_space *vm; 3560 struct i915_vma *vma; 3561 bool flush; 3562 3563 i915_check_and_clear_faults(dev); 3564 3565 /* First fill our portion of the GTT with scratch pages */ 3566 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, 3567 dev_priv->gtt.base.start, 3568 dev_priv->gtt.base.total, 3569 true); 3570 3571 /* Cache flush objects bound into GGTT and rebind them. */ 3572 vm = &dev_priv->gtt.base; 3573 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 3574 flush = false; 3575 list_for_each_entry(vma, &obj->vma_list, vma_link) { 3576 if (vma->vm != vm) 3577 continue; 3578 3579 WARN_ON(i915_vma_bind(vma, obj->cache_level, 3580 PIN_UPDATE)); 3581 3582 flush = true; 3583 } 3584 3585 if (flush) 3586 i915_gem_clflush_object(obj, obj->pin_display); 3587 } 3588 3589 if (INTEL_INFO(dev)->gen >= 8) { 3590 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 3591 chv_setup_private_ppat(dev_priv); 3592 else 3593 bdw_setup_private_ppat(dev_priv); 3594 3595 return; 3596 } 3597 3598 if (USES_PPGTT(dev)) { 3599 list_for_each_entry(vm, &dev_priv->vm_list, global_link) { 3600 /* TODO: Perhaps it shouldn't be gen6 specific */ 3601 3602 struct i915_hw_ppgtt *ppgtt = 3603 container_of(vm, struct i915_hw_ppgtt, 3604 base); 3605 3606 if (i915_is_ggtt(vm)) 3607 ppgtt = dev_priv->mm.aliasing_ppgtt; 3608 3609 gen6_write_page_range(dev_priv, &ppgtt->pd, 3610 0, ppgtt->base.total); 3611 } 3612 } 3613 3614 i915_ggtt_flush(dev_priv); 3615 } 3616 3617 static struct i915_vma * 3618 __i915_gem_vma_create(struct drm_i915_gem_object *obj, 3619 struct i915_address_space *vm, 3620 const struct i915_ggtt_view *ggtt_view) 3621 { 3622 struct i915_vma *vma; 3623 3624 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view)) 3625 return ERR_PTR(-EINVAL); 3626 3627 vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL); 3628 if (vma == NULL) 3629 return ERR_PTR(-ENOMEM); 3630 3631 INIT_LIST_HEAD(&vma->vma_link); 3632 INIT_LIST_HEAD(&vma->mm_list); 3633 INIT_LIST_HEAD(&vma->exec_list); 3634 vma->vm = vm; 3635 vma->obj = obj; 3636 3637 if (i915_is_ggtt(vm)) 3638 vma->ggtt_view = *ggtt_view; 3639 3640 list_add_tail(&vma->vma_link, &obj->vma_list); 3641 if (!i915_is_ggtt(vm)) 3642 i915_ppgtt_get(i915_vm_to_ppgtt(vm)); 3643 3644 return vma; 3645 } 3646 3647 struct i915_vma * 3648 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj, 3649 struct i915_address_space *vm) 3650 { 3651 struct i915_vma *vma; 3652 3653 vma = i915_gem_obj_to_vma(obj, vm); 3654 if (!vma) 3655 vma = __i915_gem_vma_create(obj, vm, 3656 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL); 3657 3658 return vma; 3659 } 3660 3661 struct i915_vma * 3662 i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj, 3663 const struct i915_ggtt_view *view) 3664 { 3665 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj); 3666 struct i915_vma *vma; 3667 3668 if (WARN_ON(!view)) 3669 return ERR_PTR(-EINVAL); 3670 3671 vma = i915_gem_obj_to_ggtt_view(obj, view); 3672 3673 if (IS_ERR(vma)) 3674 return vma; 3675 3676 if (!vma) 3677 vma = __i915_gem_vma_create(obj, ggtt, view); 3678 3679 return vma; 3680 3681 } 3682 3683 #ifndef __NetBSD__ 3684 static struct scatterlist * 3685 rotate_pages(dma_addr_t *in, unsigned int offset, 3686 unsigned int width, unsigned int height, 3687 struct sg_table *st, struct scatterlist *sg) 3688 { 3689 unsigned int column, row; 3690 unsigned int src_idx; 3691 3692 if (!sg) { 3693 st->nents = 0; 3694 sg = st->sgl; 3695 } 3696 3697 for (column = 0; column < width; column++) { 3698 src_idx = width * (height - 1) + column; 3699 for (row = 0; row < height; row++) { 3700 st->nents++; 3701 /* We don't need the pages, but need to initialize 3702 * the entries so the sg list can be happily traversed. 3703 * The only thing we need are DMA addresses. 3704 */ 3705 sg_set_page(sg, NULL, PAGE_SIZE, 0); 3706 sg_dma_address(sg) = in[offset + src_idx]; 3707 sg_dma_len(sg) = PAGE_SIZE; 3708 sg = sg_next(sg); 3709 src_idx -= width; 3710 } 3711 } 3712 3713 return sg; 3714 } 3715 3716 static struct sg_table * 3717 intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view, 3718 struct drm_i915_gem_object *obj) 3719 { 3720 struct intel_rotation_info *rot_info = &ggtt_view->rotation_info; 3721 unsigned int size_pages = rot_info->size >> PAGE_SHIFT; 3722 unsigned int size_pages_uv; 3723 struct sg_page_iter sg_iter; 3724 unsigned long i; 3725 dma_addr_t *page_addr_list; 3726 struct sg_table *st; 3727 unsigned int uv_start_page; 3728 struct scatterlist *sg; 3729 int ret = -ENOMEM; 3730 3731 /* Allocate a temporary list of source pages for random access. */ 3732 page_addr_list = drm_malloc_ab(obj->base.size / PAGE_SIZE, 3733 sizeof(dma_addr_t)); 3734 if (!page_addr_list) 3735 return ERR_PTR(ret); 3736 3737 /* Account for UV plane with NV12. */ 3738 if (rot_info->pixel_format == DRM_FORMAT_NV12) 3739 size_pages_uv = rot_info->size_uv >> PAGE_SHIFT; 3740 else 3741 size_pages_uv = 0; 3742 3743 /* Allocate target SG list. */ 3744 st = kmalloc(sizeof(*st), GFP_KERNEL); 3745 if (!st) 3746 goto err_st_alloc; 3747 3748 ret = sg_alloc_table(st, size_pages + size_pages_uv, GFP_KERNEL); 3749 if (ret) 3750 goto err_sg_alloc; 3751 3752 /* Populate source page list from the object. */ 3753 i = 0; 3754 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { 3755 page_addr_list[i] = sg_page_iter_dma_address(&sg_iter); 3756 i++; 3757 } 3758 3759 /* Rotate the pages. */ 3760 sg = rotate_pages(page_addr_list, 0, 3761 rot_info->width_pages, rot_info->height_pages, 3762 st, NULL); 3763 3764 /* Append the UV plane if NV12. */ 3765 if (rot_info->pixel_format == DRM_FORMAT_NV12) { 3766 uv_start_page = size_pages; 3767 3768 /* Check for tile-row un-alignment. */ 3769 if (offset_in_page(rot_info->uv_offset)) 3770 uv_start_page--; 3771 3772 rot_info->uv_start_page = uv_start_page; 3773 3774 rotate_pages(page_addr_list, uv_start_page, 3775 rot_info->width_pages_uv, 3776 rot_info->height_pages_uv, 3777 st, sg); 3778 } 3779 3780 DRM_DEBUG_KMS( 3781 "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0)).\n", 3782 obj->base.size, rot_info->pitch, rot_info->height, 3783 rot_info->pixel_format, rot_info->width_pages, 3784 rot_info->height_pages, size_pages + size_pages_uv, 3785 size_pages); 3786 3787 drm_free_large(page_addr_list); 3788 3789 return st; 3790 3791 err_sg_alloc: 3792 kfree(st); 3793 err_st_alloc: 3794 drm_free_large(page_addr_list); 3795 3796 DRM_DEBUG_KMS( 3797 "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0))\n", 3798 obj->base.size, ret, rot_info->pitch, rot_info->height, 3799 rot_info->pixel_format, rot_info->width_pages, 3800 rot_info->height_pages, size_pages + size_pages_uv, 3801 size_pages); 3802 return ERR_PTR(ret); 3803 } 3804 3805 static struct sg_table * 3806 intel_partial_pages(const struct i915_ggtt_view *view, 3807 struct drm_i915_gem_object *obj) 3808 { 3809 struct sg_table *st; 3810 struct scatterlist *sg; 3811 struct sg_page_iter obj_sg_iter; 3812 int ret = -ENOMEM; 3813 3814 st = kmalloc(sizeof(*st), GFP_KERNEL); 3815 if (!st) 3816 goto err_st_alloc; 3817 3818 ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL); 3819 if (ret) 3820 goto err_sg_alloc; 3821 3822 sg = st->sgl; 3823 st->nents = 0; 3824 for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents, 3825 view->params.partial.offset) 3826 { 3827 if (st->nents >= view->params.partial.size) 3828 break; 3829 3830 sg_set_page(sg, NULL, PAGE_SIZE, 0); 3831 sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter); 3832 sg_dma_len(sg) = PAGE_SIZE; 3833 3834 sg = sg_next(sg); 3835 st->nents++; 3836 } 3837 3838 return st; 3839 3840 err_sg_alloc: 3841 kfree(st); 3842 err_st_alloc: 3843 return ERR_PTR(ret); 3844 } 3845 #endif 3846 3847 static int 3848 i915_get_ggtt_vma_pages(struct i915_vma *vma) 3849 { 3850 int ret = 0; 3851 3852 if (vma->ggtt_view.pages) 3853 return 0; 3854 3855 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) 3856 vma->ggtt_view.pages = vma->obj->pages; 3857 #ifndef __NetBSD__ 3858 else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED) 3859 vma->ggtt_view.pages = 3860 intel_rotate_fb_obj_pages(&vma->ggtt_view, vma->obj); 3861 else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL) 3862 vma->ggtt_view.pages = 3863 intel_partial_pages(&vma->ggtt_view, vma->obj); 3864 #endif 3865 else 3866 WARN_ONCE(1, "GGTT view %u not implemented!\n", 3867 vma->ggtt_view.type); 3868 3869 if (!vma->ggtt_view.pages) { 3870 DRM_ERROR("Failed to get pages for GGTT view type %u!\n", 3871 vma->ggtt_view.type); 3872 ret = -EINVAL; 3873 } else if (IS_ERR(vma->ggtt_view.pages)) { 3874 ret = PTR_ERR(vma->ggtt_view.pages); 3875 vma->ggtt_view.pages = NULL; 3876 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n", 3877 vma->ggtt_view.type, ret); 3878 } 3879 3880 return ret; 3881 } 3882 3883 /** 3884 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. 3885 * @vma: VMA to map 3886 * @cache_level: mapping cache level 3887 * @flags: flags like global or local mapping 3888 * 3889 * DMA addresses are taken from the scatter-gather table of this object (or of 3890 * this VMA in case of non-default GGTT views) and PTE entries set up. 3891 * Note that DMA addresses are also the only part of the SG table we care about. 3892 */ 3893 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, 3894 u32 flags) 3895 { 3896 int ret; 3897 u32 bind_flags; 3898 3899 if (WARN_ON(flags == 0)) 3900 return -EINVAL; 3901 3902 bind_flags = 0; 3903 if (flags & PIN_GLOBAL) 3904 bind_flags |= GLOBAL_BIND; 3905 if (flags & PIN_USER) 3906 bind_flags |= LOCAL_BIND; 3907 3908 if (flags & PIN_UPDATE) 3909 bind_flags |= vma->bound; 3910 else 3911 bind_flags &= ~vma->bound; 3912 3913 if (bind_flags == 0) 3914 return 0; 3915 3916 if (vma->bound == 0 && vma->vm->allocate_va_range) { 3917 trace_i915_va_alloc(vma->vm, 3918 vma->node.start, 3919 vma->node.size, 3920 VM_TO_TRACE_NAME(vma->vm)); 3921 3922 /* XXX: i915_vma_pin() will fix this +- hack */ 3923 vma->pin_count++; 3924 ret = vma->vm->allocate_va_range(vma->vm, 3925 vma->node.start, 3926 vma->node.size); 3927 vma->pin_count--; 3928 if (ret) 3929 return ret; 3930 } 3931 3932 ret = vma->vm->bind_vma(vma, cache_level, bind_flags); 3933 if (ret) 3934 return ret; 3935 3936 vma->bound |= bind_flags; 3937 3938 return 0; 3939 } 3940 3941 /** 3942 * i915_ggtt_view_size - Get the size of a GGTT view. 3943 * @obj: Object the view is of. 3944 * @view: The view in question. 3945 * 3946 * @return The size of the GGTT view in bytes. 3947 */ 3948 size_t 3949 i915_ggtt_view_size(struct drm_i915_gem_object *obj, 3950 const struct i915_ggtt_view *view) 3951 { 3952 if (view->type == I915_GGTT_VIEW_NORMAL) { 3953 return obj->base.size; 3954 } else if (view->type == I915_GGTT_VIEW_ROTATED) { 3955 return view->rotation_info.size; 3956 } else if (view->type == I915_GGTT_VIEW_PARTIAL) { 3957 return view->params.partial.size << PAGE_SHIFT; 3958 } else { 3959 WARN_ONCE(1, "GGTT view %u not implemented!\n", view->type); 3960 return obj->base.size; 3961 } 3962 } 3963
Indexes created Thu Oct 23 22:10:10 GMT 2025