i915_gem_gtt.c revision 1.1.1.3
1 /* $NetBSD: i915_gem_gtt.c,v 1.1.1.3 2018/08/27 01:34:54 riastradh Exp $ */ 2 3 /* 4 * Copyright 2010 Daniel Vetter 5 * Copyright 2011-2014 Intel Corporation 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the "Software"), 9 * to deal in the Software without restriction, including without limitation 10 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 11 * and/or sell copies of the Software, and to permit persons to whom the 12 * Software is furnished to do so, subject to the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the next 15 * paragraph) shall be included in all copies or substantial portions of the 16 * Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 24 * IN THE SOFTWARE. 25 * 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: i915_gem_gtt.c,v 1.1.1.3 2018/08/27 01:34:54 riastradh Exp $"); 30 31 #include <linux/seq_file.h> 32 #include <drm/drmP.h> 33 #include <drm/i915_drm.h> 34 #include "i915_drv.h" 35 #include "i915_vgpu.h" 36 #include "i915_trace.h" 37 #include "intel_drv.h" 38 39 /** 40 * DOC: Global GTT views 41 * 42 * Background and previous state 43 * 44 * Historically objects could exists (be bound) in global GTT space only as 45 * singular instances with a view representing all of the object's backing pages 46 * in a linear fashion. This view will be called a normal view. 47 * 48 * To support multiple views of the same object, where the number of mapped 49 * pages is not equal to the backing store, or where the layout of the pages 50 * is not linear, concept of a GGTT view was added. 51 * 52 * One example of an alternative view is a stereo display driven by a single 53 * image. In this case we would have a framebuffer looking like this 54 * (2x2 pages): 55 * 56 * 12 57 * 34 58 * 59 * Above would represent a normal GGTT view as normally mapped for GPU or CPU 60 * rendering. In contrast, fed to the display engine would be an alternative 61 * view which could look something like this: 62 * 63 * 1212 64 * 3434 65 * 66 * In this example both the size and layout of pages in the alternative view is 67 * different from the normal view. 68 * 69 * Implementation and usage 70 * 71 * GGTT views are implemented using VMAs and are distinguished via enum 72 * i915_ggtt_view_type and struct i915_ggtt_view. 73 * 74 * A new flavour of core GEM functions which work with GGTT bound objects were 75 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid 76 * renaming in large amounts of code. They take the struct i915_ggtt_view 77 * parameter encapsulating all metadata required to implement a view. 78 * 79 * As a helper for callers which are only interested in the normal view, 80 * globally const i915_ggtt_view_normal singleton instance exists. All old core 81 * GEM API functions, the ones not taking the view parameter, are operating on, 82 * or with the normal GGTT view. 83 * 84 * Code wanting to add or use a new GGTT view needs to: 85 * 86 * 1. Add a new enum with a suitable name. 87 * 2. Extend the metadata in the i915_ggtt_view structure if required. 88 * 3. Add support to i915_get_vma_pages(). 89 * 90 * New views are required to build a scatter-gather table from within the 91 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and 92 * exists for the lifetime of an VMA. 93 * 94 * Core API is designed to have copy semantics which means that passed in 95 * struct i915_ggtt_view does not need to be persistent (left around after 96 * calling the core API functions). 97 * 98 */ 99 100 static int 101 i915_get_ggtt_vma_pages(struct i915_vma *vma); 102 103 const struct i915_ggtt_view i915_ggtt_view_normal; 104 const struct i915_ggtt_view i915_ggtt_view_rotated = { 105 .type = I915_GGTT_VIEW_ROTATED 106 }; 107 108 static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) 109 { 110 bool has_aliasing_ppgtt; 111 bool has_full_ppgtt; 112 113 has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6; 114 has_full_ppgtt = INTEL_INFO(dev)->gen >= 7; 115 116 if (intel_vgpu_active(dev)) 117 has_full_ppgtt = false; /* emulation is too hard */ 118 119 /* 120 * We don't allow disabling PPGTT for gen9+ as it's a requirement for 121 * execlists, the sole mechanism available to submit work. 122 */ 123 if (INTEL_INFO(dev)->gen < 9 && 124 (enable_ppgtt == 0 || !has_aliasing_ppgtt)) 125 return 0; 126 127 if (enable_ppgtt == 1) 128 return 1; 129 130 if (enable_ppgtt == 2 && has_full_ppgtt) 131 return 2; 132 133 #ifdef CONFIG_INTEL_IOMMU 134 /* Disable ppgtt on SNB if VT-d is on. */ 135 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) { 136 DRM_INFO("Disabling PPGTT because VT-d is on\n"); 137 return 0; 138 } 139 #endif 140 141 /* Early VLV doesn't have this */ 142 if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && 143 dev->pdev->revision < 0xb) { 144 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n"); 145 return 0; 146 } 147 148 if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists) 149 return 2; 150 else 151 return has_aliasing_ppgtt ? 1 : 0; 152 } 153 154 static int ppgtt_bind_vma(struct i915_vma *vma, 155 enum i915_cache_level cache_level, 156 u32 unused) 157 { 158 u32 pte_flags = 0; 159 160 /* Currently applicable only to VLV */ 161 if (vma->obj->gt_ro) 162 pte_flags |= PTE_READ_ONLY; 163 164 vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start, 165 cache_level, pte_flags); 166 167 return 0; 168 } 169 170 static void ppgtt_unbind_vma(struct i915_vma *vma) 171 { 172 vma->vm->clear_range(vma->vm, 173 vma->node.start, 174 vma->obj->base.size, 175 true); 176 } 177 178 static gen8_pte_t gen8_pte_encode(dma_addr_t addr, 179 enum i915_cache_level level, 180 bool valid) 181 { 182 gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0; 183 pte |= addr; 184 185 switch (level) { 186 case I915_CACHE_NONE: 187 pte |= PPAT_UNCACHED_INDEX; 188 break; 189 case I915_CACHE_WT: 190 pte |= PPAT_DISPLAY_ELLC_INDEX; 191 break; 192 default: 193 pte |= PPAT_CACHED_INDEX; 194 break; 195 } 196 197 return pte; 198 } 199 200 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, 201 const enum i915_cache_level level) 202 { 203 gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW; 204 pde |= addr; 205 if (level != I915_CACHE_NONE) 206 pde |= PPAT_CACHED_PDE_INDEX; 207 else 208 pde |= PPAT_UNCACHED_INDEX; 209 return pde; 210 } 211 212 #define gen8_pdpe_encode gen8_pde_encode 213 #define gen8_pml4e_encode gen8_pde_encode 214 215 static gen6_pte_t snb_pte_encode(dma_addr_t addr, 216 enum i915_cache_level level, 217 bool valid, u32 unused) 218 { 219 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 220 pte |= GEN6_PTE_ADDR_ENCODE(addr); 221 222 switch (level) { 223 case I915_CACHE_L3_LLC: 224 case I915_CACHE_LLC: 225 pte |= GEN6_PTE_CACHE_LLC; 226 break; 227 case I915_CACHE_NONE: 228 pte |= GEN6_PTE_UNCACHED; 229 break; 230 default: 231 MISSING_CASE(level); 232 } 233 234 return pte; 235 } 236 237 static gen6_pte_t ivb_pte_encode(dma_addr_t addr, 238 enum i915_cache_level level, 239 bool valid, u32 unused) 240 { 241 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 242 pte |= GEN6_PTE_ADDR_ENCODE(addr); 243 244 switch (level) { 245 case I915_CACHE_L3_LLC: 246 pte |= GEN7_PTE_CACHE_L3_LLC; 247 break; 248 case I915_CACHE_LLC: 249 pte |= GEN6_PTE_CACHE_LLC; 250 break; 251 case I915_CACHE_NONE: 252 pte |= GEN6_PTE_UNCACHED; 253 break; 254 default: 255 MISSING_CASE(level); 256 } 257 258 return pte; 259 } 260 261 static gen6_pte_t byt_pte_encode(dma_addr_t addr, 262 enum i915_cache_level level, 263 bool valid, u32 flags) 264 { 265 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 266 pte |= GEN6_PTE_ADDR_ENCODE(addr); 267 268 if (!(flags & PTE_READ_ONLY)) 269 pte |= BYT_PTE_WRITEABLE; 270 271 if (level != I915_CACHE_NONE) 272 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 273 274 return pte; 275 } 276 277 static gen6_pte_t hsw_pte_encode(dma_addr_t addr, 278 enum i915_cache_level level, 279 bool valid, u32 unused) 280 { 281 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 282 pte |= HSW_PTE_ADDR_ENCODE(addr); 283 284 if (level != I915_CACHE_NONE) 285 pte |= HSW_WB_LLC_AGE3; 286 287 return pte; 288 } 289 290 static gen6_pte_t iris_pte_encode(dma_addr_t addr, 291 enum i915_cache_level level, 292 bool valid, u32 unused) 293 { 294 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 295 pte |= HSW_PTE_ADDR_ENCODE(addr); 296 297 switch (level) { 298 case I915_CACHE_NONE: 299 break; 300 case I915_CACHE_WT: 301 pte |= HSW_WT_ELLC_LLC_AGE3; 302 break; 303 default: 304 pte |= HSW_WB_ELLC_LLC_AGE3; 305 break; 306 } 307 308 return pte; 309 } 310 311 static int __setup_page_dma(struct drm_device *dev, 312 struct i915_page_dma *p, gfp_t flags) 313 { 314 struct device *device = &dev->pdev->dev; 315 316 p->page = alloc_page(flags); 317 if (!p->page) 318 return -ENOMEM; 319 320 p->daddr = dma_map_page(device, 321 p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL); 322 323 if (dma_mapping_error(device, p->daddr)) { 324 __free_page(p->page); 325 return -EINVAL; 326 } 327 328 return 0; 329 } 330 331 static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p) 332 { 333 return __setup_page_dma(dev, p, GFP_KERNEL); 334 } 335 336 static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p) 337 { 338 if (WARN_ON(!p->page)) 339 return; 340 341 dma_unmap_page(&dev->pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL); 342 __free_page(p->page); 343 memset(p, 0, sizeof(*p)); 344 } 345 346 static void *kmap_page_dma(struct i915_page_dma *p) 347 { 348 return kmap_atomic(p->page); 349 } 350 351 /* We use the flushing unmap only with ppgtt structures: 352 * page directories, page tables and scratch pages. 353 */ 354 static void kunmap_page_dma(struct drm_device *dev, void *vaddr) 355 { 356 /* There are only few exceptions for gen >=6. chv and bxt. 357 * And we are not sure about the latter so play safe for now. 358 */ 359 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 360 drm_clflush_virt_range(vaddr, PAGE_SIZE); 361 362 kunmap_atomic(vaddr); 363 } 364 365 #define kmap_px(px) kmap_page_dma(px_base(px)) 366 #define kunmap_px(ppgtt, vaddr) kunmap_page_dma((ppgtt)->base.dev, (vaddr)) 367 368 #define setup_px(dev, px) setup_page_dma((dev), px_base(px)) 369 #define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px)) 370 #define fill_px(dev, px, v) fill_page_dma((dev), px_base(px), (v)) 371 #define fill32_px(dev, px, v) fill_page_dma_32((dev), px_base(px), (v)) 372 373 static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p, 374 const uint64_t val) 375 { 376 int i; 377 uint64_t * const vaddr = kmap_page_dma(p); 378 379 for (i = 0; i < 512; i++) 380 vaddr[i] = val; 381 382 kunmap_page_dma(dev, vaddr); 383 } 384 385 static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p, 386 const uint32_t val32) 387 { 388 uint64_t v = val32; 389 390 v = v << 32 | val32; 391 392 fill_page_dma(dev, p, v); 393 } 394 395 static struct i915_page_scratch *alloc_scratch_page(struct drm_device *dev) 396 { 397 struct i915_page_scratch *sp; 398 int ret; 399 400 sp = kzalloc(sizeof(*sp), GFP_KERNEL); 401 if (sp == NULL) 402 return ERR_PTR(-ENOMEM); 403 404 ret = __setup_page_dma(dev, px_base(sp), GFP_DMA32 | __GFP_ZERO); 405 if (ret) { 406 kfree(sp); 407 return ERR_PTR(ret); 408 } 409 410 set_pages_uc(px_page(sp), 1); 411 412 return sp; 413 } 414 415 static void free_scratch_page(struct drm_device *dev, 416 struct i915_page_scratch *sp) 417 { 418 set_pages_wb(px_page(sp), 1); 419 420 cleanup_px(dev, sp); 421 kfree(sp); 422 } 423 424 static struct i915_page_table *alloc_pt(struct drm_device *dev) 425 { 426 struct i915_page_table *pt; 427 const size_t count = INTEL_INFO(dev)->gen >= 8 ? 428 GEN8_PTES : GEN6_PTES; 429 int ret = -ENOMEM; 430 431 pt = kzalloc(sizeof(*pt), GFP_KERNEL); 432 if (!pt) 433 return ERR_PTR(-ENOMEM); 434 435 pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes), 436 GFP_KERNEL); 437 438 if (!pt->used_ptes) 439 goto fail_bitmap; 440 441 ret = setup_px(dev, pt); 442 if (ret) 443 goto fail_page_m; 444 445 return pt; 446 447 fail_page_m: 448 kfree(pt->used_ptes); 449 fail_bitmap: 450 kfree(pt); 451 452 return ERR_PTR(ret); 453 } 454 455 static void free_pt(struct drm_device *dev, struct i915_page_table *pt) 456 { 457 cleanup_px(dev, pt); 458 kfree(pt->used_ptes); 459 kfree(pt); 460 } 461 462 static void gen8_initialize_pt(struct i915_address_space *vm, 463 struct i915_page_table *pt) 464 { 465 gen8_pte_t scratch_pte; 466 467 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 468 I915_CACHE_LLC, true); 469 470 fill_px(vm->dev, pt, scratch_pte); 471 } 472 473 static void gen6_initialize_pt(struct i915_address_space *vm, 474 struct i915_page_table *pt) 475 { 476 gen6_pte_t scratch_pte; 477 478 WARN_ON(px_dma(vm->scratch_page) == 0); 479 480 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 481 I915_CACHE_LLC, true, 0); 482 483 fill32_px(vm->dev, pt, scratch_pte); 484 } 485 486 static struct i915_page_directory *alloc_pd(struct drm_device *dev) 487 { 488 struct i915_page_directory *pd; 489 int ret = -ENOMEM; 490 491 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 492 if (!pd) 493 return ERR_PTR(-ENOMEM); 494 495 pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES), 496 sizeof(*pd->used_pdes), GFP_KERNEL); 497 if (!pd->used_pdes) 498 goto fail_bitmap; 499 500 ret = setup_px(dev, pd); 501 if (ret) 502 goto fail_page_m; 503 504 return pd; 505 506 fail_page_m: 507 kfree(pd->used_pdes); 508 fail_bitmap: 509 kfree(pd); 510 511 return ERR_PTR(ret); 512 } 513 514 static void free_pd(struct drm_device *dev, struct i915_page_directory *pd) 515 { 516 if (px_page(pd)) { 517 cleanup_px(dev, pd); 518 kfree(pd->used_pdes); 519 kfree(pd); 520 } 521 } 522 523 static void gen8_initialize_pd(struct i915_address_space *vm, 524 struct i915_page_directory *pd) 525 { 526 gen8_pde_t scratch_pde; 527 528 scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC); 529 530 fill_px(vm->dev, pd, scratch_pde); 531 } 532 533 static int __pdp_init(struct drm_device *dev, 534 struct i915_page_directory_pointer *pdp) 535 { 536 size_t pdpes = I915_PDPES_PER_PDP(dev); 537 538 pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes), 539 sizeof(unsigned long), 540 GFP_KERNEL); 541 if (!pdp->used_pdpes) 542 return -ENOMEM; 543 544 pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory), 545 GFP_KERNEL); 546 if (!pdp->page_directory) { 547 kfree(pdp->used_pdpes); 548 /* the PDP might be the statically allocated top level. Keep it 549 * as clean as possible */ 550 pdp->used_pdpes = NULL; 551 return -ENOMEM; 552 } 553 554 return 0; 555 } 556 557 static void __pdp_fini(struct i915_page_directory_pointer *pdp) 558 { 559 kfree(pdp->used_pdpes); 560 kfree(pdp->page_directory); 561 pdp->page_directory = NULL; 562 } 563 564 static struct 565 i915_page_directory_pointer *alloc_pdp(struct drm_device *dev) 566 { 567 struct i915_page_directory_pointer *pdp; 568 int ret = -ENOMEM; 569 570 WARN_ON(!USES_FULL_48BIT_PPGTT(dev)); 571 572 pdp = kzalloc(sizeof(*pdp), GFP_KERNEL); 573 if (!pdp) 574 return ERR_PTR(-ENOMEM); 575 576 ret = __pdp_init(dev, pdp); 577 if (ret) 578 goto fail_bitmap; 579 580 ret = setup_px(dev, pdp); 581 if (ret) 582 goto fail_page_m; 583 584 return pdp; 585 586 fail_page_m: 587 __pdp_fini(pdp); 588 fail_bitmap: 589 kfree(pdp); 590 591 return ERR_PTR(ret); 592 } 593 594 static void free_pdp(struct drm_device *dev, 595 struct i915_page_directory_pointer *pdp) 596 { 597 __pdp_fini(pdp); 598 if (USES_FULL_48BIT_PPGTT(dev)) { 599 cleanup_px(dev, pdp); 600 kfree(pdp); 601 } 602 } 603 604 static void gen8_initialize_pdp(struct i915_address_space *vm, 605 struct i915_page_directory_pointer *pdp) 606 { 607 gen8_ppgtt_pdpe_t scratch_pdpe; 608 609 scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC); 610 611 fill_px(vm->dev, pdp, scratch_pdpe); 612 } 613 614 static void gen8_initialize_pml4(struct i915_address_space *vm, 615 struct i915_pml4 *pml4) 616 { 617 gen8_ppgtt_pml4e_t scratch_pml4e; 618 619 scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp), 620 I915_CACHE_LLC); 621 622 fill_px(vm->dev, pml4, scratch_pml4e); 623 } 624 625 static void 626 gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt, 627 struct i915_page_directory_pointer *pdp, 628 struct i915_page_directory *pd, 629 int index) 630 { 631 gen8_ppgtt_pdpe_t *page_directorypo; 632 633 if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) 634 return; 635 636 page_directorypo = kmap_px(pdp); 637 page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC); 638 kunmap_px(ppgtt, page_directorypo); 639 } 640 641 static void 642 gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt, 643 struct i915_pml4 *pml4, 644 struct i915_page_directory_pointer *pdp, 645 int index) 646 { 647 gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4); 648 649 WARN_ON(!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)); 650 pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC); 651 kunmap_px(ppgtt, pagemap); 652 } 653 654 /* Broadwell Page Directory Pointer Descriptors */ 655 static int gen8_write_pdp(struct drm_i915_gem_request *req, 656 unsigned entry, 657 dma_addr_t addr) 658 { 659 struct intel_engine_cs *ring = req->ring; 660 int ret; 661 662 BUG_ON(entry >= 4); 663 664 ret = intel_ring_begin(req, 6); 665 if (ret) 666 return ret; 667 668 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 669 intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry)); 670 intel_ring_emit(ring, upper_32_bits(addr)); 671 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 672 intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry)); 673 intel_ring_emit(ring, lower_32_bits(addr)); 674 intel_ring_advance(ring); 675 676 return 0; 677 } 678 679 static int gen8_legacy_mm_switch(struct i915_hw_ppgtt *ppgtt, 680 struct drm_i915_gem_request *req) 681 { 682 int i, ret; 683 684 for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) { 685 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 686 687 ret = gen8_write_pdp(req, i, pd_daddr); 688 if (ret) 689 return ret; 690 } 691 692 return 0; 693 } 694 695 static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt, 696 struct drm_i915_gem_request *req) 697 { 698 return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4)); 699 } 700 701 static void gen8_ppgtt_clear_pte_range(struct i915_address_space *vm, 702 struct i915_page_directory_pointer *pdp, 703 uint64_t start, 704 uint64_t length, 705 gen8_pte_t scratch_pte) 706 { 707 struct i915_hw_ppgtt *ppgtt = 708 container_of(vm, struct i915_hw_ppgtt, base); 709 gen8_pte_t *pt_vaddr; 710 unsigned pdpe = gen8_pdpe_index(start); 711 unsigned pde = gen8_pde_index(start); 712 unsigned pte = gen8_pte_index(start); 713 unsigned num_entries = length >> PAGE_SHIFT; 714 unsigned last_pte, i; 715 716 if (WARN_ON(!pdp)) 717 return; 718 719 while (num_entries) { 720 struct i915_page_directory *pd; 721 struct i915_page_table *pt; 722 723 if (WARN_ON(!pdp->page_directory[pdpe])) 724 break; 725 726 pd = pdp->page_directory[pdpe]; 727 728 if (WARN_ON(!pd->page_table[pde])) 729 break; 730 731 pt = pd->page_table[pde]; 732 733 if (WARN_ON(!px_page(pt))) 734 break; 735 736 last_pte = pte + num_entries; 737 if (last_pte > GEN8_PTES) 738 last_pte = GEN8_PTES; 739 740 pt_vaddr = kmap_px(pt); 741 742 for (i = pte; i < last_pte; i++) { 743 pt_vaddr[i] = scratch_pte; 744 num_entries--; 745 } 746 747 kunmap_px(ppgtt, pt); 748 749 pte = 0; 750 if (++pde == I915_PDES) { 751 if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) 752 break; 753 pde = 0; 754 } 755 } 756 } 757 758 static void gen8_ppgtt_clear_range(struct i915_address_space *vm, 759 uint64_t start, 760 uint64_t length, 761 bool use_scratch) 762 { 763 struct i915_hw_ppgtt *ppgtt = 764 container_of(vm, struct i915_hw_ppgtt, base); 765 gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 766 I915_CACHE_LLC, use_scratch); 767 768 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 769 gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length, 770 scratch_pte); 771 } else { 772 uint64_t templ4, pml4e; 773 struct i915_page_directory_pointer *pdp; 774 775 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) { 776 gen8_ppgtt_clear_pte_range(vm, pdp, start, length, 777 scratch_pte); 778 } 779 } 780 } 781 782 static void 783 gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm, 784 struct i915_page_directory_pointer *pdp, 785 struct sg_page_iter *sg_iter, 786 uint64_t start, 787 enum i915_cache_level cache_level) 788 { 789 struct i915_hw_ppgtt *ppgtt = 790 container_of(vm, struct i915_hw_ppgtt, base); 791 gen8_pte_t *pt_vaddr; 792 unsigned pdpe = gen8_pdpe_index(start); 793 unsigned pde = gen8_pde_index(start); 794 unsigned pte = gen8_pte_index(start); 795 796 pt_vaddr = NULL; 797 798 while (__sg_page_iter_next(sg_iter)) { 799 if (pt_vaddr == NULL) { 800 struct i915_page_directory *pd = pdp->page_directory[pdpe]; 801 struct i915_page_table *pt = pd->page_table[pde]; 802 pt_vaddr = kmap_px(pt); 803 } 804 805 pt_vaddr[pte] = 806 gen8_pte_encode(sg_page_iter_dma_address(sg_iter), 807 cache_level, true); 808 if (++pte == GEN8_PTES) { 809 kunmap_px(ppgtt, pt_vaddr); 810 pt_vaddr = NULL; 811 if (++pde == I915_PDES) { 812 if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) 813 break; 814 pde = 0; 815 } 816 pte = 0; 817 } 818 } 819 820 if (pt_vaddr) 821 kunmap_px(ppgtt, pt_vaddr); 822 } 823 824 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, 825 struct sg_table *pages, 826 uint64_t start, 827 enum i915_cache_level cache_level, 828 u32 unused) 829 { 830 struct i915_hw_ppgtt *ppgtt = 831 container_of(vm, struct i915_hw_ppgtt, base); 832 struct sg_page_iter sg_iter; 833 834 __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0); 835 836 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 837 gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start, 838 cache_level); 839 } else { 840 struct i915_page_directory_pointer *pdp; 841 uint64_t templ4, pml4e; 842 uint64_t length = (uint64_t)pages->orig_nents << PAGE_SHIFT; 843 844 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) { 845 gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter, 846 start, cache_level); 847 } 848 } 849 } 850 851 static void gen8_free_page_tables(struct drm_device *dev, 852 struct i915_page_directory *pd) 853 { 854 int i; 855 856 if (!px_page(pd)) 857 return; 858 859 for_each_set_bit(i, pd->used_pdes, I915_PDES) { 860 if (WARN_ON(!pd->page_table[i])) 861 continue; 862 863 free_pt(dev, pd->page_table[i]); 864 pd->page_table[i] = NULL; 865 } 866 } 867 868 static int gen8_init_scratch(struct i915_address_space *vm) 869 { 870 struct drm_device *dev = vm->dev; 871 872 vm->scratch_page = alloc_scratch_page(dev); 873 if (IS_ERR(vm->scratch_page)) 874 return PTR_ERR(vm->scratch_page); 875 876 vm->scratch_pt = alloc_pt(dev); 877 if (IS_ERR(vm->scratch_pt)) { 878 free_scratch_page(dev, vm->scratch_page); 879 return PTR_ERR(vm->scratch_pt); 880 } 881 882 vm->scratch_pd = alloc_pd(dev); 883 if (IS_ERR(vm->scratch_pd)) { 884 free_pt(dev, vm->scratch_pt); 885 free_scratch_page(dev, vm->scratch_page); 886 return PTR_ERR(vm->scratch_pd); 887 } 888 889 if (USES_FULL_48BIT_PPGTT(dev)) { 890 vm->scratch_pdp = alloc_pdp(dev); 891 if (IS_ERR(vm->scratch_pdp)) { 892 free_pd(dev, vm->scratch_pd); 893 free_pt(dev, vm->scratch_pt); 894 free_scratch_page(dev, vm->scratch_page); 895 return PTR_ERR(vm->scratch_pdp); 896 } 897 } 898 899 gen8_initialize_pt(vm, vm->scratch_pt); 900 gen8_initialize_pd(vm, vm->scratch_pd); 901 if (USES_FULL_48BIT_PPGTT(dev)) 902 gen8_initialize_pdp(vm, vm->scratch_pdp); 903 904 return 0; 905 } 906 907 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create) 908 { 909 enum vgt_g2v_type msg; 910 struct drm_device *dev = ppgtt->base.dev; 911 struct drm_i915_private *dev_priv = dev->dev_private; 912 unsigned int offset = vgtif_reg(pdp0_lo); 913 int i; 914 915 if (USES_FULL_48BIT_PPGTT(dev)) { 916 u64 daddr = px_dma(&ppgtt->pml4); 917 918 I915_WRITE(offset, lower_32_bits(daddr)); 919 I915_WRITE(offset + 4, upper_32_bits(daddr)); 920 921 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE : 922 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY); 923 } else { 924 for (i = 0; i < GEN8_LEGACY_PDPES; i++) { 925 u64 daddr = i915_page_dir_dma_addr(ppgtt, i); 926 927 I915_WRITE(offset, lower_32_bits(daddr)); 928 I915_WRITE(offset + 4, upper_32_bits(daddr)); 929 930 offset += 8; 931 } 932 933 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE : 934 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY); 935 } 936 937 I915_WRITE(vgtif_reg(g2v_notify), msg); 938 939 return 0; 940 } 941 942 static void gen8_free_scratch(struct i915_address_space *vm) 943 { 944 struct drm_device *dev = vm->dev; 945 946 if (USES_FULL_48BIT_PPGTT(dev)) 947 free_pdp(dev, vm->scratch_pdp); 948 free_pd(dev, vm->scratch_pd); 949 free_pt(dev, vm->scratch_pt); 950 free_scratch_page(dev, vm->scratch_page); 951 } 952 953 static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev, 954 struct i915_page_directory_pointer *pdp) 955 { 956 int i; 957 958 for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) { 959 if (WARN_ON(!pdp->page_directory[i])) 960 continue; 961 962 gen8_free_page_tables(dev, pdp->page_directory[i]); 963 free_pd(dev, pdp->page_directory[i]); 964 } 965 966 free_pdp(dev, pdp); 967 } 968 969 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt) 970 { 971 int i; 972 973 for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) { 974 if (WARN_ON(!ppgtt->pml4.pdps[i])) 975 continue; 976 977 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, ppgtt->pml4.pdps[i]); 978 } 979 980 cleanup_px(ppgtt->base.dev, &ppgtt->pml4); 981 } 982 983 static void gen8_ppgtt_cleanup(struct i915_address_space *vm) 984 { 985 struct i915_hw_ppgtt *ppgtt = 986 container_of(vm, struct i915_hw_ppgtt, base); 987 988 if (intel_vgpu_active(vm->dev)) 989 gen8_ppgtt_notify_vgt(ppgtt, false); 990 991 if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) 992 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, &ppgtt->pdp); 993 else 994 gen8_ppgtt_cleanup_4lvl(ppgtt); 995 996 gen8_free_scratch(vm); 997 } 998 999 /** 1000 * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range. 1001 * @vm: Master vm structure. 1002 * @pd: Page directory for this address range. 1003 * @start: Starting virtual address to begin allocations. 1004 * @length: Size of the allocations. 1005 * @new_pts: Bitmap set by function with new allocations. Likely used by the 1006 * caller to free on error. 1007 * 1008 * Allocate the required number of page tables. Extremely similar to 1009 * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by 1010 * the page directory boundary (instead of the page directory pointer). That 1011 * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is 1012 * possible, and likely that the caller will need to use multiple calls of this 1013 * function to achieve the appropriate allocation. 1014 * 1015 * Return: 0 if success; negative error code otherwise. 1016 */ 1017 static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm, 1018 struct i915_page_directory *pd, 1019 uint64_t start, 1020 uint64_t length, 1021 unsigned long *new_pts) 1022 { 1023 struct drm_device *dev = vm->dev; 1024 struct i915_page_table *pt; 1025 uint64_t temp; 1026 uint32_t pde; 1027 1028 gen8_for_each_pde(pt, pd, start, length, temp, pde) { 1029 /* Don't reallocate page tables */ 1030 if (test_bit(pde, pd->used_pdes)) { 1031 /* Scratch is never allocated this way */ 1032 WARN_ON(pt == vm->scratch_pt); 1033 continue; 1034 } 1035 1036 pt = alloc_pt(dev); 1037 if (IS_ERR(pt)) 1038 goto unwind_out; 1039 1040 gen8_initialize_pt(vm, pt); 1041 pd->page_table[pde] = pt; 1042 __set_bit(pde, new_pts); 1043 trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT); 1044 } 1045 1046 return 0; 1047 1048 unwind_out: 1049 for_each_set_bit(pde, new_pts, I915_PDES) 1050 free_pt(dev, pd->page_table[pde]); 1051 1052 return -ENOMEM; 1053 } 1054 1055 /** 1056 * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range. 1057 * @vm: Master vm structure. 1058 * @pdp: Page directory pointer for this address range. 1059 * @start: Starting virtual address to begin allocations. 1060 * @length: Size of the allocations. 1061 * @new_pds: Bitmap set by function with new allocations. Likely used by the 1062 * caller to free on error. 1063 * 1064 * Allocate the required number of page directories starting at the pde index of 1065 * @start, and ending at the pde index @start + @length. This function will skip 1066 * over already allocated page directories within the range, and only allocate 1067 * new ones, setting the appropriate pointer within the pdp as well as the 1068 * correct position in the bitmap @new_pds. 1069 * 1070 * The function will only allocate the pages within the range for a give page 1071 * directory pointer. In other words, if @start + @length straddles a virtually 1072 * addressed PDP boundary (512GB for 4k pages), there will be more allocations 1073 * required by the caller, This is not currently possible, and the BUG in the 1074 * code will prevent it. 1075 * 1076 * Return: 0 if success; negative error code otherwise. 1077 */ 1078 static int 1079 gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm, 1080 struct i915_page_directory_pointer *pdp, 1081 uint64_t start, 1082 uint64_t length, 1083 unsigned long *new_pds) 1084 { 1085 struct drm_device *dev = vm->dev; 1086 struct i915_page_directory *pd; 1087 uint64_t temp; 1088 uint32_t pdpe; 1089 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1090 1091 WARN_ON(!bitmap_empty(new_pds, pdpes)); 1092 1093 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { 1094 if (test_bit(pdpe, pdp->used_pdpes)) 1095 continue; 1096 1097 pd = alloc_pd(dev); 1098 if (IS_ERR(pd)) 1099 goto unwind_out; 1100 1101 gen8_initialize_pd(vm, pd); 1102 pdp->page_directory[pdpe] = pd; 1103 __set_bit(pdpe, new_pds); 1104 trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT); 1105 } 1106 1107 return 0; 1108 1109 unwind_out: 1110 for_each_set_bit(pdpe, new_pds, pdpes) 1111 free_pd(dev, pdp->page_directory[pdpe]); 1112 1113 return -ENOMEM; 1114 } 1115 1116 /** 1117 * gen8_ppgtt_alloc_page_dirpointers() - Allocate pdps for VA range. 1118 * @vm: Master vm structure. 1119 * @pml4: Page map level 4 for this address range. 1120 * @start: Starting virtual address to begin allocations. 1121 * @length: Size of the allocations. 1122 * @new_pdps: Bitmap set by function with new allocations. Likely used by the 1123 * caller to free on error. 1124 * 1125 * Allocate the required number of page directory pointers. Extremely similar to 1126 * gen8_ppgtt_alloc_page_directories() and gen8_ppgtt_alloc_pagetabs(). 1127 * The main difference is here we are limited by the pml4 boundary (instead of 1128 * the page directory pointer). 1129 * 1130 * Return: 0 if success; negative error code otherwise. 1131 */ 1132 static int 1133 gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm, 1134 struct i915_pml4 *pml4, 1135 uint64_t start, 1136 uint64_t length, 1137 unsigned long *new_pdps) 1138 { 1139 struct drm_device *dev = vm->dev; 1140 struct i915_page_directory_pointer *pdp; 1141 uint64_t temp; 1142 uint32_t pml4e; 1143 1144 WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4)); 1145 1146 gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) { 1147 if (!test_bit(pml4e, pml4->used_pml4es)) { 1148 pdp = alloc_pdp(dev); 1149 if (IS_ERR(pdp)) 1150 goto unwind_out; 1151 1152 gen8_initialize_pdp(vm, pdp); 1153 pml4->pdps[pml4e] = pdp; 1154 __set_bit(pml4e, new_pdps); 1155 trace_i915_page_directory_pointer_entry_alloc(vm, 1156 pml4e, 1157 start, 1158 GEN8_PML4E_SHIFT); 1159 } 1160 } 1161 1162 return 0; 1163 1164 unwind_out: 1165 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) 1166 free_pdp(dev, pml4->pdps[pml4e]); 1167 1168 return -ENOMEM; 1169 } 1170 1171 static void 1172 free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long *new_pts) 1173 { 1174 kfree(new_pts); 1175 kfree(new_pds); 1176 } 1177 1178 /* Fills in the page directory bitmap, and the array of page tables bitmap. Both 1179 * of these are based on the number of PDPEs in the system. 1180 */ 1181 static 1182 int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds, 1183 unsigned long **new_pts, 1184 uint32_t pdpes) 1185 { 1186 unsigned long *pds; 1187 unsigned long *pts; 1188 1189 pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_TEMPORARY); 1190 if (!pds) 1191 return -ENOMEM; 1192 1193 pts = kcalloc(pdpes, BITS_TO_LONGS(I915_PDES) * sizeof(unsigned long), 1194 GFP_TEMPORARY); 1195 if (!pts) 1196 goto err_out; 1197 1198 *new_pds = pds; 1199 *new_pts = pts; 1200 1201 return 0; 1202 1203 err_out: 1204 free_gen8_temp_bitmaps(pds, pts); 1205 return -ENOMEM; 1206 } 1207 1208 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify 1209 * the page table structures, we mark them dirty so that 1210 * context switching/execlist queuing code takes extra steps 1211 * to ensure that tlbs are flushed. 1212 */ 1213 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) 1214 { 1215 ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask; 1216 } 1217 1218 static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm, 1219 struct i915_page_directory_pointer *pdp, 1220 uint64_t start, 1221 uint64_t length) 1222 { 1223 struct i915_hw_ppgtt *ppgtt = 1224 container_of(vm, struct i915_hw_ppgtt, base); 1225 unsigned long *new_page_dirs, *new_page_tables; 1226 struct drm_device *dev = vm->dev; 1227 struct i915_page_directory *pd; 1228 const uint64_t orig_start = start; 1229 const uint64_t orig_length = length; 1230 uint64_t temp; 1231 uint32_t pdpe; 1232 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1233 int ret; 1234 1235 /* Wrap is never okay since we can only represent 48b, and we don't 1236 * actually use the other side of the canonical address space. 1237 */ 1238 if (WARN_ON(start + length < start)) 1239 return -ENODEV; 1240 1241 if (WARN_ON(start + length > vm->total)) 1242 return -ENODEV; 1243 1244 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes); 1245 if (ret) 1246 return ret; 1247 1248 /* Do the allocations first so we can easily bail out */ 1249 ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length, 1250 new_page_dirs); 1251 if (ret) { 1252 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1253 return ret; 1254 } 1255 1256 /* For every page directory referenced, allocate page tables */ 1257 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { 1258 ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length, 1259 new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES)); 1260 if (ret) 1261 goto err_out; 1262 } 1263 1264 start = orig_start; 1265 length = orig_length; 1266 1267 /* Allocations have completed successfully, so set the bitmaps, and do 1268 * the mappings. */ 1269 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { 1270 gen8_pde_t *const page_directory = kmap_px(pd); 1271 struct i915_page_table *pt; 1272 uint64_t pd_len = length; 1273 uint64_t pd_start = start; 1274 uint32_t pde; 1275 1276 /* Every pd should be allocated, we just did that above. */ 1277 WARN_ON(!pd); 1278 1279 gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) { 1280 /* Same reasoning as pd */ 1281 WARN_ON(!pt); 1282 WARN_ON(!pd_len); 1283 WARN_ON(!gen8_pte_count(pd_start, pd_len)); 1284 1285 /* Set our used ptes within the page table */ 1286 bitmap_set(pt->used_ptes, 1287 gen8_pte_index(pd_start), 1288 gen8_pte_count(pd_start, pd_len)); 1289 1290 /* Our pde is now pointing to the pagetable, pt */ 1291 __set_bit(pde, pd->used_pdes); 1292 1293 /* Map the PDE to the page table */ 1294 page_directory[pde] = gen8_pde_encode(px_dma(pt), 1295 I915_CACHE_LLC); 1296 trace_i915_page_table_entry_map(&ppgtt->base, pde, pt, 1297 gen8_pte_index(start), 1298 gen8_pte_count(start, length), 1299 GEN8_PTES); 1300 1301 /* NB: We haven't yet mapped ptes to pages. At this 1302 * point we're still relying on insert_entries() */ 1303 } 1304 1305 kunmap_px(ppgtt, page_directory); 1306 __set_bit(pdpe, pdp->used_pdpes); 1307 gen8_setup_page_directory(ppgtt, pdp, pd, pdpe); 1308 } 1309 1310 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1311 mark_tlbs_dirty(ppgtt); 1312 return 0; 1313 1314 err_out: 1315 while (pdpe--) { 1316 for_each_set_bit(temp, new_page_tables + pdpe * 1317 BITS_TO_LONGS(I915_PDES), I915_PDES) 1318 free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]); 1319 } 1320 1321 for_each_set_bit(pdpe, new_page_dirs, pdpes) 1322 free_pd(dev, pdp->page_directory[pdpe]); 1323 1324 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1325 mark_tlbs_dirty(ppgtt); 1326 return ret; 1327 } 1328 1329 static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm, 1330 struct i915_pml4 *pml4, 1331 uint64_t start, 1332 uint64_t length) 1333 { 1334 DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4); 1335 struct i915_hw_ppgtt *ppgtt = 1336 container_of(vm, struct i915_hw_ppgtt, base); 1337 struct i915_page_directory_pointer *pdp; 1338 uint64_t temp, pml4e; 1339 int ret = 0; 1340 1341 /* Do the pml4 allocations first, so we don't need to track the newly 1342 * allocated tables below the pdp */ 1343 bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4); 1344 1345 /* The pagedirectory and pagetable allocations are done in the shared 3 1346 * and 4 level code. Just allocate the pdps. 1347 */ 1348 ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length, 1349 new_pdps); 1350 if (ret) 1351 return ret; 1352 1353 WARN(bitmap_weight(new_pdps, GEN8_PML4ES_PER_PML4) > 2, 1354 "The allocation has spanned more than 512GB. " 1355 "It is highly likely this is incorrect."); 1356 1357 gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) { 1358 WARN_ON(!pdp); 1359 1360 ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length); 1361 if (ret) 1362 goto err_out; 1363 1364 gen8_setup_page_directory_pointer(ppgtt, pml4, pdp, pml4e); 1365 } 1366 1367 bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es, 1368 GEN8_PML4ES_PER_PML4); 1369 1370 return 0; 1371 1372 err_out: 1373 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) 1374 gen8_ppgtt_cleanup_3lvl(vm->dev, pml4->pdps[pml4e]); 1375 1376 return ret; 1377 } 1378 1379 static int gen8_alloc_va_range(struct i915_address_space *vm, 1380 uint64_t start, uint64_t length) 1381 { 1382 struct i915_hw_ppgtt *ppgtt = 1383 container_of(vm, struct i915_hw_ppgtt, base); 1384 1385 if (USES_FULL_48BIT_PPGTT(vm->dev)) 1386 return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length); 1387 else 1388 return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length); 1389 } 1390 1391 static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp, 1392 uint64_t start, uint64_t length, 1393 gen8_pte_t scratch_pte, 1394 struct seq_file *m) 1395 { 1396 struct i915_page_directory *pd; 1397 uint64_t temp; 1398 uint32_t pdpe; 1399 1400 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { 1401 struct i915_page_table *pt; 1402 uint64_t pd_len = length; 1403 uint64_t pd_start = start; 1404 uint32_t pde; 1405 1406 if (!test_bit(pdpe, pdp->used_pdpes)) 1407 continue; 1408 1409 seq_printf(m, "\tPDPE #%d\n", pdpe); 1410 gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) { 1411 uint32_t pte; 1412 gen8_pte_t *pt_vaddr; 1413 1414 if (!test_bit(pde, pd->used_pdes)) 1415 continue; 1416 1417 pt_vaddr = kmap_px(pt); 1418 for (pte = 0; pte < GEN8_PTES; pte += 4) { 1419 uint64_t va = 1420 (pdpe << GEN8_PDPE_SHIFT) | 1421 (pde << GEN8_PDE_SHIFT) | 1422 (pte << GEN8_PTE_SHIFT); 1423 int i; 1424 bool found = false; 1425 1426 for (i = 0; i < 4; i++) 1427 if (pt_vaddr[pte + i] != scratch_pte) 1428 found = true; 1429 if (!found) 1430 continue; 1431 1432 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte); 1433 for (i = 0; i < 4; i++) { 1434 if (pt_vaddr[pte + i] != scratch_pte) 1435 seq_printf(m, " %llx", pt_vaddr[pte + i]); 1436 else 1437 seq_puts(m, " SCRATCH "); 1438 } 1439 seq_puts(m, "\n"); 1440 } 1441 /* don't use kunmap_px, it could trigger 1442 * an unnecessary flush. 1443 */ 1444 kunmap_atomic(pt_vaddr); 1445 } 1446 } 1447 } 1448 1449 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) 1450 { 1451 struct i915_address_space *vm = &ppgtt->base; 1452 uint64_t start = ppgtt->base.start; 1453 uint64_t length = ppgtt->base.total; 1454 gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 1455 I915_CACHE_LLC, true); 1456 1457 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 1458 gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m); 1459 } else { 1460 uint64_t templ4, pml4e; 1461 struct i915_pml4 *pml4 = &ppgtt->pml4; 1462 struct i915_page_directory_pointer *pdp; 1463 1464 gen8_for_each_pml4e(pdp, pml4, start, length, templ4, pml4e) { 1465 if (!test_bit(pml4e, pml4->used_pml4es)) 1466 continue; 1467 1468 seq_printf(m, " PML4E #%llu\n", pml4e); 1469 gen8_dump_pdp(pdp, start, length, scratch_pte, m); 1470 } 1471 } 1472 } 1473 1474 static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt) 1475 { 1476 unsigned long *new_page_dirs, *new_page_tables; 1477 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1478 int ret; 1479 1480 /* We allocate temp bitmap for page tables for no gain 1481 * but as this is for init only, lets keep the things simple 1482 */ 1483 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes); 1484 if (ret) 1485 return ret; 1486 1487 /* Allocate for all pdps regardless of how the ppgtt 1488 * was defined. 1489 */ 1490 ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp, 1491 0, 1ULL << 32, 1492 new_page_dirs); 1493 if (!ret) 1494 *ppgtt->pdp.used_pdpes = *new_page_dirs; 1495 1496 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1497 1498 return ret; 1499 } 1500 1501 /* 1502 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers 1503 * with a net effect resembling a 2-level page table in normal x86 terms. Each 1504 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address 1505 * space. 1506 * 1507 */ 1508 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 1509 { 1510 int ret; 1511 1512 ret = gen8_init_scratch(&ppgtt->base); 1513 if (ret) 1514 return ret; 1515 1516 ppgtt->base.start = 0; 1517 ppgtt->base.cleanup = gen8_ppgtt_cleanup; 1518 ppgtt->base.allocate_va_range = gen8_alloc_va_range; 1519 ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; 1520 ppgtt->base.clear_range = gen8_ppgtt_clear_range; 1521 ppgtt->base.unbind_vma = ppgtt_unbind_vma; 1522 ppgtt->base.bind_vma = ppgtt_bind_vma; 1523 ppgtt->debug_dump = gen8_dump_ppgtt; 1524 1525 if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) { 1526 ret = setup_px(ppgtt->base.dev, &ppgtt->pml4); 1527 if (ret) 1528 goto free_scratch; 1529 1530 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4); 1531 1532 ppgtt->base.total = 1ULL << 48; 1533 ppgtt->switch_mm = gen8_48b_mm_switch; 1534 } else { 1535 ret = __pdp_init(ppgtt->base.dev, &ppgtt->pdp); 1536 if (ret) 1537 goto free_scratch; 1538 1539 ppgtt->base.total = 1ULL << 32; 1540 ppgtt->switch_mm = gen8_legacy_mm_switch; 1541 trace_i915_page_directory_pointer_entry_alloc(&ppgtt->base, 1542 0, 0, 1543 GEN8_PML4E_SHIFT); 1544 1545 if (intel_vgpu_active(ppgtt->base.dev)) { 1546 ret = gen8_preallocate_top_level_pdps(ppgtt); 1547 if (ret) 1548 goto free_scratch; 1549 } 1550 } 1551 1552 if (intel_vgpu_active(ppgtt->base.dev)) 1553 gen8_ppgtt_notify_vgt(ppgtt, true); 1554 1555 return 0; 1556 1557 free_scratch: 1558 gen8_free_scratch(&ppgtt->base); 1559 return ret; 1560 } 1561 1562 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) 1563 { 1564 struct i915_address_space *vm = &ppgtt->base; 1565 struct i915_page_table *unused; 1566 gen6_pte_t scratch_pte; 1567 uint32_t pd_entry; 1568 uint32_t pte, pde, temp; 1569 uint32_t start = ppgtt->base.start, length = ppgtt->base.total; 1570 1571 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 1572 I915_CACHE_LLC, true, 0); 1573 1574 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) { 1575 u32 expected; 1576 gen6_pte_t *pt_vaddr; 1577 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]); 1578 pd_entry = readl(ppgtt->pd_addr + pde); 1579 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID); 1580 1581 if (pd_entry != expected) 1582 seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n", 1583 pde, 1584 pd_entry, 1585 expected); 1586 seq_printf(m, "\tPDE: %x\n", pd_entry); 1587 1588 pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]); 1589 1590 for (pte = 0; pte < GEN6_PTES; pte+=4) { 1591 unsigned long va = 1592 (pde * PAGE_SIZE * GEN6_PTES) + 1593 (pte * PAGE_SIZE); 1594 int i; 1595 bool found = false; 1596 for (i = 0; i < 4; i++) 1597 if (pt_vaddr[pte + i] != scratch_pte) 1598 found = true; 1599 if (!found) 1600 continue; 1601 1602 seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte); 1603 for (i = 0; i < 4; i++) { 1604 if (pt_vaddr[pte + i] != scratch_pte) 1605 seq_printf(m, " %08x", pt_vaddr[pte + i]); 1606 else 1607 seq_puts(m, " SCRATCH "); 1608 } 1609 seq_puts(m, "\n"); 1610 } 1611 kunmap_px(ppgtt, pt_vaddr); 1612 } 1613 } 1614 1615 /* Write pde (index) from the page directory @pd to the page table @pt */ 1616 static void gen6_write_pde(struct i915_page_directory *pd, 1617 const int pde, struct i915_page_table *pt) 1618 { 1619 /* Caller needs to make sure the write completes if necessary */ 1620 struct i915_hw_ppgtt *ppgtt = 1621 container_of(pd, struct i915_hw_ppgtt, pd); 1622 u32 pd_entry; 1623 1624 pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt)); 1625 pd_entry |= GEN6_PDE_VALID; 1626 1627 writel(pd_entry, ppgtt->pd_addr + pde); 1628 } 1629 1630 /* Write all the page tables found in the ppgtt structure to incrementing page 1631 * directories. */ 1632 static void gen6_write_page_range(struct drm_i915_private *dev_priv, 1633 struct i915_page_directory *pd, 1634 uint32_t start, uint32_t length) 1635 { 1636 struct i915_page_table *pt; 1637 uint32_t pde, temp; 1638 1639 gen6_for_each_pde(pt, pd, start, length, temp, pde) 1640 gen6_write_pde(pd, pde, pt); 1641 1642 /* Make sure write is complete before other code can use this page 1643 * table. Also require for WC mapped PTEs */ 1644 readl(dev_priv->gtt.gsm); 1645 } 1646 1647 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt) 1648 { 1649 BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f); 1650 1651 return (ppgtt->pd.base.ggtt_offset / 64) << 16; 1652 } 1653 1654 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, 1655 struct drm_i915_gem_request *req) 1656 { 1657 struct intel_engine_cs *ring = req->ring; 1658 int ret; 1659 1660 /* NB: TLBs must be flushed and invalidated before a switch */ 1661 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1662 if (ret) 1663 return ret; 1664 1665 ret = intel_ring_begin(req, 6); 1666 if (ret) 1667 return ret; 1668 1669 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); 1670 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); 1671 intel_ring_emit(ring, PP_DIR_DCLV_2G); 1672 intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); 1673 intel_ring_emit(ring, get_pd_offset(ppgtt)); 1674 intel_ring_emit(ring, MI_NOOP); 1675 intel_ring_advance(ring); 1676 1677 return 0; 1678 } 1679 1680 static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt, 1681 struct drm_i915_gem_request *req) 1682 { 1683 struct intel_engine_cs *ring = req->ring; 1684 struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev); 1685 1686 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 1687 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); 1688 return 0; 1689 } 1690 1691 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, 1692 struct drm_i915_gem_request *req) 1693 { 1694 struct intel_engine_cs *ring = req->ring; 1695 int ret; 1696 1697 /* NB: TLBs must be flushed and invalidated before a switch */ 1698 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1699 if (ret) 1700 return ret; 1701 1702 ret = intel_ring_begin(req, 6); 1703 if (ret) 1704 return ret; 1705 1706 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); 1707 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); 1708 intel_ring_emit(ring, PP_DIR_DCLV_2G); 1709 intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); 1710 intel_ring_emit(ring, get_pd_offset(ppgtt)); 1711 intel_ring_emit(ring, MI_NOOP); 1712 intel_ring_advance(ring); 1713 1714 /* XXX: RCS is the only one to auto invalidate the TLBs? */ 1715 if (ring->id != RCS) { 1716 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1717 if (ret) 1718 return ret; 1719 } 1720 1721 return 0; 1722 } 1723 1724 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, 1725 struct drm_i915_gem_request *req) 1726 { 1727 struct intel_engine_cs *ring = req->ring; 1728 struct drm_device *dev = ppgtt->base.dev; 1729 struct drm_i915_private *dev_priv = dev->dev_private; 1730 1731 1732 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 1733 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); 1734 1735 POSTING_READ(RING_PP_DIR_DCLV(ring)); 1736 1737 return 0; 1738 } 1739 1740 static void gen8_ppgtt_enable(struct drm_device *dev) 1741 { 1742 struct drm_i915_private *dev_priv = dev->dev_private; 1743 struct intel_engine_cs *ring; 1744 int j; 1745 1746 for_each_ring(ring, dev_priv, j) { 1747 u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0; 1748 I915_WRITE(RING_MODE_GEN7(ring), 1749 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level)); 1750 } 1751 } 1752 1753 static void gen7_ppgtt_enable(struct drm_device *dev) 1754 { 1755 struct drm_i915_private *dev_priv = dev->dev_private; 1756 struct intel_engine_cs *ring; 1757 uint32_t ecochk, ecobits; 1758 int i; 1759 1760 ecobits = I915_READ(GAC_ECO_BITS); 1761 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); 1762 1763 ecochk = I915_READ(GAM_ECOCHK); 1764 if (IS_HASWELL(dev)) { 1765 ecochk |= ECOCHK_PPGTT_WB_HSW; 1766 } else { 1767 ecochk |= ECOCHK_PPGTT_LLC_IVB; 1768 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; 1769 } 1770 I915_WRITE(GAM_ECOCHK, ecochk); 1771 1772 for_each_ring(ring, dev_priv, i) { 1773 /* GFX_MODE is per-ring on gen7+ */ 1774 I915_WRITE(RING_MODE_GEN7(ring), 1775 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 1776 } 1777 } 1778 1779 static void gen6_ppgtt_enable(struct drm_device *dev) 1780 { 1781 struct drm_i915_private *dev_priv = dev->dev_private; 1782 uint32_t ecochk, gab_ctl, ecobits; 1783 1784 ecobits = I915_READ(GAC_ECO_BITS); 1785 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | 1786 ECOBITS_PPGTT_CACHE64B); 1787 1788 gab_ctl = I915_READ(GAB_CTL); 1789 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); 1790 1791 ecochk = I915_READ(GAM_ECOCHK); 1792 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B); 1793 1794 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 1795 } 1796 1797 /* PPGTT support for Sandybdrige/Gen6 and later */ 1798 static void gen6_ppgtt_clear_range(struct i915_address_space *vm, 1799 uint64_t start, 1800 uint64_t length, 1801 bool use_scratch) 1802 { 1803 struct i915_hw_ppgtt *ppgtt = 1804 container_of(vm, struct i915_hw_ppgtt, base); 1805 gen6_pte_t *pt_vaddr, scratch_pte; 1806 unsigned first_entry = start >> PAGE_SHIFT; 1807 unsigned num_entries = length >> PAGE_SHIFT; 1808 unsigned act_pt = first_entry / GEN6_PTES; 1809 unsigned first_pte = first_entry % GEN6_PTES; 1810 unsigned last_pte, i; 1811 1812 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 1813 I915_CACHE_LLC, true, 0); 1814 1815 while (num_entries) { 1816 last_pte = first_pte + num_entries; 1817 if (last_pte > GEN6_PTES) 1818 last_pte = GEN6_PTES; 1819 1820 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); 1821 1822 for (i = first_pte; i < last_pte; i++) 1823 pt_vaddr[i] = scratch_pte; 1824 1825 kunmap_px(ppgtt, pt_vaddr); 1826 1827 num_entries -= last_pte - first_pte; 1828 first_pte = 0; 1829 act_pt++; 1830 } 1831 } 1832 1833 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, 1834 struct sg_table *pages, 1835 uint64_t start, 1836 enum i915_cache_level cache_level, u32 flags) 1837 { 1838 struct i915_hw_ppgtt *ppgtt = 1839 container_of(vm, struct i915_hw_ppgtt, base); 1840 gen6_pte_t *pt_vaddr; 1841 unsigned first_entry = start >> PAGE_SHIFT; 1842 unsigned act_pt = first_entry / GEN6_PTES; 1843 unsigned act_pte = first_entry % GEN6_PTES; 1844 struct sg_page_iter sg_iter; 1845 1846 pt_vaddr = NULL; 1847 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { 1848 if (pt_vaddr == NULL) 1849 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); 1850 1851 pt_vaddr[act_pte] = 1852 vm->pte_encode(sg_page_iter_dma_address(&sg_iter), 1853 cache_level, true, flags); 1854 1855 if (++act_pte == GEN6_PTES) { 1856 kunmap_px(ppgtt, pt_vaddr); 1857 pt_vaddr = NULL; 1858 act_pt++; 1859 act_pte = 0; 1860 } 1861 } 1862 if (pt_vaddr) 1863 kunmap_px(ppgtt, pt_vaddr); 1864 } 1865 1866 static int gen6_alloc_va_range(struct i915_address_space *vm, 1867 uint64_t start_in, uint64_t length_in) 1868 { 1869 DECLARE_BITMAP(new_page_tables, I915_PDES); 1870 struct drm_device *dev = vm->dev; 1871 struct drm_i915_private *dev_priv = dev->dev_private; 1872 struct i915_hw_ppgtt *ppgtt = 1873 container_of(vm, struct i915_hw_ppgtt, base); 1874 struct i915_page_table *pt; 1875 uint32_t start, length, start_save, length_save; 1876 uint32_t pde, temp; 1877 int ret; 1878 1879 if (WARN_ON(start_in + length_in > ppgtt->base.total)) 1880 return -ENODEV; 1881 1882 start = start_save = start_in; 1883 length = length_save = length_in; 1884 1885 bitmap_zero(new_page_tables, I915_PDES); 1886 1887 /* The allocation is done in two stages so that we can bail out with 1888 * minimal amount of pain. The first stage finds new page tables that 1889 * need allocation. The second stage marks use ptes within the page 1890 * tables. 1891 */ 1892 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { 1893 if (pt != vm->scratch_pt) { 1894 WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES)); 1895 continue; 1896 } 1897 1898 /* We've already allocated a page table */ 1899 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES)); 1900 1901 pt = alloc_pt(dev); 1902 if (IS_ERR(pt)) { 1903 ret = PTR_ERR(pt); 1904 goto unwind_out; 1905 } 1906 1907 gen6_initialize_pt(vm, pt); 1908 1909 ppgtt->pd.page_table[pde] = pt; 1910 __set_bit(pde, new_page_tables); 1911 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT); 1912 } 1913 1914 start = start_save; 1915 length = length_save; 1916 1917 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { 1918 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES); 1919 1920 bitmap_zero(tmp_bitmap, GEN6_PTES); 1921 bitmap_set(tmp_bitmap, gen6_pte_index(start), 1922 gen6_pte_count(start, length)); 1923 1924 if (__test_and_clear_bit(pde, new_page_tables)) 1925 gen6_write_pde(&ppgtt->pd, pde, pt); 1926 1927 trace_i915_page_table_entry_map(vm, pde, pt, 1928 gen6_pte_index(start), 1929 gen6_pte_count(start, length), 1930 GEN6_PTES); 1931 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes, 1932 GEN6_PTES); 1933 } 1934 1935 WARN_ON(!bitmap_empty(new_page_tables, I915_PDES)); 1936 1937 /* Make sure write is complete before other code can use this page 1938 * table. Also require for WC mapped PTEs */ 1939 readl(dev_priv->gtt.gsm); 1940 1941 mark_tlbs_dirty(ppgtt); 1942 return 0; 1943 1944 unwind_out: 1945 for_each_set_bit(pde, new_page_tables, I915_PDES) { 1946 struct i915_page_table *pt = ppgtt->pd.page_table[pde]; 1947 1948 ppgtt->pd.page_table[pde] = vm->scratch_pt; 1949 free_pt(vm->dev, pt); 1950 } 1951 1952 mark_tlbs_dirty(ppgtt); 1953 return ret; 1954 } 1955 1956 static int gen6_init_scratch(struct i915_address_space *vm) 1957 { 1958 struct drm_device *dev = vm->dev; 1959 1960 vm->scratch_page = alloc_scratch_page(dev); 1961 if (IS_ERR(vm->scratch_page)) 1962 return PTR_ERR(vm->scratch_page); 1963 1964 vm->scratch_pt = alloc_pt(dev); 1965 if (IS_ERR(vm->scratch_pt)) { 1966 free_scratch_page(dev, vm->scratch_page); 1967 return PTR_ERR(vm->scratch_pt); 1968 } 1969 1970 gen6_initialize_pt(vm, vm->scratch_pt); 1971 1972 return 0; 1973 } 1974 1975 static void gen6_free_scratch(struct i915_address_space *vm) 1976 { 1977 struct drm_device *dev = vm->dev; 1978 1979 free_pt(dev, vm->scratch_pt); 1980 free_scratch_page(dev, vm->scratch_page); 1981 } 1982 1983 static void gen6_ppgtt_cleanup(struct i915_address_space *vm) 1984 { 1985 struct i915_hw_ppgtt *ppgtt = 1986 container_of(vm, struct i915_hw_ppgtt, base); 1987 struct i915_page_table *pt; 1988 uint32_t pde; 1989 1990 drm_mm_remove_node(&ppgtt->node); 1991 1992 gen6_for_all_pdes(pt, ppgtt, pde) { 1993 if (pt != vm->scratch_pt) 1994 free_pt(ppgtt->base.dev, pt); 1995 } 1996 1997 gen6_free_scratch(vm); 1998 } 1999 2000 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) 2001 { 2002 struct i915_address_space *vm = &ppgtt->base; 2003 struct drm_device *dev = ppgtt->base.dev; 2004 struct drm_i915_private *dev_priv = dev->dev_private; 2005 bool retried = false; 2006 int ret; 2007 2008 /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The 2009 * allocator works in address space sizes, so it's multiplied by page 2010 * size. We allocate at the top of the GTT to avoid fragmentation. 2011 */ 2012 BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm)); 2013 2014 ret = gen6_init_scratch(vm); 2015 if (ret) 2016 return ret; 2017 2018 alloc: 2019 ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm, 2020 &ppgtt->node, GEN6_PD_SIZE, 2021 GEN6_PD_ALIGN, 0, 2022 0, dev_priv->gtt.base.total, 2023 DRM_MM_TOPDOWN); 2024 if (ret == -ENOSPC && !retried) { 2025 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base, 2026 GEN6_PD_SIZE, GEN6_PD_ALIGN, 2027 I915_CACHE_NONE, 2028 0, dev_priv->gtt.base.total, 2029 0); 2030 if (ret) 2031 goto err_out; 2032 2033 retried = true; 2034 goto alloc; 2035 } 2036 2037 if (ret) 2038 goto err_out; 2039 2040 2041 if (ppgtt->node.start < dev_priv->gtt.mappable_end) 2042 DRM_DEBUG("Forced to use aperture for PDEs\n"); 2043 2044 return 0; 2045 2046 err_out: 2047 gen6_free_scratch(vm); 2048 return ret; 2049 } 2050 2051 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt) 2052 { 2053 return gen6_ppgtt_allocate_page_directories(ppgtt); 2054 } 2055 2056 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt, 2057 uint64_t start, uint64_t length) 2058 { 2059 struct i915_page_table *unused; 2060 uint32_t pde, temp; 2061 2062 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) 2063 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt; 2064 } 2065 2066 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 2067 { 2068 struct drm_device *dev = ppgtt->base.dev; 2069 struct drm_i915_private *dev_priv = dev->dev_private; 2070 int ret; 2071 2072 ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode; 2073 if (IS_GEN6(dev)) { 2074 ppgtt->switch_mm = gen6_mm_switch; 2075 } else if (IS_HASWELL(dev)) { 2076 ppgtt->switch_mm = hsw_mm_switch; 2077 } else if (IS_GEN7(dev)) { 2078 ppgtt->switch_mm = gen7_mm_switch; 2079 } else 2080 BUG(); 2081 2082 if (intel_vgpu_active(dev)) 2083 ppgtt->switch_mm = vgpu_mm_switch; 2084 2085 ret = gen6_ppgtt_alloc(ppgtt); 2086 if (ret) 2087 return ret; 2088 2089 ppgtt->base.allocate_va_range = gen6_alloc_va_range; 2090 ppgtt->base.clear_range = gen6_ppgtt_clear_range; 2091 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; 2092 ppgtt->base.unbind_vma = ppgtt_unbind_vma; 2093 ppgtt->base.bind_vma = ppgtt_bind_vma; 2094 ppgtt->base.cleanup = gen6_ppgtt_cleanup; 2095 ppgtt->base.start = 0; 2096 ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE; 2097 ppgtt->debug_dump = gen6_dump_ppgtt; 2098 2099 ppgtt->pd.base.ggtt_offset = 2100 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t); 2101 2102 ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm + 2103 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t); 2104 2105 gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total); 2106 2107 gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total); 2108 2109 DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n", 2110 ppgtt->node.size >> 20, 2111 ppgtt->node.start / PAGE_SIZE); 2112 2113 DRM_DEBUG("Adding PPGTT at offset %x\n", 2114 ppgtt->pd.base.ggtt_offset << 10); 2115 2116 return 0; 2117 } 2118 2119 static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) 2120 { 2121 ppgtt->base.dev = dev; 2122 2123 if (INTEL_INFO(dev)->gen < 8) 2124 return gen6_ppgtt_init(ppgtt); 2125 else 2126 return gen8_ppgtt_init(ppgtt); 2127 } 2128 2129 static void i915_address_space_init(struct i915_address_space *vm, 2130 struct drm_i915_private *dev_priv) 2131 { 2132 drm_mm_init(&vm->mm, vm->start, vm->total); 2133 vm->dev = dev_priv->dev; 2134 INIT_LIST_HEAD(&vm->active_list); 2135 INIT_LIST_HEAD(&vm->inactive_list); 2136 list_add_tail(&vm->global_link, &dev_priv->vm_list); 2137 } 2138 2139 int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) 2140 { 2141 struct drm_i915_private *dev_priv = dev->dev_private; 2142 int ret = 0; 2143 2144 ret = __hw_ppgtt_init(dev, ppgtt); 2145 if (ret == 0) { 2146 kref_init(&ppgtt->ref); 2147 i915_address_space_init(&ppgtt->base, dev_priv); 2148 } 2149 2150 return ret; 2151 } 2152 2153 int i915_ppgtt_init_hw(struct drm_device *dev) 2154 { 2155 /* In the case of execlists, PPGTT is enabled by the context descriptor 2156 * and the PDPs are contained within the context itself. We don't 2157 * need to do anything here. */ 2158 if (i915.enable_execlists) 2159 return 0; 2160 2161 if (!USES_PPGTT(dev)) 2162 return 0; 2163 2164 if (IS_GEN6(dev)) 2165 gen6_ppgtt_enable(dev); 2166 else if (IS_GEN7(dev)) 2167 gen7_ppgtt_enable(dev); 2168 else if (INTEL_INFO(dev)->gen >= 8) 2169 gen8_ppgtt_enable(dev); 2170 else 2171 MISSING_CASE(INTEL_INFO(dev)->gen); 2172 2173 return 0; 2174 } 2175 2176 int i915_ppgtt_init_ring(struct drm_i915_gem_request *req) 2177 { 2178 struct drm_i915_private *dev_priv = req->ring->dev->dev_private; 2179 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 2180 2181 if (i915.enable_execlists) 2182 return 0; 2183 2184 if (!ppgtt) 2185 return 0; 2186 2187 return ppgtt->switch_mm(ppgtt, req); 2188 } 2189 2190 struct i915_hw_ppgtt * 2191 i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv) 2192 { 2193 struct i915_hw_ppgtt *ppgtt; 2194 int ret; 2195 2196 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 2197 if (!ppgtt) 2198 return ERR_PTR(-ENOMEM); 2199 2200 ret = i915_ppgtt_init(dev, ppgtt); 2201 if (ret) { 2202 kfree(ppgtt); 2203 return ERR_PTR(ret); 2204 } 2205 2206 ppgtt->file_priv = fpriv; 2207 2208 trace_i915_ppgtt_create(&ppgtt->base); 2209 2210 return ppgtt; 2211 } 2212 2213 void i915_ppgtt_release(struct kref *kref) 2214 { 2215 struct i915_hw_ppgtt *ppgtt = 2216 container_of(kref, struct i915_hw_ppgtt, ref); 2217 2218 trace_i915_ppgtt_release(&ppgtt->base); 2219 2220 /* vmas should already be unbound */ 2221 WARN_ON(!list_empty(&ppgtt->base.active_list)); 2222 WARN_ON(!list_empty(&ppgtt->base.inactive_list)); 2223 2224 list_del(&ppgtt->base.global_link); 2225 drm_mm_takedown(&ppgtt->base.mm); 2226 2227 ppgtt->base.cleanup(&ppgtt->base); 2228 kfree(ppgtt); 2229 } 2230 2231 extern int intel_iommu_gfx_mapped; 2232 /* Certain Gen5 chipsets require require idling the GPU before 2233 * unmapping anything from the GTT when VT-d is enabled. 2234 */ 2235 static bool needs_idle_maps(struct drm_device *dev) 2236 { 2237 #ifdef CONFIG_INTEL_IOMMU 2238 /* Query intel_iommu to see if we need the workaround. Presumably that 2239 * was loaded first. 2240 */ 2241 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped) 2242 return true; 2243 #endif 2244 return false; 2245 } 2246 2247 static bool do_idling(struct drm_i915_private *dev_priv) 2248 { 2249 bool ret = dev_priv->mm.interruptible; 2250 2251 if (unlikely(dev_priv->gtt.do_idle_maps)) { 2252 dev_priv->mm.interruptible = false; 2253 if (i915_gpu_idle(dev_priv->dev)) { 2254 DRM_ERROR("Couldn't idle GPU\n"); 2255 /* Wait a bit, in hopes it avoids the hang */ 2256 udelay(10); 2257 } 2258 } 2259 2260 return ret; 2261 } 2262 2263 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) 2264 { 2265 if (unlikely(dev_priv->gtt.do_idle_maps)) 2266 dev_priv->mm.interruptible = interruptible; 2267 } 2268 2269 void i915_check_and_clear_faults(struct drm_device *dev) 2270 { 2271 struct drm_i915_private *dev_priv = dev->dev_private; 2272 struct intel_engine_cs *ring; 2273 int i; 2274 2275 if (INTEL_INFO(dev)->gen < 6) 2276 return; 2277 2278 for_each_ring(ring, dev_priv, i) { 2279 u32 fault_reg; 2280 fault_reg = I915_READ(RING_FAULT_REG(ring)); 2281 if (fault_reg & RING_FAULT_VALID) { 2282 DRM_DEBUG_DRIVER("Unexpected fault\n" 2283 "\tAddr: 0x%08lx\n" 2284 "\tAddress space: %s\n" 2285 "\tSource ID: %d\n" 2286 "\tType: %d\n", 2287 fault_reg & PAGE_MASK, 2288 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT", 2289 RING_FAULT_SRCID(fault_reg), 2290 RING_FAULT_FAULT_TYPE(fault_reg)); 2291 I915_WRITE(RING_FAULT_REG(ring), 2292 fault_reg & ~RING_FAULT_VALID); 2293 } 2294 } 2295 POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS])); 2296 } 2297 2298 static void i915_ggtt_flush(struct drm_i915_private *dev_priv) 2299 { 2300 if (INTEL_INFO(dev_priv->dev)->gen < 6) { 2301 intel_gtt_chipset_flush(); 2302 } else { 2303 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2304 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2305 } 2306 } 2307 2308 void i915_gem_suspend_gtt_mappings(struct drm_device *dev) 2309 { 2310 struct drm_i915_private *dev_priv = dev->dev_private; 2311 2312 /* Don't bother messing with faults pre GEN6 as we have little 2313 * documentation supporting that it's a good idea. 2314 */ 2315 if (INTEL_INFO(dev)->gen < 6) 2316 return; 2317 2318 i915_check_and_clear_faults(dev); 2319 2320 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, 2321 dev_priv->gtt.base.start, 2322 dev_priv->gtt.base.total, 2323 true); 2324 2325 i915_ggtt_flush(dev_priv); 2326 } 2327 2328 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) 2329 { 2330 if (!dma_map_sg(&obj->base.dev->pdev->dev, 2331 obj->pages->sgl, obj->pages->nents, 2332 PCI_DMA_BIDIRECTIONAL)) 2333 return -ENOSPC; 2334 2335 return 0; 2336 } 2337 2338 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 2339 { 2340 #ifdef writeq 2341 writeq(pte, addr); 2342 #else 2343 iowrite32((u32)pte, addr); 2344 iowrite32(pte >> 32, addr + 4); 2345 #endif 2346 } 2347 2348 static void gen8_ggtt_insert_entries(struct i915_address_space *vm, 2349 struct sg_table *st, 2350 uint64_t start, 2351 enum i915_cache_level level, u32 unused) 2352 { 2353 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2354 unsigned first_entry = start >> PAGE_SHIFT; 2355 gen8_pte_t __iomem *gtt_entries = 2356 (gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; 2357 int i = 0; 2358 struct sg_page_iter sg_iter; 2359 dma_addr_t addr = 0; /* shut up gcc */ 2360 2361 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { 2362 addr = sg_dma_address(sg_iter.sg) + 2363 (sg_iter.sg_pgoffset << PAGE_SHIFT); 2364 gen8_set_pte(>t_entries[i], 2365 gen8_pte_encode(addr, level, true)); 2366 i++; 2367 } 2368 2369 /* 2370 * XXX: This serves as a posting read to make sure that the PTE has 2371 * actually been updated. There is some concern that even though 2372 * registers and PTEs are within the same BAR that they are potentially 2373 * of NUMA access patterns. Therefore, even with the way we assume 2374 * hardware should work, we must keep this posting read for paranoia. 2375 */ 2376 if (i != 0) 2377 WARN_ON(readq(>t_entries[i-1]) 2378 != gen8_pte_encode(addr, level, true)); 2379 2380 /* This next bit makes the above posting read even more important. We 2381 * want to flush the TLBs only after we're certain all the PTE updates 2382 * have finished. 2383 */ 2384 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2385 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2386 } 2387 2388 /* 2389 * Binds an object into the global gtt with the specified cache level. The object 2390 * will be accessible to the GPU via commands whose operands reference offsets 2391 * within the global GTT as well as accessible by the GPU through the GMADR 2392 * mapped BAR (dev_priv->mm.gtt->gtt). 2393 */ 2394 static void gen6_ggtt_insert_entries(struct i915_address_space *vm, 2395 struct sg_table *st, 2396 uint64_t start, 2397 enum i915_cache_level level, u32 flags) 2398 { 2399 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2400 unsigned first_entry = start >> PAGE_SHIFT; 2401 gen6_pte_t __iomem *gtt_entries = 2402 (gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; 2403 int i = 0; 2404 struct sg_page_iter sg_iter; 2405 dma_addr_t addr = 0; 2406 2407 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { 2408 addr = sg_page_iter_dma_address(&sg_iter); 2409 iowrite32(vm->pte_encode(addr, level, true, flags), >t_entries[i]); 2410 i++; 2411 } 2412 2413 /* XXX: This serves as a posting read to make sure that the PTE has 2414 * actually been updated. There is some concern that even though 2415 * registers and PTEs are within the same BAR that they are potentially 2416 * of NUMA access patterns. Therefore, even with the way we assume 2417 * hardware should work, we must keep this posting read for paranoia. 2418 */ 2419 if (i != 0) { 2420 unsigned long gtt = readl(>t_entries[i-1]); 2421 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags)); 2422 } 2423 2424 /* This next bit makes the above posting read even more important. We 2425 * want to flush the TLBs only after we're certain all the PTE updates 2426 * have finished. 2427 */ 2428 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2429 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2430 } 2431 2432 static void gen8_ggtt_clear_range(struct i915_address_space *vm, 2433 uint64_t start, 2434 uint64_t length, 2435 bool use_scratch) 2436 { 2437 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2438 unsigned first_entry = start >> PAGE_SHIFT; 2439 unsigned num_entries = length >> PAGE_SHIFT; 2440 gen8_pte_t scratch_pte, __iomem *gtt_base = 2441 (gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; 2442 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; 2443 int i; 2444 2445 if (WARN(num_entries > max_entries, 2446 "First entry = %d; Num entries = %d (max=%d)\n", 2447 first_entry, num_entries, max_entries)) 2448 num_entries = max_entries; 2449 2450 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 2451 I915_CACHE_LLC, 2452 use_scratch); 2453 for (i = 0; i < num_entries; i++) 2454 gen8_set_pte(>t_base[i], scratch_pte); 2455 readl(gtt_base); 2456 } 2457 2458 static void gen6_ggtt_clear_range(struct i915_address_space *vm, 2459 uint64_t start, 2460 uint64_t length, 2461 bool use_scratch) 2462 { 2463 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2464 unsigned first_entry = start >> PAGE_SHIFT; 2465 unsigned num_entries = length >> PAGE_SHIFT; 2466 gen6_pte_t scratch_pte, __iomem *gtt_base = 2467 (gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; 2468 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; 2469 int i; 2470 2471 if (WARN(num_entries > max_entries, 2472 "First entry = %d; Num entries = %d (max=%d)\n", 2473 first_entry, num_entries, max_entries)) 2474 num_entries = max_entries; 2475 2476 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 2477 I915_CACHE_LLC, use_scratch, 0); 2478 2479 for (i = 0; i < num_entries; i++) 2480 iowrite32(scratch_pte, >t_base[i]); 2481 readl(gtt_base); 2482 } 2483 2484 static void i915_ggtt_insert_entries(struct i915_address_space *vm, 2485 struct sg_table *pages, 2486 uint64_t start, 2487 enum i915_cache_level cache_level, u32 unused) 2488 { 2489 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 2490 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 2491 2492 intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags); 2493 2494 } 2495 2496 static void i915_ggtt_clear_range(struct i915_address_space *vm, 2497 uint64_t start, 2498 uint64_t length, 2499 bool unused) 2500 { 2501 unsigned first_entry = start >> PAGE_SHIFT; 2502 unsigned num_entries = length >> PAGE_SHIFT; 2503 intel_gtt_clear_range(first_entry, num_entries); 2504 } 2505 2506 static int ggtt_bind_vma(struct i915_vma *vma, 2507 enum i915_cache_level cache_level, 2508 u32 flags) 2509 { 2510 struct drm_i915_gem_object *obj = vma->obj; 2511 u32 pte_flags = 0; 2512 int ret; 2513 2514 ret = i915_get_ggtt_vma_pages(vma); 2515 if (ret) 2516 return ret; 2517 2518 /* Currently applicable only to VLV */ 2519 if (obj->gt_ro) 2520 pte_flags |= PTE_READ_ONLY; 2521 2522 vma->vm->insert_entries(vma->vm, vma->ggtt_view.pages, 2523 vma->node.start, 2524 cache_level, pte_flags); 2525 2526 /* 2527 * Without aliasing PPGTT there's no difference between 2528 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally 2529 * upgrade to both bound if we bind either to avoid double-binding. 2530 */ 2531 vma->bound |= GLOBAL_BIND | LOCAL_BIND; 2532 2533 return 0; 2534 } 2535 2536 static int aliasing_gtt_bind_vma(struct i915_vma *vma, 2537 enum i915_cache_level cache_level, 2538 u32 flags) 2539 { 2540 struct drm_device *dev = vma->vm->dev; 2541 struct drm_i915_private *dev_priv = dev->dev_private; 2542 struct drm_i915_gem_object *obj = vma->obj; 2543 struct sg_table *pages = obj->pages; 2544 u32 pte_flags = 0; 2545 int ret; 2546 2547 ret = i915_get_ggtt_vma_pages(vma); 2548 if (ret) 2549 return ret; 2550 pages = vma->ggtt_view.pages; 2551 2552 /* Currently applicable only to VLV */ 2553 if (obj->gt_ro) 2554 pte_flags |= PTE_READ_ONLY; 2555 2556 2557 if (flags & GLOBAL_BIND) { 2558 vma->vm->insert_entries(vma->vm, pages, 2559 vma->node.start, 2560 cache_level, pte_flags); 2561 } 2562 2563 if (flags & LOCAL_BIND) { 2564 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; 2565 appgtt->base.insert_entries(&appgtt->base, pages, 2566 vma->node.start, 2567 cache_level, pte_flags); 2568 } 2569 2570 return 0; 2571 } 2572 2573 static void ggtt_unbind_vma(struct i915_vma *vma) 2574 { 2575 struct drm_device *dev = vma->vm->dev; 2576 struct drm_i915_private *dev_priv = dev->dev_private; 2577 struct drm_i915_gem_object *obj = vma->obj; 2578 const uint64_t size = min_t(uint64_t, 2579 obj->base.size, 2580 vma->node.size); 2581 2582 if (vma->bound & GLOBAL_BIND) { 2583 vma->vm->clear_range(vma->vm, 2584 vma->node.start, 2585 size, 2586 true); 2587 } 2588 2589 if (dev_priv->mm.aliasing_ppgtt && vma->bound & LOCAL_BIND) { 2590 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; 2591 2592 appgtt->base.clear_range(&appgtt->base, 2593 vma->node.start, 2594 size, 2595 true); 2596 } 2597 } 2598 2599 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) 2600 { 2601 struct drm_device *dev = obj->base.dev; 2602 struct drm_i915_private *dev_priv = dev->dev_private; 2603 bool interruptible; 2604 2605 interruptible = do_idling(dev_priv); 2606 2607 dma_unmap_sg(&dev->pdev->dev, obj->pages->sgl, obj->pages->nents, 2608 PCI_DMA_BIDIRECTIONAL); 2609 2610 undo_idling(dev_priv, interruptible); 2611 } 2612 2613 static void i915_gtt_color_adjust(struct drm_mm_node *node, 2614 unsigned long color, 2615 u64 *start, 2616 u64 *end) 2617 { 2618 if (node->color != color) 2619 *start += 4096; 2620 2621 if (!list_empty(&node->node_list)) { 2622 node = list_entry(node->node_list.next, 2623 struct drm_mm_node, 2624 node_list); 2625 if (node->allocated && node->color != color) 2626 *end -= 4096; 2627 } 2628 } 2629 2630 static int i915_gem_setup_global_gtt(struct drm_device *dev, 2631 u64 start, 2632 u64 mappable_end, 2633 u64 end) 2634 { 2635 /* Let GEM Manage all of the aperture. 2636 * 2637 * However, leave one page at the end still bound to the scratch page. 2638 * There are a number of places where the hardware apparently prefetches 2639 * past the end of the object, and we've seen multiple hangs with the 2640 * GPU head pointer stuck in a batchbuffer bound at the last page of the 2641 * aperture. One page should be enough to keep any prefetching inside 2642 * of the aperture. 2643 */ 2644 struct drm_i915_private *dev_priv = dev->dev_private; 2645 struct i915_address_space *ggtt_vm = &dev_priv->gtt.base; 2646 struct drm_mm_node *entry; 2647 struct drm_i915_gem_object *obj; 2648 unsigned long hole_start, hole_end; 2649 int ret; 2650 2651 BUG_ON(mappable_end > end); 2652 2653 ggtt_vm->start = start; 2654 2655 /* Subtract the guard page before address space initialization to 2656 * shrink the range used by drm_mm */ 2657 ggtt_vm->total = end - start - PAGE_SIZE; 2658 i915_address_space_init(ggtt_vm, dev_priv); 2659 ggtt_vm->total += PAGE_SIZE; 2660 2661 if (intel_vgpu_active(dev)) { 2662 ret = intel_vgt_balloon(dev); 2663 if (ret) 2664 return ret; 2665 } 2666 2667 if (!HAS_LLC(dev)) 2668 ggtt_vm->mm.color_adjust = i915_gtt_color_adjust; 2669 2670 /* Mark any preallocated objects as occupied */ 2671 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 2672 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); 2673 2674 DRM_DEBUG_KMS("reserving preallocated space: %llx + %zx\n", 2675 i915_gem_obj_ggtt_offset(obj), obj->base.size); 2676 2677 WARN_ON(i915_gem_obj_ggtt_bound(obj)); 2678 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node); 2679 if (ret) { 2680 DRM_DEBUG_KMS("Reservation failed: %i\n", ret); 2681 return ret; 2682 } 2683 vma->bound |= GLOBAL_BIND; 2684 __i915_vma_set_map_and_fenceable(vma); 2685 list_add_tail(&vma->mm_list, &ggtt_vm->inactive_list); 2686 } 2687 2688 /* Clear any non-preallocated blocks */ 2689 drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) { 2690 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", 2691 hole_start, hole_end); 2692 ggtt_vm->clear_range(ggtt_vm, hole_start, 2693 hole_end - hole_start, true); 2694 } 2695 2696 /* And finally clear the reserved guard page */ 2697 ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true); 2698 2699 if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) { 2700 struct i915_hw_ppgtt *ppgtt; 2701 2702 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 2703 if (!ppgtt) 2704 return -ENOMEM; 2705 2706 ret = __hw_ppgtt_init(dev, ppgtt); 2707 if (ret) { 2708 ppgtt->base.cleanup(&ppgtt->base); 2709 kfree(ppgtt); 2710 return ret; 2711 } 2712 2713 if (ppgtt->base.allocate_va_range) 2714 ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0, 2715 ppgtt->base.total); 2716 if (ret) { 2717 ppgtt->base.cleanup(&ppgtt->base); 2718 kfree(ppgtt); 2719 return ret; 2720 } 2721 2722 ppgtt->base.clear_range(&ppgtt->base, 2723 ppgtt->base.start, 2724 ppgtt->base.total, 2725 true); 2726 2727 dev_priv->mm.aliasing_ppgtt = ppgtt; 2728 WARN_ON(dev_priv->gtt.base.bind_vma != ggtt_bind_vma); 2729 dev_priv->gtt.base.bind_vma = aliasing_gtt_bind_vma; 2730 } 2731 2732 return 0; 2733 } 2734 2735 void i915_gem_init_global_gtt(struct drm_device *dev) 2736 { 2737 struct drm_i915_private *dev_priv = dev->dev_private; 2738 u64 gtt_size, mappable_size; 2739 2740 gtt_size = dev_priv->gtt.base.total; 2741 mappable_size = dev_priv->gtt.mappable_end; 2742 2743 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size); 2744 } 2745 2746 void i915_global_gtt_cleanup(struct drm_device *dev) 2747 { 2748 struct drm_i915_private *dev_priv = dev->dev_private; 2749 struct i915_address_space *vm = &dev_priv->gtt.base; 2750 2751 if (dev_priv->mm.aliasing_ppgtt) { 2752 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 2753 2754 ppgtt->base.cleanup(&ppgtt->base); 2755 kfree(ppgtt); 2756 } 2757 2758 if (drm_mm_initialized(&vm->mm)) { 2759 if (intel_vgpu_active(dev)) 2760 intel_vgt_deballoon(); 2761 2762 drm_mm_takedown(&vm->mm); 2763 list_del(&vm->global_link); 2764 } 2765 2766 vm->cleanup(vm); 2767 } 2768 2769 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 2770 { 2771 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 2772 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 2773 return snb_gmch_ctl << 20; 2774 } 2775 2776 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) 2777 { 2778 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; 2779 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; 2780 if (bdw_gmch_ctl) 2781 bdw_gmch_ctl = 1 << bdw_gmch_ctl; 2782 2783 #ifdef CONFIG_X86_32 2784 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */ 2785 if (bdw_gmch_ctl > 4) 2786 bdw_gmch_ctl = 4; 2787 #endif 2788 2789 return bdw_gmch_ctl << 20; 2790 } 2791 2792 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) 2793 { 2794 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; 2795 gmch_ctrl &= SNB_GMCH_GGMS_MASK; 2796 2797 if (gmch_ctrl) 2798 return 1 << (20 + gmch_ctrl); 2799 2800 return 0; 2801 } 2802 2803 static size_t gen6_get_stolen_size(u16 snb_gmch_ctl) 2804 { 2805 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; 2806 snb_gmch_ctl &= SNB_GMCH_GMS_MASK; 2807 return snb_gmch_ctl << 25; /* 32 MB units */ 2808 } 2809 2810 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl) 2811 { 2812 bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; 2813 bdw_gmch_ctl &= BDW_GMCH_GMS_MASK; 2814 return bdw_gmch_ctl << 25; /* 32 MB units */ 2815 } 2816 2817 static size_t chv_get_stolen_size(u16 gmch_ctrl) 2818 { 2819 gmch_ctrl >>= SNB_GMCH_GMS_SHIFT; 2820 gmch_ctrl &= SNB_GMCH_GMS_MASK; 2821 2822 /* 2823 * 0x0 to 0x10: 32MB increments starting at 0MB 2824 * 0x11 to 0x16: 4MB increments starting at 8MB 2825 * 0x17 to 0x1d: 4MB increments start at 36MB 2826 */ 2827 if (gmch_ctrl < 0x11) 2828 return gmch_ctrl << 25; 2829 else if (gmch_ctrl < 0x17) 2830 return (gmch_ctrl - 0x11 + 2) << 22; 2831 else 2832 return (gmch_ctrl - 0x17 + 9) << 22; 2833 } 2834 2835 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl) 2836 { 2837 gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; 2838 gen9_gmch_ctl &= BDW_GMCH_GMS_MASK; 2839 2840 if (gen9_gmch_ctl < 0xf0) 2841 return gen9_gmch_ctl << 25; /* 32 MB units */ 2842 else 2843 /* 4MB increments starting at 0xf0 for 4MB */ 2844 return (gen9_gmch_ctl - 0xf0 + 1) << 22; 2845 } 2846 2847 static int ggtt_probe_common(struct drm_device *dev, 2848 size_t gtt_size) 2849 { 2850 struct drm_i915_private *dev_priv = dev->dev_private; 2851 struct i915_page_scratch *scratch_page; 2852 phys_addr_t gtt_phys_addr; 2853 2854 /* For Modern GENs the PTEs and register space are split in the BAR */ 2855 gtt_phys_addr = pci_resource_start(dev->pdev, 0) + 2856 (pci_resource_len(dev->pdev, 0) / 2); 2857 2858 /* 2859 * On BXT writes larger than 64 bit to the GTT pagetable range will be 2860 * dropped. For WC mappings in general we have 64 byte burst writes 2861 * when the WC buffer is flushed, so we can't use it, but have to 2862 * resort to an uncached mapping. The WC issue is easily caught by the 2863 * readback check when writing GTT PTE entries. 2864 */ 2865 if (IS_BROXTON(dev)) 2866 dev_priv->gtt.gsm = ioremap_nocache(gtt_phys_addr, gtt_size); 2867 else 2868 dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size); 2869 if (!dev_priv->gtt.gsm) { 2870 DRM_ERROR("Failed to map the gtt page table\n"); 2871 return -ENOMEM; 2872 } 2873 2874 scratch_page = alloc_scratch_page(dev); 2875 if (IS_ERR(scratch_page)) { 2876 DRM_ERROR("Scratch setup failed\n"); 2877 /* iounmap will also get called at remove, but meh */ 2878 iounmap(dev_priv->gtt.gsm); 2879 return PTR_ERR(scratch_page); 2880 } 2881 2882 dev_priv->gtt.base.scratch_page = scratch_page; 2883 2884 return 0; 2885 } 2886 2887 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability 2888 * bits. When using advanced contexts each context stores its own PAT, but 2889 * writing this data shouldn't be harmful even in those cases. */ 2890 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv) 2891 { 2892 uint64_t pat; 2893 2894 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */ 2895 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */ 2896 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */ 2897 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */ 2898 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) | 2899 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) | 2900 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) | 2901 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3)); 2902 2903 if (!USES_PPGTT(dev_priv->dev)) 2904 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry, 2905 * so RTL will always use the value corresponding to 2906 * pat_sel = 000". 2907 * So let's disable cache for GGTT to avoid screen corruptions. 2908 * MOCS still can be used though. 2909 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work 2910 * before this patch, i.e. the same uncached + snooping access 2911 * like on gen6/7 seems to be in effect. 2912 * - So this just fixes blitter/render access. Again it looks 2913 * like it's not just uncached access, but uncached + snooping. 2914 * So we can still hold onto all our assumptions wrt cpu 2915 * clflushing on LLC machines. 2916 */ 2917 pat = GEN8_PPAT(0, GEN8_PPAT_UC); 2918 2919 /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b 2920 * write would work. */ 2921 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); 2922 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); 2923 } 2924 2925 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv) 2926 { 2927 uint64_t pat; 2928 2929 /* 2930 * Map WB on BDW to snooped on CHV. 2931 * 2932 * Only the snoop bit has meaning for CHV, the rest is 2933 * ignored. 2934 * 2935 * The hardware will never snoop for certain types of accesses: 2936 * - CPU GTT (GMADR->GGTT->no snoop->memory) 2937 * - PPGTT page tables 2938 * - some other special cycles 2939 * 2940 * As with BDW, we also need to consider the following for GT accesses: 2941 * "For GGTT, there is NO pat_sel[2:0] from the entry, 2942 * so RTL will always use the value corresponding to 2943 * pat_sel = 000". 2944 * Which means we must set the snoop bit in PAT entry 0 2945 * in order to keep the global status page working. 2946 */ 2947 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) | 2948 GEN8_PPAT(1, 0) | 2949 GEN8_PPAT(2, 0) | 2950 GEN8_PPAT(3, 0) | 2951 GEN8_PPAT(4, CHV_PPAT_SNOOP) | 2952 GEN8_PPAT(5, CHV_PPAT_SNOOP) | 2953 GEN8_PPAT(6, CHV_PPAT_SNOOP) | 2954 GEN8_PPAT(7, CHV_PPAT_SNOOP); 2955 2956 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); 2957 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); 2958 } 2959 2960 static int gen8_gmch_probe(struct drm_device *dev, 2961 u64 *gtt_total, 2962 size_t *stolen, 2963 phys_addr_t *mappable_base, 2964 u64 *mappable_end) 2965 { 2966 struct drm_i915_private *dev_priv = dev->dev_private; 2967 u64 gtt_size; 2968 u16 snb_gmch_ctl; 2969 int ret; 2970 2971 /* TODO: We're not aware of mappable constraints on gen8 yet */ 2972 *mappable_base = pci_resource_start(dev->pdev, 2); 2973 *mappable_end = pci_resource_len(dev->pdev, 2); 2974 2975 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39))) 2976 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39)); 2977 2978 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 2979 2980 if (INTEL_INFO(dev)->gen >= 9) { 2981 *stolen = gen9_get_stolen_size(snb_gmch_ctl); 2982 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); 2983 } else if (IS_CHERRYVIEW(dev)) { 2984 *stolen = chv_get_stolen_size(snb_gmch_ctl); 2985 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl); 2986 } else { 2987 *stolen = gen8_get_stolen_size(snb_gmch_ctl); 2988 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); 2989 } 2990 2991 *gtt_total = (gtt_size / sizeof(gen8_pte_t)) << PAGE_SHIFT; 2992 2993 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 2994 chv_setup_private_ppat(dev_priv); 2995 else 2996 bdw_setup_private_ppat(dev_priv); 2997 2998 ret = ggtt_probe_common(dev, gtt_size); 2999 3000 dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range; 3001 dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries; 3002 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 3003 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 3004 3005 return ret; 3006 } 3007 3008 static int gen6_gmch_probe(struct drm_device *dev, 3009 u64 *gtt_total, 3010 size_t *stolen, 3011 phys_addr_t *mappable_base, 3012 u64 *mappable_end) 3013 { 3014 struct drm_i915_private *dev_priv = dev->dev_private; 3015 unsigned int gtt_size; 3016 u16 snb_gmch_ctl; 3017 int ret; 3018 3019 *mappable_base = pci_resource_start(dev->pdev, 2); 3020 *mappable_end = pci_resource_len(dev->pdev, 2); 3021 3022 /* 64/512MB is the current min/max we actually know of, but this is just 3023 * a coarse sanity check. 3024 */ 3025 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { 3026 DRM_ERROR("Unknown GMADR size (%llx)\n", 3027 dev_priv->gtt.mappable_end); 3028 return -ENXIO; 3029 } 3030 3031 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) 3032 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); 3033 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 3034 3035 *stolen = gen6_get_stolen_size(snb_gmch_ctl); 3036 3037 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); 3038 *gtt_total = (gtt_size / sizeof(gen6_pte_t)) << PAGE_SHIFT; 3039 3040 ret = ggtt_probe_common(dev, gtt_size); 3041 3042 dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range; 3043 dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries; 3044 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 3045 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 3046 3047 return ret; 3048 } 3049 3050 static void gen6_gmch_remove(struct i915_address_space *vm) 3051 { 3052 3053 struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); 3054 3055 iounmap(gtt->gsm); 3056 free_scratch_page(vm->dev, vm->scratch_page); 3057 } 3058 3059 static int i915_gmch_probe(struct drm_device *dev, 3060 u64 *gtt_total, 3061 size_t *stolen, 3062 phys_addr_t *mappable_base, 3063 u64 *mappable_end) 3064 { 3065 struct drm_i915_private *dev_priv = dev->dev_private; 3066 int ret; 3067 3068 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL); 3069 if (!ret) { 3070 DRM_ERROR("failed to set up gmch\n"); 3071 return -EIO; 3072 } 3073 3074 intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); 3075 3076 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); 3077 dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries; 3078 dev_priv->gtt.base.clear_range = i915_ggtt_clear_range; 3079 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 3080 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 3081 3082 if (unlikely(dev_priv->gtt.do_idle_maps)) 3083 DRM_INFO("applying Ironlake quirks for intel_iommu\n"); 3084 3085 return 0; 3086 } 3087 3088 static void i915_gmch_remove(struct i915_address_space *vm) 3089 { 3090 intel_gmch_remove(); 3091 } 3092 3093 int i915_gem_gtt_init(struct drm_device *dev) 3094 { 3095 struct drm_i915_private *dev_priv = dev->dev_private; 3096 struct i915_gtt *gtt = &dev_priv->gtt; 3097 int ret; 3098 3099 if (INTEL_INFO(dev)->gen <= 5) { 3100 gtt->gtt_probe = i915_gmch_probe; 3101 gtt->base.cleanup = i915_gmch_remove; 3102 } else if (INTEL_INFO(dev)->gen < 8) { 3103 gtt->gtt_probe = gen6_gmch_probe; 3104 gtt->base.cleanup = gen6_gmch_remove; 3105 if (IS_HASWELL(dev) && dev_priv->ellc_size) 3106 gtt->base.pte_encode = iris_pte_encode; 3107 else if (IS_HASWELL(dev)) 3108 gtt->base.pte_encode = hsw_pte_encode; 3109 else if (IS_VALLEYVIEW(dev)) 3110 gtt->base.pte_encode = byt_pte_encode; 3111 else if (INTEL_INFO(dev)->gen >= 7) 3112 gtt->base.pte_encode = ivb_pte_encode; 3113 else 3114 gtt->base.pte_encode = snb_pte_encode; 3115 } else { 3116 dev_priv->gtt.gtt_probe = gen8_gmch_probe; 3117 dev_priv->gtt.base.cleanup = gen6_gmch_remove; 3118 } 3119 3120 gtt->base.dev = dev; 3121 3122 ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size, 3123 >t->mappable_base, >t->mappable_end); 3124 if (ret) 3125 return ret; 3126 3127 /* GMADR is the PCI mmio aperture into the global GTT. */ 3128 DRM_INFO("Memory usable by graphics device = %lluM\n", 3129 gtt->base.total >> 20); 3130 DRM_DEBUG_DRIVER("GMADR size = %lldM\n", gtt->mappable_end >> 20); 3131 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20); 3132 #ifdef CONFIG_INTEL_IOMMU 3133 if (intel_iommu_gfx_mapped) 3134 DRM_INFO("VT-d active for gfx access\n"); 3135 #endif 3136 /* 3137 * i915.enable_ppgtt is read-only, so do an early pass to validate the 3138 * user's requested state against the hardware/driver capabilities. We 3139 * do this now so that we can print out any log messages once rather 3140 * than every time we check intel_enable_ppgtt(). 3141 */ 3142 i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt); 3143 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt); 3144 3145 return 0; 3146 } 3147 3148 void i915_gem_restore_gtt_mappings(struct drm_device *dev) 3149 { 3150 struct drm_i915_private *dev_priv = dev->dev_private; 3151 struct drm_i915_gem_object *obj; 3152 struct i915_address_space *vm; 3153 struct i915_vma *vma; 3154 bool flush; 3155 3156 i915_check_and_clear_faults(dev); 3157 3158 /* First fill our portion of the GTT with scratch pages */ 3159 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, 3160 dev_priv->gtt.base.start, 3161 dev_priv->gtt.base.total, 3162 true); 3163 3164 /* Cache flush objects bound into GGTT and rebind them. */ 3165 vm = &dev_priv->gtt.base; 3166 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 3167 flush = false; 3168 list_for_each_entry(vma, &obj->vma_list, vma_link) { 3169 if (vma->vm != vm) 3170 continue; 3171 3172 WARN_ON(i915_vma_bind(vma, obj->cache_level, 3173 PIN_UPDATE)); 3174 3175 flush = true; 3176 } 3177 3178 if (flush) 3179 i915_gem_clflush_object(obj, obj->pin_display); 3180 } 3181 3182 if (INTEL_INFO(dev)->gen >= 8) { 3183 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 3184 chv_setup_private_ppat(dev_priv); 3185 else 3186 bdw_setup_private_ppat(dev_priv); 3187 3188 return; 3189 } 3190 3191 if (USES_PPGTT(dev)) { 3192 list_for_each_entry(vm, &dev_priv->vm_list, global_link) { 3193 /* TODO: Perhaps it shouldn't be gen6 specific */ 3194 3195 struct i915_hw_ppgtt *ppgtt = 3196 container_of(vm, struct i915_hw_ppgtt, 3197 base); 3198 3199 if (i915_is_ggtt(vm)) 3200 ppgtt = dev_priv->mm.aliasing_ppgtt; 3201 3202 gen6_write_page_range(dev_priv, &ppgtt->pd, 3203 0, ppgtt->base.total); 3204 } 3205 } 3206 3207 i915_ggtt_flush(dev_priv); 3208 } 3209 3210 static struct i915_vma * 3211 __i915_gem_vma_create(struct drm_i915_gem_object *obj, 3212 struct i915_address_space *vm, 3213 const struct i915_ggtt_view *ggtt_view) 3214 { 3215 struct i915_vma *vma; 3216 3217 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view)) 3218 return ERR_PTR(-EINVAL); 3219 3220 vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL); 3221 if (vma == NULL) 3222 return ERR_PTR(-ENOMEM); 3223 3224 INIT_LIST_HEAD(&vma->vma_link); 3225 INIT_LIST_HEAD(&vma->mm_list); 3226 INIT_LIST_HEAD(&vma->exec_list); 3227 vma->vm = vm; 3228 vma->obj = obj; 3229 3230 if (i915_is_ggtt(vm)) 3231 vma->ggtt_view = *ggtt_view; 3232 3233 list_add_tail(&vma->vma_link, &obj->vma_list); 3234 if (!i915_is_ggtt(vm)) 3235 i915_ppgtt_get(i915_vm_to_ppgtt(vm)); 3236 3237 return vma; 3238 } 3239 3240 struct i915_vma * 3241 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj, 3242 struct i915_address_space *vm) 3243 { 3244 struct i915_vma *vma; 3245 3246 vma = i915_gem_obj_to_vma(obj, vm); 3247 if (!vma) 3248 vma = __i915_gem_vma_create(obj, vm, 3249 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL); 3250 3251 return vma; 3252 } 3253 3254 struct i915_vma * 3255 i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj, 3256 const struct i915_ggtt_view *view) 3257 { 3258 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj); 3259 struct i915_vma *vma; 3260 3261 if (WARN_ON(!view)) 3262 return ERR_PTR(-EINVAL); 3263 3264 vma = i915_gem_obj_to_ggtt_view(obj, view); 3265 3266 if (IS_ERR(vma)) 3267 return vma; 3268 3269 if (!vma) 3270 vma = __i915_gem_vma_create(obj, ggtt, view); 3271 3272 return vma; 3273 3274 } 3275 3276 static struct scatterlist * 3277 rotate_pages(dma_addr_t *in, unsigned int offset, 3278 unsigned int width, unsigned int height, 3279 struct sg_table *st, struct scatterlist *sg) 3280 { 3281 unsigned int column, row; 3282 unsigned int src_idx; 3283 3284 if (!sg) { 3285 st->nents = 0; 3286 sg = st->sgl; 3287 } 3288 3289 for (column = 0; column < width; column++) { 3290 src_idx = width * (height - 1) + column; 3291 for (row = 0; row < height; row++) { 3292 st->nents++; 3293 /* We don't need the pages, but need to initialize 3294 * the entries so the sg list can be happily traversed. 3295 * The only thing we need are DMA addresses. 3296 */ 3297 sg_set_page(sg, NULL, PAGE_SIZE, 0); 3298 sg_dma_address(sg) = in[offset + src_idx]; 3299 sg_dma_len(sg) = PAGE_SIZE; 3300 sg = sg_next(sg); 3301 src_idx -= width; 3302 } 3303 } 3304 3305 return sg; 3306 } 3307 3308 static struct sg_table * 3309 intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view, 3310 struct drm_i915_gem_object *obj) 3311 { 3312 struct intel_rotation_info *rot_info = &ggtt_view->rotation_info; 3313 unsigned int size_pages = rot_info->size >> PAGE_SHIFT; 3314 unsigned int size_pages_uv; 3315 struct sg_page_iter sg_iter; 3316 unsigned long i; 3317 dma_addr_t *page_addr_list; 3318 struct sg_table *st; 3319 unsigned int uv_start_page; 3320 struct scatterlist *sg; 3321 int ret = -ENOMEM; 3322 3323 /* Allocate a temporary list of source pages for random access. */ 3324 page_addr_list = drm_malloc_ab(obj->base.size / PAGE_SIZE, 3325 sizeof(dma_addr_t)); 3326 if (!page_addr_list) 3327 return ERR_PTR(ret); 3328 3329 /* Account for UV plane with NV12. */ 3330 if (rot_info->pixel_format == DRM_FORMAT_NV12) 3331 size_pages_uv = rot_info->size_uv >> PAGE_SHIFT; 3332 else 3333 size_pages_uv = 0; 3334 3335 /* Allocate target SG list. */ 3336 st = kmalloc(sizeof(*st), GFP_KERNEL); 3337 if (!st) 3338 goto err_st_alloc; 3339 3340 ret = sg_alloc_table(st, size_pages + size_pages_uv, GFP_KERNEL); 3341 if (ret) 3342 goto err_sg_alloc; 3343 3344 /* Populate source page list from the object. */ 3345 i = 0; 3346 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { 3347 page_addr_list[i] = sg_page_iter_dma_address(&sg_iter); 3348 i++; 3349 } 3350 3351 /* Rotate the pages. */ 3352 sg = rotate_pages(page_addr_list, 0, 3353 rot_info->width_pages, rot_info->height_pages, 3354 st, NULL); 3355 3356 /* Append the UV plane if NV12. */ 3357 if (rot_info->pixel_format == DRM_FORMAT_NV12) { 3358 uv_start_page = size_pages; 3359 3360 /* Check for tile-row un-alignment. */ 3361 if (offset_in_page(rot_info->uv_offset)) 3362 uv_start_page--; 3363 3364 rot_info->uv_start_page = uv_start_page; 3365 3366 rotate_pages(page_addr_list, uv_start_page, 3367 rot_info->width_pages_uv, 3368 rot_info->height_pages_uv, 3369 st, sg); 3370 } 3371 3372 DRM_DEBUG_KMS( 3373 "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0)).\n", 3374 obj->base.size, rot_info->pitch, rot_info->height, 3375 rot_info->pixel_format, rot_info->width_pages, 3376 rot_info->height_pages, size_pages + size_pages_uv, 3377 size_pages); 3378 3379 drm_free_large(page_addr_list); 3380 3381 return st; 3382 3383 err_sg_alloc: 3384 kfree(st); 3385 err_st_alloc: 3386 drm_free_large(page_addr_list); 3387 3388 DRM_DEBUG_KMS( 3389 "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0))\n", 3390 obj->base.size, ret, rot_info->pitch, rot_info->height, 3391 rot_info->pixel_format, rot_info->width_pages, 3392 rot_info->height_pages, size_pages + size_pages_uv, 3393 size_pages); 3394 return ERR_PTR(ret); 3395 } 3396 3397 static struct sg_table * 3398 intel_partial_pages(const struct i915_ggtt_view *view, 3399 struct drm_i915_gem_object *obj) 3400 { 3401 struct sg_table *st; 3402 struct scatterlist *sg; 3403 struct sg_page_iter obj_sg_iter; 3404 int ret = -ENOMEM; 3405 3406 st = kmalloc(sizeof(*st), GFP_KERNEL); 3407 if (!st) 3408 goto err_st_alloc; 3409 3410 ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL); 3411 if (ret) 3412 goto err_sg_alloc; 3413 3414 sg = st->sgl; 3415 st->nents = 0; 3416 for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents, 3417 view->params.partial.offset) 3418 { 3419 if (st->nents >= view->params.partial.size) 3420 break; 3421 3422 sg_set_page(sg, NULL, PAGE_SIZE, 0); 3423 sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter); 3424 sg_dma_len(sg) = PAGE_SIZE; 3425 3426 sg = sg_next(sg); 3427 st->nents++; 3428 } 3429 3430 return st; 3431 3432 err_sg_alloc: 3433 kfree(st); 3434 err_st_alloc: 3435 return ERR_PTR(ret); 3436 } 3437 3438 static int 3439 i915_get_ggtt_vma_pages(struct i915_vma *vma) 3440 { 3441 int ret = 0; 3442 3443 if (vma->ggtt_view.pages) 3444 return 0; 3445 3446 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) 3447 vma->ggtt_view.pages = vma->obj->pages; 3448 else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED) 3449 vma->ggtt_view.pages = 3450 intel_rotate_fb_obj_pages(&vma->ggtt_view, vma->obj); 3451 else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL) 3452 vma->ggtt_view.pages = 3453 intel_partial_pages(&vma->ggtt_view, vma->obj); 3454 else 3455 WARN_ONCE(1, "GGTT view %u not implemented!\n", 3456 vma->ggtt_view.type); 3457 3458 if (!vma->ggtt_view.pages) { 3459 DRM_ERROR("Failed to get pages for GGTT view type %u!\n", 3460 vma->ggtt_view.type); 3461 ret = -EINVAL; 3462 } else if (IS_ERR(vma->ggtt_view.pages)) { 3463 ret = PTR_ERR(vma->ggtt_view.pages); 3464 vma->ggtt_view.pages = NULL; 3465 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n", 3466 vma->ggtt_view.type, ret); 3467 } 3468 3469 return ret; 3470 } 3471 3472 /** 3473 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. 3474 * @vma: VMA to map 3475 * @cache_level: mapping cache level 3476 * @flags: flags like global or local mapping 3477 * 3478 * DMA addresses are taken from the scatter-gather table of this object (or of 3479 * this VMA in case of non-default GGTT views) and PTE entries set up. 3480 * Note that DMA addresses are also the only part of the SG table we care about. 3481 */ 3482 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, 3483 u32 flags) 3484 { 3485 int ret; 3486 u32 bind_flags; 3487 3488 if (WARN_ON(flags == 0)) 3489 return -EINVAL; 3490 3491 bind_flags = 0; 3492 if (flags & PIN_GLOBAL) 3493 bind_flags |= GLOBAL_BIND; 3494 if (flags & PIN_USER) 3495 bind_flags |= LOCAL_BIND; 3496 3497 if (flags & PIN_UPDATE) 3498 bind_flags |= vma->bound; 3499 else 3500 bind_flags &= ~vma->bound; 3501 3502 if (bind_flags == 0) 3503 return 0; 3504 3505 if (vma->bound == 0 && vma->vm->allocate_va_range) { 3506 trace_i915_va_alloc(vma->vm, 3507 vma->node.start, 3508 vma->node.size, 3509 VM_TO_TRACE_NAME(vma->vm)); 3510 3511 /* XXX: i915_vma_pin() will fix this +- hack */ 3512 vma->pin_count++; 3513 ret = vma->vm->allocate_va_range(vma->vm, 3514 vma->node.start, 3515 vma->node.size); 3516 vma->pin_count--; 3517 if (ret) 3518 return ret; 3519 } 3520 3521 ret = vma->vm->bind_vma(vma, cache_level, bind_flags); 3522 if (ret) 3523 return ret; 3524 3525 vma->bound |= bind_flags; 3526 3527 return 0; 3528 } 3529 3530 /** 3531 * i915_ggtt_view_size - Get the size of a GGTT view. 3532 * @obj: Object the view is of. 3533 * @view: The view in question. 3534 * 3535 * @return The size of the GGTT view in bytes. 3536 */ 3537 size_t 3538 i915_ggtt_view_size(struct drm_i915_gem_object *obj, 3539 const struct i915_ggtt_view *view) 3540 { 3541 if (view->type == I915_GGTT_VIEW_NORMAL) { 3542 return obj->base.size; 3543 } else if (view->type == I915_GGTT_VIEW_ROTATED) { 3544 return view->rotation_info.size; 3545 } else if (view->type == I915_GGTT_VIEW_PARTIAL) { 3546 return view->params.partial.size << PAGE_SHIFT; 3547 } else { 3548 WARN_ONCE(1, "GGTT view %u not implemented!\n", view->type); 3549 return obj->base.size; 3550 } 3551 } 3552
Indexes created Wed Oct 29 03:09:52 GMT 2025