1 /* $NetBSD: i915_gem_mman.c,v 1.2 2021/12/18 23:45:30 riastradh Exp $ */ 2 3 /* 4 * SPDX-License-Identifier: MIT 5 * 6 * Copyright 2016 Intel Corporation 7 */ 8 9 #include <sys/cdefs.h> 10 __KERNEL_RCSID(0, "$NetBSD: i915_gem_mman.c,v 1.2 2021/12/18 23:45:30 riastradh Exp $"); 11 12 #include <linux/prime_numbers.h> 13 14 #include "gt/intel_engine_pm.h" 15 #include "gt/intel_gt.h" 16 #include "gt/intel_gt_pm.h" 17 #include "gem/i915_gem_region.h" 18 #include "huge_gem_object.h" 19 #include "i915_selftest.h" 20 #include "selftests/i915_random.h" 21 #include "selftests/igt_flush_test.h" 22 #include "selftests/igt_mmap.h" 23 24 struct tile { 25 unsigned int width; 26 unsigned int height; 27 unsigned int stride; 28 unsigned int size; 29 unsigned int tiling; 30 unsigned int swizzle; 31 }; 32 33 static u64 swizzle_bit(unsigned int bit, u64 offset) 34 { 35 return (offset & BIT_ULL(bit)) >> (bit - 6); 36 } 37 38 static u64 tiled_offset(const struct tile *tile, u64 v) 39 { 40 u64 x, y; 41 42 if (tile->tiling == I915_TILING_NONE) 43 return v; 44 45 y = div64_u64_rem(v, tile->stride, &x); 46 v = div64_u64_rem(y, tile->height, &y) * tile->stride * tile->height; 47 48 if (tile->tiling == I915_TILING_X) { 49 v += y * tile->width; 50 v += div64_u64_rem(x, tile->width, &x) << tile->size; 51 v += x; 52 } else if (tile->width == 128) { 53 const unsigned int ytile_span = 16; 54 const unsigned int ytile_height = 512; 55 56 v += y * ytile_span; 57 v += div64_u64_rem(x, ytile_span, &x) * ytile_height; 58 v += x; 59 } else { 60 const unsigned int ytile_span = 32; 61 const unsigned int ytile_height = 256; 62 63 v += y * ytile_span; 64 v += div64_u64_rem(x, ytile_span, &x) * ytile_height; 65 v += x; 66 } 67 68 switch (tile->swizzle) { 69 case I915_BIT_6_SWIZZLE_9: 70 v ^= swizzle_bit(9, v); 71 break; 72 case I915_BIT_6_SWIZZLE_9_10: 73 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v); 74 break; 75 case I915_BIT_6_SWIZZLE_9_11: 76 v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v); 77 break; 78 case I915_BIT_6_SWIZZLE_9_10_11: 79 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v); 80 break; 81 } 82 83 return v; 84 } 85 86 static int check_partial_mapping(struct drm_i915_gem_object *obj, 87 const struct tile *tile, 88 struct rnd_state *prng) 89 { 90 const unsigned long npages = obj->base.size / PAGE_SIZE; 91 struct i915_ggtt_view view; 92 struct i915_vma *vma; 93 unsigned long page; 94 u32 __iomem *io; 95 struct page *p; 96 unsigned int n; 97 u64 offset; 98 u32 *cpu; 99 int err; 100 101 err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride); 102 if (err) { 103 pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n", 104 tile->tiling, tile->stride, err); 105 return err; 106 } 107 108 GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling); 109 GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride); 110 111 i915_gem_object_lock(obj); 112 err = i915_gem_object_set_to_gtt_domain(obj, true); 113 i915_gem_object_unlock(obj); 114 if (err) { 115 pr_err("Failed to flush to GTT write domain; err=%d\n", err); 116 return err; 117 } 118 119 page = i915_prandom_u32_max_state(npages, prng); 120 view = compute_partial_view(obj, page, MIN_CHUNK_PAGES); 121 122 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE); 123 if (IS_ERR(vma)) { 124 pr_err("Failed to pin partial view: offset=%lu; err=%d\n", 125 page, (int)PTR_ERR(vma)); 126 return PTR_ERR(vma); 127 } 128 129 n = page - view.partial.offset; 130 GEM_BUG_ON(n >= view.partial.size); 131 132 io = i915_vma_pin_iomap(vma); 133 i915_vma_unpin(vma); 134 if (IS_ERR(io)) { 135 pr_err("Failed to iomap partial view: offset=%lu; err=%d\n", 136 page, (int)PTR_ERR(io)); 137 err = PTR_ERR(io); 138 goto out; 139 } 140 141 iowrite32(page, io + n * PAGE_SIZE / sizeof(*io)); 142 i915_vma_unpin_iomap(vma); 143 144 offset = tiled_offset(tile, page << PAGE_SHIFT); 145 if (offset >= obj->base.size) 146 goto out; 147 148 intel_gt_flush_ggtt_writes(&to_i915(obj->base.dev)->gt); 149 150 p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT); 151 cpu = kmap(p) + offset_in_page(offset); 152 drm_clflush_virt_range(cpu, sizeof(*cpu)); 153 if (*cpu != (u32)page) { 154 pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n", 155 page, n, 156 view.partial.offset, 157 view.partial.size, 158 vma->size >> PAGE_SHIFT, 159 tile->tiling ? tile_row_pages(obj) : 0, 160 vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride, 161 offset >> PAGE_SHIFT, 162 (unsigned int)offset_in_page(offset), 163 offset, 164 (u32)page, *cpu); 165 err = -EINVAL; 166 } 167 *cpu = 0; 168 drm_clflush_virt_range(cpu, sizeof(*cpu)); 169 kunmap(p); 170 171 out: 172 __i915_vma_put(vma); 173 return err; 174 } 175 176 static int check_partial_mappings(struct drm_i915_gem_object *obj, 177 const struct tile *tile, 178 unsigned long end_time) 179 { 180 const unsigned int nreal = obj->scratch / PAGE_SIZE; 181 const unsigned long npages = obj->base.size / PAGE_SIZE; 182 struct i915_vma *vma; 183 unsigned long page; 184 int err; 185 186 err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride); 187 if (err) { 188 pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n", 189 tile->tiling, tile->stride, err); 190 return err; 191 } 192 193 GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling); 194 GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride); 195 196 i915_gem_object_lock(obj); 197 err = i915_gem_object_set_to_gtt_domain(obj, true); 198 i915_gem_object_unlock(obj); 199 if (err) { 200 pr_err("Failed to flush to GTT write domain; err=%d\n", err); 201 return err; 202 } 203 204 for_each_prime_number_from(page, 1, npages) { 205 struct i915_ggtt_view view = 206 compute_partial_view(obj, page, MIN_CHUNK_PAGES); 207 u32 __iomem *io; 208 struct page *p; 209 unsigned int n; 210 u64 offset; 211 u32 *cpu; 212 213 GEM_BUG_ON(view.partial.size > nreal); 214 cond_resched(); 215 216 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE); 217 if (IS_ERR(vma)) { 218 pr_err("Failed to pin partial view: offset=%lu; err=%d\n", 219 page, (int)PTR_ERR(vma)); 220 return PTR_ERR(vma); 221 } 222 223 n = page - view.partial.offset; 224 GEM_BUG_ON(n >= view.partial.size); 225 226 io = i915_vma_pin_iomap(vma); 227 i915_vma_unpin(vma); 228 if (IS_ERR(io)) { 229 pr_err("Failed to iomap partial view: offset=%lu; err=%d\n", 230 page, (int)PTR_ERR(io)); 231 return PTR_ERR(io); 232 } 233 234 iowrite32(page, io + n * PAGE_SIZE / sizeof(*io)); 235 i915_vma_unpin_iomap(vma); 236 237 offset = tiled_offset(tile, page << PAGE_SHIFT); 238 if (offset >= obj->base.size) 239 continue; 240 241 intel_gt_flush_ggtt_writes(&to_i915(obj->base.dev)->gt); 242 243 p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT); 244 cpu = kmap(p) + offset_in_page(offset); 245 drm_clflush_virt_range(cpu, sizeof(*cpu)); 246 if (*cpu != (u32)page) { 247 pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n", 248 page, n, 249 view.partial.offset, 250 view.partial.size, 251 vma->size >> PAGE_SHIFT, 252 tile->tiling ? tile_row_pages(obj) : 0, 253 vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride, 254 offset >> PAGE_SHIFT, 255 (unsigned int)offset_in_page(offset), 256 offset, 257 (u32)page, *cpu); 258 err = -EINVAL; 259 } 260 *cpu = 0; 261 drm_clflush_virt_range(cpu, sizeof(*cpu)); 262 kunmap(p); 263 if (err) 264 return err; 265 266 __i915_vma_put(vma); 267 268 if (igt_timeout(end_time, 269 "%s: timed out after tiling=%d stride=%d\n", 270 __func__, tile->tiling, tile->stride)) 271 return -EINTR; 272 } 273 274 return 0; 275 } 276 277 static unsigned int 278 setup_tile_size(struct tile *tile, struct drm_i915_private *i915) 279 { 280 if (INTEL_GEN(i915) <= 2) { 281 tile->height = 16; 282 tile->width = 128; 283 tile->size = 11; 284 } else if (tile->tiling == I915_TILING_Y && 285 HAS_128_BYTE_Y_TILING(i915)) { 286 tile->height = 32; 287 tile->width = 128; 288 tile->size = 12; 289 } else { 290 tile->height = 8; 291 tile->width = 512; 292 tile->size = 12; 293 } 294 295 if (INTEL_GEN(i915) < 4) 296 return 8192 / tile->width; 297 else if (INTEL_GEN(i915) < 7) 298 return 128 * I965_FENCE_MAX_PITCH_VAL / tile->width; 299 else 300 return 128 * GEN7_FENCE_MAX_PITCH_VAL / tile->width; 301 } 302 303 static int igt_partial_tiling(void *arg) 304 { 305 const unsigned int nreal = 1 << 12; /* largest tile row x2 */ 306 struct drm_i915_private *i915 = arg; 307 struct drm_i915_gem_object *obj; 308 intel_wakeref_t wakeref; 309 int tiling; 310 int err; 311 312 if (!i915_ggtt_has_aperture(&i915->ggtt)) 313 return 0; 314 315 /* We want to check the page mapping and fencing of a large object 316 * mmapped through the GTT. The object we create is larger than can 317 * possibly be mmaped as a whole, and so we must use partial GGTT vma. 318 * We then check that a write through each partial GGTT vma ends up 319 * in the right set of pages within the object, and with the expected 320 * tiling, which we verify by manual swizzling. 321 */ 322 323 obj = huge_gem_object(i915, 324 nreal << PAGE_SHIFT, 325 (1 + next_prime_number(i915->ggtt.vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); 326 if (IS_ERR(obj)) 327 return PTR_ERR(obj); 328 329 err = i915_gem_object_pin_pages(obj); 330 if (err) { 331 pr_err("Failed to allocate %u pages (%lu total), err=%d\n", 332 nreal, obj->base.size / PAGE_SIZE, err); 333 goto out; 334 } 335 336 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 337 338 if (1) { 339 IGT_TIMEOUT(end); 340 struct tile tile; 341 342 tile.height = 1; 343 tile.width = 1; 344 tile.size = 0; 345 tile.stride = 0; 346 tile.swizzle = I915_BIT_6_SWIZZLE_NONE; 347 tile.tiling = I915_TILING_NONE; 348 349 err = check_partial_mappings(obj, &tile, end); 350 if (err && err != -EINTR) 351 goto out_unlock; 352 } 353 354 for (tiling = I915_TILING_X; tiling <= I915_TILING_Y; tiling++) { 355 IGT_TIMEOUT(end); 356 unsigned int max_pitch; 357 unsigned int pitch; 358 struct tile tile; 359 360 if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) 361 /* 362 * The swizzling pattern is actually unknown as it 363 * varies based on physical address of each page. 364 * See i915_gem_detect_bit_6_swizzle(). 365 */ 366 break; 367 368 tile.tiling = tiling; 369 switch (tiling) { 370 case I915_TILING_X: 371 tile.swizzle = i915->ggtt.bit_6_swizzle_x; 372 break; 373 case I915_TILING_Y: 374 tile.swizzle = i915->ggtt.bit_6_swizzle_y; 375 break; 376 } 377 378 GEM_BUG_ON(tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN); 379 if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 || 380 tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17) 381 continue; 382 383 max_pitch = setup_tile_size(&tile, i915); 384 385 for (pitch = max_pitch; pitch; pitch >>= 1) { 386 tile.stride = tile.width * pitch; 387 err = check_partial_mappings(obj, &tile, end); 388 if (err == -EINTR) 389 goto next_tiling; 390 if (err) 391 goto out_unlock; 392 393 if (pitch > 2 && INTEL_GEN(i915) >= 4) { 394 tile.stride = tile.width * (pitch - 1); 395 err = check_partial_mappings(obj, &tile, end); 396 if (err == -EINTR) 397 goto next_tiling; 398 if (err) 399 goto out_unlock; 400 } 401 402 if (pitch < max_pitch && INTEL_GEN(i915) >= 4) { 403 tile.stride = tile.width * (pitch + 1); 404 err = check_partial_mappings(obj, &tile, end); 405 if (err == -EINTR) 406 goto next_tiling; 407 if (err) 408 goto out_unlock; 409 } 410 } 411 412 if (INTEL_GEN(i915) >= 4) { 413 for_each_prime_number(pitch, max_pitch) { 414 tile.stride = tile.width * pitch; 415 err = check_partial_mappings(obj, &tile, end); 416 if (err == -EINTR) 417 goto next_tiling; 418 if (err) 419 goto out_unlock; 420 } 421 } 422 423 next_tiling: ; 424 } 425 426 out_unlock: 427 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 428 i915_gem_object_unpin_pages(obj); 429 out: 430 i915_gem_object_put(obj); 431 return err; 432 } 433 434 static int igt_smoke_tiling(void *arg) 435 { 436 const unsigned int nreal = 1 << 12; /* largest tile row x2 */ 437 struct drm_i915_private *i915 = arg; 438 struct drm_i915_gem_object *obj; 439 intel_wakeref_t wakeref; 440 I915_RND_STATE(prng); 441 unsigned long count; 442 IGT_TIMEOUT(end); 443 int err; 444 445 if (!i915_ggtt_has_aperture(&i915->ggtt)) 446 return 0; 447 448 /* 449 * igt_partial_tiling() does an exhastive check of partial tiling 450 * chunking, but will undoubtably run out of time. Here, we do a 451 * randomised search and hope over many runs of 1s with different 452 * seeds we will do a thorough check. 453 * 454 * Remember to look at the st_seed if we see a flip-flop in BAT! 455 */ 456 457 if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) 458 return 0; 459 460 obj = huge_gem_object(i915, 461 nreal << PAGE_SHIFT, 462 (1 + next_prime_number(i915->ggtt.vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); 463 if (IS_ERR(obj)) 464 return PTR_ERR(obj); 465 466 err = i915_gem_object_pin_pages(obj); 467 if (err) { 468 pr_err("Failed to allocate %u pages (%lu total), err=%d\n", 469 nreal, obj->base.size / PAGE_SIZE, err); 470 goto out; 471 } 472 473 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 474 475 count = 0; 476 do { 477 struct tile tile; 478 479 tile.tiling = 480 i915_prandom_u32_max_state(I915_TILING_Y + 1, &prng); 481 switch (tile.tiling) { 482 case I915_TILING_NONE: 483 tile.height = 1; 484 tile.width = 1; 485 tile.size = 0; 486 tile.stride = 0; 487 tile.swizzle = I915_BIT_6_SWIZZLE_NONE; 488 break; 489 490 case I915_TILING_X: 491 tile.swizzle = i915->ggtt.bit_6_swizzle_x; 492 break; 493 case I915_TILING_Y: 494 tile.swizzle = i915->ggtt.bit_6_swizzle_y; 495 break; 496 } 497 498 if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 || 499 tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17) 500 continue; 501 502 if (tile.tiling != I915_TILING_NONE) { 503 unsigned int max_pitch = setup_tile_size(&tile, i915); 504 505 tile.stride = 506 i915_prandom_u32_max_state(max_pitch, &prng); 507 tile.stride = (1 + tile.stride) * tile.width; 508 if (INTEL_GEN(i915) < 4) 509 tile.stride = rounddown_pow_of_two(tile.stride); 510 } 511 512 err = check_partial_mapping(obj, &tile, &prng); 513 if (err) 514 break; 515 516 count++; 517 } while (!__igt_timeout(end, NULL)); 518 519 pr_info("%s: Completed %lu trials\n", __func__, count); 520 521 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 522 i915_gem_object_unpin_pages(obj); 523 out: 524 i915_gem_object_put(obj); 525 return err; 526 } 527 528 static int make_obj_busy(struct drm_i915_gem_object *obj) 529 { 530 struct drm_i915_private *i915 = to_i915(obj->base.dev); 531 struct intel_engine_cs *engine; 532 533 for_each_uabi_engine(engine, i915) { 534 struct i915_request *rq; 535 struct i915_vma *vma; 536 int err; 537 538 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 539 if (IS_ERR(vma)) 540 return PTR_ERR(vma); 541 542 err = i915_vma_pin(vma, 0, 0, PIN_USER); 543 if (err) 544 return err; 545 546 rq = intel_engine_create_kernel_request(engine); 547 if (IS_ERR(rq)) { 548 i915_vma_unpin(vma); 549 return PTR_ERR(rq); 550 } 551 552 i915_vma_lock(vma); 553 err = i915_request_await_object(rq, vma->obj, true); 554 if (err == 0) 555 err = i915_vma_move_to_active(vma, rq, 556 EXEC_OBJECT_WRITE); 557 i915_vma_unlock(vma); 558 559 i915_request_add(rq); 560 i915_vma_unpin(vma); 561 if (err) 562 return err; 563 } 564 565 i915_gem_object_put(obj); /* leave it only alive via its active ref */ 566 return 0; 567 } 568 569 static bool assert_mmap_offset(struct drm_i915_private *i915, 570 unsigned long size, 571 int expected) 572 { 573 struct drm_i915_gem_object *obj; 574 struct i915_mmap_offset *mmo; 575 576 obj = i915_gem_object_create_internal(i915, size); 577 if (IS_ERR(obj)) 578 return PTR_ERR(obj); 579 580 mmo = mmap_offset_attach(obj, I915_MMAP_OFFSET_GTT, NULL); 581 i915_gem_object_put(obj); 582 583 return PTR_ERR_OR_ZERO(mmo) == expected; 584 } 585 586 static void disable_retire_worker(struct drm_i915_private *i915) 587 { 588 i915_gem_driver_unregister__shrinker(i915); 589 intel_gt_pm_get(&i915->gt); 590 cancel_delayed_work_sync(&i915->gt.requests.retire_work); 591 } 592 593 static void restore_retire_worker(struct drm_i915_private *i915) 594 { 595 igt_flush_test(i915); 596 intel_gt_pm_put(&i915->gt); 597 i915_gem_driver_register__shrinker(i915); 598 } 599 600 static void mmap_offset_lock(struct drm_i915_private *i915) 601 __acquires(&i915->drm.vma_offset_manager->vm_lock) 602 { 603 write_lock(&i915->drm.vma_offset_manager->vm_lock); 604 } 605 606 static void mmap_offset_unlock(struct drm_i915_private *i915) 607 __releases(&i915->drm.vma_offset_manager->vm_lock) 608 { 609 write_unlock(&i915->drm.vma_offset_manager->vm_lock); 610 } 611 612 static int igt_mmap_offset_exhaustion(void *arg) 613 { 614 struct drm_i915_private *i915 = arg; 615 struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm; 616 struct drm_i915_gem_object *obj; 617 struct drm_mm_node *hole, *next; 618 struct i915_mmap_offset *mmo; 619 int loop, err = 0; 620 621 /* Disable background reaper */ 622 disable_retire_worker(i915); 623 GEM_BUG_ON(!i915->gt.awake); 624 intel_gt_retire_requests(&i915->gt); 625 i915_gem_drain_freed_objects(i915); 626 627 /* Trim the device mmap space to only a page */ 628 mmap_offset_lock(i915); 629 loop = 1; /* PAGE_SIZE units */ 630 list_for_each_entry_safe(hole, next, &mm->hole_stack, hole_stack) { 631 struct drm_mm_node *resv; 632 633 resv = kzalloc(sizeof(*resv), GFP_NOWAIT); 634 if (!resv) { 635 err = -ENOMEM; 636 goto out_park; 637 } 638 639 resv->start = drm_mm_hole_node_start(hole) + loop; 640 resv->size = hole->hole_size - loop; 641 resv->color = -1ul; 642 loop = 0; 643 644 if (!resv->size) { 645 kfree(resv); 646 continue; 647 } 648 649 pr_debug("Reserving hole [%llx + %llx]\n", 650 resv->start, resv->size); 651 652 err = drm_mm_reserve_node(mm, resv); 653 if (err) { 654 pr_err("Failed to trim VMA manager, err=%d\n", err); 655 kfree(resv); 656 goto out_park; 657 } 658 } 659 GEM_BUG_ON(!list_is_singular(&mm->hole_stack)); 660 mmap_offset_unlock(i915); 661 662 /* Just fits! */ 663 if (!assert_mmap_offset(i915, PAGE_SIZE, 0)) { 664 pr_err("Unable to insert object into single page hole\n"); 665 err = -EINVAL; 666 goto out; 667 } 668 669 /* Too large */ 670 if (!assert_mmap_offset(i915, 2 * PAGE_SIZE, -ENOSPC)) { 671 pr_err("Unexpectedly succeeded in inserting too large object into single page hole\n"); 672 err = -EINVAL; 673 goto out; 674 } 675 676 /* Fill the hole, further allocation attempts should then fail */ 677 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 678 if (IS_ERR(obj)) { 679 err = PTR_ERR(obj); 680 goto out; 681 } 682 683 mmo = mmap_offset_attach(obj, I915_MMAP_OFFSET_GTT, NULL); 684 if (IS_ERR(mmo)) { 685 pr_err("Unable to insert object into reclaimed hole\n"); 686 err = PTR_ERR(mmo); 687 goto err_obj; 688 } 689 690 if (!assert_mmap_offset(i915, PAGE_SIZE, -ENOSPC)) { 691 pr_err("Unexpectedly succeeded in inserting object into no holes!\n"); 692 err = -EINVAL; 693 goto err_obj; 694 } 695 696 i915_gem_object_put(obj); 697 698 /* Now fill with busy dead objects that we expect to reap */ 699 for (loop = 0; loop < 3; loop++) { 700 if (intel_gt_is_wedged(&i915->gt)) 701 break; 702 703 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 704 if (IS_ERR(obj)) { 705 err = PTR_ERR(obj); 706 goto out; 707 } 708 709 err = make_obj_busy(obj); 710 if (err) { 711 pr_err("[loop %d] Failed to busy the object\n", loop); 712 goto err_obj; 713 } 714 } 715 716 out: 717 mmap_offset_lock(i915); 718 out_park: 719 drm_mm_for_each_node_safe(hole, next, mm) { 720 if (hole->color != -1ul) 721 continue; 722 723 drm_mm_remove_node(hole); 724 kfree(hole); 725 } 726 mmap_offset_unlock(i915); 727 restore_retire_worker(i915); 728 return err; 729 err_obj: 730 i915_gem_object_put(obj); 731 goto out; 732 } 733 734 static int gtt_set(struct drm_i915_gem_object *obj) 735 { 736 struct i915_vma *vma; 737 void __iomem *map; 738 int err = 0; 739 740 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); 741 if (IS_ERR(vma)) 742 return PTR_ERR(vma); 743 744 intel_gt_pm_get(vma->vm->gt); 745 map = i915_vma_pin_iomap(vma); 746 i915_vma_unpin(vma); 747 if (IS_ERR(map)) { 748 err = PTR_ERR(map); 749 goto out; 750 } 751 752 memset_io(map, POISON_INUSE, obj->base.size); 753 i915_vma_unpin_iomap(vma); 754 755 out: 756 intel_gt_pm_put(vma->vm->gt); 757 return err; 758 } 759 760 static int gtt_check(struct drm_i915_gem_object *obj) 761 { 762 struct i915_vma *vma; 763 void __iomem *map; 764 int err = 0; 765 766 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); 767 if (IS_ERR(vma)) 768 return PTR_ERR(vma); 769 770 intel_gt_pm_get(vma->vm->gt); 771 map = i915_vma_pin_iomap(vma); 772 i915_vma_unpin(vma); 773 if (IS_ERR(map)) { 774 err = PTR_ERR(map); 775 goto out; 776 } 777 778 if (memchr_inv((void __force *)map, POISON_FREE, obj->base.size)) { 779 pr_err("%s: Write via mmap did not land in backing store (GTT)\n", 780 obj->mm.region->name); 781 err = -EINVAL; 782 } 783 i915_vma_unpin_iomap(vma); 784 785 out: 786 intel_gt_pm_put(vma->vm->gt); 787 return err; 788 } 789 790 static int wc_set(struct drm_i915_gem_object *obj) 791 { 792 void *vaddr; 793 794 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC); 795 if (IS_ERR(vaddr)) 796 return PTR_ERR(vaddr); 797 798 memset(vaddr, POISON_INUSE, obj->base.size); 799 i915_gem_object_flush_map(obj); 800 i915_gem_object_unpin_map(obj); 801 802 return 0; 803 } 804 805 static int wc_check(struct drm_i915_gem_object *obj) 806 { 807 void *vaddr; 808 int err = 0; 809 810 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC); 811 if (IS_ERR(vaddr)) 812 return PTR_ERR(vaddr); 813 814 if (memchr_inv(vaddr, POISON_FREE, obj->base.size)) { 815 pr_err("%s: Write via mmap did not land in backing store (WC)\n", 816 obj->mm.region->name); 817 err = -EINVAL; 818 } 819 i915_gem_object_unpin_map(obj); 820 821 return err; 822 } 823 824 static bool can_mmap(struct drm_i915_gem_object *obj, enum i915_mmap_type type) 825 { 826 if (type == I915_MMAP_TYPE_GTT && 827 !i915_ggtt_has_aperture(&to_i915(obj->base.dev)->ggtt)) 828 return false; 829 830 if (type != I915_MMAP_TYPE_GTT && 831 !i915_gem_object_type_has(obj, 832 I915_GEM_OBJECT_HAS_STRUCT_PAGE | 833 I915_GEM_OBJECT_HAS_IOMEM)) 834 return false; 835 836 return true; 837 } 838 839 #define expand32(x) (((x) << 0) | ((x) << 8) | ((x) << 16) | ((x) << 24)) 840 static int __igt_mmap(struct drm_i915_private *i915, 841 struct drm_i915_gem_object *obj, 842 enum i915_mmap_type type) 843 { 844 struct i915_mmap_offset *mmo; 845 struct vm_area_struct *area; 846 unsigned long addr; 847 int err, i; 848 849 if (!can_mmap(obj, type)) 850 return 0; 851 852 err = wc_set(obj); 853 if (err == -ENXIO) 854 err = gtt_set(obj); 855 if (err) 856 return err; 857 858 mmo = mmap_offset_attach(obj, type, NULL); 859 if (IS_ERR(mmo)) 860 return PTR_ERR(mmo); 861 862 addr = igt_mmap_node(i915, &mmo->vma_node, 0, PROT_WRITE, MAP_SHARED); 863 if (IS_ERR_VALUE(addr)) 864 return addr; 865 866 pr_debug("igt_mmap(%s, %d) @ %lx\n", obj->mm.region->name, type, addr); 867 868 area = find_vma(current->mm, addr); 869 if (!area) { 870 pr_err("%s: Did not create a vm_area_struct for the mmap\n", 871 obj->mm.region->name); 872 err = -EINVAL; 873 goto out_unmap; 874 } 875 876 if (area->vm_private_data != mmo) { 877 pr_err("%s: vm_area_struct did not point back to our mmap_offset object!\n", 878 obj->mm.region->name); 879 err = -EINVAL; 880 goto out_unmap; 881 } 882 883 for (i = 0; i < obj->base.size / sizeof(u32); i++) { 884 u32 __user *ux = u64_to_user_ptr((u64)(addr + i * sizeof(*ux))); 885 u32 x; 886 887 if (get_user(x, ux)) { 888 pr_err("%s: Unable to read from mmap, offset:%zd\n", 889 obj->mm.region->name, i * sizeof(x)); 890 err = -EFAULT; 891 goto out_unmap; 892 } 893 894 if (x != expand32(POISON_INUSE)) { 895 pr_err("%s: Read incorrect value from mmap, offset:%zd, found:%x, expected:%x\n", 896 obj->mm.region->name, 897 i * sizeof(x), x, expand32(POISON_INUSE)); 898 err = -EINVAL; 899 goto out_unmap; 900 } 901 902 x = expand32(POISON_FREE); 903 if (put_user(x, ux)) { 904 pr_err("%s: Unable to write to mmap, offset:%zd\n", 905 obj->mm.region->name, i * sizeof(x)); 906 err = -EFAULT; 907 goto out_unmap; 908 } 909 } 910 911 if (type == I915_MMAP_TYPE_GTT) 912 intel_gt_flush_ggtt_writes(&i915->gt); 913 914 err = wc_check(obj); 915 if (err == -ENXIO) 916 err = gtt_check(obj); 917 out_unmap: 918 vm_munmap(addr, obj->base.size); 919 return err; 920 } 921 922 static int igt_mmap(void *arg) 923 { 924 struct drm_i915_private *i915 = arg; 925 struct intel_memory_region *mr; 926 enum intel_region_id id; 927 928 for_each_memory_region(mr, i915, id) { 929 unsigned long sizes[] = { 930 PAGE_SIZE, 931 mr->min_page_size, 932 SZ_4M, 933 }; 934 int i; 935 936 for (i = 0; i < ARRAY_SIZE(sizes); i++) { 937 struct drm_i915_gem_object *obj; 938 int err; 939 940 obj = i915_gem_object_create_region(mr, sizes[i], 0); 941 if (obj == ERR_PTR(-ENODEV)) 942 continue; 943 944 if (IS_ERR(obj)) 945 return PTR_ERR(obj); 946 947 err = __igt_mmap(i915, obj, I915_MMAP_TYPE_GTT); 948 if (err == 0) 949 err = __igt_mmap(i915, obj, I915_MMAP_TYPE_WC); 950 951 i915_gem_object_put(obj); 952 if (err) 953 return err; 954 } 955 } 956 957 return 0; 958 } 959 960 static int __igt_mmap_gpu(struct drm_i915_private *i915, 961 struct drm_i915_gem_object *obj, 962 enum i915_mmap_type type) 963 { 964 struct intel_engine_cs *engine; 965 struct i915_mmap_offset *mmo; 966 unsigned long addr; 967 u32 __user *ux; 968 u32 bbe; 969 int err; 970 971 /* 972 * Verify that the mmap access into the backing store aligns with 973 * that of the GPU, i.e. that mmap is indeed writing into the same 974 * page as being read by the GPU. 975 */ 976 977 if (!can_mmap(obj, type)) 978 return 0; 979 980 err = wc_set(obj); 981 if (err == -ENXIO) 982 err = gtt_set(obj); 983 if (err) 984 return err; 985 986 mmo = mmap_offset_attach(obj, type, NULL); 987 if (IS_ERR(mmo)) 988 return PTR_ERR(mmo); 989 990 addr = igt_mmap_node(i915, &mmo->vma_node, 0, PROT_WRITE, MAP_SHARED); 991 if (IS_ERR_VALUE(addr)) 992 return addr; 993 994 ux = u64_to_user_ptr((u64)addr); 995 bbe = MI_BATCH_BUFFER_END; 996 if (put_user(bbe, ux)) { 997 pr_err("%s: Unable to write to mmap\n", obj->mm.region->name); 998 err = -EFAULT; 999 goto out_unmap; 1000 } 1001 1002 if (type == I915_MMAP_TYPE_GTT) 1003 intel_gt_flush_ggtt_writes(&i915->gt); 1004 1005 for_each_uabi_engine(engine, i915) { 1006 struct i915_request *rq; 1007 struct i915_vma *vma; 1008 1009 vma = i915_vma_instance(obj, engine->kernel_context->vm, NULL); 1010 if (IS_ERR(vma)) { 1011 err = PTR_ERR(vma); 1012 goto out_unmap; 1013 } 1014 1015 err = i915_vma_pin(vma, 0, 0, PIN_USER); 1016 if (err) 1017 goto out_unmap; 1018 1019 rq = i915_request_create(engine->kernel_context); 1020 if (IS_ERR(rq)) { 1021 err = PTR_ERR(rq); 1022 goto out_unpin; 1023 } 1024 1025 i915_vma_lock(vma); 1026 err = i915_request_await_object(rq, vma->obj, false); 1027 if (err == 0) 1028 err = i915_vma_move_to_active(vma, rq, 0); 1029 i915_vma_unlock(vma); 1030 1031 err = engine->emit_bb_start(rq, vma->node.start, 0, 0); 1032 i915_request_get(rq); 1033 i915_request_add(rq); 1034 1035 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 1036 struct drm_printer p = 1037 drm_info_printer(engine->i915->drm.dev); 1038 1039 pr_err("%s(%s, %s): Failed to execute batch\n", 1040 __func__, engine->name, obj->mm.region->name); 1041 intel_engine_dump(engine, &p, 1042 "%s\n", engine->name); 1043 1044 intel_gt_set_wedged(engine->gt); 1045 err = -EIO; 1046 } 1047 i915_request_put(rq); 1048 1049 out_unpin: 1050 i915_vma_unpin(vma); 1051 if (err) 1052 goto out_unmap; 1053 } 1054 1055 out_unmap: 1056 vm_munmap(addr, obj->base.size); 1057 return err; 1058 } 1059 1060 static int igt_mmap_gpu(void *arg) 1061 { 1062 struct drm_i915_private *i915 = arg; 1063 struct intel_memory_region *mr; 1064 enum intel_region_id id; 1065 1066 for_each_memory_region(mr, i915, id) { 1067 struct drm_i915_gem_object *obj; 1068 int err; 1069 1070 obj = i915_gem_object_create_region(mr, PAGE_SIZE, 0); 1071 if (obj == ERR_PTR(-ENODEV)) 1072 continue; 1073 1074 if (IS_ERR(obj)) 1075 return PTR_ERR(obj); 1076 1077 err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_GTT); 1078 if (err == 0) 1079 err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_WC); 1080 1081 i915_gem_object_put(obj); 1082 if (err) 1083 return err; 1084 } 1085 1086 return 0; 1087 } 1088 1089 static int check_present_pte(pte_t *pte, unsigned long addr, void *data) 1090 { 1091 if (!pte_present(*pte) || pte_none(*pte)) { 1092 pr_err("missing PTE:%lx\n", 1093 (addr - (unsigned long)data) >> PAGE_SHIFT); 1094 return -EINVAL; 1095 } 1096 1097 return 0; 1098 } 1099 1100 static int check_absent_pte(pte_t *pte, unsigned long addr, void *data) 1101 { 1102 if (pte_present(*pte) && !pte_none(*pte)) { 1103 pr_err("present PTE:%lx; expected to be revoked\n", 1104 (addr - (unsigned long)data) >> PAGE_SHIFT); 1105 return -EINVAL; 1106 } 1107 1108 return 0; 1109 } 1110 1111 static int check_present(unsigned long addr, unsigned long len) 1112 { 1113 return apply_to_page_range(current->mm, addr, len, 1114 check_present_pte, (void *)addr); 1115 } 1116 1117 static int check_absent(unsigned long addr, unsigned long len) 1118 { 1119 return apply_to_page_range(current->mm, addr, len, 1120 check_absent_pte, (void *)addr); 1121 } 1122 1123 static int prefault_range(u64 start, u64 len) 1124 { 1125 const char __user *addr, *end; 1126 char __maybe_unused c; 1127 int err; 1128 1129 addr = u64_to_user_ptr(start); 1130 end = addr + len; 1131 1132 for (; addr < end; addr += PAGE_SIZE) { 1133 err = __get_user(c, addr); 1134 if (err) 1135 return err; 1136 } 1137 1138 return __get_user(c, end - 1); 1139 } 1140 1141 static int __igt_mmap_revoke(struct drm_i915_private *i915, 1142 struct drm_i915_gem_object *obj, 1143 enum i915_mmap_type type) 1144 { 1145 struct i915_mmap_offset *mmo; 1146 unsigned long addr; 1147 int err; 1148 1149 if (!can_mmap(obj, type)) 1150 return 0; 1151 1152 mmo = mmap_offset_attach(obj, type, NULL); 1153 if (IS_ERR(mmo)) 1154 return PTR_ERR(mmo); 1155 1156 addr = igt_mmap_node(i915, &mmo->vma_node, 0, PROT_WRITE, MAP_SHARED); 1157 if (IS_ERR_VALUE(addr)) 1158 return addr; 1159 1160 err = prefault_range(addr, obj->base.size); 1161 if (err) 1162 goto out_unmap; 1163 1164 GEM_BUG_ON(mmo->mmap_type == I915_MMAP_TYPE_GTT && 1165 !atomic_read(&obj->bind_count)); 1166 1167 err = check_present(addr, obj->base.size); 1168 if (err) { 1169 pr_err("%s: was not present\n", obj->mm.region->name); 1170 goto out_unmap; 1171 } 1172 1173 /* 1174 * After unbinding the object from the GGTT, its address may be reused 1175 * for other objects. Ergo we have to revoke the previous mmap PTE 1176 * access as it no longer points to the same object. 1177 */ 1178 err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE); 1179 if (err) { 1180 pr_err("Failed to unbind object!\n"); 1181 goto out_unmap; 1182 } 1183 GEM_BUG_ON(atomic_read(&obj->bind_count)); 1184 1185 if (type != I915_MMAP_TYPE_GTT) { 1186 __i915_gem_object_put_pages(obj); 1187 if (i915_gem_object_has_pages(obj)) { 1188 pr_err("Failed to put-pages object!\n"); 1189 err = -EINVAL; 1190 goto out_unmap; 1191 } 1192 } 1193 1194 err = check_absent(addr, obj->base.size); 1195 if (err) { 1196 pr_err("%s: was not absent\n", obj->mm.region->name); 1197 goto out_unmap; 1198 } 1199 1200 out_unmap: 1201 vm_munmap(addr, obj->base.size); 1202 return err; 1203 } 1204 1205 static int igt_mmap_revoke(void *arg) 1206 { 1207 struct drm_i915_private *i915 = arg; 1208 struct intel_memory_region *mr; 1209 enum intel_region_id id; 1210 1211 for_each_memory_region(mr, i915, id) { 1212 struct drm_i915_gem_object *obj; 1213 int err; 1214 1215 obj = i915_gem_object_create_region(mr, PAGE_SIZE, 0); 1216 if (obj == ERR_PTR(-ENODEV)) 1217 continue; 1218 1219 if (IS_ERR(obj)) 1220 return PTR_ERR(obj); 1221 1222 err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_GTT); 1223 if (err == 0) 1224 err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_WC); 1225 1226 i915_gem_object_put(obj); 1227 if (err) 1228 return err; 1229 } 1230 1231 return 0; 1232 } 1233 1234 int i915_gem_mman_live_selftests(struct drm_i915_private *i915) 1235 { 1236 static const struct i915_subtest tests[] = { 1237 SUBTEST(igt_partial_tiling), 1238 SUBTEST(igt_smoke_tiling), 1239 SUBTEST(igt_mmap_offset_exhaustion), 1240 SUBTEST(igt_mmap), 1241 SUBTEST(igt_mmap_revoke), 1242 SUBTEST(igt_mmap_gpu), 1243 }; 1244 1245 return i915_subtests(tests, i915); 1246 } 1247
Indexes created Sun Sep 28 16:09:52 GMT 2025