1 /* $NetBSD: nouveau_nvkm_subdev_mmu_vmmgp100.c,v 1.3 2021/12/19 10:51:58 riastradh Exp $ */ 2 3 /* 4 * Copyright 2017 Red Hat Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 */ 24 #include <sys/cdefs.h> 25 __KERNEL_RCSID(0, "$NetBSD: nouveau_nvkm_subdev_mmu_vmmgp100.c,v 1.3 2021/12/19 10:51:58 riastradh Exp $"); 26 27 #include "vmm.h" 28 29 #include <core/client.h> 30 #include <subdev/fb.h> 31 #include <subdev/ltc.h> 32 #include <subdev/timer.h> 33 #include <engine/gr.h> 34 35 #include <nvif/ifc00d.h> 36 #include <nvif/unpack.h> 37 38 static void 39 gp100_vmm_pfn_unmap(struct nvkm_vmm *vmm, 40 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) 41 { 42 struct device *dev = vmm->mmu->subdev.device->dev; 43 dma_addr_t addr; 44 45 nvkm_kmap(pt->memory); 46 while (ptes--) { 47 u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0); 48 u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4); 49 u64 data = (u64)datahi << 32 | datalo; 50 if ((data & (3ULL << 1)) != 0) { 51 addr = (data >> 8) << 12; 52 #ifdef __NetBSD__ 53 __USE(dev); 54 __USE(addr); 55 #else 56 dma_unmap_page(dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 57 #endif 58 } 59 ptei++; 60 } 61 nvkm_done(pt->memory); 62 } 63 64 static bool 65 gp100_vmm_pfn_clear(struct nvkm_vmm *vmm, 66 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) 67 { 68 bool dma = false; 69 nvkm_kmap(pt->memory); 70 while (ptes--) { 71 u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0); 72 u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4); 73 u64 data = (u64)datahi << 32 | datalo; 74 if ((data & BIT_ULL(0)) && (data & (3ULL << 1)) != 0) { 75 VMM_WO064(pt, vmm, ptei * 8, data & ~BIT_ULL(0)); 76 dma = true; 77 } 78 ptei++; 79 } 80 nvkm_done(pt->memory); 81 return dma; 82 } 83 84 static void 85 gp100_vmm_pgt_pfn(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 86 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 87 { 88 struct device *dev = vmm->mmu->subdev.device->dev; 89 dma_addr_t addr; 90 91 nvkm_kmap(pt->memory); 92 while (ptes--) { 93 u64 data = 0; 94 if (!(*map->pfn & NVKM_VMM_PFN_W)) 95 data |= BIT_ULL(6); /* RO. */ 96 97 if (!(*map->pfn & NVKM_VMM_PFN_VRAM)) { 98 #ifdef __NetBSD__ /* XXX */ 99 __USE(dev); 100 __USE(addr); 101 #else 102 addr = *map->pfn >> NVKM_VMM_PFN_ADDR_SHIFT; 103 addr = dma_map_page(dev, pfn_to_page(addr), 0, 104 PAGE_SIZE, DMA_BIDIRECTIONAL); 105 if (!WARN_ON(dma_mapping_error(dev, addr))) { 106 data |= addr >> 4; 107 data |= 2ULL << 1; /* SYSTEM_COHERENT_MEMORY. */ 108 data |= BIT_ULL(3); /* VOL. */ 109 data |= BIT_ULL(0); /* VALID. */ 110 } 111 #endif 112 } else { 113 data |= (*map->pfn & NVKM_VMM_PFN_ADDR) >> 4; 114 data |= BIT_ULL(0); /* VALID. */ 115 } 116 117 VMM_WO064(pt, vmm, ptei++ * 8, data); 118 map->pfn++; 119 } 120 nvkm_done(pt->memory); 121 } 122 123 static inline void 124 gp100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 125 u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) 126 { 127 u64 data = (addr >> 4) | map->type; 128 129 map->type += ptes * map->ctag; 130 131 while (ptes--) { 132 VMM_WO064(pt, vmm, ptei++ * 8, data); 133 data += map->next; 134 } 135 } 136 137 #ifndef __NetBSD__ 138 static void 139 gp100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 140 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 141 { 142 VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte); 143 } 144 #endif 145 146 static void 147 gp100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 148 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 149 { 150 if (map->page->shift == PAGE_SHIFT) { 151 VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes); 152 nvkm_kmap(pt->memory); 153 while (ptes--) { 154 const u64 data = (*map->dma++ >> 4) | map->type; 155 VMM_WO064(pt, vmm, ptei++ * 8, data); 156 map->type += map->ctag; 157 } 158 nvkm_done(pt->memory); 159 return; 160 } 161 162 VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte); 163 } 164 165 static void 166 gp100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 167 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 168 { 169 VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte); 170 } 171 172 static void 173 gp100_vmm_pgt_sparse(struct nvkm_vmm *vmm, 174 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) 175 { 176 /* VALID_FALSE + VOL tells the MMU to treat the PTE as sparse. */ 177 VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(3) /* VOL. */, ptes); 178 } 179 180 static const struct nvkm_vmm_desc_func 181 gp100_vmm_desc_spt = { 182 .unmap = gf100_vmm_pgt_unmap, 183 .sparse = gp100_vmm_pgt_sparse, 184 .mem = gp100_vmm_pgt_mem, 185 .dma = gp100_vmm_pgt_dma, 186 #ifndef __NetBSD__ 187 .sgl = gp100_vmm_pgt_sgl, 188 #endif 189 .pfn = gp100_vmm_pgt_pfn, 190 .pfn_clear = gp100_vmm_pfn_clear, 191 .pfn_unmap = gp100_vmm_pfn_unmap, 192 }; 193 194 static void 195 gp100_vmm_lpt_invalid(struct nvkm_vmm *vmm, 196 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) 197 { 198 /* VALID_FALSE + PRIV tells the MMU to ignore corresponding SPTEs. */ 199 VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(5) /* PRIV. */, ptes); 200 } 201 202 static const struct nvkm_vmm_desc_func 203 gp100_vmm_desc_lpt = { 204 .invalid = gp100_vmm_lpt_invalid, 205 .unmap = gf100_vmm_pgt_unmap, 206 .sparse = gp100_vmm_pgt_sparse, 207 .mem = gp100_vmm_pgt_mem, 208 }; 209 210 static inline void 211 gp100_vmm_pd0_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 212 u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) 213 { 214 u64 data = (addr >> 4) | map->type; 215 216 map->type += ptes * map->ctag; 217 218 while (ptes--) { 219 VMM_WO128(pt, vmm, ptei++ * 0x10, data, 0ULL); 220 data += map->next; 221 } 222 } 223 224 static void 225 gp100_vmm_pd0_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 226 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 227 { 228 VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gp100_vmm_pd0_pte); 229 } 230 231 static inline bool 232 gp100_vmm_pde(struct nvkm_mmu_pt *pt, u64 *data) 233 { 234 switch (nvkm_memory_target(pt->memory)) { 235 case NVKM_MEM_TARGET_VRAM: *data |= 1ULL << 1; break; 236 case NVKM_MEM_TARGET_HOST: *data |= 2ULL << 1; 237 *data |= BIT_ULL(3); /* VOL. */ 238 break; 239 case NVKM_MEM_TARGET_NCOH: *data |= 3ULL << 1; break; 240 default: 241 WARN_ON(1); 242 return false; 243 } 244 *data |= pt->addr >> 4; 245 return true; 246 } 247 248 static void 249 gp100_vmm_pd0_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei) 250 { 251 struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; 252 struct nvkm_mmu_pt *pd = pgd->pt[0]; 253 u64 data[2] = {}; 254 255 if (pgt->pt[0] && !gp100_vmm_pde(pgt->pt[0], &data[0])) 256 return; 257 if (pgt->pt[1] && !gp100_vmm_pde(pgt->pt[1], &data[1])) 258 return; 259 260 nvkm_kmap(pd->memory); 261 VMM_WO128(pd, vmm, pdei * 0x10, data[0], data[1]); 262 nvkm_done(pd->memory); 263 } 264 265 static void 266 gp100_vmm_pd0_sparse(struct nvkm_vmm *vmm, 267 struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes) 268 { 269 /* VALID_FALSE + VOL_BIG tells the MMU to treat the PDE as sparse. */ 270 VMM_FO128(pt, vmm, pdei * 0x10, BIT_ULL(3) /* VOL_BIG. */, 0ULL, pdes); 271 } 272 273 static void 274 gp100_vmm_pd0_unmap(struct nvkm_vmm *vmm, 275 struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes) 276 { 277 VMM_FO128(pt, vmm, pdei * 0x10, 0ULL, 0ULL, pdes); 278 } 279 280 static const struct nvkm_vmm_desc_func 281 gp100_vmm_desc_pd0 = { 282 .unmap = gp100_vmm_pd0_unmap, 283 .sparse = gp100_vmm_pd0_sparse, 284 .pde = gp100_vmm_pd0_pde, 285 .mem = gp100_vmm_pd0_mem, 286 }; 287 288 static void 289 gp100_vmm_pd1_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei) 290 { 291 struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; 292 struct nvkm_mmu_pt *pd = pgd->pt[0]; 293 u64 data = 0; 294 295 if (!gp100_vmm_pde(pgt->pt[0], &data)) 296 return; 297 298 nvkm_kmap(pd->memory); 299 VMM_WO064(pd, vmm, pdei * 8, data); 300 nvkm_done(pd->memory); 301 } 302 303 static const struct nvkm_vmm_desc_func 304 gp100_vmm_desc_pd1 = { 305 .unmap = gf100_vmm_pgt_unmap, 306 .sparse = gp100_vmm_pgt_sparse, 307 .pde = gp100_vmm_pd1_pde, 308 }; 309 310 const struct nvkm_vmm_desc 311 gp100_vmm_desc_16[] = { 312 { LPT, 5, 8, 0x0100, &gp100_vmm_desc_lpt }, 313 { PGD, 8, 16, 0x1000, &gp100_vmm_desc_pd0 }, 314 { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 }, 315 { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 }, 316 { PGD, 2, 8, 0x1000, &gp100_vmm_desc_pd1 }, 317 {} 318 }; 319 320 const struct nvkm_vmm_desc 321 gp100_vmm_desc_12[] = { 322 { SPT, 9, 8, 0x1000, &gp100_vmm_desc_spt }, 323 { PGD, 8, 16, 0x1000, &gp100_vmm_desc_pd0 }, 324 { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 }, 325 { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 }, 326 { PGD, 2, 8, 0x1000, &gp100_vmm_desc_pd1 }, 327 {} 328 }; 329 330 int 331 gp100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc, 332 struct nvkm_vmm_map *map) 333 { 334 const enum nvkm_memory_target target = nvkm_memory_target(map->memory); 335 const struct nvkm_vmm_page *page = map->page; 336 union { 337 struct gp100_vmm_map_vn vn; 338 struct gp100_vmm_map_v0 v0; 339 } *args = argv; 340 struct nvkm_device *device = vmm->mmu->subdev.device; 341 struct nvkm_memory *memory = map->memory; 342 u8 kind, kind_inv, priv, ro, vol; 343 int kindn, aper, ret = -ENOSYS; 344 const u8 *kindm; 345 346 map->next = (1ULL << page->shift) >> 4; 347 map->type = 0; 348 349 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { 350 vol = !!args->v0.vol; 351 ro = !!args->v0.ro; 352 priv = !!args->v0.priv; 353 kind = args->v0.kind; 354 } else 355 if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { 356 vol = target == NVKM_MEM_TARGET_HOST; 357 ro = 0; 358 priv = 0; 359 kind = 0x00; 360 } else { 361 VMM_DEBUG(vmm, "args"); 362 return ret; 363 } 364 365 aper = vmm->func->aper(target); 366 if (WARN_ON(aper < 0)) 367 return aper; 368 369 kindm = vmm->mmu->func->kind(vmm->mmu, &kindn, &kind_inv); 370 if (kind >= kindn || kindm[kind] == kind_inv) { 371 VMM_DEBUG(vmm, "kind %02x", kind); 372 return -EINVAL; 373 } 374 375 if (kindm[kind] != kind) { 376 u64 tags = nvkm_memory_size(memory) >> 16; 377 if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) { 378 VMM_DEBUG(vmm, "comp %d %02x", aper, page->type); 379 return -EINVAL; 380 } 381 382 ret = nvkm_memory_tags_get(memory, device, tags, 383 nvkm_ltc_tags_clear, 384 &map->tags); 385 if (ret) { 386 VMM_DEBUG(vmm, "comp %d", ret); 387 return ret; 388 } 389 390 if (map->tags->mn) { 391 tags = map->tags->mn->offset + (map->offset >> 16); 392 map->ctag |= ((1ULL << page->shift) >> 16) << 36; 393 map->type |= tags << 36; 394 map->next |= map->ctag; 395 } else { 396 kind = kindm[kind]; 397 } 398 } 399 400 map->type |= BIT(0); 401 map->type |= (u64)aper << 1; 402 map->type |= (u64) vol << 3; 403 map->type |= (u64)priv << 5; 404 map->type |= (u64) ro << 6; 405 map->type |= (u64)kind << 56; 406 return 0; 407 } 408 409 static int 410 gp100_vmm_fault_cancel(struct nvkm_vmm *vmm, void *argv, u32 argc) 411 { 412 struct nvkm_device *device = vmm->mmu->subdev.device; 413 union { 414 struct gp100_vmm_fault_cancel_v0 v0; 415 } *args = argv; 416 int ret = -ENOSYS; 417 u32 inst, aper; 418 419 if ((ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) 420 return ret; 421 422 /* Translate MaxwellFaultBufferA instance pointer to the same 423 * format as the NV_GR_FECS_CURRENT_CTX register. 424 */ 425 aper = (args->v0.inst >> 8) & 3; 426 args->v0.inst >>= 12; 427 args->v0.inst |= aper << 28; 428 args->v0.inst |= 0x80000000; 429 430 if (!WARN_ON(nvkm_gr_ctxsw_pause(device))) { 431 if ((inst = nvkm_gr_ctxsw_inst(device)) == args->v0.inst) { 432 gf100_vmm_invalidate(vmm, 0x0000001b 433 /* CANCEL_TARGETED. */ | 434 (args->v0.hub << 20) | 435 (args->v0.gpc << 15) | 436 (args->v0.client << 9)); 437 } 438 WARN_ON(nvkm_gr_ctxsw_resume(device)); 439 } 440 441 return 0; 442 } 443 444 static int 445 gp100_vmm_fault_replay(struct nvkm_vmm *vmm, void *argv, u32 argc) 446 { 447 union { 448 struct gp100_vmm_fault_replay_vn vn; 449 } *args = argv; 450 int ret = -ENOSYS; 451 452 if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { 453 gf100_vmm_invalidate(vmm, 0x0000000b); /* REPLAY_GLOBAL. */ 454 } 455 456 return ret; 457 } 458 459 int 460 gp100_vmm_mthd(struct nvkm_vmm *vmm, 461 struct nvkm_client *client, u32 mthd, void *argv, u32 argc) 462 { 463 if (client->super) { 464 switch (mthd) { 465 case GP100_VMM_VN_FAULT_REPLAY: 466 return gp100_vmm_fault_replay(vmm, argv, argc); 467 case GP100_VMM_VN_FAULT_CANCEL: 468 return gp100_vmm_fault_cancel(vmm, argv, argc); 469 default: 470 break; 471 } 472 } 473 return -EINVAL; 474 } 475 476 void 477 gp100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr) 478 { 479 struct nvkm_device *device = vmm->mmu->subdev.device; 480 nvkm_wr32(device, 0x100cb8, lower_32_bits(addr)); 481 nvkm_wr32(device, 0x100cec, upper_32_bits(addr)); 482 } 483 484 void 485 gp100_vmm_flush(struct nvkm_vmm *vmm, int depth) 486 { 487 u32 type = (5 /* CACHE_LEVEL_UP_TO_PDE3 */ - depth) << 24; 488 type = 0; /*XXX: need to confirm stuff works with depth enabled... */ 489 if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) 490 type |= 0x00000004; /* HUB_ONLY */ 491 type |= 0x00000001; /* PAGE_ALL */ 492 gf100_vmm_invalidate(vmm, type); 493 } 494 495 int 496 gp100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) 497 { 498 u64 base = BIT_ULL(10) /* VER2 */ | BIT_ULL(11) /* 64KiB */; 499 if (vmm->replay) { 500 base |= BIT_ULL(4); /* FAULT_REPLAY_TEX */ 501 base |= BIT_ULL(5); /* FAULT_REPLAY_GCC */ 502 } 503 return gf100_vmm_join_(vmm, inst, base); 504 } 505 506 static const struct nvkm_vmm_func 507 gp100_vmm = { 508 .join = gp100_vmm_join, 509 .part = gf100_vmm_part, 510 .aper = gf100_vmm_aper, 511 .valid = gp100_vmm_valid, 512 .flush = gp100_vmm_flush, 513 .mthd = gp100_vmm_mthd, 514 .invalidate_pdb = gp100_vmm_invalidate_pdb, 515 .page = { 516 { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, 517 { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, 518 { 29, &gp100_vmm_desc_16[2], NVKM_VMM_PAGE_Sxxx }, 519 { 21, &gp100_vmm_desc_16[1], NVKM_VMM_PAGE_SVxC }, 520 { 16, &gp100_vmm_desc_16[0], NVKM_VMM_PAGE_SVxC }, 521 { 12, &gp100_vmm_desc_12[0], NVKM_VMM_PAGE_SVHx }, 522 {} 523 } 524 }; 525 526 int 527 gp100_vmm_new_(const struct nvkm_vmm_func *func, 528 struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, 529 void *argv, u32 argc, struct lock_class_key *key, 530 const char *name, struct nvkm_vmm **pvmm) 531 { 532 union { 533 struct gp100_vmm_vn vn; 534 struct gp100_vmm_v0 v0; 535 } *args = argv; 536 int ret = -ENOSYS; 537 bool replay; 538 539 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { 540 replay = args->v0.fault_replay != 0; 541 } else 542 if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { 543 replay = false; 544 } else 545 return ret; 546 547 ret = nvkm_vmm_new_(func, mmu, 0, managed, addr, size, key, name, pvmm); 548 if (ret) 549 return ret; 550 551 (*pvmm)->replay = replay; 552 return 0; 553 } 554 555 int 556 gp100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, 557 void *argv, u32 argc, struct lock_class_key *key, 558 const char *name, struct nvkm_vmm **pvmm) 559 { 560 return gp100_vmm_new_(&gp100_vmm, mmu, managed, addr, size, 561 argv, argc, key, name, pvmm); 562 } 563
Indexes created Sat Sep 20 22:09:52 GMT 2025