1 /* $NetBSD: nouveau_nvkm_subdev_mmu_base.c,v 1.8 2021/12/19 10:51:58 riastradh Exp $ */ 2 3 /* 4 * Copyright 2010 Red Hat Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Ben Skeggs 25 */ 26 #include <sys/cdefs.h> 27 __KERNEL_RCSID(0, "$NetBSD: nouveau_nvkm_subdev_mmu_base.c,v 1.8 2021/12/19 10:51:58 riastradh Exp $"); 28 29 #include "ummu.h" 30 #include "vmm.h" 31 32 #include <subdev/bar.h> 33 #include <subdev/fb.h> 34 35 #include <nvif/if500d.h> 36 #include <nvif/if900d.h> 37 38 #include <linux/nbsd-namespace.h> 39 40 struct nvkm_mmu_ptp { 41 struct nvkm_mmu_pt *pt; 42 struct list_head head; 43 u8 shift; 44 u16 mask; 45 u16 free; 46 }; 47 48 static void 49 nvkm_mmu_ptp_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt *pt) 50 { 51 const int slot = pt->base >> pt->ptp->shift; 52 struct nvkm_mmu_ptp *ptp = pt->ptp; 53 54 /* If there were no free slots in the parent allocation before, 55 * there will be now, so return PTP to the cache. 56 */ 57 if (!ptp->free) 58 list_add(&ptp->head, &mmu->ptp.list); 59 ptp->free |= BIT(slot); 60 61 /* If there's no more sub-allocations, destroy PTP. */ 62 if (ptp->free == ptp->mask) { 63 nvkm_mmu_ptc_put(mmu, force, &ptp->pt); 64 list_del(&ptp->head); 65 kfree(ptp); 66 } 67 68 kfree(pt); 69 } 70 71 static struct nvkm_mmu_pt * 72 nvkm_mmu_ptp_get(struct nvkm_mmu *mmu, u32 size, bool zero) 73 { 74 struct nvkm_mmu_pt *pt; 75 struct nvkm_mmu_ptp *ptp; 76 int slot; 77 78 if (!(pt = kzalloc(sizeof(*pt), GFP_KERNEL))) 79 return NULL; 80 81 ptp = list_first_entry_or_null(&mmu->ptp.list, typeof(*ptp), head); 82 if (!ptp) { 83 /* Need to allocate a new parent to sub-allocate from. */ 84 if (!(ptp = kmalloc(sizeof(*ptp), GFP_KERNEL))) { 85 kfree(pt); 86 return NULL; 87 } 88 89 ptp->pt = nvkm_mmu_ptc_get(mmu, 0x1000, 0x1000, false); 90 if (!ptp->pt) { 91 kfree(ptp); 92 kfree(pt); 93 return NULL; 94 } 95 96 ptp->shift = order_base_2(size); 97 slot = nvkm_memory_size(ptp->pt->memory) >> ptp->shift; 98 ptp->mask = (1 << slot) - 1; 99 ptp->free = ptp->mask; 100 list_add(&ptp->head, &mmu->ptp.list); 101 } 102 pt->ptp = ptp; 103 pt->sub = true; 104 105 /* Sub-allocate from parent object, removing PTP from cache 106 * if there's no more free slots left. 107 */ 108 slot = __ffs(ptp->free); 109 ptp->free &= ~BIT(slot); 110 if (!ptp->free) 111 list_del(&ptp->head); 112 113 pt->memory = pt->ptp->pt->memory; 114 pt->base = slot << ptp->shift; 115 pt->addr = pt->ptp->pt->addr + pt->base; 116 return pt; 117 } 118 119 struct nvkm_mmu_ptc { 120 struct list_head head; 121 struct list_head item; 122 u32 size; 123 u32 refs; 124 }; 125 126 static inline struct nvkm_mmu_ptc * 127 nvkm_mmu_ptc_find(struct nvkm_mmu *mmu, u32 size) 128 { 129 struct nvkm_mmu_ptc *ptc; 130 131 list_for_each_entry(ptc, &mmu->ptc.list, head) { 132 if (ptc->size == size) 133 return ptc; 134 } 135 136 ptc = kmalloc(sizeof(*ptc), GFP_KERNEL); 137 if (ptc) { 138 INIT_LIST_HEAD(&ptc->item); 139 ptc->size = size; 140 ptc->refs = 0; 141 list_add(&ptc->head, &mmu->ptc.list); 142 } 143 144 return ptc; 145 } 146 147 void 148 nvkm_mmu_ptc_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt **ppt) 149 { 150 struct nvkm_mmu_pt *pt = *ppt; 151 if (pt) { 152 /* Handle sub-allocated page tables. */ 153 if (pt->sub) { 154 mutex_lock(&mmu->ptp.mutex); 155 nvkm_mmu_ptp_put(mmu, force, pt); 156 mutex_unlock(&mmu->ptp.mutex); 157 return; 158 } 159 160 /* Either cache or free the object. */ 161 mutex_lock(&mmu->ptc.mutex); 162 if (pt->ptc->refs < 8 /* Heuristic. */ && !force) { 163 list_add_tail(&pt->head, &pt->ptc->item); 164 pt->ptc->refs++; 165 } else { 166 nvkm_memory_unref(&pt->memory); 167 kfree(pt); 168 } 169 mutex_unlock(&mmu->ptc.mutex); 170 } 171 } 172 173 struct nvkm_mmu_pt * 174 nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero) 175 { 176 struct nvkm_mmu_ptc *ptc; 177 struct nvkm_mmu_pt *pt; 178 int ret; 179 180 /* Sub-allocated page table (ie. GP100 LPT). */ 181 if (align < 0x1000) { 182 mutex_lock(&mmu->ptp.mutex); 183 pt = nvkm_mmu_ptp_get(mmu, align, zero); 184 mutex_unlock(&mmu->ptp.mutex); 185 return pt; 186 } 187 188 /* Lookup cache for this page table size. */ 189 mutex_lock(&mmu->ptc.mutex); 190 ptc = nvkm_mmu_ptc_find(mmu, size); 191 if (!ptc) { 192 mutex_unlock(&mmu->ptc.mutex); 193 return NULL; 194 } 195 196 /* If there's a free PT in the cache, reuse it. */ 197 pt = list_first_entry_or_null(&ptc->item, typeof(*pt), head); 198 if (pt) { 199 if (zero) 200 nvkm_fo64(pt->memory, 0, 0, size >> 3); 201 list_del(&pt->head); 202 ptc->refs--; 203 mutex_unlock(&mmu->ptc.mutex); 204 return pt; 205 } 206 mutex_unlock(&mmu->ptc.mutex); 207 208 /* No such luck, we need to allocate. */ 209 if (!(pt = kmalloc(sizeof(*pt), GFP_KERNEL))) 210 return NULL; 211 pt->ptc = ptc; 212 pt->sub = false; 213 214 ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST, 215 size, align, zero, &pt->memory); 216 if (ret) { 217 kfree(pt); 218 return NULL; 219 } 220 221 pt->base = 0; 222 pt->addr = nvkm_memory_addr(pt->memory); 223 return pt; 224 } 225 226 void 227 nvkm_mmu_ptc_dump(struct nvkm_mmu *mmu) 228 { 229 struct nvkm_mmu_ptc *ptc; 230 list_for_each_entry(ptc, &mmu->ptc.list, head) { 231 struct nvkm_mmu_pt *pt, *tt; 232 list_for_each_entry_safe(pt, tt, &ptc->item, head) { 233 nvkm_memory_unref(&pt->memory); 234 list_del(&pt->head); 235 kfree(pt); 236 } 237 } 238 } 239 240 static void 241 nvkm_mmu_ptc_fini(struct nvkm_mmu *mmu) 242 { 243 struct nvkm_mmu_ptc *ptc, *ptct; 244 245 list_for_each_entry_safe(ptc, ptct, &mmu->ptc.list, head) { 246 WARN_ON(!list_empty(&ptc->item)); 247 list_del(&ptc->head); 248 kfree(ptc); 249 } 250 251 mutex_destroy(&mmu->ptp.mutex); 252 mutex_destroy(&mmu->ptc.mutex); 253 } 254 255 static void 256 nvkm_mmu_ptc_init(struct nvkm_mmu *mmu) 257 { 258 mutex_init(&mmu->ptc.mutex); 259 INIT_LIST_HEAD(&mmu->ptc.list); 260 mutex_init(&mmu->ptp.mutex); 261 INIT_LIST_HEAD(&mmu->ptp.list); 262 } 263 264 static void 265 nvkm_mmu_type(struct nvkm_mmu *mmu, int heap, u8 type) 266 { 267 if (heap >= 0 && !WARN_ON(mmu->type_nr == ARRAY_SIZE(mmu->type))) { 268 mmu->type[mmu->type_nr].type = type | mmu->heap[heap].type; 269 mmu->type[mmu->type_nr].heap = heap; 270 mmu->type_nr++; 271 } 272 } 273 274 static int 275 nvkm_mmu_heap(struct nvkm_mmu *mmu, u8 type, u64 size) 276 { 277 if (size) { 278 if (!WARN_ON(mmu->heap_nr == ARRAY_SIZE(mmu->heap))) { 279 mmu->heap[mmu->heap_nr].type = type; 280 mmu->heap[mmu->heap_nr].size = size; 281 return mmu->heap_nr++; 282 } 283 } 284 return -EINVAL; 285 } 286 287 static void 288 nvkm_mmu_host(struct nvkm_mmu *mmu) 289 { 290 struct nvkm_device *device = mmu->subdev.device; 291 u8 type = NVKM_MEM_KIND * !!mmu->func->kind_sys; 292 int heap; 293 294 /* Non-mappable system memory. */ 295 heap = nvkm_mmu_heap(mmu, NVKM_MEM_HOST, ~0ULL); 296 nvkm_mmu_type(mmu, heap, type); 297 298 /* Non-coherent, cached, system memory. 299 * 300 * Block-linear mappings of system memory must be done through 301 * BAR1, and cannot be supported on systems where we're unable 302 * to map BAR1 with write-combining. 303 */ 304 type |= NVKM_MEM_MAPPABLE; 305 if (!device->bar || device->bar->iomap_uncached) 306 nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND); 307 else 308 nvkm_mmu_type(mmu, heap, type); 309 310 /* Coherent, cached, system memory. 311 * 312 * Unsupported on systems that aren't able to support snooped 313 * mappings, and also for block-linear mappings which must be 314 * done through BAR1. 315 */ 316 type |= NVKM_MEM_COHERENT; 317 if (device->func->cpu_coherent) 318 nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND); 319 320 /* Uncached system memory. */ 321 nvkm_mmu_type(mmu, heap, type |= NVKM_MEM_UNCACHED); 322 } 323 324 static void 325 nvkm_mmu_vram(struct nvkm_mmu *mmu) 326 { 327 struct nvkm_device *device = mmu->subdev.device; 328 struct nvkm_mm *mm = &device->fb->ram->vram; 329 const u32 sizeN = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NORMAL); 330 const u32 sizeU = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NOMAP); 331 const u32 sizeM = nvkm_mm_heap_size(mm, NVKM_RAM_MM_MIXED); 332 u8 type = NVKM_MEM_KIND * !!mmu->func->kind; 333 u8 heap = NVKM_MEM_VRAM; 334 int heapM, heapN, heapU; 335 336 /* Mixed-memory doesn't support compression or display. */ 337 heapM = nvkm_mmu_heap(mmu, heap, sizeM << NVKM_RAM_MM_SHIFT); 338 339 heap |= NVKM_MEM_COMP; 340 heap |= NVKM_MEM_DISP; 341 heapN = nvkm_mmu_heap(mmu, heap, sizeN << NVKM_RAM_MM_SHIFT); 342 heapU = nvkm_mmu_heap(mmu, heap, sizeU << NVKM_RAM_MM_SHIFT); 343 344 /* Add non-mappable VRAM types first so that they're preferred 345 * over anything else. Mixed-memory will be slower than other 346 * heaps, it's prioritised last. 347 */ 348 nvkm_mmu_type(mmu, heapU, type); 349 nvkm_mmu_type(mmu, heapN, type); 350 nvkm_mmu_type(mmu, heapM, type); 351 352 /* Add host memory types next, under the assumption that users 353 * wanting mappable memory want to use them as staging buffers 354 * or the like. 355 */ 356 nvkm_mmu_host(mmu); 357 358 /* Mappable VRAM types go last, as they're basically the worst 359 * possible type to ask for unless there's no other choice. 360 */ 361 if (device->bar) { 362 /* Write-combined BAR1 access. */ 363 type |= NVKM_MEM_MAPPABLE; 364 if (!device->bar->iomap_uncached) { 365 nvkm_mmu_type(mmu, heapN, type); 366 nvkm_mmu_type(mmu, heapM, type); 367 } 368 369 /* Uncached BAR1 access. */ 370 type |= NVKM_MEM_COHERENT; 371 type |= NVKM_MEM_UNCACHED; 372 nvkm_mmu_type(mmu, heapN, type); 373 nvkm_mmu_type(mmu, heapM, type); 374 } 375 } 376 377 static int 378 nvkm_mmu_oneinit(struct nvkm_subdev *subdev) 379 { 380 struct nvkm_mmu *mmu = nvkm_mmu(subdev); 381 382 /* Determine available memory types. */ 383 if (mmu->subdev.device->fb && mmu->subdev.device->fb->ram) 384 nvkm_mmu_vram(mmu); 385 else 386 nvkm_mmu_host(mmu); 387 388 if (mmu->func->vmm.global) { 389 int ret = nvkm_vmm_new(subdev->device, 0, 0, NULL, 0, NULL, 390 "gart", &mmu->vmm); 391 if (ret) 392 return ret; 393 } 394 395 return 0; 396 } 397 398 static int 399 nvkm_mmu_init(struct nvkm_subdev *subdev) 400 { 401 struct nvkm_mmu *mmu = nvkm_mmu(subdev); 402 if (mmu->func->init) 403 mmu->func->init(mmu); 404 return 0; 405 } 406 407 static void * 408 nvkm_mmu_dtor(struct nvkm_subdev *subdev) 409 { 410 struct nvkm_mmu *mmu = nvkm_mmu(subdev); 411 412 nvkm_vmm_unref(&mmu->vmm); 413 414 nvkm_mmu_ptc_fini(mmu); 415 return mmu; 416 } 417 418 static const struct nvkm_subdev_func 419 nvkm_mmu = { 420 .dtor = nvkm_mmu_dtor, 421 .oneinit = nvkm_mmu_oneinit, 422 .init = nvkm_mmu_init, 423 }; 424 425 void 426 nvkm_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device, 427 int index, struct nvkm_mmu *mmu) 428 { 429 nvkm_subdev_ctor(&nvkm_mmu, device, index, &mmu->subdev); 430 mmu->func = func; 431 mmu->dma_bits = func->dma_bits; 432 nvkm_mmu_ptc_init(mmu); 433 mmu->user.ctor = nvkm_ummu_new; 434 mmu->user.base = func->mmu.user; 435 } 436 437 int 438 nvkm_mmu_new_(const struct nvkm_mmu_func *func, struct nvkm_device *device, 439 int index, struct nvkm_mmu **pmmu) 440 { 441 if (!(*pmmu = kzalloc(sizeof(**pmmu), GFP_KERNEL))) 442 return -ENOMEM; 443 nvkm_mmu_ctor(func, device, index, *pmmu); 444 return 0; 445 } 446
Indexes created Tue Sep 30 11:09:46 GMT 2025