1 /* $NetBSD: vmm.h,v 1.3 2021/12/19 10:51:58 riastradh Exp $ */ 2 3 #ifndef __NVKM_VMM_H__ 4 #define __NVKM_VMM_H__ 5 #include "priv.h" 6 #include <core/memory.h> 7 enum nvkm_memory_target; 8 9 struct nvkm_vmm_pt { 10 /* Some GPUs have a mapping level with a dual page tables to 11 * support large and small pages in the same address-range. 12 * 13 * We track the state of both page tables in one place, which 14 * is why there's multiple PT pointers/refcounts here. 15 */ 16 struct nvkm_mmu_pt *pt[2]; 17 u32 refs[2]; 18 19 /* Page size handled by this PT. 20 * 21 * Tesla backend needs to know this when writinge PDEs, 22 * otherwise unnecessary. 23 */ 24 u8 page; 25 26 /* Entire page table sparse. 27 * 28 * Used to propagate sparseness to child page tables. 29 */ 30 bool sparse:1; 31 32 /* Tracking for page directories. 33 * 34 * The array is indexed by PDE, and will either point to the 35 * child page table, or indicate the PDE is marked as sparse. 36 **/ 37 #define NVKM_VMM_PDE_INVALID(pde) IS_ERR_OR_NULL(pde) 38 #define NVKM_VMM_PDE_SPARSED(pde) IS_ERR(pde) 39 #define NVKM_VMM_PDE_SPARSE ERR_PTR(-EBUSY) 40 struct nvkm_vmm_pt **pde; 41 42 /* Tracking for dual page tables. 43 * 44 * There's one entry for each LPTE, keeping track of whether 45 * there are valid SPTEs in the same address-range. 46 * 47 * This information is used to manage LPTE state transitions. 48 */ 49 #define NVKM_VMM_PTE_SPARSE 0x80 50 #define NVKM_VMM_PTE_VALID 0x40 51 #define NVKM_VMM_PTE_SPTES 0x3f 52 u8 pte[]; 53 }; 54 55 typedef void (*nvkm_vmm_pxe_func)(struct nvkm_vmm *, 56 struct nvkm_mmu_pt *, u32 ptei, u32 ptes); 57 typedef void (*nvkm_vmm_pde_func)(struct nvkm_vmm *, 58 struct nvkm_vmm_pt *, u32 pdei); 59 typedef void (*nvkm_vmm_pte_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *, 60 u32 ptei, u32 ptes, struct nvkm_vmm_map *); 61 62 struct nvkm_vmm_desc_func { 63 nvkm_vmm_pxe_func invalid; 64 nvkm_vmm_pxe_func unmap; 65 nvkm_vmm_pxe_func sparse; 66 67 nvkm_vmm_pde_func pde; 68 69 nvkm_vmm_pte_func mem; 70 nvkm_vmm_pte_func dma; 71 nvkm_vmm_pte_func sgl; 72 73 nvkm_vmm_pte_func pfn; 74 bool (*pfn_clear)(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32 ptei, u32 ptes); 75 nvkm_vmm_pxe_func pfn_unmap; 76 }; 77 78 extern const struct nvkm_vmm_desc_func gf100_vmm_pgd; 79 void gf100_vmm_pgd_pde(struct nvkm_vmm *, struct nvkm_vmm_pt *, u32); 80 extern const struct nvkm_vmm_desc_func gf100_vmm_pgt; 81 void gf100_vmm_pgt_unmap(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32); 82 void gf100_vmm_pgt_mem(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, 83 struct nvkm_vmm_map *); 84 void gf100_vmm_pgt_dma(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, 85 struct nvkm_vmm_map *); 86 void gf100_vmm_pgt_sgl(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32, 87 struct nvkm_vmm_map *); 88 89 void gk104_vmm_lpt_invalid(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32); 90 91 struct nvkm_vmm_desc { 92 enum { 93 PGD, 94 PGT, 95 SPT, 96 LPT, 97 } type; 98 u8 bits; /* VMA bits covered by PT. */ 99 u8 size; /* Bytes-per-PTE. */ 100 u32 align; /* PT address alignment. */ 101 const struct nvkm_vmm_desc_func *func; 102 }; 103 104 extern const struct nvkm_vmm_desc nv50_vmm_desc_12[]; 105 extern const struct nvkm_vmm_desc nv50_vmm_desc_16[]; 106 107 extern const struct nvkm_vmm_desc gk104_vmm_desc_16_12[]; 108 extern const struct nvkm_vmm_desc gk104_vmm_desc_16_16[]; 109 extern const struct nvkm_vmm_desc gk104_vmm_desc_17_12[]; 110 extern const struct nvkm_vmm_desc gk104_vmm_desc_17_17[]; 111 112 extern const struct nvkm_vmm_desc gm200_vmm_desc_16_12[]; 113 extern const struct nvkm_vmm_desc gm200_vmm_desc_16_16[]; 114 extern const struct nvkm_vmm_desc gm200_vmm_desc_17_12[]; 115 extern const struct nvkm_vmm_desc gm200_vmm_desc_17_17[]; 116 117 extern const struct nvkm_vmm_desc gp100_vmm_desc_12[]; 118 extern const struct nvkm_vmm_desc gp100_vmm_desc_16[]; 119 120 struct nvkm_vmm_page { 121 u8 shift; 122 const struct nvkm_vmm_desc *desc; 123 #define NVKM_VMM_PAGE_SPARSE 0x01 124 #define NVKM_VMM_PAGE_VRAM 0x02 125 #define NVKM_VMM_PAGE_HOST 0x04 126 #define NVKM_VMM_PAGE_COMP 0x08 127 #define NVKM_VMM_PAGE_Sxxx (NVKM_VMM_PAGE_SPARSE) 128 #define NVKM_VMM_PAGE_xVxx (NVKM_VMM_PAGE_VRAM) 129 #define NVKM_VMM_PAGE_SVxx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_VRAM) 130 #define NVKM_VMM_PAGE_xxHx (NVKM_VMM_PAGE_HOST) 131 #define NVKM_VMM_PAGE_SxHx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_HOST) 132 #define NVKM_VMM_PAGE_xVHx (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_HOST) 133 #define NVKM_VMM_PAGE_SVHx (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_HOST) 134 #define NVKM_VMM_PAGE_xVxC (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_COMP) 135 #define NVKM_VMM_PAGE_SVxC (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_COMP) 136 #define NVKM_VMM_PAGE_xxHC (NVKM_VMM_PAGE_xxHx | NVKM_VMM_PAGE_COMP) 137 #define NVKM_VMM_PAGE_SxHC (NVKM_VMM_PAGE_SxHx | NVKM_VMM_PAGE_COMP) 138 u8 type; 139 }; 140 141 struct nvkm_vmm_func { 142 int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst); 143 void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst); 144 145 int (*aper)(enum nvkm_memory_target); 146 int (*valid)(struct nvkm_vmm *, void *argv, u32 argc, 147 struct nvkm_vmm_map *); 148 void (*flush)(struct nvkm_vmm *, int depth); 149 150 int (*mthd)(struct nvkm_vmm *, struct nvkm_client *, 151 u32 mthd, void *argv, u32 argc); 152 153 void (*invalidate_pdb)(struct nvkm_vmm *, u64 addr); 154 155 u64 page_block; 156 const struct nvkm_vmm_page page[]; 157 }; 158 159 struct nvkm_vmm_join { 160 struct nvkm_memory *inst; 161 struct list_head head; 162 }; 163 164 int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, 165 u32 pd_header, bool managed, u64 addr, u64 size, 166 struct lock_class_key *, const char *name, 167 struct nvkm_vmm **); 168 int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *, 169 u32 pd_header, bool managed, u64 addr, u64 size, 170 struct lock_class_key *, const char *name, struct nvkm_vmm *); 171 struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr); 172 struct nvkm_vma *nvkm_vmm_node_split(struct nvkm_vmm *, struct nvkm_vma *, 173 u64 addr, u64 size); 174 int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref, 175 bool sparse, u8 page, u8 align, u64 size, 176 struct nvkm_vma **pvma); 177 void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *); 178 void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *, bool pfn); 179 void nvkm_vmm_unmap_region(struct nvkm_vmm *, struct nvkm_vma *); 180 181 #define NVKM_VMM_PFN_ADDR 0xfffffffffffff000ULL 182 #define NVKM_VMM_PFN_ADDR_SHIFT 12 183 #define NVKM_VMM_PFN_APER 0x00000000000000f0ULL 184 #define NVKM_VMM_PFN_HOST 0x0000000000000000ULL 185 #define NVKM_VMM_PFN_VRAM 0x0000000000000010ULL 186 #define NVKM_VMM_PFN_W 0x0000000000000002ULL 187 #define NVKM_VMM_PFN_V 0x0000000000000001ULL 188 #define NVKM_VMM_PFN_NONE 0x0000000000000000ULL 189 190 int nvkm_vmm_pfn_map(struct nvkm_vmm *, u8 page, u64 addr, u64 size, u64 *pfn); 191 int nvkm_vmm_pfn_unmap(struct nvkm_vmm *, u64 addr, u64 size); 192 193 struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail); 194 195 int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32, 196 bool, u64, u64, void *, u32, struct lock_class_key *, 197 const char *, struct nvkm_vmm **); 198 int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 199 200 int nv50_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 201 void nv50_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); 202 int nv50_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 203 void nv50_vmm_flush(struct nvkm_vmm *, int); 204 205 int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, 206 struct nvkm_mmu *, bool, u64, u64, void *, u32, 207 struct lock_class_key *, const char *, struct nvkm_vmm **); 208 int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); 209 int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 210 void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *); 211 int gf100_vmm_aper(enum nvkm_memory_target); 212 int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 213 void gf100_vmm_flush(struct nvkm_vmm *, int); 214 void gf100_vmm_invalidate(struct nvkm_vmm *, u32 type); 215 void gf100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr); 216 217 int gk20a_vmm_aper(enum nvkm_memory_target); 218 219 int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *, 220 struct nvkm_mmu *, bool, u64, u64, void *, u32, 221 struct lock_class_key *, const char *, struct nvkm_vmm **); 222 int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base); 223 int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 224 225 int gp100_vmm_new_(const struct nvkm_vmm_func *, 226 struct nvkm_mmu *, bool, u64, u64, void *, u32, 227 struct lock_class_key *, const char *, struct nvkm_vmm **); 228 int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 229 int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *); 230 void gp100_vmm_flush(struct nvkm_vmm *, int); 231 int gp100_vmm_mthd(struct nvkm_vmm *, struct nvkm_client *, u32, void *, u32); 232 void gp100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr); 233 234 int gv100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *); 235 236 int nv04_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 237 struct lock_class_key *, const char *, struct nvkm_vmm **); 238 int nv41_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 239 struct lock_class_key *, const char *, struct nvkm_vmm **); 240 int nv44_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 241 struct lock_class_key *, const char *, struct nvkm_vmm **); 242 int nv50_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 243 struct lock_class_key *, const char *, struct nvkm_vmm **); 244 int mcp77_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 245 struct lock_class_key *, const char *, struct nvkm_vmm **); 246 int g84_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 247 struct lock_class_key *, const char *, struct nvkm_vmm **); 248 int gf100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 249 struct lock_class_key *, const char *, struct nvkm_vmm **); 250 int gk104_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 251 struct lock_class_key *, const char *, struct nvkm_vmm **); 252 int gk20a_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 253 struct lock_class_key *, const char *, struct nvkm_vmm **); 254 int gm200_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32, 255 struct lock_class_key *, const char *, 256 struct nvkm_vmm **); 257 int gm200_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 258 struct lock_class_key *, const char *, 259 struct nvkm_vmm **); 260 int gm20b_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32, 261 struct lock_class_key *, const char *, 262 struct nvkm_vmm **); 263 int gm20b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 264 struct lock_class_key *, const char *, 265 struct nvkm_vmm **); 266 int gp100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 267 struct lock_class_key *, const char *, 268 struct nvkm_vmm **); 269 int gp10b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 270 struct lock_class_key *, const char *, 271 struct nvkm_vmm **); 272 int gv100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 273 struct lock_class_key *, const char *, 274 struct nvkm_vmm **); 275 int tu102_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32, 276 struct lock_class_key *, const char *, 277 struct nvkm_vmm **); 278 279 #define VMM_PRINT(l,v,p,f,a...) do { \ 280 struct nvkm_vmm *_vmm = (v); \ 281 if (CONFIG_NOUVEAU_DEBUG >= (l) && _vmm->debug >= (l)) { \ 282 nvkm_printk_(&_vmm->mmu->subdev, 0, p, "%s: "f"\n", \ 283 _vmm->name, ##a); \ 284 } \ 285 } while(0) 286 #define VMM_DEBUG(v,f,a...) VMM_PRINT(NV_DBG_DEBUG, (v), info, f, ##a) 287 #define VMM_TRACE(v,f,a...) VMM_PRINT(NV_DBG_TRACE, (v), info, f, ##a) 288 #define VMM_SPAM(v,f,a...) VMM_PRINT(NV_DBG_SPAM , (v), dbg, f, ##a) 289 290 #define VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,BASE,SIZE,NEXT) do { \ 291 nvkm_kmap((PT)->memory); \ 292 while (PTEN) { \ 293 u64 _ptes = ((SIZE) - MAP->off) >> MAP->page->shift; \ 294 u64 _addr = ((BASE) + MAP->off); \ 295 \ 296 if (_ptes > PTEN) { \ 297 MAP->off += PTEN << MAP->page->shift; \ 298 _ptes = PTEN; \ 299 } else { \ 300 MAP->off = 0; \ 301 NEXT; \ 302 } \ 303 \ 304 VMM_SPAM(VMM, "ITER %08x %08x PTE(s)", PTEI, (u32)_ptes); \ 305 \ 306 FILL(VMM, PT, PTEI, _ptes, MAP, _addr); \ 307 PTEI += _ptes; \ 308 PTEN -= _ptes; \ 309 }; \ 310 nvkm_done((PT)->memory); \ 311 } while(0) 312 313 #define VMM_MAP_ITER_MEM(VMM,PT,PTEI,PTEN,MAP,FILL) \ 314 VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 315 ((u64)MAP->mem->offset << NVKM_RAM_MM_SHIFT), \ 316 ((u64)MAP->mem->length << NVKM_RAM_MM_SHIFT), \ 317 (MAP->mem = MAP->mem->next)) 318 #define VMM_MAP_ITER_DMA(VMM,PT,PTEI,PTEN,MAP,FILL) \ 319 VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 320 *MAP->dma, PAGE_SIZE, MAP->dma++) 321 #define VMM_MAP_ITER_SGL(VMM,PT,PTEI,PTEN,MAP,FILL) \ 322 VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \ 323 sg_dma_address(MAP->sgl), sg_dma_len(MAP->sgl), \ 324 (MAP->sgl = sg_next(MAP->sgl))) 325 326 #define VMM_FO(m,o,d,c,b) nvkm_fo##b((m)->memory, (o), (d), (c)) 327 #define VMM_WO(m,o,d,c,b) nvkm_wo##b((m)->memory, (o), (d)) 328 #define VMM_XO(m,v,o,d,c,b,fn,f,a...) do { \ 329 const u32 _pteo = (o); u##b _data = (d); \ 330 VMM_SPAM((v), " %010"PRIx64" "f, (u64)(m)->addr + _pteo, _data, ##a); \ 331 VMM_##fn((m), (m)->base + _pteo, _data, (c), b); \ 332 } while(0) 333 334 #define VMM_WO032(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 32, WO, "%08x") 335 #define VMM_FO032(m,v,o,d,c) \ 336 VMM_XO((m),(v),(o),(d),(c), 32, FO, "%08x %08x", (c)) 337 338 #define VMM_WO064(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 64, WO, "%016"PRIx64"") 339 #define VMM_FO064(m,v,o,d,c) \ 340 VMM_XO((m),(v),(o),(d),(c), 64, FO, "%016"PRIx64" %08x", (c)) 341 342 #define VMM_XO128(m,v,o,lo,hi,c,f,a...) do { \ 343 u32 _pteo = (o), _ptes = (c); \ 344 const u64 _addr = (m)->addr + _pteo; \ 345 VMM_SPAM((v), " %010"PRIx64" %016"PRIx64"%016"PRIx64""f, _addr, (u64)(hi), (u64)(lo), ##a); \ 346 while (_ptes--) { \ 347 nvkm_wo64((m)->memory, (m)->base + _pteo + 0, (lo)); \ 348 nvkm_wo64((m)->memory, (m)->base + _pteo + 8, (hi)); \ 349 _pteo += 0x10; \ 350 } \ 351 } while(0) 352 353 #define VMM_WO128(m,v,o,lo,hi) VMM_XO128((m),(v),(o),(lo),(hi), 1, "") 354 #define VMM_FO128(m,v,o,lo,hi,c) do { \ 355 nvkm_kmap((m)->memory); \ 356 VMM_XO128((m),(v),(o),(lo),(hi),(c), " %08x", (c)); \ 357 nvkm_done((m)->memory); \ 358 } while(0) 359 #endif 360
Indexes created Sun Sep 21 14:09:52 GMT 2025