1 1.19 riastrad /* $NetBSD: ttm_tt.c,v 1.19 2022/06/26 17:53:06 riastradh Exp $ */ 2 1.11 riastrad 3 1.13 riastrad /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 4 1.1 riastrad /************************************************************************** 5 1.1 riastrad * 6 1.1 riastrad * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 7 1.1 riastrad * All Rights Reserved. 8 1.1 riastrad * 9 1.1 riastrad * Permission is hereby granted, free of charge, to any person obtaining a 10 1.1 riastrad * copy of this software and associated documentation files (the 11 1.1 riastrad * "Software"), to deal in the Software without restriction, including 12 1.1 riastrad * without limitation the rights to use, copy, modify, merge, publish, 13 1.1 riastrad * distribute, sub license, and/or sell copies of the Software, and to 14 1.1 riastrad * permit persons to whom the Software is furnished to do so, subject to 15 1.1 riastrad * the following conditions: 16 1.1 riastrad * 17 1.1 riastrad * The above copyright notice and this permission notice (including the 18 1.1 riastrad * next paragraph) shall be included in all copies or substantial portions 19 1.1 riastrad * of the Software. 20 1.1 riastrad * 21 1.1 riastrad * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 22 1.1 riastrad * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 23 1.1 riastrad * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 24 1.1 riastrad * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 25 1.1 riastrad * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 26 1.1 riastrad * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 27 1.1 riastrad * USE OR OTHER DEALINGS IN THE SOFTWARE. 28 1.1 riastrad * 29 1.1 riastrad **************************************************************************/ 30 1.1 riastrad /* 31 1.1 riastrad * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 32 1.1 riastrad */ 33 1.1 riastrad 34 1.11 riastrad #include <sys/cdefs.h> 35 1.19 riastrad __KERNEL_RCSID(0, "$NetBSD: ttm_tt.c,v 1.19 2022/06/26 17:53:06 riastradh Exp $"); 36 1.11 riastrad 37 1.1 riastrad #define pr_fmt(fmt) "[TTM] " fmt 38 1.1 riastrad 39 1.1 riastrad #include <linux/sched.h> 40 1.1 riastrad #include <linux/pagemap.h> 41 1.1 riastrad #include <linux/shmem_fs.h> 42 1.1 riastrad #include <linux/file.h> 43 1.1 riastrad #include <drm/drm_cache.h> 44 1.15 riastrad #include <drm/drm_mem_util.h> 45 1.1 riastrad #include <drm/ttm/ttm_bo_driver.h> 46 1.1 riastrad #include <drm/ttm/ttm_page_alloc.h> 47 1.2 riastrad #include <drm/bus_dma_hacks.h> 48 1.13 riastrad #include <drm/ttm/ttm_set_memory.h> 49 1.13 riastrad 50 1.13 riastrad /** 51 1.13 riastrad * Allocates a ttm structure for the given BO. 52 1.13 riastrad */ 53 1.13 riastrad int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) 54 1.13 riastrad { 55 1.13 riastrad struct ttm_bo_device *bdev = bo->bdev; 56 1.13 riastrad uint32_t page_flags = 0; 57 1.13 riastrad 58 1.13 riastrad dma_resv_assert_held(bo->base.resv); 59 1.13 riastrad 60 1.13 riastrad if (bdev->need_dma32) 61 1.13 riastrad page_flags |= TTM_PAGE_FLAG_DMA32; 62 1.13 riastrad 63 1.13 riastrad if (bdev->no_retry) 64 1.13 riastrad page_flags |= TTM_PAGE_FLAG_NO_RETRY; 65 1.13 riastrad 66 1.13 riastrad switch (bo->type) { 67 1.13 riastrad case ttm_bo_type_device: 68 1.13 riastrad if (zero_alloc) 69 1.13 riastrad page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 70 1.13 riastrad break; 71 1.13 riastrad case ttm_bo_type_kernel: 72 1.13 riastrad break; 73 1.13 riastrad case ttm_bo_type_sg: 74 1.13 riastrad page_flags |= TTM_PAGE_FLAG_SG; 75 1.13 riastrad break; 76 1.13 riastrad default: 77 1.13 riastrad bo->ttm = NULL; 78 1.13 riastrad pr_err("Illegal buffer object type\n"); 79 1.13 riastrad return -EINVAL; 80 1.13 riastrad } 81 1.13 riastrad 82 1.13 riastrad bo->ttm = bdev->driver->ttm_tt_create(bo, page_flags); 83 1.13 riastrad if (unlikely(bo->ttm == NULL)) 84 1.13 riastrad return -ENOMEM; 85 1.13 riastrad 86 1.13 riastrad return 0; 87 1.13 riastrad } 88 1.1 riastrad 89 1.1 riastrad /** 90 1.1 riastrad * Allocates storage for pointers to the pages that back the ttm. 91 1.1 riastrad */ 92 1.13 riastrad static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm) 93 1.1 riastrad { 94 1.13 riastrad ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*), 95 1.13 riastrad GFP_KERNEL | __GFP_ZERO); 96 1.13 riastrad if (!ttm->pages) 97 1.13 riastrad return -ENOMEM; 98 1.13 riastrad return 0; 99 1.1 riastrad } 100 1.1 riastrad 101 1.18 riastrad static int ttm_sg_tt_alloc_page_directory(struct ttm_dma_tt *); 102 1.18 riastrad 103 1.13 riastrad static int ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) 104 1.1 riastrad { 105 1.18 riastrad #ifdef __NetBSD__ 106 1.18 riastrad int r; 107 1.18 riastrad 108 1.18 riastrad /* Create array of pages at ttm->ttm.pages. */ 109 1.18 riastrad r = ttm_tt_alloc_page_directory(&ttm->ttm); 110 1.18 riastrad if (r) 111 1.18 riastrad return r; 112 1.18 riastrad 113 1.18 riastrad /* Create bus DMA map at ttm->dma_address. */ 114 1.18 riastrad r = ttm_sg_tt_alloc_page_directory(ttm); 115 1.19 riastrad if (r) { 116 1.19 riastrad kvfree(ttm->ttm.pages); 117 1.19 riastrad ttm->ttm.pages = NULL; 118 1.18 riastrad return r; 119 1.19 riastrad } 120 1.18 riastrad 121 1.18 riastrad /* Success! */ 122 1.18 riastrad return 0; 123 1.18 riastrad #else 124 1.13 riastrad ttm->ttm.pages = kvmalloc_array(ttm->ttm.num_pages, 125 1.11 riastrad sizeof(*ttm->ttm.pages) + 126 1.13 riastrad sizeof(*ttm->dma_address), 127 1.13 riastrad GFP_KERNEL | __GFP_ZERO); 128 1.13 riastrad if (!ttm->ttm.pages) 129 1.13 riastrad return -ENOMEM; 130 1.13 riastrad ttm->dma_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages); 131 1.13 riastrad return 0; 132 1.18 riastrad #endif 133 1.13 riastrad } 134 1.13 riastrad 135 1.13 riastrad static int ttm_sg_tt_alloc_page_directory(struct ttm_dma_tt *ttm) 136 1.13 riastrad { 137 1.18 riastrad #ifdef __NetBSD__ 138 1.18 riastrad ttm->dma_address = NULL; 139 1.18 riastrad /* XXX errno NetBSD->Linux */ 140 1.18 riastrad return -bus_dmamap_create(ttm->ttm.bdev->dmat, 141 1.18 riastrad ttm->ttm.num_pages << PAGE_SHIFT, ttm->ttm.num_pages, PAGE_SIZE, 0, 142 1.18 riastrad BUS_DMA_WAITOK, &ttm->dma_address); 143 1.18 riastrad #else 144 1.13 riastrad ttm->dma_address = kvmalloc_array(ttm->ttm.num_pages, 145 1.13 riastrad sizeof(*ttm->dma_address), 146 1.13 riastrad GFP_KERNEL | __GFP_ZERO); 147 1.13 riastrad if (!ttm->dma_address) 148 1.13 riastrad return -ENOMEM; 149 1.13 riastrad return 0; 150 1.18 riastrad #endif 151 1.1 riastrad } 152 1.1 riastrad 153 1.13 riastrad static int ttm_tt_set_page_caching(struct page *p, 154 1.13 riastrad enum ttm_caching_state c_old, 155 1.13 riastrad enum ttm_caching_state c_new) 156 1.1 riastrad { 157 1.2 riastrad #ifdef __NetBSD__ 158 1.2 riastrad return 0; 159 1.2 riastrad #else 160 1.1 riastrad int ret = 0; 161 1.1 riastrad 162 1.1 riastrad if (PageHighMem(p)) 163 1.1 riastrad return 0; 164 1.1 riastrad 165 1.1 riastrad if (c_old != tt_cached) { 166 1.1 riastrad /* p isn't in the default caching state, set it to 167 1.1 riastrad * writeback first to free its current memtype. */ 168 1.1 riastrad 169 1.13 riastrad ret = ttm_set_pages_wb(p, 1); 170 1.1 riastrad if (ret) 171 1.1 riastrad return ret; 172 1.1 riastrad } 173 1.1 riastrad 174 1.1 riastrad if (c_new == tt_wc) 175 1.13 riastrad ret = ttm_set_pages_wc(p, 1); 176 1.1 riastrad else if (c_new == tt_uncached) 177 1.13 riastrad ret = ttm_set_pages_uc(p, 1); 178 1.1 riastrad 179 1.1 riastrad return ret; 180 1.2 riastrad #endif 181 1.1 riastrad } 182 1.1 riastrad 183 1.1 riastrad /* 184 1.1 riastrad * Change caching policy for the linear kernel map 185 1.1 riastrad * for range of pages in a ttm. 186 1.1 riastrad */ 187 1.1 riastrad 188 1.1 riastrad static int ttm_tt_set_caching(struct ttm_tt *ttm, 189 1.1 riastrad enum ttm_caching_state c_state) 190 1.1 riastrad { 191 1.1 riastrad int i, j; 192 1.1 riastrad struct page *cur_page; 193 1.1 riastrad int ret; 194 1.1 riastrad 195 1.1 riastrad if (ttm->caching_state == c_state) 196 1.1 riastrad return 0; 197 1.1 riastrad 198 1.1 riastrad if (ttm->state == tt_unpopulated) { 199 1.1 riastrad /* Change caching but don't populate */ 200 1.1 riastrad ttm->caching_state = c_state; 201 1.1 riastrad return 0; 202 1.1 riastrad } 203 1.1 riastrad 204 1.1 riastrad if (ttm->caching_state == tt_cached) 205 1.1 riastrad drm_clflush_pages(ttm->pages, ttm->num_pages); 206 1.1 riastrad 207 1.1 riastrad for (i = 0; i < ttm->num_pages; ++i) { 208 1.1 riastrad cur_page = ttm->pages[i]; 209 1.1 riastrad if (likely(cur_page != NULL)) { 210 1.1 riastrad ret = ttm_tt_set_page_caching(cur_page, 211 1.1 riastrad ttm->caching_state, 212 1.1 riastrad c_state); 213 1.1 riastrad if (unlikely(ret != 0)) 214 1.1 riastrad goto out_err; 215 1.1 riastrad } 216 1.1 riastrad } 217 1.1 riastrad 218 1.1 riastrad ttm->caching_state = c_state; 219 1.1 riastrad 220 1.1 riastrad return 0; 221 1.1 riastrad 222 1.1 riastrad out_err: 223 1.1 riastrad for (j = 0; j < i; ++j) { 224 1.1 riastrad cur_page = ttm->pages[j]; 225 1.1 riastrad if (likely(cur_page != NULL)) { 226 1.1 riastrad (void)ttm_tt_set_page_caching(cur_page, c_state, 227 1.1 riastrad ttm->caching_state); 228 1.1 riastrad } 229 1.1 riastrad } 230 1.1 riastrad 231 1.1 riastrad return ret; 232 1.1 riastrad } 233 1.1 riastrad 234 1.1 riastrad int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) 235 1.1 riastrad { 236 1.1 riastrad enum ttm_caching_state state; 237 1.1 riastrad 238 1.1 riastrad if (placement & TTM_PL_FLAG_WC) 239 1.1 riastrad state = tt_wc; 240 1.1 riastrad else if (placement & TTM_PL_FLAG_UNCACHED) 241 1.1 riastrad state = tt_uncached; 242 1.1 riastrad else 243 1.1 riastrad state = tt_cached; 244 1.1 riastrad 245 1.1 riastrad return ttm_tt_set_caching(ttm, state); 246 1.1 riastrad } 247 1.1 riastrad EXPORT_SYMBOL(ttm_tt_set_placement_caching); 248 1.1 riastrad 249 1.1 riastrad void ttm_tt_destroy(struct ttm_tt *ttm) 250 1.1 riastrad { 251 1.13 riastrad if (ttm == NULL) 252 1.1 riastrad return; 253 1.1 riastrad 254 1.13 riastrad ttm_tt_unbind(ttm); 255 1.1 riastrad 256 1.2 riastrad if (ttm->state == tt_unbound) 257 1.2 riastrad ttm_tt_unpopulate(ttm); 258 1.1 riastrad 259 1.4 riastrad #ifndef __NetBSD__ 260 1.1 riastrad if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && 261 1.1 riastrad ttm->swap_storage) 262 1.1 riastrad fput(ttm->swap_storage); 263 1.1 riastrad 264 1.1 riastrad ttm->swap_storage = NULL; 265 1.6 riastrad #endif 266 1.1 riastrad ttm->func->destroy(ttm); 267 1.1 riastrad } 268 1.1 riastrad 269 1.13 riastrad static void ttm_tt_init_fields(struct ttm_tt *ttm, 270 1.13 riastrad struct ttm_buffer_object *bo, 271 1.13 riastrad uint32_t page_flags) 272 1.13 riastrad { 273 1.13 riastrad ttm->bdev = bo->bdev; 274 1.13 riastrad ttm->num_pages = bo->num_pages; 275 1.1 riastrad ttm->caching_state = tt_cached; 276 1.1 riastrad ttm->page_flags = page_flags; 277 1.1 riastrad ttm->state = tt_unpopulated; 278 1.2 riastrad #ifdef __NetBSD__ 279 1.14 riastrad WARN(bo->num_pages == 0, 280 1.14 riastrad "zero-size allocation in %s, please file a NetBSD PR", 281 1.8 riastrad __func__); /* paranoia -- can't prove in five minutes */ 282 1.16 riastrad ttm->swap_storage = uao_create(PAGE_SIZE * MAX(1, bo->num_pages), 0); 283 1.14 riastrad uao_set_pgfl(ttm->swap_storage, bus_dmamem_pgfl(ttm->bdev->dmat)); 284 1.2 riastrad #else 285 1.1 riastrad ttm->swap_storage = NULL; 286 1.2 riastrad #endif 287 1.13 riastrad ttm->sg = bo->sg; 288 1.13 riastrad } 289 1.13 riastrad 290 1.13 riastrad int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, 291 1.13 riastrad uint32_t page_flags) 292 1.13 riastrad { 293 1.13 riastrad ttm_tt_init_fields(ttm, bo, page_flags); 294 1.1 riastrad 295 1.13 riastrad if (ttm_tt_alloc_page_directory(ttm)) { 296 1.1 riastrad ttm_tt_destroy(ttm); 297 1.1 riastrad pr_err("Failed allocating page table\n"); 298 1.1 riastrad return -ENOMEM; 299 1.1 riastrad } 300 1.1 riastrad return 0; 301 1.1 riastrad } 302 1.1 riastrad EXPORT_SYMBOL(ttm_tt_init); 303 1.1 riastrad 304 1.1 riastrad void ttm_tt_fini(struct ttm_tt *ttm) 305 1.1 riastrad { 306 1.13 riastrad kvfree(ttm->pages); 307 1.13 riastrad ttm->pages = NULL; 308 1.4 riastrad #ifdef __NetBSD__ 309 1.2 riastrad uao_detach(ttm->swap_storage); 310 1.2 riastrad ttm->swap_storage = NULL; 311 1.4 riastrad #endif 312 1.1 riastrad } 313 1.1 riastrad EXPORT_SYMBOL(ttm_tt_fini); 314 1.1 riastrad 315 1.13 riastrad int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_buffer_object *bo, 316 1.13 riastrad uint32_t page_flags) 317 1.1 riastrad { 318 1.1 riastrad struct ttm_tt *ttm = &ttm_dma->ttm; 319 1.1 riastrad 320 1.13 riastrad ttm_tt_init_fields(ttm, bo, page_flags); 321 1.1 riastrad 322 1.1 riastrad INIT_LIST_HEAD(&ttm_dma->pages_list); 323 1.13 riastrad if (ttm_dma_tt_alloc_page_directory(ttm_dma)) { 324 1.13 riastrad ttm_tt_destroy(ttm); 325 1.13 riastrad pr_err("Failed allocating page table\n"); 326 1.13 riastrad return -ENOMEM; 327 1.13 riastrad } 328 1.2 riastrad return 0; 329 1.13 riastrad } 330 1.13 riastrad EXPORT_SYMBOL(ttm_dma_tt_init); 331 1.13 riastrad 332 1.13 riastrad int ttm_sg_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_buffer_object *bo, 333 1.13 riastrad uint32_t page_flags) 334 1.13 riastrad { 335 1.13 riastrad struct ttm_tt *ttm = &ttm_dma->ttm; 336 1.13 riastrad int ret; 337 1.13 riastrad 338 1.13 riastrad ttm_tt_init_fields(ttm, bo, page_flags); 339 1.13 riastrad 340 1.13 riastrad INIT_LIST_HEAD(&ttm_dma->pages_list); 341 1.13 riastrad if (page_flags & TTM_PAGE_FLAG_SG) 342 1.13 riastrad ret = ttm_sg_tt_alloc_page_directory(ttm_dma); 343 1.13 riastrad else 344 1.13 riastrad ret = ttm_dma_tt_alloc_page_directory(ttm_dma); 345 1.13 riastrad if (ret) { 346 1.1 riastrad ttm_tt_destroy(ttm); 347 1.1 riastrad pr_err("Failed allocating page table\n"); 348 1.1 riastrad return -ENOMEM; 349 1.1 riastrad } 350 1.1 riastrad return 0; 351 1.1 riastrad } 352 1.13 riastrad EXPORT_SYMBOL(ttm_sg_tt_init); 353 1.1 riastrad 354 1.1 riastrad void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) 355 1.1 riastrad { 356 1.1 riastrad struct ttm_tt *ttm = &ttm_dma->ttm; 357 1.1 riastrad 358 1.4 riastrad #ifdef __NetBSD__ 359 1.18 riastrad if (ttm_dma->dma_address) { 360 1.18 riastrad bus_dmamap_destroy(ttm->bdev->dmat, ttm_dma->dma_address); 361 1.18 riastrad ttm_dma->dma_address = NULL; 362 1.18 riastrad } 363 1.18 riastrad ttm_tt_fini(ttm); 364 1.18 riastrad #else 365 1.13 riastrad if (ttm->pages) 366 1.13 riastrad kvfree(ttm->pages); 367 1.13 riastrad else 368 1.13 riastrad kvfree(ttm_dma->dma_address); 369 1.13 riastrad ttm->pages = NULL; 370 1.18 riastrad ttm_dma->dma_address = NULL; 371 1.4 riastrad #endif 372 1.1 riastrad } 373 1.1 riastrad EXPORT_SYMBOL(ttm_dma_tt_fini); 374 1.1 riastrad 375 1.1 riastrad void ttm_tt_unbind(struct ttm_tt *ttm) 376 1.1 riastrad { 377 1.5 rjs int ret __diagused; 378 1.1 riastrad 379 1.1 riastrad if (ttm->state == tt_bound) { 380 1.1 riastrad ret = ttm->func->unbind(ttm); 381 1.1 riastrad BUG_ON(ret); 382 1.1 riastrad ttm->state = tt_unbound; 383 1.1 riastrad } 384 1.1 riastrad } 385 1.1 riastrad 386 1.13 riastrad int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem, 387 1.13 riastrad struct ttm_operation_ctx *ctx) 388 1.1 riastrad { 389 1.1 riastrad int ret = 0; 390 1.1 riastrad 391 1.1 riastrad if (!ttm) 392 1.1 riastrad return -EINVAL; 393 1.1 riastrad 394 1.1 riastrad if (ttm->state == tt_bound) 395 1.1 riastrad return 0; 396 1.1 riastrad 397 1.13 riastrad ret = ttm_tt_populate(ttm, ctx); 398 1.1 riastrad if (ret) 399 1.1 riastrad return ret; 400 1.1 riastrad 401 1.1 riastrad ret = ttm->func->bind(ttm, bo_mem); 402 1.1 riastrad if (unlikely(ret != 0)) 403 1.1 riastrad return ret; 404 1.1 riastrad 405 1.1 riastrad ttm->state = tt_bound; 406 1.1 riastrad 407 1.1 riastrad return 0; 408 1.1 riastrad } 409 1.1 riastrad EXPORT_SYMBOL(ttm_tt_bind); 410 1.1 riastrad 411 1.7 riastrad #ifdef __NetBSD__ 412 1.7 riastrad /* 413 1.7 riastrad * ttm_tt_wire(ttm) 414 1.7 riastrad * 415 1.7 riastrad * Wire the uvm pages of ttm and fill the ttm page array. ttm 416 1.9 maya * must be unpopulated, and must be marked swapped. This does not 417 1.9 maya * change either state -- the caller is expected to include it 418 1.9 maya * among other operations for such a state transition. 419 1.7 riastrad */ 420 1.7 riastrad int 421 1.7 riastrad ttm_tt_wire(struct ttm_tt *ttm) 422 1.1 riastrad { 423 1.2 riastrad struct uvm_object *uobj = ttm->swap_storage; 424 1.17 riastrad struct vm_page *vm_page; 425 1.2 riastrad unsigned i; 426 1.2 riastrad int error; 427 1.2 riastrad 428 1.9 maya KASSERTMSG((ttm->state == tt_unpopulated), 429 1.9 maya "ttm_tt %p must be unpopulated for wiring, but state=%d", 430 1.7 riastrad ttm, (int)ttm->state); 431 1.7 riastrad KASSERT(ISSET(ttm->page_flags, TTM_PAGE_FLAG_SWAPPED)); 432 1.2 riastrad KASSERT(uobj != NULL); 433 1.7 riastrad 434 1.2 riastrad error = uvm_obj_wirepages(uobj, 0, (ttm->num_pages << PAGE_SHIFT), 435 1.17 riastrad NULL); 436 1.2 riastrad if (error) 437 1.2 riastrad /* XXX errno NetBSD->Linux */ 438 1.2 riastrad return -error; 439 1.2 riastrad 440 1.17 riastrad rw_enter(uobj->vmobjlock, RW_READER); 441 1.17 riastrad for (i = 0; i < ttm->num_pages; i++) { 442 1.17 riastrad vm_page = uvm_pagelookup(uobj, ptoa(i)); 443 1.17 riastrad ttm->pages[i] = container_of(vm_page, struct page, p_vmp); 444 1.2 riastrad } 445 1.17 riastrad rw_exit(uobj->vmobjlock); 446 1.2 riastrad 447 1.2 riastrad /* Success! */ 448 1.2 riastrad return 0; 449 1.7 riastrad } 450 1.7 riastrad 451 1.7 riastrad /* 452 1.7 riastrad * ttm_tt_unwire(ttm) 453 1.7 riastrad * 454 1.7 riastrad * Nullify the ttm page array and unwire the uvm pages of ttm. 455 1.7 riastrad * ttm must be unbound and must be marked swapped. This does not 456 1.7 riastrad * change either state -- the caller is expected to include it 457 1.7 riastrad * among other operations for such a state transition. 458 1.7 riastrad */ 459 1.7 riastrad void 460 1.7 riastrad ttm_tt_unwire(struct ttm_tt *ttm) 461 1.7 riastrad { 462 1.7 riastrad struct uvm_object *uobj = ttm->swap_storage; 463 1.7 riastrad unsigned i; 464 1.7 riastrad 465 1.7 riastrad KASSERTMSG((ttm->state == tt_unbound), 466 1.7 riastrad "ttm_tt %p must be unbound for unwiring, but state=%d", 467 1.7 riastrad ttm, (int)ttm->state); 468 1.7 riastrad KASSERT(!ISSET(ttm->page_flags, TTM_PAGE_FLAG_SWAPPED)); 469 1.7 riastrad KASSERT(uobj != NULL); 470 1.7 riastrad 471 1.7 riastrad uvm_obj_unwirepages(uobj, 0, (ttm->num_pages << PAGE_SHIFT)); 472 1.7 riastrad for (i = 0; i < ttm->num_pages; i++) 473 1.7 riastrad ttm->pages[i] = NULL; 474 1.7 riastrad } 475 1.7 riastrad #endif 476 1.7 riastrad 477 1.7 riastrad #ifndef __NetBSD__ 478 1.7 riastrad int ttm_tt_swapin(struct ttm_tt *ttm) 479 1.7 riastrad { 480 1.1 riastrad struct address_space *swap_space; 481 1.1 riastrad struct file *swap_storage; 482 1.1 riastrad struct page *from_page; 483 1.1 riastrad struct page *to_page; 484 1.1 riastrad int i; 485 1.1 riastrad int ret = -ENOMEM; 486 1.1 riastrad 487 1.1 riastrad swap_storage = ttm->swap_storage; 488 1.1 riastrad BUG_ON(swap_storage == NULL); 489 1.1 riastrad 490 1.13 riastrad swap_space = swap_storage->f_mapping; 491 1.1 riastrad 492 1.1 riastrad for (i = 0; i < ttm->num_pages; ++i) { 493 1.13 riastrad gfp_t gfp_mask = mapping_gfp_mask(swap_space); 494 1.13 riastrad 495 1.13 riastrad gfp_mask |= (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY ? __GFP_RETRY_MAYFAIL : 0); 496 1.13 riastrad from_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_mask); 497 1.13 riastrad 498 1.1 riastrad if (IS_ERR(from_page)) { 499 1.1 riastrad ret = PTR_ERR(from_page); 500 1.1 riastrad goto out_err; 501 1.1 riastrad } 502 1.1 riastrad to_page = ttm->pages[i]; 503 1.1 riastrad if (unlikely(to_page == NULL)) 504 1.1 riastrad goto out_err; 505 1.1 riastrad 506 1.1 riastrad copy_highpage(to_page, from_page); 507 1.13 riastrad put_page(from_page); 508 1.1 riastrad } 509 1.1 riastrad 510 1.1 riastrad if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP)) 511 1.1 riastrad fput(swap_storage); 512 1.1 riastrad ttm->swap_storage = NULL; 513 1.1 riastrad ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; 514 1.1 riastrad 515 1.1 riastrad return 0; 516 1.1 riastrad out_err: 517 1.1 riastrad return ret; 518 1.7 riastrad } 519 1.2 riastrad #endif 520 1.1 riastrad 521 1.2 riastrad int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) 522 1.2 riastrad { 523 1.7 riastrad #ifdef __NetBSD__ 524 1.2 riastrad 525 1.7 riastrad KASSERTMSG((ttm->state == tt_unpopulated || ttm->state == tt_unbound), 526 1.7 riastrad "ttm_tt %p must be unpopulated or unbound for swapout," 527 1.7 riastrad " but state=%d", 528 1.7 riastrad ttm, (int)ttm->state); 529 1.7 riastrad KASSERTMSG((ttm->caching_state == tt_cached), 530 1.7 riastrad "ttm_tt %p must be cached for swapout, but caching_state=%d", 531 1.7 riastrad ttm, (int)ttm->caching_state); 532 1.2 riastrad KASSERT(persistent_swap_storage == NULL); 533 1.2 riastrad 534 1.7 riastrad ttm->bdev->driver->ttm_tt_swapout(ttm); 535 1.2 riastrad return 0; 536 1.2 riastrad #else 537 1.1 riastrad struct address_space *swap_space; 538 1.1 riastrad struct file *swap_storage; 539 1.1 riastrad struct page *from_page; 540 1.1 riastrad struct page *to_page; 541 1.1 riastrad int i; 542 1.1 riastrad int ret = -ENOMEM; 543 1.1 riastrad 544 1.1 riastrad BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated); 545 1.1 riastrad BUG_ON(ttm->caching_state != tt_cached); 546 1.1 riastrad 547 1.1 riastrad if (!persistent_swap_storage) { 548 1.1 riastrad swap_storage = shmem_file_setup("ttm swap", 549 1.1 riastrad ttm->num_pages << PAGE_SHIFT, 550 1.1 riastrad 0); 551 1.11 riastrad if (IS_ERR(swap_storage)) { 552 1.1 riastrad pr_err("Failed allocating swap storage\n"); 553 1.1 riastrad return PTR_ERR(swap_storage); 554 1.1 riastrad } 555 1.13 riastrad } else { 556 1.1 riastrad swap_storage = persistent_swap_storage; 557 1.13 riastrad } 558 1.1 riastrad 559 1.13 riastrad swap_space = swap_storage->f_mapping; 560 1.1 riastrad 561 1.1 riastrad for (i = 0; i < ttm->num_pages; ++i) { 562 1.13 riastrad gfp_t gfp_mask = mapping_gfp_mask(swap_space); 563 1.13 riastrad 564 1.13 riastrad gfp_mask |= (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY ? __GFP_RETRY_MAYFAIL : 0); 565 1.13 riastrad 566 1.1 riastrad from_page = ttm->pages[i]; 567 1.1 riastrad if (unlikely(from_page == NULL)) 568 1.1 riastrad continue; 569 1.13 riastrad 570 1.13 riastrad to_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_mask); 571 1.11 riastrad if (IS_ERR(to_page)) { 572 1.1 riastrad ret = PTR_ERR(to_page); 573 1.1 riastrad goto out_err; 574 1.1 riastrad } 575 1.1 riastrad copy_highpage(to_page, from_page); 576 1.1 riastrad set_page_dirty(to_page); 577 1.1 riastrad mark_page_accessed(to_page); 578 1.13 riastrad put_page(to_page); 579 1.1 riastrad } 580 1.1 riastrad 581 1.2 riastrad ttm_tt_unpopulate(ttm); 582 1.1 riastrad ttm->swap_storage = swap_storage; 583 1.1 riastrad ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; 584 1.1 riastrad if (persistent_swap_storage) 585 1.1 riastrad ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP; 586 1.1 riastrad 587 1.1 riastrad return 0; 588 1.1 riastrad out_err: 589 1.1 riastrad if (!persistent_swap_storage) 590 1.1 riastrad fput(swap_storage); 591 1.1 riastrad 592 1.1 riastrad return ret; 593 1.7 riastrad #endif 594 1.1 riastrad } 595 1.2 riastrad 596 1.13 riastrad static void ttm_tt_add_mapping(struct ttm_tt *ttm) 597 1.13 riastrad { 598 1.14 riastrad #ifndef __NetBSD__ 599 1.13 riastrad pgoff_t i; 600 1.13 riastrad 601 1.13 riastrad if (ttm->page_flags & TTM_PAGE_FLAG_SG) 602 1.13 riastrad return; 603 1.13 riastrad 604 1.13 riastrad for (i = 0; i < ttm->num_pages; ++i) 605 1.13 riastrad ttm->pages[i]->mapping = ttm->bdev->dev_mapping; 606 1.14 riastrad #endif 607 1.13 riastrad } 608 1.13 riastrad 609 1.13 riastrad int ttm_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) 610 1.13 riastrad { 611 1.13 riastrad int ret; 612 1.13 riastrad 613 1.13 riastrad if (ttm->state != tt_unpopulated) 614 1.13 riastrad return 0; 615 1.13 riastrad 616 1.13 riastrad if (ttm->bdev->driver->ttm_tt_populate) 617 1.13 riastrad ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx); 618 1.13 riastrad else 619 1.14 riastrad #ifdef __NetBSD__ 620 1.14 riastrad panic("no ttm population"); 621 1.14 riastrad #else 622 1.13 riastrad ret = ttm_pool_populate(ttm, ctx); 623 1.14 riastrad #endif 624 1.13 riastrad if (!ret) 625 1.13 riastrad ttm_tt_add_mapping(ttm); 626 1.13 riastrad return ret; 627 1.13 riastrad } 628 1.13 riastrad 629 1.2 riastrad static void ttm_tt_clear_mapping(struct ttm_tt *ttm) 630 1.2 riastrad { 631 1.2 riastrad #ifndef __NetBSD__ 632 1.2 riastrad pgoff_t i; 633 1.2 riastrad struct page **page = ttm->pages; 634 1.2 riastrad 635 1.2 riastrad if (ttm->page_flags & TTM_PAGE_FLAG_SG) 636 1.2 riastrad return; 637 1.2 riastrad 638 1.2 riastrad for (i = 0; i < ttm->num_pages; ++i) { 639 1.2 riastrad (*page)->mapping = NULL; 640 1.2 riastrad (*page++)->index = 0; 641 1.2 riastrad } 642 1.2 riastrad #endif 643 1.2 riastrad } 644 1.2 riastrad 645 1.2 riastrad void ttm_tt_unpopulate(struct ttm_tt *ttm) 646 1.2 riastrad { 647 1.2 riastrad if (ttm->state == tt_unpopulated) 648 1.2 riastrad return; 649 1.2 riastrad 650 1.2 riastrad ttm_tt_clear_mapping(ttm); 651 1.13 riastrad if (ttm->bdev->driver->ttm_tt_unpopulate) 652 1.13 riastrad ttm->bdev->driver->ttm_tt_unpopulate(ttm); 653 1.13 riastrad else 654 1.14 riastrad #ifdef __NetBSD__ 655 1.14 riastrad panic("no ttm pool unpopulation"); 656 1.14 riastrad #else 657 1.13 riastrad ttm_pool_unpopulate(ttm); 658 1.14 riastrad #endif 659 1.2 riastrad } 660