Home | History | Annotate | Line # | Download | only in ttm
      1  1.19  riastrad /*	$NetBSD: ttm_tt.c,v 1.19 2022/06/26 17:53:06 riastradh Exp $	*/
      2  1.11  riastrad 
      3  1.13  riastrad /* SPDX-License-Identifier: GPL-2.0 OR MIT */
      4   1.1  riastrad /**************************************************************************
      5   1.1  riastrad  *
      6   1.1  riastrad  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
      7   1.1  riastrad  * All Rights Reserved.
      8   1.1  riastrad  *
      9   1.1  riastrad  * Permission is hereby granted, free of charge, to any person obtaining a
     10   1.1  riastrad  * copy of this software and associated documentation files (the
     11   1.1  riastrad  * "Software"), to deal in the Software without restriction, including
     12   1.1  riastrad  * without limitation the rights to use, copy, modify, merge, publish,
     13   1.1  riastrad  * distribute, sub license, and/or sell copies of the Software, and to
     14   1.1  riastrad  * permit persons to whom the Software is furnished to do so, subject to
     15   1.1  riastrad  * the following conditions:
     16   1.1  riastrad  *
     17   1.1  riastrad  * The above copyright notice and this permission notice (including the
     18   1.1  riastrad  * next paragraph) shall be included in all copies or substantial portions
     19   1.1  riastrad  * of the Software.
     20   1.1  riastrad  *
     21   1.1  riastrad  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     22   1.1  riastrad  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     23   1.1  riastrad  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
     24   1.1  riastrad  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
     25   1.1  riastrad  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
     26   1.1  riastrad  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
     27   1.1  riastrad  * USE OR OTHER DEALINGS IN THE SOFTWARE.
     28   1.1  riastrad  *
     29   1.1  riastrad  **************************************************************************/
     30   1.1  riastrad /*
     31   1.1  riastrad  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
     32   1.1  riastrad  */
     33   1.1  riastrad 
     34  1.11  riastrad #include <sys/cdefs.h>
     35  1.19  riastrad __KERNEL_RCSID(0, "$NetBSD: ttm_tt.c,v 1.19 2022/06/26 17:53:06 riastradh Exp $");
     36  1.11  riastrad 
     37   1.1  riastrad #define pr_fmt(fmt) "[TTM] " fmt
     38   1.1  riastrad 
     39   1.1  riastrad #include <linux/sched.h>
     40   1.1  riastrad #include <linux/pagemap.h>
     41   1.1  riastrad #include <linux/shmem_fs.h>
     42   1.1  riastrad #include <linux/file.h>
     43   1.1  riastrad #include <drm/drm_cache.h>
     44  1.15  riastrad #include <drm/drm_mem_util.h>
     45   1.1  riastrad #include <drm/ttm/ttm_bo_driver.h>
     46   1.1  riastrad #include <drm/ttm/ttm_page_alloc.h>
     47   1.2  riastrad #include <drm/bus_dma_hacks.h>
     48  1.13  riastrad #include <drm/ttm/ttm_set_memory.h>
     49  1.13  riastrad 
     50  1.13  riastrad /**
     51  1.13  riastrad  * Allocates a ttm structure for the given BO.
     52  1.13  riastrad  */
     53  1.13  riastrad int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc)
     54  1.13  riastrad {
     55  1.13  riastrad 	struct ttm_bo_device *bdev = bo->bdev;
     56  1.13  riastrad 	uint32_t page_flags = 0;
     57  1.13  riastrad 
     58  1.13  riastrad 	dma_resv_assert_held(bo->base.resv);
     59  1.13  riastrad 
     60  1.13  riastrad 	if (bdev->need_dma32)
     61  1.13  riastrad 		page_flags |= TTM_PAGE_FLAG_DMA32;
     62  1.13  riastrad 
     63  1.13  riastrad 	if (bdev->no_retry)
     64  1.13  riastrad 		page_flags |= TTM_PAGE_FLAG_NO_RETRY;
     65  1.13  riastrad 
     66  1.13  riastrad 	switch (bo->type) {
     67  1.13  riastrad 	case ttm_bo_type_device:
     68  1.13  riastrad 		if (zero_alloc)
     69  1.13  riastrad 			page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC;
     70  1.13  riastrad 		break;
     71  1.13  riastrad 	case ttm_bo_type_kernel:
     72  1.13  riastrad 		break;
     73  1.13  riastrad 	case ttm_bo_type_sg:
     74  1.13  riastrad 		page_flags |= TTM_PAGE_FLAG_SG;
     75  1.13  riastrad 		break;
     76  1.13  riastrad 	default:
     77  1.13  riastrad 		bo->ttm = NULL;
     78  1.13  riastrad 		pr_err("Illegal buffer object type\n");
     79  1.13  riastrad 		return -EINVAL;
     80  1.13  riastrad 	}
     81  1.13  riastrad 
     82  1.13  riastrad 	bo->ttm = bdev->driver->ttm_tt_create(bo, page_flags);
     83  1.13  riastrad 	if (unlikely(bo->ttm == NULL))
     84  1.13  riastrad 		return -ENOMEM;
     85  1.13  riastrad 
     86  1.13  riastrad 	return 0;
     87  1.13  riastrad }
     88   1.1  riastrad 
     89   1.1  riastrad /**
     90   1.1  riastrad  * Allocates storage for pointers to the pages that back the ttm.
     91   1.1  riastrad  */
     92  1.13  riastrad static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
     93   1.1  riastrad {
     94  1.13  riastrad 	ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
     95  1.13  riastrad 			GFP_KERNEL | __GFP_ZERO);
     96  1.13  riastrad 	if (!ttm->pages)
     97  1.13  riastrad 		return -ENOMEM;
     98  1.13  riastrad 	return 0;
     99   1.1  riastrad }
    100   1.1  riastrad 
    101  1.18  riastrad static int ttm_sg_tt_alloc_page_directory(struct ttm_dma_tt *);
    102  1.18  riastrad 
    103  1.13  riastrad static int ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm)
    104   1.1  riastrad {
    105  1.18  riastrad #ifdef __NetBSD__
    106  1.18  riastrad 	int r;
    107  1.18  riastrad 
    108  1.18  riastrad 	/* Create array of pages at ttm->ttm.pages.  */
    109  1.18  riastrad 	r = ttm_tt_alloc_page_directory(&ttm->ttm);
    110  1.18  riastrad 	if (r)
    111  1.18  riastrad 		return r;
    112  1.18  riastrad 
    113  1.18  riastrad 	/* Create bus DMA map at ttm->dma_address.  */
    114  1.18  riastrad 	r = ttm_sg_tt_alloc_page_directory(ttm);
    115  1.19  riastrad 	if (r) {
    116  1.19  riastrad 		kvfree(ttm->ttm.pages);
    117  1.19  riastrad 		ttm->ttm.pages = NULL;
    118  1.18  riastrad 		return r;
    119  1.19  riastrad 	}
    120  1.18  riastrad 
    121  1.18  riastrad 	/* Success!  */
    122  1.18  riastrad 	return 0;
    123  1.18  riastrad #else
    124  1.13  riastrad 	ttm->ttm.pages = kvmalloc_array(ttm->ttm.num_pages,
    125  1.11  riastrad 					  sizeof(*ttm->ttm.pages) +
    126  1.13  riastrad 					  sizeof(*ttm->dma_address),
    127  1.13  riastrad 					  GFP_KERNEL | __GFP_ZERO);
    128  1.13  riastrad 	if (!ttm->ttm.pages)
    129  1.13  riastrad 		return -ENOMEM;
    130  1.13  riastrad 	ttm->dma_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages);
    131  1.13  riastrad 	return 0;
    132  1.18  riastrad #endif
    133  1.13  riastrad }
    134  1.13  riastrad 
    135  1.13  riastrad static int ttm_sg_tt_alloc_page_directory(struct ttm_dma_tt *ttm)
    136  1.13  riastrad {
    137  1.18  riastrad #ifdef __NetBSD__
    138  1.18  riastrad 	ttm->dma_address = NULL;
    139  1.18  riastrad 	/* XXX errno NetBSD->Linux */
    140  1.18  riastrad 	return -bus_dmamap_create(ttm->ttm.bdev->dmat,
    141  1.18  riastrad 	    ttm->ttm.num_pages << PAGE_SHIFT, ttm->ttm.num_pages, PAGE_SIZE, 0,
    142  1.18  riastrad 	    BUS_DMA_WAITOK, &ttm->dma_address);
    143  1.18  riastrad #else
    144  1.13  riastrad 	ttm->dma_address = kvmalloc_array(ttm->ttm.num_pages,
    145  1.13  riastrad 					  sizeof(*ttm->dma_address),
    146  1.13  riastrad 					  GFP_KERNEL | __GFP_ZERO);
    147  1.13  riastrad 	if (!ttm->dma_address)
    148  1.13  riastrad 		return -ENOMEM;
    149  1.13  riastrad 	return 0;
    150  1.18  riastrad #endif
    151   1.1  riastrad }
    152   1.1  riastrad 
    153  1.13  riastrad static int ttm_tt_set_page_caching(struct page *p,
    154  1.13  riastrad 				   enum ttm_caching_state c_old,
    155  1.13  riastrad 				   enum ttm_caching_state c_new)
    156   1.1  riastrad {
    157   1.2  riastrad #ifdef __NetBSD__
    158   1.2  riastrad 	return 0;
    159   1.2  riastrad #else
    160   1.1  riastrad 	int ret = 0;
    161   1.1  riastrad 
    162   1.1  riastrad 	if (PageHighMem(p))
    163   1.1  riastrad 		return 0;
    164   1.1  riastrad 
    165   1.1  riastrad 	if (c_old != tt_cached) {
    166   1.1  riastrad 		/* p isn't in the default caching state, set it to
    167   1.1  riastrad 		 * writeback first to free its current memtype. */
    168   1.1  riastrad 
    169  1.13  riastrad 		ret = ttm_set_pages_wb(p, 1);
    170   1.1  riastrad 		if (ret)
    171   1.1  riastrad 			return ret;
    172   1.1  riastrad 	}
    173   1.1  riastrad 
    174   1.1  riastrad 	if (c_new == tt_wc)
    175  1.13  riastrad 		ret = ttm_set_pages_wc(p, 1);
    176   1.1  riastrad 	else if (c_new == tt_uncached)
    177  1.13  riastrad 		ret = ttm_set_pages_uc(p, 1);
    178   1.1  riastrad 
    179   1.1  riastrad 	return ret;
    180   1.2  riastrad #endif
    181   1.1  riastrad }
    182   1.1  riastrad 
    183   1.1  riastrad /*
    184   1.1  riastrad  * Change caching policy for the linear kernel map
    185   1.1  riastrad  * for range of pages in a ttm.
    186   1.1  riastrad  */
    187   1.1  riastrad 
    188   1.1  riastrad static int ttm_tt_set_caching(struct ttm_tt *ttm,
    189   1.1  riastrad 			      enum ttm_caching_state c_state)
    190   1.1  riastrad {
    191   1.1  riastrad 	int i, j;
    192   1.1  riastrad 	struct page *cur_page;
    193   1.1  riastrad 	int ret;
    194   1.1  riastrad 
    195   1.1  riastrad 	if (ttm->caching_state == c_state)
    196   1.1  riastrad 		return 0;
    197   1.1  riastrad 
    198   1.1  riastrad 	if (ttm->state == tt_unpopulated) {
    199   1.1  riastrad 		/* Change caching but don't populate */
    200   1.1  riastrad 		ttm->caching_state = c_state;
    201   1.1  riastrad 		return 0;
    202   1.1  riastrad 	}
    203   1.1  riastrad 
    204   1.1  riastrad 	if (ttm->caching_state == tt_cached)
    205   1.1  riastrad 		drm_clflush_pages(ttm->pages, ttm->num_pages);
    206   1.1  riastrad 
    207   1.1  riastrad 	for (i = 0; i < ttm->num_pages; ++i) {
    208   1.1  riastrad 		cur_page = ttm->pages[i];
    209   1.1  riastrad 		if (likely(cur_page != NULL)) {
    210   1.1  riastrad 			ret = ttm_tt_set_page_caching(cur_page,
    211   1.1  riastrad 						      ttm->caching_state,
    212   1.1  riastrad 						      c_state);
    213   1.1  riastrad 			if (unlikely(ret != 0))
    214   1.1  riastrad 				goto out_err;
    215   1.1  riastrad 		}
    216   1.1  riastrad 	}
    217   1.1  riastrad 
    218   1.1  riastrad 	ttm->caching_state = c_state;
    219   1.1  riastrad 
    220   1.1  riastrad 	return 0;
    221   1.1  riastrad 
    222   1.1  riastrad out_err:
    223   1.1  riastrad 	for (j = 0; j < i; ++j) {
    224   1.1  riastrad 		cur_page = ttm->pages[j];
    225   1.1  riastrad 		if (likely(cur_page != NULL)) {
    226   1.1  riastrad 			(void)ttm_tt_set_page_caching(cur_page, c_state,
    227   1.1  riastrad 						      ttm->caching_state);
    228   1.1  riastrad 		}
    229   1.1  riastrad 	}
    230   1.1  riastrad 
    231   1.1  riastrad 	return ret;
    232   1.1  riastrad }
    233   1.1  riastrad 
    234   1.1  riastrad int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement)
    235   1.1  riastrad {
    236   1.1  riastrad 	enum ttm_caching_state state;
    237   1.1  riastrad 
    238   1.1  riastrad 	if (placement & TTM_PL_FLAG_WC)
    239   1.1  riastrad 		state = tt_wc;
    240   1.1  riastrad 	else if (placement & TTM_PL_FLAG_UNCACHED)
    241   1.1  riastrad 		state = tt_uncached;
    242   1.1  riastrad 	else
    243   1.1  riastrad 		state = tt_cached;
    244   1.1  riastrad 
    245   1.1  riastrad 	return ttm_tt_set_caching(ttm, state);
    246   1.1  riastrad }
    247   1.1  riastrad EXPORT_SYMBOL(ttm_tt_set_placement_caching);
    248   1.1  riastrad 
    249   1.1  riastrad void ttm_tt_destroy(struct ttm_tt *ttm)
    250   1.1  riastrad {
    251  1.13  riastrad 	if (ttm == NULL)
    252   1.1  riastrad 		return;
    253   1.1  riastrad 
    254  1.13  riastrad 	ttm_tt_unbind(ttm);
    255   1.1  riastrad 
    256   1.2  riastrad 	if (ttm->state == tt_unbound)
    257   1.2  riastrad 		ttm_tt_unpopulate(ttm);
    258   1.1  riastrad 
    259   1.4  riastrad #ifndef __NetBSD__
    260   1.1  riastrad 	if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) &&
    261   1.1  riastrad 	    ttm->swap_storage)
    262   1.1  riastrad 		fput(ttm->swap_storage);
    263   1.1  riastrad 
    264   1.1  riastrad 	ttm->swap_storage = NULL;
    265   1.6  riastrad #endif
    266   1.1  riastrad 	ttm->func->destroy(ttm);
    267   1.1  riastrad }
    268   1.1  riastrad 
    269  1.13  riastrad static void ttm_tt_init_fields(struct ttm_tt *ttm,
    270  1.13  riastrad 			       struct ttm_buffer_object *bo,
    271  1.13  riastrad 			       uint32_t page_flags)
    272  1.13  riastrad {
    273  1.13  riastrad 	ttm->bdev = bo->bdev;
    274  1.13  riastrad 	ttm->num_pages = bo->num_pages;
    275   1.1  riastrad 	ttm->caching_state = tt_cached;
    276   1.1  riastrad 	ttm->page_flags = page_flags;
    277   1.1  riastrad 	ttm->state = tt_unpopulated;
    278   1.2  riastrad #ifdef __NetBSD__
    279  1.14  riastrad 	WARN(bo->num_pages == 0,
    280  1.14  riastrad 	    "zero-size allocation in %s, please file a NetBSD PR",
    281   1.8  riastrad 	    __func__);	/* paranoia -- can't prove in five minutes */
    282  1.16  riastrad 	ttm->swap_storage = uao_create(PAGE_SIZE * MAX(1, bo->num_pages), 0);
    283  1.14  riastrad 	uao_set_pgfl(ttm->swap_storage, bus_dmamem_pgfl(ttm->bdev->dmat));
    284   1.2  riastrad #else
    285   1.1  riastrad 	ttm->swap_storage = NULL;
    286   1.2  riastrad #endif
    287  1.13  riastrad 	ttm->sg = bo->sg;
    288  1.13  riastrad }
    289  1.13  riastrad 
    290  1.13  riastrad int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo,
    291  1.13  riastrad 		uint32_t page_flags)
    292  1.13  riastrad {
    293  1.13  riastrad 	ttm_tt_init_fields(ttm, bo, page_flags);
    294   1.1  riastrad 
    295  1.13  riastrad 	if (ttm_tt_alloc_page_directory(ttm)) {
    296   1.1  riastrad 		ttm_tt_destroy(ttm);
    297   1.1  riastrad 		pr_err("Failed allocating page table\n");
    298   1.1  riastrad 		return -ENOMEM;
    299   1.1  riastrad 	}
    300   1.1  riastrad 	return 0;
    301   1.1  riastrad }
    302   1.1  riastrad EXPORT_SYMBOL(ttm_tt_init);
    303   1.1  riastrad 
    304   1.1  riastrad void ttm_tt_fini(struct ttm_tt *ttm)
    305   1.1  riastrad {
    306  1.13  riastrad 	kvfree(ttm->pages);
    307  1.13  riastrad 	ttm->pages = NULL;
    308   1.4  riastrad #ifdef __NetBSD__
    309   1.2  riastrad 	uao_detach(ttm->swap_storage);
    310   1.2  riastrad 	ttm->swap_storage = NULL;
    311   1.4  riastrad #endif
    312   1.1  riastrad }
    313   1.1  riastrad EXPORT_SYMBOL(ttm_tt_fini);
    314   1.1  riastrad 
    315  1.13  riastrad int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_buffer_object *bo,
    316  1.13  riastrad 		    uint32_t page_flags)
    317   1.1  riastrad {
    318   1.1  riastrad 	struct ttm_tt *ttm = &ttm_dma->ttm;
    319   1.1  riastrad 
    320  1.13  riastrad 	ttm_tt_init_fields(ttm, bo, page_flags);
    321   1.1  riastrad 
    322   1.1  riastrad 	INIT_LIST_HEAD(&ttm_dma->pages_list);
    323  1.13  riastrad 	if (ttm_dma_tt_alloc_page_directory(ttm_dma)) {
    324  1.13  riastrad 		ttm_tt_destroy(ttm);
    325  1.13  riastrad 		pr_err("Failed allocating page table\n");
    326  1.13  riastrad 		return -ENOMEM;
    327  1.13  riastrad 	}
    328   1.2  riastrad 	return 0;
    329  1.13  riastrad }
    330  1.13  riastrad EXPORT_SYMBOL(ttm_dma_tt_init);
    331  1.13  riastrad 
    332  1.13  riastrad int ttm_sg_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_buffer_object *bo,
    333  1.13  riastrad 		   uint32_t page_flags)
    334  1.13  riastrad {
    335  1.13  riastrad 	struct ttm_tt *ttm = &ttm_dma->ttm;
    336  1.13  riastrad 	int ret;
    337  1.13  riastrad 
    338  1.13  riastrad 	ttm_tt_init_fields(ttm, bo, page_flags);
    339  1.13  riastrad 
    340  1.13  riastrad 	INIT_LIST_HEAD(&ttm_dma->pages_list);
    341  1.13  riastrad 	if (page_flags & TTM_PAGE_FLAG_SG)
    342  1.13  riastrad 		ret = ttm_sg_tt_alloc_page_directory(ttm_dma);
    343  1.13  riastrad 	else
    344  1.13  riastrad 		ret = ttm_dma_tt_alloc_page_directory(ttm_dma);
    345  1.13  riastrad 	if (ret) {
    346   1.1  riastrad 		ttm_tt_destroy(ttm);
    347   1.1  riastrad 		pr_err("Failed allocating page table\n");
    348   1.1  riastrad 		return -ENOMEM;
    349   1.1  riastrad 	}
    350   1.1  riastrad 	return 0;
    351   1.1  riastrad }
    352  1.13  riastrad EXPORT_SYMBOL(ttm_sg_tt_init);
    353   1.1  riastrad 
    354   1.1  riastrad void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma)
    355   1.1  riastrad {
    356   1.1  riastrad 	struct ttm_tt *ttm = &ttm_dma->ttm;
    357   1.1  riastrad 
    358   1.4  riastrad #ifdef __NetBSD__
    359  1.18  riastrad 	if (ttm_dma->dma_address) {
    360  1.18  riastrad 		bus_dmamap_destroy(ttm->bdev->dmat, ttm_dma->dma_address);
    361  1.18  riastrad 		ttm_dma->dma_address = NULL;
    362  1.18  riastrad 	}
    363  1.18  riastrad 	ttm_tt_fini(ttm);
    364  1.18  riastrad #else
    365  1.13  riastrad 	if (ttm->pages)
    366  1.13  riastrad 		kvfree(ttm->pages);
    367  1.13  riastrad 	else
    368  1.13  riastrad 		kvfree(ttm_dma->dma_address);
    369  1.13  riastrad 	ttm->pages = NULL;
    370  1.18  riastrad 	ttm_dma->dma_address = NULL;
    371   1.4  riastrad #endif
    372   1.1  riastrad }
    373   1.1  riastrad EXPORT_SYMBOL(ttm_dma_tt_fini);
    374   1.1  riastrad 
    375   1.1  riastrad void ttm_tt_unbind(struct ttm_tt *ttm)
    376   1.1  riastrad {
    377   1.5       rjs 	int ret __diagused;
    378   1.1  riastrad 
    379   1.1  riastrad 	if (ttm->state == tt_bound) {
    380   1.1  riastrad 		ret = ttm->func->unbind(ttm);
    381   1.1  riastrad 		BUG_ON(ret);
    382   1.1  riastrad 		ttm->state = tt_unbound;
    383   1.1  riastrad 	}
    384   1.1  riastrad }
    385   1.1  riastrad 
    386  1.13  riastrad int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem,
    387  1.13  riastrad 		struct ttm_operation_ctx *ctx)
    388   1.1  riastrad {
    389   1.1  riastrad 	int ret = 0;
    390   1.1  riastrad 
    391   1.1  riastrad 	if (!ttm)
    392   1.1  riastrad 		return -EINVAL;
    393   1.1  riastrad 
    394   1.1  riastrad 	if (ttm->state == tt_bound)
    395   1.1  riastrad 		return 0;
    396   1.1  riastrad 
    397  1.13  riastrad 	ret = ttm_tt_populate(ttm, ctx);
    398   1.1  riastrad 	if (ret)
    399   1.1  riastrad 		return ret;
    400   1.1  riastrad 
    401   1.1  riastrad 	ret = ttm->func->bind(ttm, bo_mem);
    402   1.1  riastrad 	if (unlikely(ret != 0))
    403   1.1  riastrad 		return ret;
    404   1.1  riastrad 
    405   1.1  riastrad 	ttm->state = tt_bound;
    406   1.1  riastrad 
    407   1.1  riastrad 	return 0;
    408   1.1  riastrad }
    409   1.1  riastrad EXPORT_SYMBOL(ttm_tt_bind);
    410   1.1  riastrad 
    411   1.7  riastrad #ifdef __NetBSD__
    412   1.7  riastrad /*
    413   1.7  riastrad  * ttm_tt_wire(ttm)
    414   1.7  riastrad  *
    415   1.7  riastrad  *	Wire the uvm pages of ttm and fill the ttm page array.  ttm
    416   1.9      maya  *	must be unpopulated, and must be marked swapped.  This does not
    417   1.9      maya  *	change either state -- the caller is expected to include it
    418   1.9      maya  *	among other operations for such a state transition.
    419   1.7  riastrad  */
    420   1.7  riastrad int
    421   1.7  riastrad ttm_tt_wire(struct ttm_tt *ttm)
    422   1.1  riastrad {
    423   1.2  riastrad 	struct uvm_object *uobj = ttm->swap_storage;
    424  1.17  riastrad 	struct vm_page *vm_page;
    425   1.2  riastrad 	unsigned i;
    426   1.2  riastrad 	int error;
    427   1.2  riastrad 
    428   1.9      maya 	KASSERTMSG((ttm->state == tt_unpopulated),
    429   1.9      maya 	    "ttm_tt %p must be unpopulated for wiring, but state=%d",
    430   1.7  riastrad 	    ttm, (int)ttm->state);
    431   1.7  riastrad 	KASSERT(ISSET(ttm->page_flags, TTM_PAGE_FLAG_SWAPPED));
    432   1.2  riastrad 	KASSERT(uobj != NULL);
    433   1.7  riastrad 
    434   1.2  riastrad 	error = uvm_obj_wirepages(uobj, 0, (ttm->num_pages << PAGE_SHIFT),
    435  1.17  riastrad 	    NULL);
    436   1.2  riastrad 	if (error)
    437   1.2  riastrad 		/* XXX errno NetBSD->Linux */
    438   1.2  riastrad 		return -error;
    439   1.2  riastrad 
    440  1.17  riastrad 	rw_enter(uobj->vmobjlock, RW_READER);
    441  1.17  riastrad 	for (i = 0; i < ttm->num_pages; i++) {
    442  1.17  riastrad 		vm_page = uvm_pagelookup(uobj, ptoa(i));
    443  1.17  riastrad 		ttm->pages[i] = container_of(vm_page, struct page, p_vmp);
    444   1.2  riastrad 	}
    445  1.17  riastrad 	rw_exit(uobj->vmobjlock);
    446   1.2  riastrad 
    447   1.2  riastrad 	/* Success!  */
    448   1.2  riastrad 	return 0;
    449   1.7  riastrad }
    450   1.7  riastrad 
    451   1.7  riastrad /*
    452   1.7  riastrad  * ttm_tt_unwire(ttm)
    453   1.7  riastrad  *
    454   1.7  riastrad  *	Nullify the ttm page array and unwire the uvm pages of ttm.
    455   1.7  riastrad  *	ttm must be unbound and must be marked swapped.  This does not
    456   1.7  riastrad  *	change either state -- the caller is expected to include it
    457   1.7  riastrad  *	among other operations for such a state transition.
    458   1.7  riastrad  */
    459   1.7  riastrad void
    460   1.7  riastrad ttm_tt_unwire(struct ttm_tt *ttm)
    461   1.7  riastrad {
    462   1.7  riastrad 	struct uvm_object *uobj = ttm->swap_storage;
    463   1.7  riastrad 	unsigned i;
    464   1.7  riastrad 
    465   1.7  riastrad 	KASSERTMSG((ttm->state == tt_unbound),
    466   1.7  riastrad 	    "ttm_tt %p must be unbound for unwiring, but state=%d",
    467   1.7  riastrad 	    ttm, (int)ttm->state);
    468   1.7  riastrad 	KASSERT(!ISSET(ttm->page_flags, TTM_PAGE_FLAG_SWAPPED));
    469   1.7  riastrad 	KASSERT(uobj != NULL);
    470   1.7  riastrad 
    471   1.7  riastrad 	uvm_obj_unwirepages(uobj, 0, (ttm->num_pages << PAGE_SHIFT));
    472   1.7  riastrad 	for (i = 0; i < ttm->num_pages; i++)
    473   1.7  riastrad 		ttm->pages[i] = NULL;
    474   1.7  riastrad }
    475   1.7  riastrad #endif
    476   1.7  riastrad 
    477   1.7  riastrad #ifndef __NetBSD__
    478   1.7  riastrad int ttm_tt_swapin(struct ttm_tt *ttm)
    479   1.7  riastrad {
    480   1.1  riastrad 	struct address_space *swap_space;
    481   1.1  riastrad 	struct file *swap_storage;
    482   1.1  riastrad 	struct page *from_page;
    483   1.1  riastrad 	struct page *to_page;
    484   1.1  riastrad 	int i;
    485   1.1  riastrad 	int ret = -ENOMEM;
    486   1.1  riastrad 
    487   1.1  riastrad 	swap_storage = ttm->swap_storage;
    488   1.1  riastrad 	BUG_ON(swap_storage == NULL);
    489   1.1  riastrad 
    490  1.13  riastrad 	swap_space = swap_storage->f_mapping;
    491   1.1  riastrad 
    492   1.1  riastrad 	for (i = 0; i < ttm->num_pages; ++i) {
    493  1.13  riastrad 		gfp_t gfp_mask = mapping_gfp_mask(swap_space);
    494  1.13  riastrad 
    495  1.13  riastrad 		gfp_mask |= (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY ? __GFP_RETRY_MAYFAIL : 0);
    496  1.13  riastrad 		from_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_mask);
    497  1.13  riastrad 
    498   1.1  riastrad 		if (IS_ERR(from_page)) {
    499   1.1  riastrad 			ret = PTR_ERR(from_page);
    500   1.1  riastrad 			goto out_err;
    501   1.1  riastrad 		}
    502   1.1  riastrad 		to_page = ttm->pages[i];
    503   1.1  riastrad 		if (unlikely(to_page == NULL))
    504   1.1  riastrad 			goto out_err;
    505   1.1  riastrad 
    506   1.1  riastrad 		copy_highpage(to_page, from_page);
    507  1.13  riastrad 		put_page(from_page);
    508   1.1  riastrad 	}
    509   1.1  riastrad 
    510   1.1  riastrad 	if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP))
    511   1.1  riastrad 		fput(swap_storage);
    512   1.1  riastrad 	ttm->swap_storage = NULL;
    513   1.1  riastrad 	ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;
    514   1.1  riastrad 
    515   1.1  riastrad 	return 0;
    516   1.1  riastrad out_err:
    517   1.1  riastrad 	return ret;
    518   1.7  riastrad }
    519   1.2  riastrad #endif
    520   1.1  riastrad 
    521   1.2  riastrad int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage)
    522   1.2  riastrad {
    523   1.7  riastrad #ifdef __NetBSD__
    524   1.2  riastrad 
    525   1.7  riastrad 	KASSERTMSG((ttm->state == tt_unpopulated || ttm->state == tt_unbound),
    526   1.7  riastrad 	    "ttm_tt %p must be unpopulated or unbound for swapout,"
    527   1.7  riastrad 	    " but state=%d",
    528   1.7  riastrad 	    ttm, (int)ttm->state);
    529   1.7  riastrad 	KASSERTMSG((ttm->caching_state == tt_cached),
    530   1.7  riastrad 	    "ttm_tt %p must be cached for swapout, but caching_state=%d",
    531   1.7  riastrad 	    ttm, (int)ttm->caching_state);
    532   1.2  riastrad 	KASSERT(persistent_swap_storage == NULL);
    533   1.2  riastrad 
    534   1.7  riastrad 	ttm->bdev->driver->ttm_tt_swapout(ttm);
    535   1.2  riastrad 	return 0;
    536   1.2  riastrad #else
    537   1.1  riastrad 	struct address_space *swap_space;
    538   1.1  riastrad 	struct file *swap_storage;
    539   1.1  riastrad 	struct page *from_page;
    540   1.1  riastrad 	struct page *to_page;
    541   1.1  riastrad 	int i;
    542   1.1  riastrad 	int ret = -ENOMEM;
    543   1.1  riastrad 
    544   1.1  riastrad 	BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated);
    545   1.1  riastrad 	BUG_ON(ttm->caching_state != tt_cached);
    546   1.1  riastrad 
    547   1.1  riastrad 	if (!persistent_swap_storage) {
    548   1.1  riastrad 		swap_storage = shmem_file_setup("ttm swap",
    549   1.1  riastrad 						ttm->num_pages << PAGE_SHIFT,
    550   1.1  riastrad 						0);
    551  1.11  riastrad 		if (IS_ERR(swap_storage)) {
    552   1.1  riastrad 			pr_err("Failed allocating swap storage\n");
    553   1.1  riastrad 			return PTR_ERR(swap_storage);
    554   1.1  riastrad 		}
    555  1.13  riastrad 	} else {
    556   1.1  riastrad 		swap_storage = persistent_swap_storage;
    557  1.13  riastrad 	}
    558   1.1  riastrad 
    559  1.13  riastrad 	swap_space = swap_storage->f_mapping;
    560   1.1  riastrad 
    561   1.1  riastrad 	for (i = 0; i < ttm->num_pages; ++i) {
    562  1.13  riastrad 		gfp_t gfp_mask = mapping_gfp_mask(swap_space);
    563  1.13  riastrad 
    564  1.13  riastrad 		gfp_mask |= (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY ? __GFP_RETRY_MAYFAIL : 0);
    565  1.13  riastrad 
    566   1.1  riastrad 		from_page = ttm->pages[i];
    567   1.1  riastrad 		if (unlikely(from_page == NULL))
    568   1.1  riastrad 			continue;
    569  1.13  riastrad 
    570  1.13  riastrad 		to_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_mask);
    571  1.11  riastrad 		if (IS_ERR(to_page)) {
    572   1.1  riastrad 			ret = PTR_ERR(to_page);
    573   1.1  riastrad 			goto out_err;
    574   1.1  riastrad 		}
    575   1.1  riastrad 		copy_highpage(to_page, from_page);
    576   1.1  riastrad 		set_page_dirty(to_page);
    577   1.1  riastrad 		mark_page_accessed(to_page);
    578  1.13  riastrad 		put_page(to_page);
    579   1.1  riastrad 	}
    580   1.1  riastrad 
    581   1.2  riastrad 	ttm_tt_unpopulate(ttm);
    582   1.1  riastrad 	ttm->swap_storage = swap_storage;
    583   1.1  riastrad 	ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
    584   1.1  riastrad 	if (persistent_swap_storage)
    585   1.1  riastrad 		ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP;
    586   1.1  riastrad 
    587   1.1  riastrad 	return 0;
    588   1.1  riastrad out_err:
    589   1.1  riastrad 	if (!persistent_swap_storage)
    590   1.1  riastrad 		fput(swap_storage);
    591   1.1  riastrad 
    592   1.1  riastrad 	return ret;
    593   1.7  riastrad #endif
    594   1.1  riastrad }
    595   1.2  riastrad 
    596  1.13  riastrad static void ttm_tt_add_mapping(struct ttm_tt *ttm)
    597  1.13  riastrad {
    598  1.14  riastrad #ifndef __NetBSD__
    599  1.13  riastrad 	pgoff_t i;
    600  1.13  riastrad 
    601  1.13  riastrad 	if (ttm->page_flags & TTM_PAGE_FLAG_SG)
    602  1.13  riastrad 		return;
    603  1.13  riastrad 
    604  1.13  riastrad 	for (i = 0; i < ttm->num_pages; ++i)
    605  1.13  riastrad 		ttm->pages[i]->mapping = ttm->bdev->dev_mapping;
    606  1.14  riastrad #endif
    607  1.13  riastrad }
    608  1.13  riastrad 
    609  1.13  riastrad int ttm_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
    610  1.13  riastrad {
    611  1.13  riastrad 	int ret;
    612  1.13  riastrad 
    613  1.13  riastrad 	if (ttm->state != tt_unpopulated)
    614  1.13  riastrad 		return 0;
    615  1.13  riastrad 
    616  1.13  riastrad 	if (ttm->bdev->driver->ttm_tt_populate)
    617  1.13  riastrad 		ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx);
    618  1.13  riastrad 	else
    619  1.14  riastrad #ifdef __NetBSD__
    620  1.14  riastrad 		panic("no ttm population");
    621  1.14  riastrad #else
    622  1.13  riastrad 		ret = ttm_pool_populate(ttm, ctx);
    623  1.14  riastrad #endif
    624  1.13  riastrad 	if (!ret)
    625  1.13  riastrad 		ttm_tt_add_mapping(ttm);
    626  1.13  riastrad 	return ret;
    627  1.13  riastrad }
    628  1.13  riastrad 
    629   1.2  riastrad static void ttm_tt_clear_mapping(struct ttm_tt *ttm)
    630   1.2  riastrad {
    631   1.2  riastrad #ifndef __NetBSD__
    632   1.2  riastrad 	pgoff_t i;
    633   1.2  riastrad 	struct page **page = ttm->pages;
    634   1.2  riastrad 
    635   1.2  riastrad 	if (ttm->page_flags & TTM_PAGE_FLAG_SG)
    636   1.2  riastrad 		return;
    637   1.2  riastrad 
    638   1.2  riastrad 	for (i = 0; i < ttm->num_pages; ++i) {
    639   1.2  riastrad 		(*page)->mapping = NULL;
    640   1.2  riastrad 		(*page++)->index = 0;
    641   1.2  riastrad 	}
    642   1.2  riastrad #endif
    643   1.2  riastrad }
    644   1.2  riastrad 
    645   1.2  riastrad void ttm_tt_unpopulate(struct ttm_tt *ttm)
    646   1.2  riastrad {
    647   1.2  riastrad 	if (ttm->state == tt_unpopulated)
    648   1.2  riastrad 		return;
    649   1.2  riastrad 
    650   1.2  riastrad 	ttm_tt_clear_mapping(ttm);
    651  1.13  riastrad 	if (ttm->bdev->driver->ttm_tt_unpopulate)
    652  1.13  riastrad 		ttm->bdev->driver->ttm_tt_unpopulate(ttm);
    653  1.13  riastrad 	else
    654  1.14  riastrad #ifdef __NetBSD__
    655  1.14  riastrad 		panic("no ttm pool unpopulation");
    656  1.14  riastrad #else
    657  1.13  riastrad 		ttm_pool_unpopulate(ttm);
    658  1.14  riastrad #endif
    659   1.2  riastrad }
    660