Home | History | Annotate | Line # | Download | only in ttm
ttm_bo_vm.c revision 1.2.4.1
      1  1.2.4.1    martin /*	$NetBSD: ttm_bo_vm.c,v 1.2.4.1 2014/08/18 07:49:10 martin Exp $	*/
      2      1.1  riastrad 
      3      1.1  riastrad /*-
      4      1.1  riastrad  * Copyright (c) 2014 The NetBSD Foundation, Inc.
      5      1.1  riastrad  * All rights reserved.
      6      1.1  riastrad  *
      7      1.1  riastrad  * This code is derived from software contributed to The NetBSD Foundation
      8      1.1  riastrad  * by Taylor R. Campbell.
      9      1.1  riastrad  *
     10      1.1  riastrad  * Redistribution and use in source and binary forms, with or without
     11      1.1  riastrad  * modification, are permitted provided that the following conditions
     12      1.1  riastrad  * are met:
     13      1.1  riastrad  * 1. Redistributions of source code must retain the above copyright
     14      1.1  riastrad  *    notice, this list of conditions and the following disclaimer.
     15      1.1  riastrad  * 2. Redistributions in binary form must reproduce the above copyright
     16      1.1  riastrad  *    notice, this list of conditions and the following disclaimer in the
     17      1.1  riastrad  *    documentation and/or other materials provided with the distribution.
     18      1.1  riastrad  *
     19      1.1  riastrad  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20      1.1  riastrad  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21      1.1  riastrad  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22      1.1  riastrad  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23      1.1  riastrad  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24      1.1  riastrad  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25      1.1  riastrad  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26      1.1  riastrad  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27      1.1  riastrad  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28      1.1  riastrad  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29      1.1  riastrad  * POSSIBILITY OF SUCH DAMAGE.
     30      1.1  riastrad  */
     31      1.1  riastrad 
     32      1.1  riastrad #include <sys/cdefs.h>
     33  1.2.4.1    martin __KERNEL_RCSID(0, "$NetBSD: ttm_bo_vm.c,v 1.2.4.1 2014/08/18 07:49:10 martin Exp $");
     34      1.1  riastrad 
     35      1.1  riastrad #include <sys/types.h>
     36      1.1  riastrad 
     37      1.1  riastrad #include <uvm/uvm.h>
     38      1.1  riastrad #include <uvm/uvm_extern.h>
     39      1.1  riastrad #include <uvm/uvm_fault.h>
     40      1.1  riastrad 
     41      1.1  riastrad #include <linux/bitops.h>
     42      1.1  riastrad 
     43      1.1  riastrad #include <drm/drm_vma_manager.h>
     44      1.1  riastrad 
     45      1.1  riastrad #include <ttm/ttm_bo_driver.h>
     46      1.1  riastrad 
     47      1.1  riastrad static int	ttm_bo_uvm_fault_idle(struct ttm_buffer_object *,
     48      1.1  riastrad 		    struct uvm_faultinfo *, struct uvm_object *);
     49      1.1  riastrad static int	ttm_bo_uvm_lookup(struct ttm_bo_device *, unsigned long,
     50      1.1  riastrad 		    unsigned long, struct ttm_buffer_object **);
     51      1.1  riastrad 
     52      1.1  riastrad void
     53      1.1  riastrad ttm_bo_uvm_reference(struct uvm_object *uobj)
     54      1.1  riastrad {
     55      1.1  riastrad 	struct ttm_buffer_object *const bo = container_of(uobj,
     56      1.1  riastrad 	    struct ttm_buffer_object, uvmobj);
     57      1.1  riastrad 
     58      1.1  riastrad 	(void)ttm_bo_reference(bo);
     59      1.1  riastrad }
     60      1.1  riastrad 
     61      1.1  riastrad void
     62      1.1  riastrad ttm_bo_uvm_detach(struct uvm_object *uobj)
     63      1.1  riastrad {
     64      1.1  riastrad 	struct ttm_buffer_object *bo = container_of(uobj,
     65      1.1  riastrad 	    struct ttm_buffer_object, uvmobj);
     66      1.1  riastrad 
     67      1.1  riastrad 	ttm_bo_unref(&bo);
     68      1.1  riastrad 	KASSERT(bo == NULL);
     69      1.1  riastrad }
     70      1.1  riastrad 
     71      1.1  riastrad int
     72      1.1  riastrad ttm_bo_uvm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr,
     73      1.1  riastrad     struct vm_page **pps, int npages, int centeridx, vm_prot_t access_type,
     74      1.1  riastrad     int flags)
     75      1.1  riastrad {
     76      1.1  riastrad 	struct uvm_object *const uobj = ufi->entry->object.uvm_obj;
     77      1.1  riastrad 	struct ttm_buffer_object *const bo = container_of(uobj,
     78      1.1  riastrad 	    struct ttm_buffer_object, uvmobj);
     79      1.1  riastrad 	struct ttm_bo_device *const bdev = bo->bdev;
     80      1.1  riastrad 	struct ttm_mem_type_manager *man =
     81      1.1  riastrad 	    &bdev->man[bo->mem.mem_type];
     82      1.1  riastrad 	union {
     83      1.1  riastrad 		bus_addr_t base;
     84      1.1  riastrad 		struct ttm_tt *ttm;
     85      1.1  riastrad 	} u;
     86      1.1  riastrad 	size_t size __diagused;
     87      1.1  riastrad 	voff_t uoffset;		/* offset in bytes into bo */
     88      1.1  riastrad 	unsigned startpage;	/* offset in pages into bo */
     89      1.1  riastrad 	unsigned i;
     90      1.1  riastrad 	vm_prot_t vm_prot;	/* VM_PROT_* */
     91      1.1  riastrad 	pgprot_t pgprot;	/* VM_PROT_* | PMAP_* cacheability flags */
     92  1.2.4.1    martin 	unsigned mmapflags;
     93      1.1  riastrad 	int ret;
     94      1.1  riastrad 
     95      1.1  riastrad 	/* Copy-on-write mappings make no sense for the graphics aperture.  */
     96      1.1  riastrad 	if (UVM_ET_ISCOPYONWRITE(ufi->entry)) {
     97      1.1  riastrad 		ret = -EIO;
     98      1.1  riastrad 		goto out0;
     99      1.1  riastrad 	}
    100      1.1  riastrad 
    101      1.1  riastrad 	/* Try to lock the buffer.  */
    102      1.1  riastrad 	ret = ttm_bo_reserve(bo, true, true, false, NULL);
    103      1.1  riastrad 	if (ret) {
    104      1.1  riastrad 		if (ret != -EBUSY)
    105      1.1  riastrad 			goto out0;
    106      1.1  riastrad 		/*
    107      1.1  riastrad 		 * It's currently locked.  Unlock the fault, wait for
    108      1.1  riastrad 		 * it, and start over.
    109      1.1  riastrad 		 */
    110      1.1  riastrad 		uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj);
    111      1.1  riastrad 		(void)ttm_bo_wait_unreserved(bo);
    112      1.1  riastrad 		return -ERESTART;
    113      1.1  riastrad 	}
    114      1.1  riastrad 
    115      1.1  riastrad 	/* drm prime buffers are not mappable.  XXX Catch this earlier?  */
    116      1.1  riastrad 	if (bo->ttm && ISSET(bo->ttm->page_flags, TTM_PAGE_FLAG_SG)) {
    117      1.1  riastrad 		ret = -EINVAL;
    118      1.1  riastrad 		goto out0;
    119      1.1  riastrad 	}
    120      1.1  riastrad 
    121      1.1  riastrad 	/* Notify the driver of a fault if it wants.  */
    122      1.1  riastrad 	if (bdev->driver->fault_reserve_notify) {
    123      1.1  riastrad 		ret = (*bdev->driver->fault_reserve_notify)(bo);
    124      1.1  riastrad 		if (ret) {
    125      1.1  riastrad 			if (ret == -ERESTART)
    126      1.1  riastrad 				ret = -EIO;
    127      1.1  riastrad 			goto out0;
    128      1.1  riastrad 		}
    129      1.1  riastrad 	}
    130      1.1  riastrad 
    131      1.1  riastrad 	ret = ttm_bo_uvm_fault_idle(bo, ufi, uobj);
    132      1.1  riastrad 	if (ret) {
    133      1.1  riastrad 		/* Unlocks if it restarts.  */
    134      1.1  riastrad 		KASSERT(ret == -ERESTART);
    135      1.1  riastrad 		/* XXX errno Linux->NetBSD */
    136      1.1  riastrad 		return -ret;
    137      1.1  riastrad 	}
    138      1.1  riastrad 
    139      1.1  riastrad 	ret = ttm_mem_io_lock(man, true);
    140      1.1  riastrad 	if (ret) {
    141      1.1  riastrad 		ret = -EIO;
    142      1.1  riastrad 		goto out1;
    143      1.1  riastrad 	}
    144      1.1  riastrad 	ret = ttm_mem_io_reserve_vm(bo);
    145      1.1  riastrad 	if (ret) {
    146      1.1  riastrad 		ret = -EIO;
    147      1.1  riastrad 		goto out2;
    148      1.1  riastrad 	}
    149      1.1  riastrad 
    150      1.1  riastrad 	vm_prot = ufi->entry->protection;
    151      1.1  riastrad 	if (bo->mem.bus.is_iomem) {
    152      1.1  riastrad 		u.base = (bo->mem.bus.base + bo->mem.bus.offset);
    153      1.1  riastrad 		size = bo->mem.bus.size;
    154      1.1  riastrad 		pgprot = ttm_io_prot(bo->mem.placement, vm_prot);
    155      1.1  riastrad 	} else {
    156      1.1  riastrad 		u.ttm = bo->ttm;
    157      1.1  riastrad 		size = (bo->ttm->num_pages << PAGE_SHIFT);
    158      1.1  riastrad 		if (ISSET(bo->mem.placement, TTM_PL_FLAG_CACHED))
    159      1.1  riastrad 			pgprot = vm_prot;
    160      1.1  riastrad 		else
    161      1.1  riastrad 			pgprot = ttm_io_prot(bo->mem.placement, vm_prot);
    162      1.1  riastrad 		if ((*u.ttm->bdev->driver->ttm_tt_populate)(u.ttm)) {
    163      1.1  riastrad 			ret = -ENOMEM;
    164      1.1  riastrad 			goto out2;
    165      1.1  riastrad 		}
    166      1.1  riastrad 	}
    167      1.1  riastrad 
    168      1.1  riastrad 	KASSERT(ufi->entry->start <= vaddr);
    169      1.1  riastrad 	KASSERT((ufi->entry->offset & (PAGE_SIZE - 1)) == 0);
    170      1.1  riastrad 	KASSERT(ufi->entry->offset <= size);
    171      1.1  riastrad 	KASSERT((vaddr - ufi->entry->start) <= (size - ufi->entry->offset));
    172      1.1  riastrad 	KASSERT(npages <= ((size - ufi->entry->offset) -
    173      1.1  riastrad 		(vaddr - ufi->entry->start)));
    174      1.1  riastrad 	uoffset = (ufi->entry->offset + (vaddr - ufi->entry->start));
    175      1.1  riastrad 	startpage = (uoffset >> PAGE_SHIFT);
    176      1.1  riastrad 	for (i = 0; i < npages; i++) {
    177      1.1  riastrad 		paddr_t paddr;
    178      1.1  riastrad 
    179      1.1  riastrad 		/* XXX PGO_ALLPAGES?  */
    180      1.1  riastrad 		if (pps[i] == PGO_DONTCARE)
    181      1.1  riastrad 			continue;
    182  1.2.4.1    martin 		if (bo->mem.bus.is_iomem) {
    183  1.2.4.1    martin 			const paddr_t cookie = bus_space_mmap(bdev->memt,
    184  1.2.4.1    martin 			    u.base, ((startpage + i) << PAGE_SHIFT), vm_prot,
    185  1.2.4.1    martin 			    0);
    186  1.2.4.1    martin 
    187  1.2.4.1    martin 			paddr = pmap_phys_address(cookie);
    188  1.2.4.1    martin 			mmapflags = pmap_mmap_flags(cookie);
    189  1.2.4.1    martin 		} else {
    190      1.1  riastrad 			paddr = page_to_phys(u.ttm->pages[startpage + i]);
    191  1.2.4.1    martin 			mmapflags = 0;
    192  1.2.4.1    martin 		}
    193      1.2  riastrad 		ret = -pmap_enter(ufi->orig_map->pmap, vaddr + i*PAGE_SIZE,
    194  1.2.4.1    martin 		    paddr, vm_prot, (PMAP_CANFAIL | pgprot | mmapflags));
    195      1.1  riastrad 		if (ret)
    196      1.1  riastrad 			goto out3;
    197      1.1  riastrad 	}
    198      1.1  riastrad 
    199      1.1  riastrad out3:	pmap_update(ufi->orig_map->pmap);
    200      1.1  riastrad out2:	ttm_mem_io_unlock(man);
    201      1.1  riastrad out1:	ttm_bo_unreserve(bo);
    202      1.1  riastrad out0:	uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj);
    203      1.1  riastrad 	/* XXX errno Linux->NetBSD */
    204      1.1  riastrad 	return -ret;
    205      1.1  riastrad }
    206      1.1  riastrad 
    207      1.1  riastrad static int
    208      1.1  riastrad ttm_bo_uvm_fault_idle(struct ttm_buffer_object *bo, struct uvm_faultinfo *ufi,
    209      1.1  riastrad     struct uvm_object *uobj)
    210      1.1  riastrad {
    211      1.1  riastrad 	struct ttm_bo_device *const bdev = bo->bdev;
    212      1.1  riastrad 	int ret = 0;
    213      1.1  riastrad 
    214      1.1  riastrad 	spin_lock(&bdev->fence_lock);
    215      1.1  riastrad 	if (__predict_true(!test_bit(TTM_BO_PRIV_FLAG_MOVING,
    216      1.1  riastrad 		    &bo->priv_flags)))
    217      1.1  riastrad 		goto out;
    218      1.1  riastrad 	if (ttm_bo_wait(bo, false, false, true) == 0)
    219      1.1  riastrad 		goto out;
    220      1.1  riastrad 
    221      1.1  riastrad 	uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj);
    222      1.1  riastrad 	(void)ttm_bo_wait(bo, false, true, false);
    223      1.1  riastrad 	ret = -ERESTART;
    224      1.1  riastrad 
    225      1.1  riastrad out:	spin_unlock(&bdev->fence_lock);
    226      1.1  riastrad 	return ret;
    227      1.1  riastrad }
    228      1.1  riastrad 
    229      1.1  riastrad int
    230      1.1  riastrad ttm_bo_mmap_object(struct ttm_bo_device *bdev, off_t offset, size_t size,
    231      1.1  riastrad     vm_prot_t prot, struct uvm_object **uobjp, voff_t *uoffsetp,
    232      1.1  riastrad     struct file *file)
    233      1.1  riastrad {
    234      1.1  riastrad 	const unsigned long startpage = (offset >> PAGE_SHIFT);
    235      1.1  riastrad 	const unsigned long npages = (size >> PAGE_SHIFT);
    236      1.1  riastrad 	struct ttm_buffer_object *bo;
    237      1.1  riastrad 	int ret;
    238      1.1  riastrad 
    239      1.1  riastrad 	KASSERT(0 == (offset & (PAGE_SIZE - 1)));
    240      1.1  riastrad 	KASSERT(0 == (size & (PAGE_SIZE - 1)));
    241      1.1  riastrad 
    242      1.1  riastrad 	ret = ttm_bo_uvm_lookup(bdev, startpage, npages, &bo);
    243      1.1  riastrad 	if (ret)
    244      1.1  riastrad 		goto fail0;
    245      1.1  riastrad 	KASSERT(drm_vma_node_start(&bo->vma_node) <= offset);
    246      1.1  riastrad 	/* XXX Just assert this?  */
    247      1.1  riastrad 	if (__predict_false(bdev->driver->verify_access == NULL)) {
    248      1.1  riastrad 		ret = -EPERM;
    249      1.1  riastrad 		goto fail1;
    250      1.1  riastrad 	}
    251      1.1  riastrad 	ret = (*bdev->driver->verify_access)(bo, file);
    252      1.1  riastrad 	if (ret)
    253      1.1  riastrad 		goto fail1;
    254      1.1  riastrad 
    255      1.1  riastrad 	/* Success!  */
    256      1.1  riastrad 	*uobjp = &bo->uvmobj;
    257      1.1  riastrad 	*uoffsetp = (offset -
    258      1.1  riastrad 	    (drm_vma_node_start(&bo->vma_node) << PAGE_SHIFT));
    259      1.1  riastrad 	return 0;
    260      1.1  riastrad 
    261      1.1  riastrad fail1:	ttm_bo_unref(&bo);
    262      1.1  riastrad fail0:	KASSERT(ret);
    263      1.1  riastrad 	return ret;
    264      1.1  riastrad }
    265      1.1  riastrad 
    266      1.1  riastrad static int
    267      1.1  riastrad ttm_bo_uvm_lookup(struct ttm_bo_device *bdev, unsigned long startpage,
    268      1.1  riastrad     unsigned long npages, struct ttm_buffer_object **bop)
    269      1.1  riastrad {
    270      1.1  riastrad 	struct ttm_buffer_object *bo = NULL;
    271      1.1  riastrad 	struct drm_vma_offset_node *node;
    272      1.1  riastrad 
    273      1.1  riastrad 	drm_vma_offset_lock_lookup(&bdev->vma_manager);
    274      1.1  riastrad 	node = drm_vma_offset_lookup_locked(&bdev->vma_manager, startpage,
    275      1.1  riastrad 	    npages);
    276      1.1  riastrad 	if (node != NULL) {
    277      1.1  riastrad 		bo = container_of(node, struct ttm_buffer_object, vma_node);
    278      1.1  riastrad 		if (!kref_get_unless_zero(&bo->kref))
    279      1.1  riastrad 			bo = NULL;
    280      1.1  riastrad 	}
    281      1.1  riastrad 	drm_vma_offset_unlock_lookup(&bdev->vma_manager);
    282      1.1  riastrad 
    283      1.1  riastrad 	if (bo == NULL)
    284      1.1  riastrad 		return -ENOENT;
    285      1.1  riastrad 
    286      1.1  riastrad 	*bop = bo;
    287      1.1  riastrad 	return 0;
    288      1.1  riastrad }
    289