linux_kmap.c revision 1.4.10.3
1 1.4.10.2 tls /* $NetBSD: linux_kmap.c,v 1.4.10.3 2017/12/03 11:38:00 jdolecek Exp $ */ 2 1.4.10.2 tls 3 1.4.10.2 tls /*- 4 1.4.10.2 tls * Copyright (c) 2013 The NetBSD Foundation, Inc. 5 1.4.10.2 tls * All rights reserved. 6 1.4.10.2 tls * 7 1.4.10.2 tls * This code is derived from software contributed to The NetBSD Foundation 8 1.4.10.2 tls * by Taylor R. Campbell. 9 1.4.10.2 tls * 10 1.4.10.2 tls * Redistribution and use in source and binary forms, with or without 11 1.4.10.2 tls * modification, are permitted provided that the following conditions 12 1.4.10.2 tls * are met: 13 1.4.10.2 tls * 1. Redistributions of source code must retain the above copyright 14 1.4.10.2 tls * notice, this list of conditions and the following disclaimer. 15 1.4.10.2 tls * 2. Redistributions in binary form must reproduce the above copyright 16 1.4.10.2 tls * notice, this list of conditions and the following disclaimer in the 17 1.4.10.2 tls * documentation and/or other materials provided with the distribution. 18 1.4.10.2 tls * 19 1.4.10.2 tls * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 1.4.10.2 tls * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 1.4.10.2 tls * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 1.4.10.2 tls * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 1.4.10.2 tls * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 1.4.10.2 tls * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 1.4.10.2 tls * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 1.4.10.2 tls * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 1.4.10.2 tls * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 1.4.10.2 tls * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 1.4.10.2 tls * POSSIBILITY OF SUCH DAMAGE. 30 1.4.10.2 tls */ 31 1.4.10.2 tls 32 1.4.10.2 tls #include <sys/cdefs.h> 33 1.4.10.2 tls __KERNEL_RCSID(0, "$NetBSD: linux_kmap.c,v 1.4.10.3 2017/12/03 11:38:00 jdolecek Exp $"); 34 1.4.10.2 tls 35 1.4.10.2 tls #include <sys/types.h> 36 1.4.10.2 tls #include <sys/kmem.h> 37 1.4.10.2 tls #include <sys/mutex.h> 38 1.4.10.2 tls #include <sys/rbtree.h> 39 1.4.10.2 tls 40 1.4.10.3 jdolecek #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 41 1.4.10.3 jdolecek #include <dev/mm.h> 42 1.4.10.3 jdolecek #endif 43 1.4.10.3 jdolecek 44 1.4.10.2 tls #include <uvm/uvm_extern.h> 45 1.4.10.2 tls 46 1.4.10.2 tls #include <linux/highmem.h> 47 1.4.10.2 tls 48 1.4.10.2 tls /* 49 1.4.10.2 tls * XXX Kludgerific implementation of Linux kmap_atomic, which is 50 1.4.10.2 tls * required not to fail. To accomodate this, we reserve one page of 51 1.4.10.2 tls * kva at boot (or load) and limit the system to at most kmap_atomic in 52 1.4.10.2 tls * use at a time. 53 1.4.10.2 tls */ 54 1.4.10.2 tls 55 1.4.10.2 tls static kmutex_t linux_kmap_atomic_lock; 56 1.4.10.2 tls static vaddr_t linux_kmap_atomic_vaddr; 57 1.4.10.2 tls 58 1.4.10.2 tls static kmutex_t linux_kmap_lock; 59 1.4.10.2 tls static rb_tree_t linux_kmap_entries; 60 1.4.10.2 tls 61 1.4.10.2 tls struct linux_kmap_entry { 62 1.4.10.2 tls paddr_t lke_paddr; 63 1.4.10.2 tls vaddr_t lke_vaddr; 64 1.4.10.2 tls unsigned int lke_refcnt; 65 1.4.10.2 tls rb_node_t lke_node; 66 1.4.10.2 tls }; 67 1.4.10.2 tls 68 1.4.10.2 tls static int 69 1.4.10.2 tls lke_compare_nodes(void *ctx __unused, const void *an, const void *bn) 70 1.4.10.2 tls { 71 1.4.10.2 tls const struct linux_kmap_entry *const a = an; 72 1.4.10.2 tls const struct linux_kmap_entry *const b = bn; 73 1.4.10.2 tls 74 1.4.10.2 tls if (a->lke_paddr < b->lke_paddr) 75 1.4.10.2 tls return -1; 76 1.4.10.2 tls else if (a->lke_paddr > b->lke_paddr) 77 1.4.10.2 tls return +1; 78 1.4.10.2 tls else 79 1.4.10.2 tls return 0; 80 1.4.10.2 tls } 81 1.4.10.2 tls 82 1.4.10.2 tls static int 83 1.4.10.2 tls lke_compare_key(void *ctx __unused, const void *node, const void *key) 84 1.4.10.2 tls { 85 1.4.10.2 tls const struct linux_kmap_entry *const lke = node; 86 1.4.10.2 tls const paddr_t *const paddrp = key; 87 1.4.10.2 tls 88 1.4.10.2 tls if (lke->lke_paddr < *paddrp) 89 1.4.10.2 tls return -1; 90 1.4.10.2 tls else if (lke->lke_paddr > *paddrp) 91 1.4.10.2 tls return +1; 92 1.4.10.2 tls else 93 1.4.10.2 tls return 0; 94 1.4.10.2 tls } 95 1.4.10.2 tls 96 1.4.10.2 tls static const rb_tree_ops_t linux_kmap_entry_ops = { 97 1.4.10.2 tls .rbto_compare_nodes = &lke_compare_nodes, 98 1.4.10.2 tls .rbto_compare_key = &lke_compare_key, 99 1.4.10.2 tls .rbto_node_offset = offsetof(struct linux_kmap_entry, lke_node), 100 1.4.10.2 tls .rbto_context = NULL, 101 1.4.10.2 tls }; 102 1.4.10.2 tls 103 1.4.10.2 tls int 104 1.4.10.2 tls linux_kmap_init(void) 105 1.4.10.2 tls { 106 1.4.10.2 tls 107 1.4.10.3 jdolecek /* IPL_VM since interrupt handlers use kmap_atomic. */ 108 1.4.10.2 tls mutex_init(&linux_kmap_atomic_lock, MUTEX_DEFAULT, IPL_VM); 109 1.4.10.2 tls 110 1.4.10.2 tls linux_kmap_atomic_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 111 1.4.10.2 tls (UVM_KMF_VAONLY | UVM_KMF_WAITVA)); 112 1.4.10.2 tls 113 1.4.10.2 tls KASSERT(linux_kmap_atomic_vaddr != 0); 114 1.4.10.2 tls KASSERT(!pmap_extract(pmap_kernel(), linux_kmap_atomic_vaddr, NULL)); 115 1.4.10.2 tls 116 1.4.10.3 jdolecek mutex_init(&linux_kmap_lock, MUTEX_DEFAULT, IPL_NONE); 117 1.4.10.2 tls rb_tree_init(&linux_kmap_entries, &linux_kmap_entry_ops); 118 1.4.10.2 tls 119 1.4.10.2 tls return 0; 120 1.4.10.2 tls } 121 1.4.10.2 tls 122 1.4.10.2 tls void 123 1.4.10.2 tls linux_kmap_fini(void) 124 1.4.10.2 tls { 125 1.4.10.2 tls 126 1.4.10.3 jdolecek KASSERT(RB_TREE_MIN(&linux_kmap_entries) == NULL); 127 1.4.10.3 jdolecek #if 0 /* XXX no rb_tree_destroy */ 128 1.4.10.2 tls rb_tree_destroy(&linux_kmap_entries); 129 1.4.10.2 tls #endif 130 1.4.10.2 tls mutex_destroy(&linux_kmap_lock); 131 1.4.10.2 tls 132 1.4.10.2 tls KASSERT(linux_kmap_atomic_vaddr != 0); 133 1.4.10.2 tls KASSERT(!pmap_extract(pmap_kernel(), linux_kmap_atomic_vaddr, NULL)); 134 1.4.10.2 tls 135 1.4.10.2 tls uvm_km_free(kernel_map, linux_kmap_atomic_vaddr, PAGE_SIZE, 136 1.4.10.2 tls (UVM_KMF_VAONLY | UVM_KMF_WAITVA)); 137 1.4.10.2 tls 138 1.4.10.2 tls mutex_destroy(&linux_kmap_atomic_lock); 139 1.4.10.2 tls } 140 1.4.10.2 tls 141 1.4.10.2 tls void * 142 1.4.10.2 tls kmap_atomic(struct page *page) 143 1.4.10.2 tls { 144 1.4.10.2 tls const paddr_t paddr = uvm_vm_page_to_phys(&page->p_vmp); 145 1.4.10.3 jdolecek vaddr_t vaddr; 146 1.4.10.2 tls 147 1.4.10.3 jdolecek #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 148 1.4.10.3 jdolecek if (mm_md_direct_mapped_phys(paddr, &vaddr)) 149 1.4.10.3 jdolecek return (void *)vaddr; 150 1.4.10.3 jdolecek #endif 151 1.4.10.2 tls 152 1.4.10.3 jdolecek mutex_spin_enter(&linux_kmap_atomic_lock); 153 1.4.10.2 tls KASSERT(linux_kmap_atomic_vaddr != 0); 154 1.4.10.2 tls KASSERT(!pmap_extract(pmap_kernel(), linux_kmap_atomic_vaddr, NULL)); 155 1.4.10.3 jdolecek vaddr = linux_kmap_atomic_vaddr; 156 1.4.10.3 jdolecek pmap_kenter_pa(vaddr, paddr, (VM_PROT_READ | VM_PROT_WRITE), 0); 157 1.4.10.2 tls pmap_update(pmap_kernel()); 158 1.4.10.2 tls 159 1.4.10.2 tls return (void *)vaddr; 160 1.4.10.2 tls } 161 1.4.10.2 tls 162 1.4.10.2 tls void 163 1.4.10.2 tls kunmap_atomic(void *addr) 164 1.4.10.2 tls { 165 1.4.10.2 tls const vaddr_t vaddr = (vaddr_t)addr; 166 1.4.10.2 tls 167 1.4.10.3 jdolecek #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 168 1.4.10.3 jdolecek { 169 1.4.10.3 jdolecek paddr_t paddr; 170 1.4.10.3 jdolecek vaddr_t vaddr1; 171 1.4.10.3 jdolecek bool ok __diagused; 172 1.4.10.3 jdolecek 173 1.4.10.3 jdolecek ok = pmap_extract(pmap_kernel(), vaddr, &paddr); 174 1.4.10.3 jdolecek KASSERT(ok); 175 1.4.10.3 jdolecek if (mm_md_direct_mapped_phys(paddr, &vaddr1) && vaddr1 == vaddr) 176 1.4.10.3 jdolecek return; 177 1.4.10.3 jdolecek } 178 1.4.10.3 jdolecek #endif 179 1.4.10.3 jdolecek 180 1.4.10.2 tls KASSERT(mutex_owned(&linux_kmap_atomic_lock)); 181 1.4.10.2 tls KASSERT(linux_kmap_atomic_vaddr == vaddr); 182 1.4.10.2 tls KASSERT(pmap_extract(pmap_kernel(), vaddr, NULL)); 183 1.4.10.2 tls 184 1.4.10.2 tls pmap_kremove(vaddr, PAGE_SIZE); 185 1.4.10.2 tls pmap_update(pmap_kernel()); 186 1.4.10.2 tls 187 1.4.10.2 tls mutex_spin_exit(&linux_kmap_atomic_lock); 188 1.4.10.2 tls } 189 1.4.10.2 tls 190 1.4.10.2 tls void * 191 1.4.10.2 tls kmap(struct page *page) 192 1.4.10.2 tls { 193 1.4.10.2 tls const paddr_t paddr = VM_PAGE_TO_PHYS(&page->p_vmp); 194 1.4.10.3 jdolecek vaddr_t vaddr; 195 1.4.10.3 jdolecek 196 1.4.10.3 jdolecek ASSERT_SLEEPABLE(); 197 1.4.10.3 jdolecek 198 1.4.10.3 jdolecek #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 199 1.4.10.3 jdolecek if (mm_md_direct_mapped_phys(paddr, &vaddr)) 200 1.4.10.3 jdolecek return (void *)vaddr; 201 1.4.10.3 jdolecek #endif 202 1.4.10.3 jdolecek 203 1.4.10.3 jdolecek vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 204 1.4.10.2 tls (UVM_KMF_VAONLY | UVM_KMF_WAITVA)); 205 1.4.10.2 tls KASSERT(vaddr != 0); 206 1.4.10.2 tls 207 1.4.10.2 tls struct linux_kmap_entry *const lke = kmem_alloc(sizeof(*lke), 208 1.4.10.2 tls KM_SLEEP); 209 1.4.10.2 tls lke->lke_paddr = paddr; 210 1.4.10.2 tls lke->lke_vaddr = vaddr; 211 1.4.10.2 tls 212 1.4.10.3 jdolecek mutex_enter(&linux_kmap_lock); 213 1.4.10.3 jdolecek struct linux_kmap_entry *const collision __diagused = 214 1.4.10.2 tls rb_tree_insert_node(&linux_kmap_entries, lke); 215 1.4.10.2 tls KASSERT(collision == lke); 216 1.4.10.3 jdolecek mutex_exit(&linux_kmap_lock); 217 1.4.10.2 tls 218 1.4.10.2 tls KASSERT(!pmap_extract(pmap_kernel(), vaddr, NULL)); 219 1.4.10.3 jdolecek pmap_kenter_pa(vaddr, paddr, (VM_PROT_READ | VM_PROT_WRITE), 0); 220 1.4.10.2 tls pmap_update(pmap_kernel()); 221 1.4.10.2 tls 222 1.4.10.2 tls return (void *)vaddr; 223 1.4.10.2 tls } 224 1.4.10.2 tls 225 1.4.10.2 tls void 226 1.4.10.2 tls kunmap(struct page *page) 227 1.4.10.2 tls { 228 1.4.10.2 tls const paddr_t paddr = VM_PAGE_TO_PHYS(&page->p_vmp); 229 1.4.10.2 tls 230 1.4.10.3 jdolecek ASSERT_SLEEPABLE(); 231 1.4.10.3 jdolecek 232 1.4.10.3 jdolecek #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 233 1.4.10.3 jdolecek { 234 1.4.10.3 jdolecek vaddr_t vaddr1; 235 1.4.10.3 jdolecek 236 1.4.10.3 jdolecek if (mm_md_direct_mapped_phys(paddr, &vaddr1)) 237 1.4.10.3 jdolecek return; 238 1.4.10.3 jdolecek } 239 1.4.10.3 jdolecek #endif 240 1.4.10.3 jdolecek 241 1.4.10.3 jdolecek mutex_enter(&linux_kmap_lock); 242 1.4.10.2 tls struct linux_kmap_entry *const lke = 243 1.4.10.2 tls rb_tree_find_node(&linux_kmap_entries, &paddr); 244 1.4.10.2 tls KASSERT(lke != NULL); 245 1.4.10.2 tls rb_tree_remove_node(&linux_kmap_entries, lke); 246 1.4.10.3 jdolecek mutex_exit(&linux_kmap_lock); 247 1.4.10.2 tls 248 1.4.10.2 tls const vaddr_t vaddr = lke->lke_vaddr; 249 1.4.10.2 tls kmem_free(lke, sizeof(*lke)); 250 1.4.10.2 tls 251 1.4.10.2 tls KASSERT(pmap_extract(pmap_kernel(), vaddr, NULL)); 252 1.4.10.2 tls 253 1.4.10.2 tls pmap_kremove(vaddr, PAGE_SIZE); 254 1.4.10.2 tls pmap_update(pmap_kernel()); 255 1.4.10.2 tls 256 1.4.10.2 tls uvm_km_free(kernel_map, vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 257 1.4.10.2 tls } 258
Indexes created Fri Sep 26 08:10:20 GMT 2025